@hraness/ghostget 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +458 -0
- package/LICENSE +21 -0
- package/README.md +1359 -0
- package/bunfig.toml +2 -0
- package/dist/apple-photos-client.js +704 -0
- package/dist/beeper-client.js +4602 -0
- package/dist/client.js +1605 -0
- package/dist/index-26yq8q16.js +31 -0
- package/dist/index-9t13b4k7.js +508 -0
- package/dist/index-axwm9b3v.js +5 -0
- package/dist/index-gwk7rbyj.js +30 -0
- package/dist/index.js +701 -0
- package/dist/messaging.js +1130 -0
- package/dist/omni-client.js +1406 -0
- package/dist/whatsapp-client.js +355 -0
- package/docs/imessage-direct-provider.md +106 -0
- package/docs/rental-listings.md +94 -0
- package/package.json +600 -0
- package/skills/ghostget/SKILL.md +251 -0
- package/skills/ghostget/agents/openai.yaml +4 -0
- package/skills/ghostget/references/adapter-contract.md +181 -0
- package/skills/ghostget/references/apple-photos.md +115 -0
- package/skills/ghostget/references/article-drafts.md +235 -0
- package/skills/ghostget/references/code-owned-provider-scaffold.md +134 -0
- package/skills/ghostget/references/cross-posting-video.md +139 -0
- package/skills/ghostget/references/cross-posting.md +82 -0
- package/skills/ghostget/references/derivation.md +390 -0
- package/skills/ghostget/references/hraness-social-profile-stats.json +269 -0
- package/skills/ghostget/references/install.md +66 -0
- package/skills/ghostget/references/linkedin-adapter.md +442 -0
- package/skills/ghostget/references/messaging.md +168 -0
- package/skills/ghostget/references/meta-comet-contract.md +153 -0
- package/skills/ghostget/references/platform-patterns.md +122 -0
- package/skills/ghostget/references/provider-plugins.md +496 -0
- package/skills/ghostget/references/reddit-flair.md +78 -0
- package/skills/ghostget/references/safety-and-state.md +214 -0
- package/skills/ghostget/references/settlement-and-duplicate-cleanup.md +79 -0
- package/skills/ghostget/references/social-platform-routing.md +66 -0
- package/skills/ghostget/references/social-profile-stats.md +195 -0
- package/skills/ghostget/references/social-video-platform-routing.md +89 -0
- package/skills/ghostget/references/whatsapp-adapter.md +101 -0
- package/skills/ghostget/references/x-adapter.md +332 -0
- package/skills/ghostget/references/x-ai-disclosure.md +71 -0
- package/src/apple-photos-cli.ts +639 -0
- package/src/apple-photos-client-types.ts +168 -0
- package/src/apple-photos-client.ts +201 -0
- package/src/apple-photos-contact-evidence.ts +838 -0
- package/src/apple-photos-local-source.ts +1131 -0
- package/src/args.ts +2053 -0
- package/src/article-draft-document.ts +417 -0
- package/src/article-draft-embeds.ts +123 -0
- package/src/article-draft-images.ts +214 -0
- package/src/assets/adapter-template/wrench-adapter.json +40 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.json +1500 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v1.0.0.json +129 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v1.1.0.json +207 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v2.0.0.json +1463 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v2.1.0.json +1463 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v2.2.0.json +1475 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v2.3.0.json +1488 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.json +684 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.0.0.json +581 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.1.0.json +581 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.2.0.json +581 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.3.0.json +615 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.4.0.json +684 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.5.0.json +684 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.6.0.json +684 -0
- package/src/assets/adapters/clasificados/wrench-web-adapter.json +54 -0
- package/src/assets/adapters/facebook/wrench-web-adapter.json +750 -0
- package/src/assets/adapters/facebook/wrench-web-adapter.v1.0.0.json +706 -0
- package/src/assets/adapters/facebook/wrench-web-adapter.v1.1.0.json +703 -0
- package/src/assets/adapters/facebook/wrench-web-adapter.v1.2.0.json +750 -0
- package/src/assets/adapters/facebook-group/wrench-web-adapter.json +682 -0
- package/src/assets/adapters/facebook-group/wrench-web-adapter.v1.0.0.json +583 -0
- package/src/assets/adapters/facebook-group/wrench-web-adapter.v1.1.0.json +682 -0
- package/src/assets/adapters/facebook-marketplace/wrench-web-adapter.json +437 -0
- package/src/assets/adapters/facebook-marketplace/wrench-web-adapter.v1.0.0.json +383 -0
- package/src/assets/adapters/facebook-marketplace/wrench-web-adapter.v1.1.0.json +437 -0
- package/src/assets/adapters/facebook-marketplace/wrench-web-adapter.v1.1.1.json +437 -0
- package/src/assets/adapters/facebook-page/wrench-web-adapter.json +880 -0
- package/src/assets/adapters/facebook-page/wrench-web-adapter.v1.0.0.json +740 -0
- package/src/assets/adapters/facebook-page/wrench-web-adapter.v1.1.0.json +880 -0
- package/src/assets/adapters/github/wrench-web-adapter.json +69 -0
- package/src/assets/adapters/github/wrench-web-adapter.v1.0.0.json +42 -0
- package/src/assets/adapters/github/wrench-web-adapter.v1.1.0.json +69 -0
- package/src/assets/adapters/gmail/wrench-adapter.json +161 -0
- package/src/assets/adapters/gmail/wrench-adapter.v1.2.0.json +157 -0
- package/src/assets/adapters/hacker-news/wrench-web-adapter.json +315 -0
- package/src/assets/adapters/hacker-news/wrench-web-adapter.v1.0.0.json +315 -0
- package/src/assets/adapters/imessage/wrench-web-adapter.json +236 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.json +719 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.0.0.json +627 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.1.0.json +609 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.2.0.json +640 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.3.0.json +671 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.4.0.json +710 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.5.0.json +710 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.6.0.json +719 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.7.0.json +719 -0
- package/src/assets/adapters/linkedin/wrench-adapter.json +442 -0
- package/src/assets/adapters/linkedin/wrench-adapter.v0.4.0.json +124 -0
- package/src/assets/adapters/linkedin/wrench-adapter.v1.0.0.json +411 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.json +914 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.0.0.json +604 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.1.0.json +604 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.10.0.json +809 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.11.0.json +809 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.12.0.json +820 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.13.0.json +819 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.14.0.json +819 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.15.0.json +819 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.16.0.json +874 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.17.0.json +874 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.18.0.json +874 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.19.0.json +874 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.2.0.json +731 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.3.0.json +768 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.4.0.json +808 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.5.0.json +808 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.6.0.json +808 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.7.0.json +825 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.8.0.json +792 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.9.0.json +809 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.json +855 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.0.0.json +577 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.1.0.json +604 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.10.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.11.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.12.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.2.0.json +653 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.3.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.4.0.json +713 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.5.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.6.0.json +713 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.7.0.json +713 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.8.0.json +713 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.9.0.json +713 -0
- package/src/assets/adapters/substack/wrench-web-adapter.json +936 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.0.0.json +817 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.1.0.json +812 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.2.0.json +812 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.3.0.json +902 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.4.0.json +936 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.5.0.json +936 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.6.0.json +936 -0
- package/src/assets/adapters/threads/wrench-web-adapter.json +692 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.0.0.json +629 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.1.0.json +623 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.2.0.json +622 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.3.0.json +622 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.4.0.json +622 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.5.0.json +699 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.6.0.json +693 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.7.0.json +692 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.json +762 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.v1.0.0.json +681 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.v1.1.0.json +708 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.v1.2.0.json +762 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.v1.3.0.json +762 -0
- package/src/assets/adapters/twitch/wrench-web-adapter.json +42 -0
- package/src/assets/adapters/twitch/wrench-web-adapter.v1.0.0.json +42 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.json +400 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.v1.0.0.json +355 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.v1.1.0.json +386 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.v1.2.0.json +386 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.v1.3.0.json +400 -0
- package/src/assets/adapters/x/wrench-adapter.json +749 -0
- package/src/assets/adapters/x/wrench-adapter.v1.0.0.json +700 -0
- package/src/assets/adapters/x/wrench-adapter.v1.1.0.json +707 -0
- package/src/assets/adapters/x/wrench-web-adapter.json +712 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.1.0.json +579 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.10.0.json +685 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.11.0.json +712 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.12.0.json +712 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.13.0.json +712 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.2.0.json +578 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.3.0.json +610 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.4.0.json +617 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.5.0.json +634 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.6.0.json +632 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.7.0.json +649 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.8.0.json +649 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.9.0.json +683 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.json +561 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.v1.0.0.json +465 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.v1.1.0.json +519 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.v1.2.0.json +561 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.v1.3.0.json +561 -0
- package/src/assets/code-owned-provider-template/plugin.test.ts.template +54 -0
- package/src/assets/code-owned-provider-template/plugin.ts.template +102 -0
- package/src/assets/code-owned-provider-template/runtime.internal.test.ts.template +44 -0
- package/src/assets/code-owned-provider-template/runtime.ts.template +59 -0
- package/src/assets/code-owned-provider-template/wrench-adapter.json.template +33 -0
- package/src/assets/fonts/nebula-sans/LICENSE.txt +96 -0
- package/src/assets/fonts/nebula-sans/NebulaSans-Book.woff2 +0 -0
- package/src/assets/fonts/nebula-sans/PROVENANCE.md +14 -0
- package/src/assets.d.ts +4 -0
- package/src/auth.ts +1157 -0
- package/src/beeper-client-types.ts +169 -0
- package/src/beeper-client.ts +250 -0
- package/src/beeper-contact-interactions-cli.ts +180 -0
- package/src/beeper-contact-interactions.ts +1612 -0
- package/src/beeper-message-bundle-v1.ts +257 -0
- package/src/beeper-message-like-me-cli.ts +84 -0
- package/src/beeper-message-like-me-export.ts +2274 -0
- package/src/beeper-message-like-me-recovery.ts +931 -0
- package/src/beeper-message-like-me-source.ts +4034 -0
- package/src/browser-admission.ts +928 -0
- package/src/browser-snapshots.ts +260 -0
- package/src/browser.ts +3873 -0
- package/src/canonical-json.ts +60 -0
- package/src/catalog-cli.ts +638 -0
- package/src/cli.ts +208 -0
- package/src/client-types.ts +301 -0
- package/src/client.ts +2726 -0
- package/src/confirmed-write-failure.ts +44 -0
- package/src/confirmed-write-model.ts +72 -0
- package/src/confirmed-write-platform.ts +1055 -0
- package/src/confirmed-write-program.ts +224 -0
- package/src/confirmed-write-runtime.ts +16 -0
- package/src/cursor-token.ts +228 -0
- package/src/derivation-dnr-readiness.ts +367 -0
- package/src/derivation-file-chooser.ts +384 -0
- package/src/derivation-network-boundary.ts +835 -0
- package/src/derivation-network-guard.ts +782 -0
- package/src/derivation-network-proxy-helper.ts +440 -0
- package/src/derivation-network-proxy.ts +385 -0
- package/src/derive-command-helper.ts +693 -0
- package/src/derive-fixtures.ts +343 -0
- package/src/derive-review.ts +980 -0
- package/src/derive.ts +4754 -0
- package/src/fixtures/beeper-message-like-me-v1/accounts.ndjson +2 -0
- package/src/fixtures/beeper-message-like-me-v1/conversations.ndjson +1 -0
- package/src/fixtures/beeper-message-like-me-v1/manifest.json +1 -0
- package/src/fixtures/beeper-message-like-me-v1/messages.ndjson +2 -0
- package/src/fixtures/beeper-message-like-me-v1/participants.ndjson +3 -0
- package/src/fixtures/beeper-message-like-me-v1/reactions.ndjson +1 -0
- package/src/fixtures/beeper-message-like-me-v1/tombstones.ndjson +1 -0
- package/src/ghostget.ts +3672 -0
- package/src/gmail-capture.ts +1244 -0
- package/src/har-internal.ts +1242 -0
- package/src/har.ts +743 -0
- package/src/index.ts +218 -0
- package/src/invocation-read-platform.ts +55 -0
- package/src/invocation-read-program.ts +93 -0
- package/src/invocation-read-runtime.ts +11 -0
- package/src/linked-device-lifecycle-admission.ts +450 -0
- package/src/linked-device-lifecycle-journal.ts +1560 -0
- package/src/linked-device-lifecycle-runtime.ts +1602 -0
- package/src/local-cli-admission.ts +135 -0
- package/src/local-cli-contracts.ts +271 -0
- package/src/local-cli-execution.ts +98 -0
- package/src/local-cli-surface-contract.ts +1510 -0
- package/src/local-cli-tool-identity.ts +336 -0
- package/src/media/archive.ts +2325 -0
- package/src/media/args.ts +303 -0
- package/src/media/cli.ts +287 -0
- package/src/media/doctor.ts +475 -0
- package/src/media/ffmpeg.ts +1330 -0
- package/src/media/http-capture.ts +711 -0
- package/src/media/http-probe.ts +593 -0
- package/src/media/http.ts +387 -0
- package/src/media/index.ts +10 -0
- package/src/media/local-transcription.ts +613 -0
- package/src/media/lock.ts +321 -0
- package/src/media/manifest.ts +1712 -0
- package/src/media/metadata.ts +906 -0
- package/src/media/process.ts +723 -0
- package/src/media/revision.ts +166 -0
- package/src/media/runtime-closure.ts +1063 -0
- package/src/media/source-router.ts +187 -0
- package/src/media/transcriber-config.ts +1122 -0
- package/src/media/transcript-persistence-model.ts +15 -0
- package/src/media/transcript-persistence-platform.ts +89 -0
- package/src/media/transcript-persistence-program.ts +39 -0
- package/src/media/transcript-persistence-runtime.ts +24 -0
- package/src/media/transcript.ts +616 -0
- package/src/media/utf8-order.ts +30 -0
- package/src/media/whisper-cpp.ts +464 -0
- package/src/media/whisper-language.ts +58 -0
- package/src/media/yt-dlp.ts +439 -0
- package/src/message-like-me-agentic-messaging.ts +882 -0
- package/src/messaging-action-store.ts +793 -0
- package/src/messaging-runtime.ts +2136 -0
- package/src/messaging-store.ts +539 -0
- package/src/messaging-types.ts +1663 -0
- package/src/messaging.ts +334 -0
- package/src/model.ts +2671 -0
- package/src/oauth-google.ts +888 -0
- package/src/omni-client-types.ts +306 -0
- package/src/omni-client.ts +2225 -0
- package/src/omni-limits.ts +34 -0
- package/src/omni-model.ts +1614 -0
- package/src/omni-request.ts +379 -0
- package/src/omni-runtime.ts +1632 -0
- package/src/operation-deadline.ts +189 -0
- package/src/path-helper.ts +1656 -0
- package/src/pinned-https.ts +194 -0
- package/src/plan-assets.ts +832 -0
- package/src/platform-catalog.ts +1832 -0
- package/src/plugins/beeper-linked-device/plugin.ts +310 -0
- package/src/plugins/bluesky-web/plugin.ts +178 -0
- package/src/plugins/clasificados-web/plugin.ts +66 -0
- package/src/plugins/github-web/plugin.ts +80 -0
- package/src/plugins/gmail-official/plugin.ts +110 -0
- package/src/plugins/hacker-news-web/plugin.ts +52 -0
- package/src/plugins/imessage-direct/plugin.ts +195 -0
- package/src/plugins/imessage-direct/vendor/0001-fix-keep-AppleScript-send-payloads-out-of-child-argv.patch +1430 -0
- package/src/plugins/imessage-direct/vendor/0002-feat-rpc-add-exact-chat-lookup.patch +225 -0
- package/src/plugins/imessage-direct/vendor/provenance.json +80 -0
- package/src/plugins/linkedin-official/plugin.ts +84 -0
- package/src/plugins/linkedin-web/plugin.ts +412 -0
- package/src/plugins/meta-web/plugin.ts +317 -0
- package/src/plugins/reddit-web/flair.ts +492 -0
- package/src/plugins/reddit-web/plugin.ts +477 -0
- package/src/plugins/substack-web/plugin.ts +124 -0
- package/src/plugins/tiktok-web/plugin.ts +67 -0
- package/src/plugins/twitch-web/plugin.ts +54 -0
- package/src/plugins/whatsapp-linked-device/plugin.ts +105 -0
- package/src/plugins/x-official/plugin.ts +64 -0
- package/src/plugins/x-web/plugin.ts +488 -0
- package/src/plugins/youtube-web/plugin.ts +101 -0
- package/src/portable-run-recovery.ts +466 -0
- package/src/process-identity.ts +501 -0
- package/src/profile-clone-helper.ts +104 -0
- package/src/provider-catalog-views.ts +51 -0
- package/src/provider-context.ts +69 -0
- package/src/provider-contract-definitions-x.ts +45 -0
- package/src/provider-contract-definitions.ts +948 -0
- package/src/provider-contract-input-gmail.ts +176 -0
- package/src/provider-contract-input-linkedin.ts +77 -0
- package/src/provider-contract-input-x.ts +304 -0
- package/src/provider-contract-planning.ts +23 -0
- package/src/provider-contract-semantic-identity.ts +54 -0
- package/src/provider-contracts.ts +251 -0
- package/src/provider-http.ts +723 -0
- package/src/provider-plugin-auth.ts +36 -0
- package/src/provider-plugin-builtins.ts +229 -0
- package/src/provider-plugin-cleanup-barrier.ts +229 -0
- package/src/provider-plugin-cleanup-execution.ts +79 -0
- package/src/provider-plugin-cleanup-resource.ts +470 -0
- package/src/provider-plugin-contract-identity.ts +707 -0
- package/src/provider-plugin-host.ts +1972 -0
- package/src/provider-plugin-identifiers.ts +60 -0
- package/src/provider-plugin-invocation-lease.ts +889 -0
- package/src/provider-plugin-lifecycle-kernel.ts +566 -0
- package/src/provider-plugin-lifecycle.ts +1455 -0
- package/src/provider-plugin-module-analysis.ts +336 -0
- package/src/provider-plugin-package.ts +2530 -0
- package/src/provider-plugin-portable-authority.ts +106 -0
- package/src/provider-plugin-portable-catalog.ts +94 -0
- package/src/provider-plugin-portable-identity.ts +315 -0
- package/src/provider-plugin-portable-runtime.ts +2057 -0
- package/src/provider-plugin-protocol.ts +1625 -0
- package/src/provider-plugin-registry.ts +3342 -0
- package/src/provider-plugin-store.ts +2155 -0
- package/src/provider-plugin-test-fixture.ts +5 -0
- package/src/provider-plugin.ts +4907 -0
- package/src/provider-plugins.generated.ts +43 -0
- package/src/provider-plugins.ts +8 -0
- package/src/provider-subject.ts +41 -0
- package/src/provider.ts +315 -0
- package/src/providers/beeper-local-runtime.ts +5426 -0
- package/src/providers/beeper-local.ts +2739 -0
- package/src/providers/beeper-messaging.ts +462 -0
- package/src/providers/beeper-omni.ts +1159 -0
- package/src/providers/bluesky-web-runtime.ts +2895 -0
- package/src/providers/bluesky-web.ts +1614 -0
- package/src/providers/clasificados-web-runtime.ts +255 -0
- package/src/providers/clasificados-web.ts +503 -0
- package/src/providers/contact-projection.ts +301 -0
- package/src/providers/github-read-model.ts +94 -0
- package/src/providers/github-read-platform.ts +165 -0
- package/src/providers/github-read-program.ts +97 -0
- package/src/providers/github-web-runtime.ts +320 -0
- package/src/providers/github-web.ts +403 -0
- package/src/providers/gmail-api.ts +2834 -0
- package/src/providers/gmail-omni.ts +830 -0
- package/src/providers/gmail.ts +1130 -0
- package/src/providers/hacker-news-web-runtime.ts +220 -0
- package/src/providers/hacker-news-web.ts +809 -0
- package/src/providers/imessage-direct-install.ts +354 -0
- package/src/providers/imessage-direct-messaging.ts +532 -0
- package/src/providers/imessage-direct-omni.ts +400 -0
- package/src/providers/imessage-direct-runtime.ts +1498 -0
- package/src/providers/imessage-direct.ts +351 -0
- package/src/providers/instagram-video-foundations.ts +913 -0
- package/src/providers/instagram-web-profile-browser.ts +600 -0
- package/src/providers/iso-bmff.ts +338 -0
- package/src/providers/linkedin-company-platform.ts +42 -0
- package/src/providers/linkedin-company-program.ts +88 -0
- package/src/providers/linkedin-contact-failure.ts +84 -0
- package/src/providers/linkedin-contact-platform.ts +45 -0
- package/src/providers/linkedin-contact-program.ts +91 -0
- package/src/providers/linkedin-profile-activity-failure.ts +76 -0
- package/src/providers/linkedin-profile-activity-platform.ts +41 -0
- package/src/providers/linkedin-profile-activity-program.ts +67 -0
- package/src/providers/linkedin-read-failure.ts +52 -0
- package/src/providers/linkedin-self-platform.ts +53 -0
- package/src/providers/linkedin-self-program.ts +122 -0
- package/src/providers/linkedin-web-article-browser.ts +794 -0
- package/src/providers/linkedin-web-bootstrap.ts +146 -0
- package/src/providers/linkedin-web-contact.ts +830 -0
- package/src/providers/linkedin-web-feed-browser.ts +832 -0
- package/src/providers/linkedin-web-feed.ts +864 -0
- package/src/providers/linkedin-web-post-browser.ts +653 -0
- package/src/providers/linkedin-web-profile-browser.ts +683 -0
- package/src/providers/linkedin-web-runtime.ts +2388 -0
- package/src/providers/linkedin-web.ts +3305 -0
- package/src/providers/linkedin.ts +1885 -0
- package/src/providers/meta-bootstrap.ts +1043 -0
- package/src/providers/meta-facebook-group.ts +690 -0
- package/src/providers/meta-marketplace-relay.ts +747 -0
- package/src/providers/meta-omni.ts +373 -0
- package/src/providers/meta-relay-bundle.ts +1049 -0
- package/src/providers/meta-web-descriptors.ts +1740 -0
- package/src/providers/meta-web-runtime.ts +3388 -0
- package/src/providers/meta-web.ts +2931 -0
- package/src/providers/read-failure.ts +139 -0
- package/src/providers/reddit-omni.ts +462 -0
- package/src/providers/reddit-web-runtime.ts +1929 -0
- package/src/providers/reddit-web.ts +1611 -0
- package/src/providers/rental-listings.ts +460 -0
- package/src/providers/substack-omni.ts +406 -0
- package/src/providers/substack-video-mp4.ts +33 -0
- package/src/providers/substack-web-runtime.ts +2812 -0
- package/src/providers/substack-web.ts +1502 -0
- package/src/providers/tiktok-video-mp4.ts +33 -0
- package/src/providers/tiktok-web-runtime.ts +770 -0
- package/src/providers/tiktok-web.ts +2786 -0
- package/src/providers/twitch-web-runtime.ts +226 -0
- package/src/providers/twitch-web.ts +274 -0
- package/src/providers/whatsapp-account-identity.ts +55 -0
- package/src/providers/whatsapp-contact-projection-helper.ts +874 -0
- package/src/providers/whatsapp-contact-projection-protocol.ts +403 -0
- package/src/providers/whatsapp-interaction-projection-helper.ts +1452 -0
- package/src/providers/whatsapp-interaction-projection-protocol.ts +462 -0
- package/src/providers/whatsapp-message-export-projection-protocol.ts +532 -0
- package/src/providers/whatsapp-messaging.ts +132 -0
- package/src/providers/whatsapp-omni.ts +586 -0
- package/src/providers/whatsapp-web-runtime.ts +3108 -0
- package/src/providers/whatsapp-web.ts +1037 -0
- package/src/providers/x-image-provenance.ts +523 -0
- package/src/providers/x-made-with-ai.ts +106 -0
- package/src/providers/x-transaction-id.ts +413 -0
- package/src/providers/x-web-runtime.ts +3485 -0
- package/src/providers/x-web.ts +2332 -0
- package/src/providers/x.ts +1889 -0
- package/src/providers/youtube-web-runtime.ts +1400 -0
- package/src/providers/youtube-web.ts +1221 -0
- package/src/read-admission-runtime.ts +91 -0
- package/src/read-client.ts +521 -0
- package/src/read-effect-platform.ts +7 -0
- package/src/read-effect-runtime.ts +14 -0
- package/src/read-effect.ts +52 -0
- package/src/read-projection-admission.ts +866 -0
- package/src/read-projections.ts +3151 -0
- package/src/recovery.ts +1492 -0
- package/src/reviewed-template.ts +146 -0
- package/src/run-journal.ts +1272 -0
- package/src/runtime.ts +5116 -0
- package/src/scripts/generate-provider-plugin-catalog.ts +121 -0
- package/src/scripts/install-whatsapp-protocol.sh +232 -0
- package/src/scripts/recover-install-stages.ts +1491 -0
- package/src/scripts/resolve-state-home.ts +30 -0
- package/src/scripts/scaffold-web-provider.ts +973 -0
- package/src/scripts/sync-bundled-adapters.ts +695 -0
- package/src/session-secrets.ts +1183 -0
- package/src/state-helper.bunfig.toml +2 -0
- package/src/state-helper.ts +2914 -0
- package/src/storage.ts +3842 -0
- package/src/transport-policy.ts +2 -0
- package/src/usage.ts +182 -0
- package/src/version.ts +2 -0
- package/src/web-session-authentication-policy.ts +136 -0
- package/src/web-session-cleanup-admission.ts +2344 -0
- package/src/web-session-cleanup-join.ts +99 -0
- package/src/web-session-client.ts +996 -0
- package/src/web-session-contract-definitions.ts +820 -0
- package/src/web-session-contract-planning.ts +42 -0
- package/src/web-session-contracts.ts +228 -0
- package/src/web-session-cookies.ts +500 -0
- package/src/web-session-execution.ts +385 -0
- package/src/web-session-http.ts +373 -0
- package/src/web-session-read-errors.ts +19 -0
- package/src/web-session-read-runtime.ts +124 -0
- package/src/web-session-recovery.ts +818 -0
- package/src/web-session-sites.ts +58 -0
- package/src/web-session-template.ts +994 -0
- package/src/web-session.ts +93 -0
- package/src/whatsapp-client-binding.ts +21 -0
- package/src/whatsapp-client-types.ts +75 -0
- package/src/whatsapp-client.ts +395 -0
- package/src/whatsapp-export-coordinate.ts +17 -0
- package/src/whatsapp-message-bundle-v2.ts +195 -0
- package/src/whatsapp-message-like-me-cli.ts +195 -0
- package/src/whatsapp-message-like-me-export.ts +1072 -0
- package/src/whatsapp-message-like-me-source.ts +1012 -0
- package/tsconfig.json +67 -0
|
@@ -0,0 +1,4034 @@
|
|
|
1
|
+
import { constants, type Stats } from "node:fs";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import {
|
|
4
|
+
chmod,
|
|
5
|
+
lstat,
|
|
6
|
+
mkdir,
|
|
7
|
+
mkdtemp,
|
|
8
|
+
open,
|
|
9
|
+
readdir,
|
|
10
|
+
readlink,
|
|
11
|
+
realpath,
|
|
12
|
+
rmdir,
|
|
13
|
+
statfs,
|
|
14
|
+
unlink,
|
|
15
|
+
} from "node:fs/promises";
|
|
16
|
+
import { tmpdir } from "node:os";
|
|
17
|
+
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
18
|
+
|
|
19
|
+
import type { GhostgetAuth } from "./auth";
|
|
20
|
+
import { removePrivateDirectoryTree } from "./storage";
|
|
21
|
+
import { canonicalJson, sha256 } from "./canonical-json";
|
|
22
|
+
import {
|
|
23
|
+
createBeeperMessageLikeMeDirectoryLease,
|
|
24
|
+
releaseBeeperMessageLikeMeDirectoryLease,
|
|
25
|
+
updateBeeperMessageLikeMeDirectoryLease,
|
|
26
|
+
type BeeperMessageLikeMeDirectoryLease,
|
|
27
|
+
} from "./beeper-message-like-me-recovery";
|
|
28
|
+
import {
|
|
29
|
+
BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS,
|
|
30
|
+
BEEPER_MESSAGE_LIKE_ME_MAX_TOTAL_BYTES,
|
|
31
|
+
} from "./beeper-message-like-me-export";
|
|
32
|
+
import type {
|
|
33
|
+
BeeperMessageLikeMeAccount,
|
|
34
|
+
BeeperMessageLikeMeAttachment,
|
|
35
|
+
BeeperMessageLikeMeBundleProgress,
|
|
36
|
+
BeeperMessageLikeMeConversation,
|
|
37
|
+
BeeperMessageLikeMeExportSource,
|
|
38
|
+
BeeperMessageLikeMeMessage,
|
|
39
|
+
BeeperMessageLikeMeParticipant,
|
|
40
|
+
BeeperMessageLikeMeReaction,
|
|
41
|
+
BeeperMessageLikeMeRecord,
|
|
42
|
+
BeeperMessageLikeMeTombstone,
|
|
43
|
+
} from "./beeper-message-like-me-export";
|
|
44
|
+
import {
|
|
45
|
+
BEEPER_CLI_PIN,
|
|
46
|
+
beeperCliArtifactForRuntime,
|
|
47
|
+
planBeeperAccountsListCommand,
|
|
48
|
+
planBeeperMessageLikeMeExportCommand,
|
|
49
|
+
planBeeperTargetStatusCommand,
|
|
50
|
+
} from "./providers/beeper-local";
|
|
51
|
+
import {
|
|
52
|
+
beeperSubjectFromAccountsAndTarget,
|
|
53
|
+
parseBeeperTargetRealmProof,
|
|
54
|
+
parseBeeperCliEnvelope,
|
|
55
|
+
parseBeeperExportAccounts,
|
|
56
|
+
parseBeeperExportConversation,
|
|
57
|
+
parseBeeperExportMessages,
|
|
58
|
+
materializePinnedBeeperCliBinary,
|
|
59
|
+
resolvePinnedBeeperCliBinary,
|
|
60
|
+
validateBeeperCliStore,
|
|
61
|
+
type BeeperAccountProjection,
|
|
62
|
+
type BeeperAttachmentProjection,
|
|
63
|
+
type BeeperCliInvocationResult,
|
|
64
|
+
type BeeperConversationProjection,
|
|
65
|
+
type BeeperMessageProjection,
|
|
66
|
+
type BeeperReactionProjection,
|
|
67
|
+
type BeeperTargetRealmProof,
|
|
68
|
+
type BeeperUserProjection,
|
|
69
|
+
} from "./providers/beeper-local-runtime";
|
|
70
|
+
|
|
71
|
+
const PRIVATE_DIRECTORY_MODE = 0o700;
|
|
72
|
+
const PRIVATE_FILE_MODE = 0o600;
|
|
73
|
+
const MAX_STDOUT_BYTES = 64 * 1024;
|
|
74
|
+
const MAX_STDERR_BYTES = 64 * 1024;
|
|
75
|
+
const ACCOUNT_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
76
|
+
const MAX_ACCOUNTS_LIST_BYTES = 8 * 1024 * 1024;
|
|
77
|
+
const MAX_BEEPER_CONFIG_BYTES = 4 * 1024 * 1024;
|
|
78
|
+
const MAX_ACCOUNTS_JSON_BYTES = 32 * 1024 * 1024;
|
|
79
|
+
const MAX_CHATS_JSON_BYTES = 64 * 1024 * 1024;
|
|
80
|
+
const MAX_CHAT_JSON_BYTES = 32 * 1024 * 1024;
|
|
81
|
+
const MAX_MESSAGES_JSON_BYTES = 64 * 1024 * 1024;
|
|
82
|
+
const MAX_EXPORT_CHATS = 100_000;
|
|
83
|
+
const MAX_EXPORT_MESSAGES_PER_CHAT = 1_000_000;
|
|
84
|
+
const MAX_RAW_WORKING_BYTES = 4 * 1024 * 1024 * 1024;
|
|
85
|
+
const MIN_FREE_FILESYSTEM_BYTES = 2 * 1024 * 1024 * 1024;
|
|
86
|
+
const MAX_RAW_WORKING_ENTRIES = 1_000_000;
|
|
87
|
+
const MAX_RAW_CACHE_SYMLINK_TARGET_BYTES = 512;
|
|
88
|
+
const RAW_WORKING_MONITOR_INTERVAL_MS = 500;
|
|
89
|
+
const RAW_WORKING_RECOVERY_GRACE_MS = 5 * 60 * 1_000;
|
|
90
|
+
const DEFAULT_MAX_PARTICIPANTS = 500;
|
|
91
|
+
const MAX_PARTICIPANT_OCCURRENCES = 250_000;
|
|
92
|
+
const DEFAULT_TIMEOUT_MS = 6 * 60 * 60 * 1_000;
|
|
93
|
+
|
|
94
|
+
type JsonRecord = Readonly<Record<string, unknown>>;
|
|
95
|
+
|
|
96
|
+
export type BeeperMessageLikeMeSourceLimits = Readonly<{
|
|
97
|
+
limitChats?: number;
|
|
98
|
+
limitMessages?: number;
|
|
99
|
+
maxParticipants?: number;
|
|
100
|
+
timeoutMs?: number;
|
|
101
|
+
}>;
|
|
102
|
+
|
|
103
|
+
export type BeeperExportCliInvocation = Readonly<{
|
|
104
|
+
binary: string;
|
|
105
|
+
arguments: readonly string[];
|
|
106
|
+
environment: Readonly<Record<string, string>>;
|
|
107
|
+
timeoutMs: number;
|
|
108
|
+
maxOutputBytes: number;
|
|
109
|
+
maxStderrBytes: number;
|
|
110
|
+
workingRoot?: string;
|
|
111
|
+
maxWorkingBytes?: number;
|
|
112
|
+
onHeartbeat?: (elapsedSeconds: number) => void;
|
|
113
|
+
signal?: AbortSignal;
|
|
114
|
+
directoryLease?: BeeperMessageLikeMeDirectoryLease;
|
|
115
|
+
}>;
|
|
116
|
+
|
|
117
|
+
export type BeeperMessageLikeMeProgress =
|
|
118
|
+
| BeeperMessageLikeMeBundleProgress
|
|
119
|
+
| Readonly<{
|
|
120
|
+
phase: "recovery-started";
|
|
121
|
+
}>
|
|
122
|
+
| Readonly<{
|
|
123
|
+
phase: "recovery-completed";
|
|
124
|
+
recovered: number;
|
|
125
|
+
published: number;
|
|
126
|
+
}>
|
|
127
|
+
| Readonly<{
|
|
128
|
+
phase: "preparing";
|
|
129
|
+
}>
|
|
130
|
+
| Readonly<{
|
|
131
|
+
phase: "accounts-discovered";
|
|
132
|
+
accounts: number;
|
|
133
|
+
}>
|
|
134
|
+
| Readonly<{
|
|
135
|
+
phase: "accounts-progress";
|
|
136
|
+
stage: "discovering" | "verifying";
|
|
137
|
+
elapsedSeconds: number;
|
|
138
|
+
}>
|
|
139
|
+
| Readonly<{
|
|
140
|
+
phase: "account-started";
|
|
141
|
+
account: number;
|
|
142
|
+
accounts: number;
|
|
143
|
+
}>
|
|
144
|
+
| Readonly<{
|
|
145
|
+
phase: "account-validating";
|
|
146
|
+
account: number;
|
|
147
|
+
accounts: number;
|
|
148
|
+
elapsedSeconds: number;
|
|
149
|
+
}>
|
|
150
|
+
| Readonly<{
|
|
151
|
+
phase: "account-progress";
|
|
152
|
+
account: number;
|
|
153
|
+
accounts: number;
|
|
154
|
+
elapsedSeconds: number;
|
|
155
|
+
}>
|
|
156
|
+
| Readonly<{
|
|
157
|
+
phase: "account-skipped";
|
|
158
|
+
account: number;
|
|
159
|
+
accounts: number;
|
|
160
|
+
reason: "chat-limit-reached";
|
|
161
|
+
}>
|
|
162
|
+
| Readonly<{
|
|
163
|
+
phase: "account-completed";
|
|
164
|
+
account: number;
|
|
165
|
+
accounts: number;
|
|
166
|
+
chats: number;
|
|
167
|
+
messages: number;
|
|
168
|
+
}>
|
|
169
|
+
| Readonly<{
|
|
170
|
+
phase: "accounts-verifying";
|
|
171
|
+
accounts: number;
|
|
172
|
+
}>
|
|
173
|
+
| Readonly<{
|
|
174
|
+
phase: "conversion-progress";
|
|
175
|
+
elapsedSeconds: number;
|
|
176
|
+
}>
|
|
177
|
+
| Readonly<{
|
|
178
|
+
phase: "conversion-started";
|
|
179
|
+
accounts: number;
|
|
180
|
+
chats: number;
|
|
181
|
+
messages: number;
|
|
182
|
+
}>;
|
|
183
|
+
|
|
184
|
+
export type BeeperMessageLikeMeSourceDependencies = Readonly<{
|
|
185
|
+
/** Test-only seam. Production resolves the exact pinned binary hash. */
|
|
186
|
+
binaryPath?: string;
|
|
187
|
+
/** Test-only seam for exercising the fixed production bundle record cap. */
|
|
188
|
+
maxBundleRecords?: number;
|
|
189
|
+
/** Test-only seam for exercising the fixed production bundle byte cap. */
|
|
190
|
+
maxBundleBytes?: number;
|
|
191
|
+
/** Test-only seam for exercising the per-chat JSON allocation cap. */
|
|
192
|
+
maxMessagesJsonBytes?: number;
|
|
193
|
+
/** Test-only seam for exercising the participant-occurrence work cap. */
|
|
194
|
+
maxParticipantOccurrences?: number;
|
|
195
|
+
/** Test-only seam for mutating a fixture between bounded alias passes. */
|
|
196
|
+
onSelfAliasRefinementPass?: (pass: number) => Promise<void>;
|
|
197
|
+
runCli?: (
|
|
198
|
+
invocation: BeeperExportCliInvocation,
|
|
199
|
+
) => Promise<BeeperCliInvocationResult>;
|
|
200
|
+
createWorkingDirectory?: () => Promise<string>;
|
|
201
|
+
removeWorkingDirectory?: (path: string) => Promise<void>;
|
|
202
|
+
}>;
|
|
203
|
+
|
|
204
|
+
export type BeeperMessageLikeMeSourceRequest = Readonly<{
|
|
205
|
+
auth: GhostgetAuth;
|
|
206
|
+
limits?: BeeperMessageLikeMeSourceLimits;
|
|
207
|
+
signal?: AbortSignal;
|
|
208
|
+
onProgress?: (progress: BeeperMessageLikeMeProgress) => void;
|
|
209
|
+
/**
|
|
210
|
+
* Internal projection seam for content-free derivatives of this exact
|
|
211
|
+
* admitted history. Coordinates never become bundle fields, so the v1
|
|
212
|
+
* Message Like Me bytes remain unchanged.
|
|
213
|
+
*/
|
|
214
|
+
onRecordCoordinate?: (
|
|
215
|
+
record: BeeperMessageLikeMeRecord,
|
|
216
|
+
coordinate: BeeperMessageLikeMeSourceCoordinate,
|
|
217
|
+
) => void;
|
|
218
|
+
environment?: Readonly<Record<string, string | undefined>>;
|
|
219
|
+
dependencies?: BeeperMessageLikeMeSourceDependencies;
|
|
220
|
+
}>;
|
|
221
|
+
|
|
222
|
+
export type BeeperMessageLikeMeSourceCoordinate =
|
|
223
|
+
| Readonly<{
|
|
224
|
+
kind: "account";
|
|
225
|
+
accountId: string;
|
|
226
|
+
}>
|
|
227
|
+
| Readonly<{
|
|
228
|
+
kind: "participant";
|
|
229
|
+
accountId: string;
|
|
230
|
+
participantId: string;
|
|
231
|
+
}>
|
|
232
|
+
| Readonly<{
|
|
233
|
+
kind: "conversation";
|
|
234
|
+
accountId: string;
|
|
235
|
+
conversationId: string;
|
|
236
|
+
}>
|
|
237
|
+
| Readonly<{
|
|
238
|
+
kind: "message";
|
|
239
|
+
accountId: string;
|
|
240
|
+
conversationId: string;
|
|
241
|
+
messageId: string;
|
|
242
|
+
}>;
|
|
243
|
+
|
|
244
|
+
type ParsedLimits = Readonly<{
|
|
245
|
+
limitChats: number | null;
|
|
246
|
+
limitMessages: number | null;
|
|
247
|
+
maxParticipants: number;
|
|
248
|
+
timeoutMs: number;
|
|
249
|
+
}>;
|
|
250
|
+
|
|
251
|
+
type ParticipantFact = Readonly<{
|
|
252
|
+
readonly id: string;
|
|
253
|
+
readonly accountId: string;
|
|
254
|
+
/** Exact account-scoped participant coordinate from the official export. */
|
|
255
|
+
readonly sourceId: string;
|
|
256
|
+
readonly providerId: string;
|
|
257
|
+
readonly displayName: string | null;
|
|
258
|
+
readonly handle: string | null;
|
|
259
|
+
readonly isSelf: boolean | null;
|
|
260
|
+
}>;
|
|
261
|
+
|
|
262
|
+
type PrivateDirectoryIdentity = Readonly<{
|
|
263
|
+
device: number;
|
|
264
|
+
inode: number;
|
|
265
|
+
}>;
|
|
266
|
+
|
|
267
|
+
type ConversationScan = Readonly<{
|
|
268
|
+
chat: Pick<
|
|
269
|
+
BeeperConversationProjection,
|
|
270
|
+
"id" | "accountId" | "lastActivity" | "title" | "type"
|
|
271
|
+
>;
|
|
272
|
+
root: string;
|
|
273
|
+
messagesPath: string;
|
|
274
|
+
messagesSha256: string;
|
|
275
|
+
observedAt: string;
|
|
276
|
+
participantIds: readonly string[];
|
|
277
|
+
participantFactChanges: readonly ParticipantFact[];
|
|
278
|
+
participantsComplete: boolean;
|
|
279
|
+
startedAt: string | null;
|
|
280
|
+
lastMessageAt: string | null;
|
|
281
|
+
messageCount: number;
|
|
282
|
+
reactionCount: number;
|
|
283
|
+
reactionProviderIdNonUniqueGroups: number;
|
|
284
|
+
tombstoneCount: number;
|
|
285
|
+
nonParticipantRecordBytes: number;
|
|
286
|
+
}>;
|
|
287
|
+
|
|
288
|
+
type ValidatedShardChat = Readonly<{
|
|
289
|
+
chat: Pick<BeeperConversationProjection, "id" | "accountId">;
|
|
290
|
+
root: string;
|
|
291
|
+
chatPath: string;
|
|
292
|
+
chatSha256: string;
|
|
293
|
+
messagesPath: string;
|
|
294
|
+
expectedMessageCount: number;
|
|
295
|
+
observedAt: string;
|
|
296
|
+
}>;
|
|
297
|
+
|
|
298
|
+
type ValidatedAccountShard = Readonly<{
|
|
299
|
+
accountId: string;
|
|
300
|
+
completedAt: string;
|
|
301
|
+
chats: readonly ValidatedShardChat[];
|
|
302
|
+
messageCount: number;
|
|
303
|
+
}>;
|
|
304
|
+
|
|
305
|
+
type SelfAliasPrepass = Readonly<{
|
|
306
|
+
aliasesByAccount: ReadonlyMap<string, ReadonlySet<string>>;
|
|
307
|
+
coveredChats: readonly ValidatedShardChat[];
|
|
308
|
+
bundleByteLimitReached: boolean;
|
|
309
|
+
bundleRecordLimitReached: boolean;
|
|
310
|
+
evidenceLimitReached: boolean;
|
|
311
|
+
messagesSha256ByPath: ReadonlyMap<string, string>;
|
|
312
|
+
participantOccurrenceLimitReached: boolean;
|
|
313
|
+
provisionalRecordBytes: number;
|
|
314
|
+
provisionalRecordCount: number;
|
|
315
|
+
}>;
|
|
316
|
+
|
|
317
|
+
type BeeperCliStoreSnapshot = Readonly<{
|
|
318
|
+
config: JsonRecord;
|
|
319
|
+
desktopTarget: JsonRecord;
|
|
320
|
+
targetBaseUrl: string;
|
|
321
|
+
}>;
|
|
322
|
+
|
|
323
|
+
type OperationPrivateBeeperStore = Readonly<{
|
|
324
|
+
path: string;
|
|
325
|
+
identity: PrivateDirectoryIdentity;
|
|
326
|
+
}>;
|
|
327
|
+
|
|
328
|
+
type ExportManifest = Readonly<{
|
|
329
|
+
accounts: unknown;
|
|
330
|
+
attachmentCount: number;
|
|
331
|
+
chatCount: number;
|
|
332
|
+
completedAt: string;
|
|
333
|
+
createdAt: string;
|
|
334
|
+
messageCount: number;
|
|
335
|
+
version: 1;
|
|
336
|
+
}>;
|
|
337
|
+
|
|
338
|
+
function fail(message: string): never {
|
|
339
|
+
throw new Error(`Beeper Message Like Me source: ${message}`);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function isErrno(error: unknown, code: string): boolean {
|
|
343
|
+
return typeof error === "object" && error !== null && "code" in error
|
|
344
|
+
&& (error as { readonly code?: unknown }).code === code;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function record(value: unknown, label: string): JsonRecord {
|
|
348
|
+
if (
|
|
349
|
+
typeof value !== "object"
|
|
350
|
+
|| value === null
|
|
351
|
+
|| Array.isArray(value)
|
|
352
|
+
|| (
|
|
353
|
+
Object.getPrototypeOf(value) !== Object.prototype
|
|
354
|
+
&& Object.getPrototypeOf(value) !== null
|
|
355
|
+
)
|
|
356
|
+
) return fail(`${label} must be a plain object`);
|
|
357
|
+
return value as JsonRecord;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
function exactKeys(
|
|
361
|
+
value: JsonRecord,
|
|
362
|
+
required: readonly string[],
|
|
363
|
+
optional: readonly string[],
|
|
364
|
+
label: string,
|
|
365
|
+
): void {
|
|
366
|
+
const remaining = new Set(Object.keys(value));
|
|
367
|
+
for (const key of required) {
|
|
368
|
+
if (!remaining.delete(key)) fail(`${label} omitted a required field`);
|
|
369
|
+
}
|
|
370
|
+
for (const key of optional) remaining.delete(key);
|
|
371
|
+
if (remaining.size > 0) fail(`${label} contains an unreviewed field`);
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
function array(value: unknown, label: string, maximum: number): readonly unknown[] {
|
|
375
|
+
if (!Array.isArray(value) || value.length > maximum) {
|
|
376
|
+
return fail(`${label} must be an array inside its reviewed bound`);
|
|
377
|
+
}
|
|
378
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
379
|
+
if (!Object.hasOwn(value, index)) fail(`${label} must not be sparse`);
|
|
380
|
+
}
|
|
381
|
+
return value;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
function integer(value: unknown, label: string, maximum: number): number {
|
|
385
|
+
if (
|
|
386
|
+
typeof value !== "number"
|
|
387
|
+
|| !Number.isSafeInteger(value)
|
|
388
|
+
|| value < 0
|
|
389
|
+
|| value > maximum
|
|
390
|
+
) return fail(`${label} must be a bounded non-negative integer`);
|
|
391
|
+
return value;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function positiveInteger(
|
|
395
|
+
value: unknown,
|
|
396
|
+
label: string,
|
|
397
|
+
maximum: number,
|
|
398
|
+
): number {
|
|
399
|
+
const parsed = integer(value, label, maximum);
|
|
400
|
+
if (parsed < 1) return fail(`${label} must be positive`);
|
|
401
|
+
return parsed;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function string(value: unknown, label: string, maximum: number): string {
|
|
405
|
+
if (
|
|
406
|
+
typeof value !== "string"
|
|
407
|
+
|| value.length < 1
|
|
408
|
+
|| Buffer.byteLength(value, "utf8") > maximum
|
|
409
|
+
|| /[\0\r\n]/u.test(value)
|
|
410
|
+
) return fail(`${label} must be bounded text`);
|
|
411
|
+
return value;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
function timestamp(value: unknown, label: string): string {
|
|
415
|
+
const source = string(value, label, 64);
|
|
416
|
+
const milliseconds = Date.parse(source);
|
|
417
|
+
if (!Number.isFinite(milliseconds)) return fail(`${label} must be a timestamp`);
|
|
418
|
+
return new Date(milliseconds).toISOString();
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
function parseLimits(value: BeeperMessageLikeMeSourceLimits | undefined): ParsedLimits {
|
|
422
|
+
return Object.freeze({
|
|
423
|
+
limitChats: value?.limitChats === undefined
|
|
424
|
+
? null
|
|
425
|
+
: positiveInteger(value.limitChats, "limitChats", MAX_EXPORT_CHATS),
|
|
426
|
+
limitMessages: value?.limitMessages === undefined
|
|
427
|
+
? null
|
|
428
|
+
: positiveInteger(
|
|
429
|
+
value.limitMessages,
|
|
430
|
+
"limitMessages",
|
|
431
|
+
MAX_EXPORT_MESSAGES_PER_CHAT,
|
|
432
|
+
),
|
|
433
|
+
maxParticipants: value?.maxParticipants === undefined
|
|
434
|
+
? DEFAULT_MAX_PARTICIPANTS
|
|
435
|
+
: positiveInteger(value.maxParticipants, "maxParticipants", 2_000),
|
|
436
|
+
timeoutMs: value?.timeoutMs === undefined
|
|
437
|
+
? DEFAULT_TIMEOUT_MS
|
|
438
|
+
: positiveInteger(value.timeoutMs, "timeoutMs", DEFAULT_TIMEOUT_MS),
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
function requireAuth(auth: GhostgetAuth): Extract<GhostgetAuth, {
|
|
443
|
+
readonly kind: "linked-device-store";
|
|
444
|
+
}> & Readonly<{ provider: "beeper"; subject: string }> {
|
|
445
|
+
if (
|
|
446
|
+
auth.kind !== "linked-device-store"
|
|
447
|
+
|| auth.provider !== "beeper"
|
|
448
|
+
|| auth.subject === undefined
|
|
449
|
+
|| !/^beeper:local:[a-f0-9]{64}$/u.test(auth.subject)
|
|
450
|
+
) return fail("export requires an account-bound Beeper linked-device-store auth locator");
|
|
451
|
+
return auth as Extract<GhostgetAuth, {
|
|
452
|
+
readonly kind: "linked-device-store";
|
|
453
|
+
}> & Readonly<{ provider: "beeper"; subject: string }>;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
function throwIfAborted(signal: AbortSignal | undefined): void {
|
|
457
|
+
if (signal?.aborted === true) fail("export was cancelled");
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function digest(parts: readonly string[]): string {
|
|
461
|
+
return sha256(canonicalJson(parts));
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
function bundleRecordBytes(record: BeeperMessageLikeMeRecord): number {
|
|
465
|
+
return Buffer.byteLength(canonicalJson(record), "utf8") + 1;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
function localId(kind: string, ...parts: readonly string[]): string {
|
|
469
|
+
return `${kind}:${digest(parts)}`;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
function providerId(kind: string, ...parts: readonly string[]): string {
|
|
473
|
+
return `beeper-${kind}:${digest(parts)}`;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
function normalizeNetwork(value: string | null, fallback: string): string {
|
|
477
|
+
const normalized = (value ?? fallback)
|
|
478
|
+
.normalize("NFKD")
|
|
479
|
+
.toLowerCase()
|
|
480
|
+
.replace(/[^a-z0-9._+-]+/gu, "-")
|
|
481
|
+
.replace(/^[^a-z0-9]+|[^a-z0-9]+$/gu, "") || "unknown";
|
|
482
|
+
if (Buffer.byteLength(normalized, "utf8") <= 64) return normalized;
|
|
483
|
+
return `${normalized.slice(0, 47).replace(/[^a-z0-9]+$/u, "")}-${digest([normalized]).slice(0, 16)}`;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
function preferredHandle(user: BeeperUserProjection): string | null {
|
|
487
|
+
return user.phoneNumber ?? user.email ?? user.username;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
function safeBaseName(value: string | null): string | null {
|
|
491
|
+
if (value === null) return null;
|
|
492
|
+
const name = basename(value.replaceAll("\\", "/"));
|
|
493
|
+
if (name.length < 1 || name === "." || name === "..") return null;
|
|
494
|
+
return name;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
function attachment(
|
|
498
|
+
value: BeeperAttachmentProjection,
|
|
499
|
+
): BeeperMessageLikeMeAttachment {
|
|
500
|
+
const kind: BeeperMessageLikeMeAttachment["kind"] = value.isSticker === true
|
|
501
|
+
? "sticker"
|
|
502
|
+
: value.type === "img"
|
|
503
|
+
? "image"
|
|
504
|
+
: value.type === "video"
|
|
505
|
+
? "video"
|
|
506
|
+
: value.type === "audio"
|
|
507
|
+
? "audio"
|
|
508
|
+
: value.fileName !== null
|
|
509
|
+
? "document"
|
|
510
|
+
: "unknown";
|
|
511
|
+
return Object.freeze({
|
|
512
|
+
kind,
|
|
513
|
+
mimeType: value.mimeType,
|
|
514
|
+
name: safeBaseName(value.fileName),
|
|
515
|
+
sizeBytes: value.fileSizeBytes,
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
function reactionBody(value: string): string {
|
|
520
|
+
if (
|
|
521
|
+
/[\\/]/u.test(value)
|
|
522
|
+
|| /^[a-z][a-z0-9+.-]*:/iu.test(value)
|
|
523
|
+
|| Buffer.byteLength(value, "utf8") > 128
|
|
524
|
+
) return "custom-reaction";
|
|
525
|
+
if (/^:[A-Za-z0-9_+-]{1,64}:$/u.test(value)) return value;
|
|
526
|
+
if (/^[^\p{L}\p{N}]*\p{Extended_Pictographic}[^\p{L}\p{N}]*$/u.test(value)) {
|
|
527
|
+
return value;
|
|
528
|
+
}
|
|
529
|
+
if (["+1", "-1", "like", "love", "laugh", "sad", "angry"].includes(value)) {
|
|
530
|
+
return value;
|
|
531
|
+
}
|
|
532
|
+
return "custom-reaction";
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
function safeSegment(value: string): string {
|
|
536
|
+
const normalized = value
|
|
537
|
+
.replace(/[^a-zA-Z0-9._-]+/gu, "_")
|
|
538
|
+
.replace(/^_+|_+$/gu, "");
|
|
539
|
+
return normalized.slice(0, 120) || "item";
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
async function assertOwnedDirectory(path: string, root?: string): Promise<string> {
|
|
543
|
+
const canonical = await realpath(path);
|
|
544
|
+
if (canonical !== path) return fail("export directory traversed a symbolic link");
|
|
545
|
+
if (root !== undefined && canonical !== root && !canonical.startsWith(`${root}${sep}`)) {
|
|
546
|
+
return fail("export directory escaped private staging");
|
|
547
|
+
}
|
|
548
|
+
const metadata = await lstat(canonical);
|
|
549
|
+
if (
|
|
550
|
+
!metadata.isDirectory()
|
|
551
|
+
|| metadata.isSymbolicLink()
|
|
552
|
+
|| metadata.uid !== process.getuid?.()
|
|
553
|
+
|| (metadata.mode & 0o022) !== 0
|
|
554
|
+
) return fail("export directory is not an owned physical directory");
|
|
555
|
+
return canonical;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
async function assertPrivateOwnedDirectory(
|
|
559
|
+
path: string,
|
|
560
|
+
root?: string,
|
|
561
|
+
): Promise<string> {
|
|
562
|
+
const canonical = await assertOwnedDirectory(path, root);
|
|
563
|
+
const metadata = await lstat(canonical);
|
|
564
|
+
if ((metadata.mode & 0o777) !== PRIVATE_DIRECTORY_MODE) {
|
|
565
|
+
return fail("private export root permissions changed");
|
|
566
|
+
}
|
|
567
|
+
return canonical;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
async function readOwnedJsonDocument(
|
|
571
|
+
path: string,
|
|
572
|
+
root: string,
|
|
573
|
+
maximumBytes: number,
|
|
574
|
+
privateFile = false,
|
|
575
|
+
signal?: AbortSignal,
|
|
576
|
+
): Promise<Readonly<{ value: unknown; sha256: string }>> {
|
|
577
|
+
throwIfAborted(signal);
|
|
578
|
+
if (!path.startsWith(`${root}${sep}`)) return fail("export file escaped private staging");
|
|
579
|
+
let handle;
|
|
580
|
+
try {
|
|
581
|
+
handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
582
|
+
} catch (error) {
|
|
583
|
+
if (isErrno(error, "ENOENT")) return fail("official export omitted a required file");
|
|
584
|
+
if (isErrno(error, "ELOOP")) return fail("official export file must not be a symbolic link");
|
|
585
|
+
return fail("official export file could not be opened safely");
|
|
586
|
+
}
|
|
587
|
+
try {
|
|
588
|
+
const before = await handle.stat();
|
|
589
|
+
if (
|
|
590
|
+
!before.isFile()
|
|
591
|
+
|| before.uid !== process.getuid?.()
|
|
592
|
+
|| before.nlink !== 1
|
|
593
|
+
|| (before.mode & 0o022) !== 0
|
|
594
|
+
|| (privateFile && (before.mode & 0o077) !== 0)
|
|
595
|
+
|| before.size < 2
|
|
596
|
+
|| before.size > maximumBytes
|
|
597
|
+
) return fail("official export file is outside its ownership or size bound");
|
|
598
|
+
const bytes = Buffer.allocUnsafe(before.size);
|
|
599
|
+
let offset = 0;
|
|
600
|
+
while (offset < bytes.byteLength) {
|
|
601
|
+
throwIfAborted(signal);
|
|
602
|
+
const result = await handle.read(bytes, offset, bytes.byteLength - offset, offset);
|
|
603
|
+
if (result.bytesRead === 0) break;
|
|
604
|
+
offset += result.bytesRead;
|
|
605
|
+
}
|
|
606
|
+
const extra = Buffer.allocUnsafe(1);
|
|
607
|
+
const overflow = await handle.read(extra, 0, 1, offset);
|
|
608
|
+
const after = await handle.stat();
|
|
609
|
+
if (
|
|
610
|
+
offset !== bytes.byteLength
|
|
611
|
+
|| overflow.bytesRead !== 0
|
|
612
|
+
|| before.dev !== after.dev
|
|
613
|
+
|| before.ino !== after.ino
|
|
614
|
+
|| after.nlink !== 1
|
|
615
|
+
|| before.size !== after.size
|
|
616
|
+
|| before.mtimeMs !== after.mtimeMs
|
|
617
|
+
|| before.ctimeMs !== after.ctimeMs
|
|
618
|
+
) return fail("official export file changed while being read");
|
|
619
|
+
const pathMetadata = await lstat(path);
|
|
620
|
+
throwIfAborted(signal);
|
|
621
|
+
if (
|
|
622
|
+
pathMetadata.isSymbolicLink()
|
|
623
|
+
|| pathMetadata.nlink !== 1
|
|
624
|
+
|| pathMetadata.dev !== after.dev
|
|
625
|
+
|| pathMetadata.ino !== after.ino
|
|
626
|
+
) return fail("official export file changed while being read");
|
|
627
|
+
let value: unknown;
|
|
628
|
+
try {
|
|
629
|
+
const decoded = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
630
|
+
value = JSON.parse(decoded) as unknown;
|
|
631
|
+
} catch {
|
|
632
|
+
return fail("official export file is not valid UTF-8 JSON");
|
|
633
|
+
}
|
|
634
|
+
throwIfAborted(signal);
|
|
635
|
+
return Object.freeze({
|
|
636
|
+
value,
|
|
637
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
638
|
+
});
|
|
639
|
+
} finally {
|
|
640
|
+
await handle.close();
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
async function readOwnedJson(
|
|
645
|
+
path: string,
|
|
646
|
+
root: string,
|
|
647
|
+
maximumBytes: number,
|
|
648
|
+
privateFile = false,
|
|
649
|
+
signal?: AbortSignal,
|
|
650
|
+
): Promise<unknown> {
|
|
651
|
+
return (await readOwnedJsonDocument(
|
|
652
|
+
path,
|
|
653
|
+
root,
|
|
654
|
+
maximumBytes,
|
|
655
|
+
privateFile,
|
|
656
|
+
signal,
|
|
657
|
+
)).value;
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
async function ownedFileSize(path: string, root: string): Promise<number> {
|
|
661
|
+
if (!path.startsWith(`${root}${sep}`)) return fail("export file escaped private staging");
|
|
662
|
+
let handle;
|
|
663
|
+
try {
|
|
664
|
+
handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
665
|
+
} catch (error) {
|
|
666
|
+
if (isErrno(error, "ENOENT")) return fail("official export omitted a required file");
|
|
667
|
+
if (isErrno(error, "ELOOP")) return fail("official export file must not be a symbolic link");
|
|
668
|
+
return fail("official export file could not be opened safely");
|
|
669
|
+
}
|
|
670
|
+
try {
|
|
671
|
+
const metadata = await handle.stat();
|
|
672
|
+
const pathMetadata = await lstat(path);
|
|
673
|
+
if (
|
|
674
|
+
!metadata.isFile()
|
|
675
|
+
|| metadata.uid !== process.getuid?.()
|
|
676
|
+
|| metadata.nlink !== 1
|
|
677
|
+
|| (metadata.mode & 0o022) !== 0
|
|
678
|
+
|| metadata.size < 2
|
|
679
|
+
|| pathMetadata.isSymbolicLink()
|
|
680
|
+
|| pathMetadata.dev !== metadata.dev
|
|
681
|
+
|| pathMetadata.ino !== metadata.ino
|
|
682
|
+
|| pathMetadata.nlink !== 1
|
|
683
|
+
) return fail("official export file is outside its ownership or size bound");
|
|
684
|
+
return metadata.size;
|
|
685
|
+
} finally {
|
|
686
|
+
await handle.close();
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
function parseManifest(value: unknown): ExportManifest {
|
|
691
|
+
const source = record(value, "official export manifest");
|
|
692
|
+
exactKeys(source, [
|
|
693
|
+
"accounts",
|
|
694
|
+
"attachmentCount",
|
|
695
|
+
"chatCount",
|
|
696
|
+
"completedAt",
|
|
697
|
+
"createdAt",
|
|
698
|
+
"messageCount",
|
|
699
|
+
"version",
|
|
700
|
+
], [], "official export manifest");
|
|
701
|
+
if (source.version !== 1) return fail("official export manifest version is unsupported");
|
|
702
|
+
const createdAt = timestamp(source.createdAt, "official export manifest createdAt");
|
|
703
|
+
const completedAt = timestamp(source.completedAt, "official export manifest completedAt");
|
|
704
|
+
if (completedAt < createdAt) {
|
|
705
|
+
return fail("official export manifest completed before it started");
|
|
706
|
+
}
|
|
707
|
+
return Object.freeze({
|
|
708
|
+
accounts: source.accounts,
|
|
709
|
+
attachmentCount: integer(
|
|
710
|
+
source.attachmentCount,
|
|
711
|
+
"official export manifest attachmentCount",
|
|
712
|
+
Number.MAX_SAFE_INTEGER,
|
|
713
|
+
),
|
|
714
|
+
chatCount: integer(
|
|
715
|
+
source.chatCount,
|
|
716
|
+
"official export manifest chatCount",
|
|
717
|
+
MAX_EXPORT_CHATS,
|
|
718
|
+
),
|
|
719
|
+
completedAt,
|
|
720
|
+
createdAt,
|
|
721
|
+
messageCount: integer(
|
|
722
|
+
source.messageCount,
|
|
723
|
+
"official export manifest messageCount",
|
|
724
|
+
Number.MAX_SAFE_INTEGER,
|
|
725
|
+
),
|
|
726
|
+
version: 1,
|
|
727
|
+
});
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
async function readBoundedStream(
|
|
731
|
+
stream: ReadableStream<Uint8Array>,
|
|
732
|
+
maximum: number,
|
|
733
|
+
label: string,
|
|
734
|
+
): Promise<string> {
|
|
735
|
+
const reader = stream.getReader();
|
|
736
|
+
const chunks: Uint8Array[] = [];
|
|
737
|
+
let size = 0;
|
|
738
|
+
try {
|
|
739
|
+
for (;;) {
|
|
740
|
+
const item = await reader.read();
|
|
741
|
+
if (item.done) break;
|
|
742
|
+
if (item.value.byteLength > maximum - size) return fail(`${label} exceeded its byte bound`);
|
|
743
|
+
chunks.push(item.value.slice());
|
|
744
|
+
size += item.value.byteLength;
|
|
745
|
+
}
|
|
746
|
+
} finally {
|
|
747
|
+
reader.releaseLock();
|
|
748
|
+
}
|
|
749
|
+
const output = new Uint8Array(size);
|
|
750
|
+
let offset = 0;
|
|
751
|
+
for (const chunk of chunks) {
|
|
752
|
+
output.set(chunk, offset);
|
|
753
|
+
offset += chunk.byteLength;
|
|
754
|
+
}
|
|
755
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(output);
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
function pathInside(root: string, path: string): boolean {
|
|
759
|
+
return path.startsWith(`${root}${sep}`);
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
function sameFilesystemObject(left: Stats, right: Stats): boolean {
|
|
763
|
+
return left.dev === right.dev
|
|
764
|
+
&& left.ino === right.ino
|
|
765
|
+
&& left.uid === right.uid
|
|
766
|
+
&& left.mode === right.mode
|
|
767
|
+
&& left.nlink === right.nlink
|
|
768
|
+
&& left.size === right.size
|
|
769
|
+
&& left.ctimeMs === right.ctimeMs
|
|
770
|
+
&& left.mtimeMs === right.mtimeMs;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
class BeeperRawStagingInvariantError extends Error {}
|
|
774
|
+
|
|
775
|
+
function rawStagingFail(message: string): never {
|
|
776
|
+
throw new BeeperRawStagingInvariantError(
|
|
777
|
+
`Beeper Message Like Me source: ${message}`,
|
|
778
|
+
);
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
function unsafeCacheSymlink(): never {
|
|
782
|
+
return rawStagingFail(
|
|
783
|
+
"raw export staging contained an unsafe CLI payload-cache symbolic link",
|
|
784
|
+
);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
async function assertSafeCliPayloadCacheSymlink(
|
|
788
|
+
path: string,
|
|
789
|
+
directory: string,
|
|
790
|
+
cacheRoot: string,
|
|
791
|
+
linkMetadata: Stats,
|
|
792
|
+
uid: number,
|
|
793
|
+
): Promise<void> {
|
|
794
|
+
if (!pathInside(cacheRoot, path)) {
|
|
795
|
+
return rawStagingFail(
|
|
796
|
+
"raw export staging contained a symbolic link outside its CLI payload cache",
|
|
797
|
+
);
|
|
798
|
+
}
|
|
799
|
+
if (
|
|
800
|
+
linkMetadata.uid !== uid
|
|
801
|
+
|| linkMetadata.nlink !== 1
|
|
802
|
+
|| /[\u0000-\u001f\u007f\ufffd]/u.test(basename(path))
|
|
803
|
+
) return unsafeCacheSymlink();
|
|
804
|
+
|
|
805
|
+
let target: string;
|
|
806
|
+
try {
|
|
807
|
+
target = await readlink(path);
|
|
808
|
+
} catch {
|
|
809
|
+
return unsafeCacheSymlink();
|
|
810
|
+
}
|
|
811
|
+
if (
|
|
812
|
+
target.length === 0
|
|
813
|
+
|| isAbsolute(target)
|
|
814
|
+
|| Buffer.byteLength(target, "utf8") > MAX_RAW_CACHE_SYMLINK_TARGET_BYTES
|
|
815
|
+
|| /[\u0000-\u001f\u007f\ufffd]/u.test(target)
|
|
816
|
+
) return unsafeCacheSymlink();
|
|
817
|
+
const lexicalTarget = resolve(directory, target);
|
|
818
|
+
if (!pathInside(cacheRoot, lexicalTarget)) return unsafeCacheSymlink();
|
|
819
|
+
|
|
820
|
+
const targetParent = dirname(lexicalTarget);
|
|
821
|
+
const targetParentRelative = relative(cacheRoot, targetParent);
|
|
822
|
+
const linkParentRelative = relative(cacheRoot, directory);
|
|
823
|
+
if (
|
|
824
|
+
dirname(path) !== directory
|
|
825
|
+
|| targetParentRelative === ".."
|
|
826
|
+
|| targetParentRelative.startsWith(`..${sep}`)
|
|
827
|
+
|| isAbsolute(targetParentRelative)
|
|
828
|
+
|| linkParentRelative === ".."
|
|
829
|
+
|| linkParentRelative.startsWith(`..${sep}`)
|
|
830
|
+
|| isAbsolute(linkParentRelative)
|
|
831
|
+
) return unsafeCacheSymlink();
|
|
832
|
+
const parentSnapshots: { readonly path: string; readonly metadata: Stats }[] = [];
|
|
833
|
+
try {
|
|
834
|
+
const cacheMetadata = await lstat(cacheRoot);
|
|
835
|
+
if (
|
|
836
|
+
!cacheMetadata.isDirectory()
|
|
837
|
+
|| cacheMetadata.isSymbolicLink()
|
|
838
|
+
|| cacheMetadata.uid !== uid
|
|
839
|
+
|| (cacheMetadata.mode & 0o022) !== 0
|
|
840
|
+
) return unsafeCacheSymlink();
|
|
841
|
+
parentSnapshots.push({ path: cacheRoot, metadata: cacheMetadata });
|
|
842
|
+
for (const parentRelative of [targetParentRelative, linkParentRelative]) {
|
|
843
|
+
if (parentRelative === "") continue;
|
|
844
|
+
let currentParent = cacheRoot;
|
|
845
|
+
for (const segment of parentRelative.split(sep)) {
|
|
846
|
+
if (segment === "" || segment === "." || segment === "..") {
|
|
847
|
+
return unsafeCacheSymlink();
|
|
848
|
+
}
|
|
849
|
+
currentParent = join(currentParent, segment);
|
|
850
|
+
if (parentSnapshots.some((snapshot) => snapshot.path === currentParent)) {
|
|
851
|
+
continue;
|
|
852
|
+
}
|
|
853
|
+
const metadata = await lstat(currentParent);
|
|
854
|
+
if (
|
|
855
|
+
!metadata.isDirectory()
|
|
856
|
+
|| metadata.isSymbolicLink()
|
|
857
|
+
|| metadata.uid !== uid
|
|
858
|
+
|| (metadata.mode & 0o022) !== 0
|
|
859
|
+
) return unsafeCacheSymlink();
|
|
860
|
+
parentSnapshots.push({ path: currentParent, metadata });
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
} catch {
|
|
864
|
+
return unsafeCacheSymlink();
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
let targetHandle;
|
|
868
|
+
try {
|
|
869
|
+
targetHandle = await open(
|
|
870
|
+
lexicalTarget,
|
|
871
|
+
constants.O_RDONLY | constants.O_NOFOLLOW,
|
|
872
|
+
);
|
|
873
|
+
} catch {
|
|
874
|
+
return unsafeCacheSymlink();
|
|
875
|
+
}
|
|
876
|
+
try {
|
|
877
|
+
const targetBefore = await targetHandle.stat();
|
|
878
|
+
if (
|
|
879
|
+
!targetBefore.isFile()
|
|
880
|
+
|| targetBefore.uid !== uid
|
|
881
|
+
|| targetBefore.nlink !== 1
|
|
882
|
+
|| (targetBefore.mode & 0o022) !== 0
|
|
883
|
+
) return unsafeCacheSymlink();
|
|
884
|
+
let targetPathMetadata: Stats;
|
|
885
|
+
let targetCanonical: string;
|
|
886
|
+
let linkCanonical: string;
|
|
887
|
+
let linkAfter: Stats;
|
|
888
|
+
let targetAfterText: string;
|
|
889
|
+
try {
|
|
890
|
+
targetPathMetadata = await lstat(lexicalTarget);
|
|
891
|
+
if (
|
|
892
|
+
!targetPathMetadata.isFile()
|
|
893
|
+
|| targetPathMetadata.isSymbolicLink()
|
|
894
|
+
|| targetPathMetadata.uid !== uid
|
|
895
|
+
|| targetPathMetadata.nlink !== 1
|
|
896
|
+
|| (targetPathMetadata.mode & 0o022) !== 0
|
|
897
|
+
|| !sameFilesystemObject(targetBefore, targetPathMetadata)
|
|
898
|
+
) return unsafeCacheSymlink();
|
|
899
|
+
targetCanonical = await realpath(lexicalTarget);
|
|
900
|
+
linkCanonical = await realpath(path);
|
|
901
|
+
linkAfter = await lstat(path);
|
|
902
|
+
targetAfterText = await readlink(path);
|
|
903
|
+
} catch {
|
|
904
|
+
return unsafeCacheSymlink();
|
|
905
|
+
}
|
|
906
|
+
const targetAfter = await targetHandle.stat();
|
|
907
|
+
if (
|
|
908
|
+
!sameFilesystemObject(targetBefore, targetAfter)
|
|
909
|
+
|| targetCanonical !== lexicalTarget
|
|
910
|
+
|| linkCanonical !== lexicalTarget
|
|
911
|
+
|| !sameFilesystemObject(linkMetadata, linkAfter)
|
|
912
|
+
|| targetAfterText !== target
|
|
913
|
+
) return unsafeCacheSymlink();
|
|
914
|
+
for (const snapshot of parentSnapshots) {
|
|
915
|
+
let current: Stats;
|
|
916
|
+
try {
|
|
917
|
+
current = await lstat(snapshot.path);
|
|
918
|
+
} catch {
|
|
919
|
+
return unsafeCacheSymlink();
|
|
920
|
+
}
|
|
921
|
+
if (
|
|
922
|
+
!current.isDirectory()
|
|
923
|
+
|| current.isSymbolicLink()
|
|
924
|
+
|| !sameFilesystemObject(snapshot.metadata, current)
|
|
925
|
+
|| await realpath(snapshot.path).catch(() => null) !== snapshot.path
|
|
926
|
+
) return unsafeCacheSymlink();
|
|
927
|
+
}
|
|
928
|
+
} finally {
|
|
929
|
+
await targetHandle.close();
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
/** @internal Exported only for focused safety tests. */
|
|
934
|
+
export async function enforceBeeperRawWorkingBudget(
|
|
935
|
+
root: string,
|
|
936
|
+
maximumBytes: number,
|
|
937
|
+
minimumFreeBytes = MIN_FREE_FILESYSTEM_BYTES,
|
|
938
|
+
signal?: AbortSignal,
|
|
939
|
+
): Promise<Readonly<{ bytes: number; entries: number }>> {
|
|
940
|
+
throwIfAborted(signal);
|
|
941
|
+
if (
|
|
942
|
+
!isAbsolute(root)
|
|
943
|
+
|| !Number.isSafeInteger(maximumBytes)
|
|
944
|
+
|| maximumBytes < 1
|
|
945
|
+
|| maximumBytes > MAX_RAW_WORKING_BYTES
|
|
946
|
+
|| !Number.isSafeInteger(minimumFreeBytes)
|
|
947
|
+
|| minimumFreeBytes < 0
|
|
948
|
+
) return rawStagingFail("raw export staging budget was invalid");
|
|
949
|
+
const canonicalRoot = await assertPrivateOwnedDirectory(await realpath(root));
|
|
950
|
+
const cacheRoot = join(canonicalRoot, "cli-payload-cache");
|
|
951
|
+
const uid = process.getuid?.();
|
|
952
|
+
if (uid === undefined) {
|
|
953
|
+
return rawStagingFail("raw export staging requires a POSIX user identity");
|
|
954
|
+
}
|
|
955
|
+
const stack = [canonicalRoot];
|
|
956
|
+
let bytes = 0;
|
|
957
|
+
let entries = 0;
|
|
958
|
+
while (stack.length > 0) {
|
|
959
|
+
throwIfAborted(signal);
|
|
960
|
+
const directory = stack.pop();
|
|
961
|
+
if (directory === undefined) break;
|
|
962
|
+
let names: readonly string[];
|
|
963
|
+
try {
|
|
964
|
+
names = (await readdir(directory)).sort();
|
|
965
|
+
} catch (error) {
|
|
966
|
+
if (isErrno(error, "ENOENT")) continue;
|
|
967
|
+
return rawStagingFail("raw export staging could not be inspected safely");
|
|
968
|
+
}
|
|
969
|
+
for (const name of names) {
|
|
970
|
+
throwIfAborted(signal);
|
|
971
|
+
const path = join(directory, name);
|
|
972
|
+
let metadata;
|
|
973
|
+
try {
|
|
974
|
+
metadata = await lstat(path);
|
|
975
|
+
} catch (error) {
|
|
976
|
+
if (isErrno(error, "ENOENT")) continue;
|
|
977
|
+
return rawStagingFail("raw export staging could not be inspected safely");
|
|
978
|
+
}
|
|
979
|
+
entries += 1;
|
|
980
|
+
if (entries > MAX_RAW_WORKING_ENTRIES) {
|
|
981
|
+
return rawStagingFail("raw export staging exceeded its entry budget");
|
|
982
|
+
}
|
|
983
|
+
const allocatedBytes = metadata.blocks * 512;
|
|
984
|
+
const entryBytes = Math.max(metadata.size, allocatedBytes);
|
|
985
|
+
if (
|
|
986
|
+
!Number.isSafeInteger(entryBytes)
|
|
987
|
+
|| entryBytes < 0
|
|
988
|
+
|| entryBytes > maximumBytes - bytes
|
|
989
|
+
) return rawStagingFail("raw export staging exceeded its byte budget");
|
|
990
|
+
bytes += entryBytes;
|
|
991
|
+
if (metadata.isSymbolicLink()) {
|
|
992
|
+
await assertSafeCliPayloadCacheSymlink(
|
|
993
|
+
path,
|
|
994
|
+
directory,
|
|
995
|
+
cacheRoot,
|
|
996
|
+
metadata,
|
|
997
|
+
uid,
|
|
998
|
+
);
|
|
999
|
+
} else if (
|
|
1000
|
+
metadata.uid !== uid
|
|
1001
|
+
|| (metadata.mode & 0o022) !== 0
|
|
1002
|
+
) {
|
|
1003
|
+
return rawStagingFail(
|
|
1004
|
+
"raw export staging contained an unsafe filesystem entry",
|
|
1005
|
+
);
|
|
1006
|
+
} else if (metadata.isDirectory()) {
|
|
1007
|
+
const canonical = await realpath(path).catch(() => null);
|
|
1008
|
+
if (
|
|
1009
|
+
canonical !== path
|
|
1010
|
+
|| !canonical.startsWith(`${canonicalRoot}${sep}`)
|
|
1011
|
+
) {
|
|
1012
|
+
return rawStagingFail(
|
|
1013
|
+
"raw export staging directory changed during inspection",
|
|
1014
|
+
);
|
|
1015
|
+
}
|
|
1016
|
+
stack.push(path);
|
|
1017
|
+
} else if (!metadata.isFile() || metadata.nlink !== 1) {
|
|
1018
|
+
return rawStagingFail(
|
|
1019
|
+
"raw export staging contained an unsafe filesystem entry",
|
|
1020
|
+
);
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
let filesystem;
|
|
1025
|
+
try {
|
|
1026
|
+
filesystem = await statfs(canonicalRoot);
|
|
1027
|
+
} catch {
|
|
1028
|
+
return rawStagingFail(
|
|
1029
|
+
"raw export filesystem capacity could not be inspected",
|
|
1030
|
+
);
|
|
1031
|
+
}
|
|
1032
|
+
const availableBytes = filesystem.bavail * filesystem.bsize;
|
|
1033
|
+
if (
|
|
1034
|
+
!Number.isSafeInteger(availableBytes)
|
|
1035
|
+
|| availableBytes < minimumFreeBytes
|
|
1036
|
+
) return rawStagingFail("raw export filesystem reserve would be exhausted");
|
|
1037
|
+
return Object.freeze({ bytes, entries });
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
async function enforceBeeperRawFilesystemReserve(
|
|
1041
|
+
root: string,
|
|
1042
|
+
signal: AbortSignal | undefined,
|
|
1043
|
+
): Promise<void> {
|
|
1044
|
+
throwIfAborted(signal);
|
|
1045
|
+
let filesystem;
|
|
1046
|
+
try {
|
|
1047
|
+
filesystem = await statfs(root);
|
|
1048
|
+
} catch {
|
|
1049
|
+
return fail("raw export filesystem capacity could not be inspected");
|
|
1050
|
+
}
|
|
1051
|
+
throwIfAborted(signal);
|
|
1052
|
+
const availableBytes = filesystem.bavail * filesystem.bsize;
|
|
1053
|
+
if (
|
|
1054
|
+
!Number.isSafeInteger(availableBytes)
|
|
1055
|
+
|| availableBytes < MIN_FREE_FILESYSTEM_BYTES
|
|
1056
|
+
) return fail("raw export filesystem reserve would be exhausted");
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
/** @internal Exported only for focused process-lifecycle safety tests. */
|
|
1060
|
+
export async function runExportCli(
|
|
1061
|
+
invocation: BeeperExportCliInvocation,
|
|
1062
|
+
): Promise<BeeperCliInvocationResult> {
|
|
1063
|
+
throwIfAborted(invocation.signal);
|
|
1064
|
+
if (
|
|
1065
|
+
(invocation.workingRoot === undefined)
|
|
1066
|
+
!== (invocation.maxWorkingBytes === undefined)
|
|
1067
|
+
) return fail("raw export staging monitor was incompletely configured");
|
|
1068
|
+
if (
|
|
1069
|
+
invocation.workingRoot !== undefined
|
|
1070
|
+
&& invocation.maxWorkingBytes !== undefined
|
|
1071
|
+
) {
|
|
1072
|
+
try {
|
|
1073
|
+
await enforceBeeperRawWorkingBudget(
|
|
1074
|
+
invocation.workingRoot,
|
|
1075
|
+
invocation.maxWorkingBytes,
|
|
1076
|
+
MIN_FREE_FILESYSTEM_BYTES,
|
|
1077
|
+
invocation.signal,
|
|
1078
|
+
);
|
|
1079
|
+
} catch (error) {
|
|
1080
|
+
throwIfAborted(invocation.signal);
|
|
1081
|
+
if (error instanceof BeeperRawStagingInvariantError) throw error;
|
|
1082
|
+
return fail("official export raw staging safety check failed");
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
if (invocation.directoryLease !== undefined) {
|
|
1086
|
+
updateBeeperMessageLikeMeDirectoryLease(
|
|
1087
|
+
invocation.directoryLease,
|
|
1088
|
+
"launching",
|
|
1089
|
+
);
|
|
1090
|
+
}
|
|
1091
|
+
let leaseSettlementAttempted = false;
|
|
1092
|
+
const settleDirectoryLease = (): void => {
|
|
1093
|
+
if (
|
|
1094
|
+
invocation.directoryLease === undefined
|
|
1095
|
+
|| leaseSettlementAttempted
|
|
1096
|
+
) return;
|
|
1097
|
+
leaseSettlementAttempted = true;
|
|
1098
|
+
updateBeeperMessageLikeMeDirectoryLease(
|
|
1099
|
+
invocation.directoryLease,
|
|
1100
|
+
"settled",
|
|
1101
|
+
);
|
|
1102
|
+
};
|
|
1103
|
+
const child = (() => {
|
|
1104
|
+
try {
|
|
1105
|
+
return Bun.spawn([
|
|
1106
|
+
"/bin/sh",
|
|
1107
|
+
"-c",
|
|
1108
|
+
"umask 077\nexec \"$@\"",
|
|
1109
|
+
"wrench-beeper-export",
|
|
1110
|
+
invocation.binary,
|
|
1111
|
+
...invocation.arguments,
|
|
1112
|
+
], {
|
|
1113
|
+
env: { ...invocation.environment },
|
|
1114
|
+
stdin: "ignore",
|
|
1115
|
+
stdout: "pipe",
|
|
1116
|
+
stderr: "pipe",
|
|
1117
|
+
detached: true,
|
|
1118
|
+
});
|
|
1119
|
+
} catch (error) {
|
|
1120
|
+
try {
|
|
1121
|
+
settleDirectoryLease();
|
|
1122
|
+
} catch (settlementError) {
|
|
1123
|
+
throw new AggregateError(
|
|
1124
|
+
[error, settlementError],
|
|
1125
|
+
"Beeper export launch and recovery lifecycle settlement both failed",
|
|
1126
|
+
);
|
|
1127
|
+
}
|
|
1128
|
+
throw error;
|
|
1129
|
+
}
|
|
1130
|
+
})();
|
|
1131
|
+
let timedOut = false;
|
|
1132
|
+
let cancelled = false;
|
|
1133
|
+
let heartbeatFailed = false;
|
|
1134
|
+
let workingBudgetFailed = false;
|
|
1135
|
+
let workingBudgetInspection: Promise<void> | undefined;
|
|
1136
|
+
let filesystemInspection: Promise<void> | undefined;
|
|
1137
|
+
let forceKill: ReturnType<typeof setTimeout> | null = null;
|
|
1138
|
+
const signalGroup = (signal: "SIGTERM" | "SIGKILL"): void => {
|
|
1139
|
+
try {
|
|
1140
|
+
process.kill(-child.pid, signal);
|
|
1141
|
+
} catch {
|
|
1142
|
+
// The complete child process group already exited.
|
|
1143
|
+
}
|
|
1144
|
+
};
|
|
1145
|
+
if (invocation.directoryLease !== undefined) {
|
|
1146
|
+
try {
|
|
1147
|
+
updateBeeperMessageLikeMeDirectoryLease(
|
|
1148
|
+
invocation.directoryLease,
|
|
1149
|
+
"running",
|
|
1150
|
+
child.pid,
|
|
1151
|
+
);
|
|
1152
|
+
} catch (error) {
|
|
1153
|
+
signalGroup("SIGKILL");
|
|
1154
|
+
let reapError: unknown;
|
|
1155
|
+
try {
|
|
1156
|
+
await child.exited;
|
|
1157
|
+
} catch (childError) {
|
|
1158
|
+
reapError = childError;
|
|
1159
|
+
}
|
|
1160
|
+
let settlementError: unknown;
|
|
1161
|
+
try {
|
|
1162
|
+
settleDirectoryLease();
|
|
1163
|
+
} catch (leaseError) {
|
|
1164
|
+
settlementError = leaseError;
|
|
1165
|
+
}
|
|
1166
|
+
const failures = [error, reapError, settlementError].filter(
|
|
1167
|
+
(failure) => failure !== undefined,
|
|
1168
|
+
);
|
|
1169
|
+
if (failures.length > 1) {
|
|
1170
|
+
throw new AggregateError(
|
|
1171
|
+
failures,
|
|
1172
|
+
"Beeper export recovery lifecycle failed after launch",
|
|
1173
|
+
);
|
|
1174
|
+
}
|
|
1175
|
+
throw error;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
const terminate = (): void => {
|
|
1179
|
+
signalGroup("SIGTERM");
|
|
1180
|
+
if (forceKill === null) {
|
|
1181
|
+
forceKill = setTimeout(() => signalGroup("SIGKILL"), 2_000);
|
|
1182
|
+
}
|
|
1183
|
+
};
|
|
1184
|
+
const inspectWorkingBudget = (): void => {
|
|
1185
|
+
if (
|
|
1186
|
+
invocation.workingRoot === undefined
|
|
1187
|
+
|| invocation.maxWorkingBytes === undefined
|
|
1188
|
+
|| workingBudgetInspection !== undefined
|
|
1189
|
+
) return;
|
|
1190
|
+
workingBudgetInspection = (async () => {
|
|
1191
|
+
try {
|
|
1192
|
+
await enforceBeeperRawWorkingBudget(
|
|
1193
|
+
invocation.workingRoot!,
|
|
1194
|
+
invocation.maxWorkingBytes!,
|
|
1195
|
+
MIN_FREE_FILESYSTEM_BYTES,
|
|
1196
|
+
invocation.signal,
|
|
1197
|
+
);
|
|
1198
|
+
} catch {
|
|
1199
|
+
workingBudgetFailed = true;
|
|
1200
|
+
terminate();
|
|
1201
|
+
}
|
|
1202
|
+
})().finally(() => {
|
|
1203
|
+
workingBudgetInspection = undefined;
|
|
1204
|
+
});
|
|
1205
|
+
};
|
|
1206
|
+
const inspectFilesystemReserve = (): void => {
|
|
1207
|
+
if (invocation.workingRoot === undefined || filesystemInspection !== undefined) return;
|
|
1208
|
+
filesystemInspection = (async () => {
|
|
1209
|
+
try {
|
|
1210
|
+
await enforceBeeperRawFilesystemReserve(
|
|
1211
|
+
invocation.workingRoot!,
|
|
1212
|
+
invocation.signal,
|
|
1213
|
+
);
|
|
1214
|
+
} catch {
|
|
1215
|
+
workingBudgetFailed = true;
|
|
1216
|
+
terminate();
|
|
1217
|
+
}
|
|
1218
|
+
})().finally(() => {
|
|
1219
|
+
filesystemInspection = undefined;
|
|
1220
|
+
});
|
|
1221
|
+
};
|
|
1222
|
+
const onAbort = (): void => {
|
|
1223
|
+
cancelled = true;
|
|
1224
|
+
terminate();
|
|
1225
|
+
};
|
|
1226
|
+
invocation.signal?.addEventListener("abort", onAbort, { once: true });
|
|
1227
|
+
const timeout = setTimeout(() => {
|
|
1228
|
+
timedOut = true;
|
|
1229
|
+
terminate();
|
|
1230
|
+
}, invocation.timeoutMs);
|
|
1231
|
+
const startedAt = Date.now();
|
|
1232
|
+
const heartbeat = invocation.onHeartbeat === undefined
|
|
1233
|
+
? null
|
|
1234
|
+
: setInterval(() => {
|
|
1235
|
+
try {
|
|
1236
|
+
invocation.onHeartbeat?.(
|
|
1237
|
+
Math.max(1, Math.floor((Date.now() - startedAt) / 1_000)),
|
|
1238
|
+
);
|
|
1239
|
+
} catch {
|
|
1240
|
+
heartbeatFailed = true;
|
|
1241
|
+
terminate();
|
|
1242
|
+
}
|
|
1243
|
+
}, ACCOUNT_HEARTBEAT_INTERVAL_MS);
|
|
1244
|
+
const workingBudgetMonitor = invocation.workingRoot === undefined
|
|
1245
|
+
? null
|
|
1246
|
+
: setInterval(() => {
|
|
1247
|
+
inspectWorkingBudget();
|
|
1248
|
+
inspectFilesystemReserve();
|
|
1249
|
+
}, RAW_WORKING_MONITOR_INTERVAL_MS);
|
|
1250
|
+
inspectWorkingBudget();
|
|
1251
|
+
inspectFilesystemReserve();
|
|
1252
|
+
try {
|
|
1253
|
+
const [exitCode, stdout, stderr] = await Promise.all([
|
|
1254
|
+
child.exited,
|
|
1255
|
+
readBoundedStream(child.stdout, invocation.maxOutputBytes, "Beeper export stdout"),
|
|
1256
|
+
readBoundedStream(
|
|
1257
|
+
child.stderr,
|
|
1258
|
+
invocation.maxStderrBytes,
|
|
1259
|
+
"Beeper export stderr",
|
|
1260
|
+
),
|
|
1261
|
+
]);
|
|
1262
|
+
settleDirectoryLease();
|
|
1263
|
+
if (workingBudgetInspection !== undefined) await workingBudgetInspection;
|
|
1264
|
+
if (filesystemInspection !== undefined) await filesystemInspection;
|
|
1265
|
+
if (
|
|
1266
|
+
!workingBudgetFailed
|
|
1267
|
+
&& invocation.workingRoot !== undefined
|
|
1268
|
+
&& invocation.maxWorkingBytes !== undefined
|
|
1269
|
+
) {
|
|
1270
|
+
try {
|
|
1271
|
+
await enforceBeeperRawWorkingBudget(
|
|
1272
|
+
invocation.workingRoot,
|
|
1273
|
+
invocation.maxWorkingBytes,
|
|
1274
|
+
MIN_FREE_FILESYSTEM_BYTES,
|
|
1275
|
+
invocation.signal,
|
|
1276
|
+
);
|
|
1277
|
+
} catch {
|
|
1278
|
+
workingBudgetFailed = true;
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
if (cancelled) return fail("official export was cancelled");
|
|
1282
|
+
if (timedOut) return fail("official export timed out");
|
|
1283
|
+
if (heartbeatFailed) return fail("export progress reporting failed");
|
|
1284
|
+
if (workingBudgetFailed) {
|
|
1285
|
+
return fail("official export exceeded its raw staging safety budget");
|
|
1286
|
+
}
|
|
1287
|
+
return Object.freeze({ exitCode, stdout, stderr });
|
|
1288
|
+
} catch (error) {
|
|
1289
|
+
signalGroup("SIGKILL");
|
|
1290
|
+
let reapError: unknown;
|
|
1291
|
+
try {
|
|
1292
|
+
await child.exited;
|
|
1293
|
+
} catch (childError) {
|
|
1294
|
+
reapError = childError;
|
|
1295
|
+
}
|
|
1296
|
+
let settlementError: unknown;
|
|
1297
|
+
try {
|
|
1298
|
+
settleDirectoryLease();
|
|
1299
|
+
} catch (leaseError) {
|
|
1300
|
+
settlementError = leaseError;
|
|
1301
|
+
}
|
|
1302
|
+
const failures = [error, reapError, settlementError].filter(
|
|
1303
|
+
(failure) => failure !== undefined,
|
|
1304
|
+
);
|
|
1305
|
+
if (failures.length > 1) {
|
|
1306
|
+
throw new AggregateError(
|
|
1307
|
+
failures,
|
|
1308
|
+
"Beeper export and recovery lifecycle settlement both failed",
|
|
1309
|
+
);
|
|
1310
|
+
}
|
|
1311
|
+
throw error;
|
|
1312
|
+
} finally {
|
|
1313
|
+
clearTimeout(timeout);
|
|
1314
|
+
if (heartbeat !== null) clearInterval(heartbeat);
|
|
1315
|
+
if (workingBudgetMonitor !== null) clearInterval(workingBudgetMonitor);
|
|
1316
|
+
const pendingWorkingBudgetInspection = workingBudgetInspection;
|
|
1317
|
+
if (pendingWorkingBudgetInspection !== undefined) {
|
|
1318
|
+
await pendingWorkingBudgetInspection;
|
|
1319
|
+
}
|
|
1320
|
+
const pendingFilesystemInspection = filesystemInspection;
|
|
1321
|
+
if (pendingFilesystemInspection !== undefined) {
|
|
1322
|
+
await pendingFilesystemInspection;
|
|
1323
|
+
}
|
|
1324
|
+
if (forceKill !== null) clearTimeout(forceKill);
|
|
1325
|
+
invocation.signal?.removeEventListener("abort", onAbort);
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
function environmentForExport(
|
|
1330
|
+
configDirectory: string,
|
|
1331
|
+
cacheDirectory: string,
|
|
1332
|
+
): Readonly<Record<string, string>> {
|
|
1333
|
+
return Object.freeze({
|
|
1334
|
+
PATH: "/usr/bin:/bin",
|
|
1335
|
+
LANG: "C.UTF-8",
|
|
1336
|
+
CI: "1",
|
|
1337
|
+
BEEPER_CLI_CONFIG_DIR: configDirectory,
|
|
1338
|
+
BEEPER_CLI_BINARY_CACHE_DIR: cacheDirectory,
|
|
1339
|
+
BEEPER_READONLY: "1",
|
|
1340
|
+
BEEPER_QUIET: "1",
|
|
1341
|
+
BEEPER_SKIP_UPDATE_CHECK: "1",
|
|
1342
|
+
NO_UPDATE_NOTIFIER: "1",
|
|
1343
|
+
});
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
function remainingTimeoutMs(deadlineMs: number): number {
|
|
1347
|
+
const remaining = deadlineMs - Date.now();
|
|
1348
|
+
if (remaining <= 0) return fail("official export exceeded its command-wide timeout");
|
|
1349
|
+
return Math.max(1, Math.floor(remaining));
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
async function withProgressHeartbeat<T>(
|
|
1353
|
+
operation: () => Promise<T>,
|
|
1354
|
+
onHeartbeat: ((elapsedSeconds: number) => void) | undefined,
|
|
1355
|
+
): Promise<T> {
|
|
1356
|
+
if (onHeartbeat === undefined) return operation();
|
|
1357
|
+
const startedAt = Date.now();
|
|
1358
|
+
let heartbeatFailed = false;
|
|
1359
|
+
const heartbeat = setInterval(() => {
|
|
1360
|
+
try {
|
|
1361
|
+
onHeartbeat(Math.max(1, Math.floor((Date.now() - startedAt) / 1_000)));
|
|
1362
|
+
} catch {
|
|
1363
|
+
heartbeatFailed = true;
|
|
1364
|
+
}
|
|
1365
|
+
}, ACCOUNT_HEARTBEAT_INTERVAL_MS);
|
|
1366
|
+
try {
|
|
1367
|
+
const value = await operation();
|
|
1368
|
+
if (heartbeatFailed) return fail("export progress reporting failed");
|
|
1369
|
+
return value;
|
|
1370
|
+
} finally {
|
|
1371
|
+
clearInterval(heartbeat);
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
function parseCliJson(stdout: string, label: string): unknown {
|
|
1376
|
+
const source = stdout.trim();
|
|
1377
|
+
if (source.length === 0) return fail(`${label} omitted JSON output`);
|
|
1378
|
+
let value: unknown;
|
|
1379
|
+
try {
|
|
1380
|
+
value = JSON.parse(source) as unknown;
|
|
1381
|
+
} catch {
|
|
1382
|
+
return fail(`${label} returned malformed JSON`);
|
|
1383
|
+
}
|
|
1384
|
+
try {
|
|
1385
|
+
return parseBeeperCliEnvelope(value, label);
|
|
1386
|
+
} catch {
|
|
1387
|
+
return fail(`${label} returned an invalid success envelope`);
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
async function enumerateAccounts(
|
|
1392
|
+
binary: string,
|
|
1393
|
+
environment: Readonly<Record<string, string>>,
|
|
1394
|
+
deadlineMs: number,
|
|
1395
|
+
run: (invocation: BeeperExportCliInvocation) => Promise<BeeperCliInvocationResult>,
|
|
1396
|
+
directoryLease: BeeperMessageLikeMeDirectoryLease | undefined,
|
|
1397
|
+
signal: AbortSignal | undefined,
|
|
1398
|
+
onHeartbeat?: (elapsedSeconds: number) => void,
|
|
1399
|
+
): Promise<readonly BeeperAccountProjection[]> {
|
|
1400
|
+
const timeoutMs = remainingTimeoutMs(deadlineMs);
|
|
1401
|
+
const command = planBeeperAccountsListCommand(timeoutMs);
|
|
1402
|
+
const result = await run({
|
|
1403
|
+
binary,
|
|
1404
|
+
arguments: command.argv,
|
|
1405
|
+
environment,
|
|
1406
|
+
timeoutMs,
|
|
1407
|
+
maxOutputBytes: MAX_ACCOUNTS_LIST_BYTES,
|
|
1408
|
+
maxStderrBytes: MAX_STDERR_BYTES,
|
|
1409
|
+
...(directoryLease === undefined ? {} : { directoryLease }),
|
|
1410
|
+
...(onHeartbeat === undefined ? {} : { onHeartbeat }),
|
|
1411
|
+
...(signal === undefined ? {} : { signal }),
|
|
1412
|
+
});
|
|
1413
|
+
throwIfAborted(signal);
|
|
1414
|
+
if (result.exitCode !== 0 || result.stderr.trim().length !== 0) {
|
|
1415
|
+
return fail("official account enumeration failed");
|
|
1416
|
+
}
|
|
1417
|
+
try {
|
|
1418
|
+
return parseBeeperExportAccounts(parseCliJson(result.stdout, "official account enumeration"));
|
|
1419
|
+
} catch {
|
|
1420
|
+
return fail("official account enumeration returned an unsupported projection");
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
async function proveDesktopTargetRealm(
|
|
1425
|
+
binary: string,
|
|
1426
|
+
environment: Readonly<Record<string, string>>,
|
|
1427
|
+
targetBaseUrl: string,
|
|
1428
|
+
deadlineMs: number,
|
|
1429
|
+
run: (invocation: BeeperExportCliInvocation) => Promise<BeeperCliInvocationResult>,
|
|
1430
|
+
directoryLease: BeeperMessageLikeMeDirectoryLease | undefined,
|
|
1431
|
+
signal: AbortSignal | undefined,
|
|
1432
|
+
): Promise<BeeperTargetRealmProof> {
|
|
1433
|
+
const timeoutMs = remainingTimeoutMs(deadlineMs);
|
|
1434
|
+
const command = planBeeperTargetStatusCommand(timeoutMs);
|
|
1435
|
+
const result = await run({
|
|
1436
|
+
binary,
|
|
1437
|
+
arguments: command.argv,
|
|
1438
|
+
environment,
|
|
1439
|
+
timeoutMs,
|
|
1440
|
+
maxOutputBytes: 64 * 1024,
|
|
1441
|
+
maxStderrBytes: MAX_STDERR_BYTES,
|
|
1442
|
+
...(directoryLease === undefined ? {} : { directoryLease }),
|
|
1443
|
+
...(signal === undefined ? {} : { signal }),
|
|
1444
|
+
});
|
|
1445
|
+
throwIfAborted(signal);
|
|
1446
|
+
if (result.exitCode !== 0 || result.stderr.trim().length !== 0) {
|
|
1447
|
+
return fail("official Desktop target realm proof failed");
|
|
1448
|
+
}
|
|
1449
|
+
try {
|
|
1450
|
+
return parseBeeperTargetRealmProof(
|
|
1451
|
+
parseCliJson(result.stdout, "official Desktop target realm proof"),
|
|
1452
|
+
targetBaseUrl,
|
|
1453
|
+
);
|
|
1454
|
+
} catch {
|
|
1455
|
+
return fail("official Desktop target realm proof returned an unsupported projection");
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
function accountOutputRealm(
|
|
1460
|
+
account: BeeperAccountProjection,
|
|
1461
|
+
): Readonly<Record<string, unknown>> {
|
|
1462
|
+
return Object.freeze({
|
|
1463
|
+
accountId: account.accountId,
|
|
1464
|
+
bridge: account.bridge,
|
|
1465
|
+
network: account.network,
|
|
1466
|
+
user: Object.freeze({
|
|
1467
|
+
id: account.user.id,
|
|
1468
|
+
displayName: account.selectorAliases.displayName,
|
|
1469
|
+
name: account.selectorAliases.name,
|
|
1470
|
+
fullName: account.user.fullName,
|
|
1471
|
+
username: account.user.username,
|
|
1472
|
+
phoneNumber: account.user.phoneNumber,
|
|
1473
|
+
email: account.user.email,
|
|
1474
|
+
isSelf: account.user.isSelf,
|
|
1475
|
+
}),
|
|
1476
|
+
});
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
function outputRealmDigest(accounts: readonly BeeperAccountProjection[]): string {
|
|
1480
|
+
return sha256(canonicalJson([...accounts]
|
|
1481
|
+
.sort((left, right) => left.accountId.localeCompare(right.accountId))
|
|
1482
|
+
.map(accountOutputRealm)));
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
function assertOutputRealm(
|
|
1486
|
+
accounts: readonly BeeperAccountProjection[],
|
|
1487
|
+
expectedDigest: string,
|
|
1488
|
+
): void {
|
|
1489
|
+
if (outputRealmDigest(accounts) !== expectedDigest) {
|
|
1490
|
+
fail("connected Beeper account inventory changed during export");
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
function normalizeOfficialAccountSelector(value: string | null): string {
|
|
1495
|
+
return (value ?? "").trim().toLowerCase().replace(/[\s._-]+/gu, "");
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
/** @internal Exported only for pinned-selector safety tests. */
|
|
1499
|
+
export function assertUniqueOfficialAccountSelector(
|
|
1500
|
+
selected: BeeperAccountProjection,
|
|
1501
|
+
accounts: readonly BeeperAccountProjection[],
|
|
1502
|
+
): void {
|
|
1503
|
+
const wanted = normalizeOfficialAccountSelector(selected.accountId);
|
|
1504
|
+
const matches = accounts.filter((account) => [
|
|
1505
|
+
account.accountId,
|
|
1506
|
+
account.network,
|
|
1507
|
+
account.bridge.type,
|
|
1508
|
+
account.bridge.id,
|
|
1509
|
+
account.user.id,
|
|
1510
|
+
account.user.username,
|
|
1511
|
+
account.selectorAliases.displayName,
|
|
1512
|
+
account.selectorAliases.name,
|
|
1513
|
+
account.user.email,
|
|
1514
|
+
].some((candidate) => normalizeOfficialAccountSelector(candidate) === wanted));
|
|
1515
|
+
if (matches.length !== 1 || matches[0]?.accountId !== selected.accountId) {
|
|
1516
|
+
fail("one Beeper account ID is ambiguous under the pinned CLI selector rules");
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
async function readCliStoreSnapshot(
|
|
1521
|
+
configDirectory: string,
|
|
1522
|
+
): Promise<BeeperCliStoreSnapshot> {
|
|
1523
|
+
const assertLocalDesktopUrl = (value: unknown, label: string): string | undefined => {
|
|
1524
|
+
if (value === undefined) return;
|
|
1525
|
+
const source = string(value, label, 2_048);
|
|
1526
|
+
let parsed: URL;
|
|
1527
|
+
try {
|
|
1528
|
+
parsed = new URL(source);
|
|
1529
|
+
} catch {
|
|
1530
|
+
return fail(`${label} is no longer a reviewed loopback Desktop URL`);
|
|
1531
|
+
}
|
|
1532
|
+
const port = Number(parsed.port);
|
|
1533
|
+
if (
|
|
1534
|
+
parsed.protocol !== "http:"
|
|
1535
|
+
|| parsed.hostname !== "127.0.0.1"
|
|
1536
|
+
|| parsed.username !== ""
|
|
1537
|
+
|| parsed.password !== ""
|
|
1538
|
+
|| parsed.pathname !== "/"
|
|
1539
|
+
|| parsed.search !== ""
|
|
1540
|
+
|| parsed.hash !== ""
|
|
1541
|
+
|| !Number.isSafeInteger(port)
|
|
1542
|
+
|| port < 23_373
|
|
1543
|
+
|| port > 23_392
|
|
1544
|
+
) return fail(`${label} is no longer a reviewed loopback Desktop URL`);
|
|
1545
|
+
return source;
|
|
1546
|
+
};
|
|
1547
|
+
const storedAuth = (value: unknown, label: string): JsonRecord => {
|
|
1548
|
+
const auth = record(value, label);
|
|
1549
|
+
exactKeys(auth, ["accessToken", "tokenType"], [
|
|
1550
|
+
"clientID",
|
|
1551
|
+
"expiresAt",
|
|
1552
|
+
"scope",
|
|
1553
|
+
"source",
|
|
1554
|
+
], label);
|
|
1555
|
+
const accessToken = string(auth.accessToken, `${label}.accessToken`, 64 * 1024);
|
|
1556
|
+
if (auth.tokenType !== "Bearer") return fail(`${label}.tokenType is unsupported`);
|
|
1557
|
+
const optionalText = (key: "clientID" | "scope", maximum: number): string | undefined =>
|
|
1558
|
+
auth[key] === undefined ? undefined : string(auth[key], `${label}.${key}`, maximum);
|
|
1559
|
+
const clientID = optionalText("clientID", 2_048);
|
|
1560
|
+
const scope = optionalText("scope", 2_048);
|
|
1561
|
+
const expiresAt = auth.expiresAt === undefined
|
|
1562
|
+
? undefined
|
|
1563
|
+
: timestamp(auth.expiresAt, `${label}.expiresAt`);
|
|
1564
|
+
const source = auth.source === undefined
|
|
1565
|
+
? undefined
|
|
1566
|
+
: string(auth.source, `${label}.source`, 64);
|
|
1567
|
+
if (
|
|
1568
|
+
source !== undefined
|
|
1569
|
+
&& !["desktop-db", "desktop-cache", "desktop-oauth", "remote-oauth", "manual"].includes(source)
|
|
1570
|
+
) return fail(`${label}.source is unsupported`);
|
|
1571
|
+
return Object.freeze({
|
|
1572
|
+
accessToken,
|
|
1573
|
+
tokenType: "Bearer",
|
|
1574
|
+
...(clientID === undefined ? {} : { clientID }),
|
|
1575
|
+
...(expiresAt === undefined ? {} : { expiresAt }),
|
|
1576
|
+
...(scope === undefined ? {} : { scope }),
|
|
1577
|
+
...(source === undefined ? {} : { source }),
|
|
1578
|
+
});
|
|
1579
|
+
};
|
|
1580
|
+
const assertPrivateSourceFile = async (path: string, label: string): Promise<void> => {
|
|
1581
|
+
const metadata = await lstat(path);
|
|
1582
|
+
if (
|
|
1583
|
+
!metadata.isFile()
|
|
1584
|
+
|| metadata.isSymbolicLink()
|
|
1585
|
+
|| metadata.uid !== process.getuid?.()
|
|
1586
|
+
|| metadata.nlink !== 1
|
|
1587
|
+
|| (metadata.mode & 0o077) !== 0
|
|
1588
|
+
|| await realpath(path) !== path
|
|
1589
|
+
) return fail(`${label} is no longer one private physical file`);
|
|
1590
|
+
};
|
|
1591
|
+
const configPath = join(configDirectory, "config.json");
|
|
1592
|
+
await assertPrivateSourceFile(configPath, "Beeper CLI config");
|
|
1593
|
+
const config = record(
|
|
1594
|
+
await readOwnedJson(
|
|
1595
|
+
configPath,
|
|
1596
|
+
configDirectory,
|
|
1597
|
+
MAX_BEEPER_CONFIG_BYTES,
|
|
1598
|
+
true,
|
|
1599
|
+
),
|
|
1600
|
+
"Beeper CLI config",
|
|
1601
|
+
);
|
|
1602
|
+
exactKeys(
|
|
1603
|
+
config,
|
|
1604
|
+
[],
|
|
1605
|
+
["auth", "baseURL", "defaultAccount", "defaultTarget"],
|
|
1606
|
+
"Beeper CLI config",
|
|
1607
|
+
);
|
|
1608
|
+
if (config.defaultTarget !== "desktop") {
|
|
1609
|
+
return fail("Beeper CLI config no longer selects the fixed Desktop target");
|
|
1610
|
+
}
|
|
1611
|
+
const configBaseUrl = assertLocalDesktopUrl(
|
|
1612
|
+
config.baseURL,
|
|
1613
|
+
"Beeper CLI config.baseURL",
|
|
1614
|
+
);
|
|
1615
|
+
const configAuth = config.auth === undefined
|
|
1616
|
+
? undefined
|
|
1617
|
+
: storedAuth(config.auth, "Beeper CLI config.auth");
|
|
1618
|
+
const targetsRoot = await assertOwnedDirectory(
|
|
1619
|
+
join(configDirectory, "targets"),
|
|
1620
|
+
configDirectory,
|
|
1621
|
+
);
|
|
1622
|
+
const targetPath = join(targetsRoot, "desktop.json");
|
|
1623
|
+
await assertPrivateSourceFile(targetPath, "Beeper Desktop target");
|
|
1624
|
+
const desktopTarget = record(
|
|
1625
|
+
await readOwnedJson(
|
|
1626
|
+
targetPath,
|
|
1627
|
+
configDirectory,
|
|
1628
|
+
MAX_BEEPER_CONFIG_BYTES,
|
|
1629
|
+
true,
|
|
1630
|
+
),
|
|
1631
|
+
"Beeper Desktop target",
|
|
1632
|
+
);
|
|
1633
|
+
exactKeys(desktopTarget, ["id", "type", "baseURL"], [
|
|
1634
|
+
"auth",
|
|
1635
|
+
"dataDir",
|
|
1636
|
+
"managed",
|
|
1637
|
+
"name",
|
|
1638
|
+
"port",
|
|
1639
|
+
"profile",
|
|
1640
|
+
"runtime",
|
|
1641
|
+
"serverEnv",
|
|
1642
|
+
], "Beeper Desktop target");
|
|
1643
|
+
if (desktopTarget.id !== "desktop" || desktopTarget.type !== "desktop") {
|
|
1644
|
+
return fail("Beeper CLI target no longer identifies the fixed Desktop realm");
|
|
1645
|
+
}
|
|
1646
|
+
const targetBaseUrl = assertLocalDesktopUrl(
|
|
1647
|
+
desktopTarget.baseURL,
|
|
1648
|
+
"Beeper Desktop target.baseURL",
|
|
1649
|
+
);
|
|
1650
|
+
if (targetBaseUrl === undefined) return fail("Beeper Desktop target omitted baseURL");
|
|
1651
|
+
const targetBasePort = Number(new URL(targetBaseUrl).port);
|
|
1652
|
+
if (
|
|
1653
|
+
(desktopTarget.managed !== undefined && desktopTarget.managed !== false)
|
|
1654
|
+
|| desktopTarget.dataDir !== undefined
|
|
1655
|
+
|| desktopTarget.profile !== undefined
|
|
1656
|
+
|| desktopTarget.serverEnv !== undefined
|
|
1657
|
+
) return fail("Beeper Desktop target contains an active endpoint override");
|
|
1658
|
+
if (desktopTarget.port !== undefined) {
|
|
1659
|
+
const port = positiveInteger(
|
|
1660
|
+
desktopTarget.port,
|
|
1661
|
+
"Beeper Desktop target.port",
|
|
1662
|
+
23_392,
|
|
1663
|
+
);
|
|
1664
|
+
if (port < 23_373 || port !== targetBasePort) {
|
|
1665
|
+
return fail("Beeper Desktop target.port did not match its exact base URL");
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
if (desktopTarget.runtime !== undefined) {
|
|
1669
|
+
const runtime = record(desktopTarget.runtime, "Beeper Desktop target.runtime");
|
|
1670
|
+
exactKeys(runtime, ["install", "port"], [], "Beeper Desktop target.runtime");
|
|
1671
|
+
if (runtime.install !== "desktop") {
|
|
1672
|
+
return fail("Beeper Desktop target.runtime.install is unsupported");
|
|
1673
|
+
}
|
|
1674
|
+
const port = positiveInteger(
|
|
1675
|
+
runtime.port,
|
|
1676
|
+
"Beeper Desktop target.runtime.port",
|
|
1677
|
+
23_392,
|
|
1678
|
+
);
|
|
1679
|
+
if (port < 23_373 || port !== targetBasePort) {
|
|
1680
|
+
return fail("Beeper Desktop target.runtime.port did not match its exact base URL");
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
if (desktopTarget.name !== undefined) {
|
|
1684
|
+
string(desktopTarget.name, "Beeper Desktop target.name", 2_048);
|
|
1685
|
+
}
|
|
1686
|
+
const targetAuth = desktopTarget.auth === undefined
|
|
1687
|
+
? undefined
|
|
1688
|
+
: storedAuth(desktopTarget.auth, "Beeper Desktop target.auth");
|
|
1689
|
+
const effectiveAuth = targetAuth
|
|
1690
|
+
?? (configAuth !== undefined
|
|
1691
|
+
&& (configBaseUrl === undefined || configBaseUrl === targetBaseUrl)
|
|
1692
|
+
? configAuth
|
|
1693
|
+
: undefined);
|
|
1694
|
+
if (effectiveAuth === undefined) {
|
|
1695
|
+
return fail("Beeper Desktop target has no effective stored access token");
|
|
1696
|
+
}
|
|
1697
|
+
return Object.freeze({
|
|
1698
|
+
config: Object.freeze({
|
|
1699
|
+
baseURL: targetBaseUrl,
|
|
1700
|
+
defaultTarget: "desktop",
|
|
1701
|
+
}),
|
|
1702
|
+
desktopTarget: Object.freeze({
|
|
1703
|
+
auth: effectiveAuth,
|
|
1704
|
+
baseURL: targetBaseUrl,
|
|
1705
|
+
id: "desktop",
|
|
1706
|
+
managed: false,
|
|
1707
|
+
type: "desktop",
|
|
1708
|
+
}),
|
|
1709
|
+
targetBaseUrl,
|
|
1710
|
+
});
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
async function writePrivateJsonExclusive(path: string, value: unknown): Promise<void> {
|
|
1714
|
+
const bytes = Buffer.from(`${JSON.stringify(value)}\n`, "utf8");
|
|
1715
|
+
if (bytes.byteLength < 2 || bytes.byteLength > MAX_BEEPER_CONFIG_BYTES) {
|
|
1716
|
+
return fail("operation-private Beeper config exceeded its size bound");
|
|
1717
|
+
}
|
|
1718
|
+
const handle = await open(
|
|
1719
|
+
path,
|
|
1720
|
+
constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW,
|
|
1721
|
+
PRIVATE_FILE_MODE,
|
|
1722
|
+
);
|
|
1723
|
+
try {
|
|
1724
|
+
await handle.chmod(PRIVATE_FILE_MODE);
|
|
1725
|
+
let offset = 0;
|
|
1726
|
+
while (offset < bytes.byteLength) {
|
|
1727
|
+
const result = await handle.write(bytes, offset, bytes.byteLength - offset, offset);
|
|
1728
|
+
if (result.bytesWritten === 0) return fail("operation-private Beeper config stopped accepting bytes");
|
|
1729
|
+
offset += result.bytesWritten;
|
|
1730
|
+
}
|
|
1731
|
+
await handle.sync();
|
|
1732
|
+
const metadata = await handle.stat();
|
|
1733
|
+
if (
|
|
1734
|
+
!metadata.isFile()
|
|
1735
|
+
|| metadata.uid !== process.getuid?.()
|
|
1736
|
+
|| metadata.nlink !== 1
|
|
1737
|
+
|| (metadata.mode & 0o777) !== PRIVATE_FILE_MODE
|
|
1738
|
+
|| metadata.size !== bytes.byteLength
|
|
1739
|
+
) return fail("operation-private Beeper config was not written privately");
|
|
1740
|
+
} finally {
|
|
1741
|
+
await handle.close();
|
|
1742
|
+
}
|
|
1743
|
+
const metadata = await lstat(path);
|
|
1744
|
+
if (metadata.isSymbolicLink() || metadata.nlink !== 1) {
|
|
1745
|
+
return fail("operation-private Beeper config changed after creation");
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
async function createOperationPrivateBeeperStore(
|
|
1750
|
+
root: string,
|
|
1751
|
+
snapshot: BeeperCliStoreSnapshot,
|
|
1752
|
+
accountId?: string,
|
|
1753
|
+
): Promise<OperationPrivateBeeperStore> {
|
|
1754
|
+
await mkdir(root, { mode: PRIVATE_DIRECTORY_MODE });
|
|
1755
|
+
await chmod(root, PRIVATE_DIRECTORY_MODE);
|
|
1756
|
+
const targets = join(root, "targets");
|
|
1757
|
+
await mkdir(targets, { mode: PRIVATE_DIRECTORY_MODE });
|
|
1758
|
+
await chmod(targets, PRIVATE_DIRECTORY_MODE);
|
|
1759
|
+
await assertPrivateOwnedDirectory(root);
|
|
1760
|
+
await assertPrivateOwnedDirectory(targets, root);
|
|
1761
|
+
const config = Object.freeze({
|
|
1762
|
+
...snapshot.config,
|
|
1763
|
+
...(accountId === undefined ? {} : { defaultAccount: accountId }),
|
|
1764
|
+
});
|
|
1765
|
+
await writePrivateJsonExclusive(join(root, "config.json"), config);
|
|
1766
|
+
await writePrivateJsonExclusive(join(targets, "desktop.json"), snapshot.desktopTarget);
|
|
1767
|
+
try {
|
|
1768
|
+
const path = await validateBeeperCliStore(root);
|
|
1769
|
+
const metadata = await lstat(path);
|
|
1770
|
+
return Object.freeze({
|
|
1771
|
+
path,
|
|
1772
|
+
identity: Object.freeze({ device: metadata.dev, inode: metadata.ino }),
|
|
1773
|
+
});
|
|
1774
|
+
} catch {
|
|
1775
|
+
return fail("operation-private Beeper selector store did not validate");
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
|
|
1779
|
+
async function validateAccountShard(
|
|
1780
|
+
rawRoot: string,
|
|
1781
|
+
canonicalWorking: string,
|
|
1782
|
+
selected: BeeperAccountProjection,
|
|
1783
|
+
baselineRealmDigest: string,
|
|
1784
|
+
signal: AbortSignal | undefined,
|
|
1785
|
+
): Promise<ValidatedAccountShard> {
|
|
1786
|
+
throwIfAborted(signal);
|
|
1787
|
+
await chmod(rawRoot, PRIVATE_DIRECTORY_MODE);
|
|
1788
|
+
await assertPrivateOwnedDirectory(rawRoot, canonicalWorking);
|
|
1789
|
+
const accounts = parseBeeperExportAccounts(await readOwnedJson(
|
|
1790
|
+
join(rawRoot, "accounts.json"),
|
|
1791
|
+
rawRoot,
|
|
1792
|
+
MAX_ACCOUNTS_JSON_BYTES,
|
|
1793
|
+
false,
|
|
1794
|
+
signal,
|
|
1795
|
+
));
|
|
1796
|
+
throwIfAborted(signal);
|
|
1797
|
+
assertOutputRealm(accounts, baselineRealmDigest);
|
|
1798
|
+
const manifest = parseManifest(await readOwnedJson(
|
|
1799
|
+
join(rawRoot, "manifest.json"),
|
|
1800
|
+
rawRoot,
|
|
1801
|
+
MAX_ACCOUNTS_JSON_BYTES,
|
|
1802
|
+
false,
|
|
1803
|
+
signal,
|
|
1804
|
+
));
|
|
1805
|
+
const manifestAccounts = parseBeeperExportAccounts(manifest.accounts);
|
|
1806
|
+
if (
|
|
1807
|
+
manifest.attachmentCount !== 0
|
|
1808
|
+
|| canonicalJson(manifestAccounts) !== canonicalJson(accounts)
|
|
1809
|
+
|| outputRealmDigest(manifestAccounts) !== outputRealmDigest(accounts)
|
|
1810
|
+
) return fail("official export manifest did not prove a no-attachment account snapshot");
|
|
1811
|
+
const listedValues = array(
|
|
1812
|
+
await readOwnedJson(
|
|
1813
|
+
join(rawRoot, "chats.json"),
|
|
1814
|
+
rawRoot,
|
|
1815
|
+
MAX_CHATS_JSON_BYTES,
|
|
1816
|
+
false,
|
|
1817
|
+
signal,
|
|
1818
|
+
),
|
|
1819
|
+
"official export chats",
|
|
1820
|
+
MAX_EXPORT_CHATS,
|
|
1821
|
+
);
|
|
1822
|
+
const listedChats: Array<Pick<BeeperConversationProjection, "id" | "accountId">> = [];
|
|
1823
|
+
for (const value of listedValues) {
|
|
1824
|
+
throwIfAborted(signal);
|
|
1825
|
+
const chat = parseBeeperExportConversation(value, accounts);
|
|
1826
|
+
listedChats.push(Object.freeze({ id: chat.id, accountId: chat.accountId }));
|
|
1827
|
+
}
|
|
1828
|
+
if (manifest.chatCount !== listedChats.length) {
|
|
1829
|
+
return fail("official export manifest chat count did not match chats.json");
|
|
1830
|
+
}
|
|
1831
|
+
if (listedChats.some((chat) => chat.accountId !== selected.accountId)) {
|
|
1832
|
+
return fail("official per-account export crossed its selected account boundary");
|
|
1833
|
+
}
|
|
1834
|
+
await assertExactDirectoryEntries(rawRoot, [
|
|
1835
|
+
...(listedChats.length === 0 ? [] : [".beeper-export-state.json"]),
|
|
1836
|
+
"accounts.json",
|
|
1837
|
+
"chats",
|
|
1838
|
+
"chats.json",
|
|
1839
|
+
"manifest.json",
|
|
1840
|
+
], "official export root");
|
|
1841
|
+
throwIfAborted(signal);
|
|
1842
|
+
const chatSegments = listedChats.map((chat) => safeSegment(chat.id));
|
|
1843
|
+
if (new Set(chatSegments).size !== chatSegments.length) {
|
|
1844
|
+
return fail("official export chat directory names collided");
|
|
1845
|
+
}
|
|
1846
|
+
const chatsRoot = await assertOwnedDirectory(join(rawRoot, "chats"), rawRoot);
|
|
1847
|
+
await assertExactDirectoryEntries(chatsRoot, chatSegments, "official export chats");
|
|
1848
|
+
const stateMessageCounts = listedChats.length === 0
|
|
1849
|
+
? new Map<string, number>()
|
|
1850
|
+
: parseOfficialState(await readOwnedJson(
|
|
1851
|
+
join(rawRoot, ".beeper-export-state.json"),
|
|
1852
|
+
rawRoot,
|
|
1853
|
+
MAX_CHATS_JSON_BYTES,
|
|
1854
|
+
false,
|
|
1855
|
+
signal,
|
|
1856
|
+
), listedChats.map((chat) => chat.id), manifest.createdAt);
|
|
1857
|
+
let officialMessageCount = 0;
|
|
1858
|
+
const chats: ValidatedShardChat[] = [];
|
|
1859
|
+
for (const [index, listedChat] of listedChats.entries()) {
|
|
1860
|
+
throwIfAborted(signal);
|
|
1861
|
+
const segment = chatSegments[index];
|
|
1862
|
+
if (segment === undefined) return fail("official export chat segment disappeared");
|
|
1863
|
+
const directory = await assertOwnedDirectory(join(chatsRoot, segment), rawRoot);
|
|
1864
|
+
await assertExactDirectoryEntries(directory, [
|
|
1865
|
+
"attachments",
|
|
1866
|
+
"chat.json",
|
|
1867
|
+
"messages.html",
|
|
1868
|
+
"messages.json",
|
|
1869
|
+
"messages.markdown",
|
|
1870
|
+
], "official export chat directory");
|
|
1871
|
+
const attachmentsDirectory = await assertOwnedDirectory(
|
|
1872
|
+
join(directory, "attachments"),
|
|
1873
|
+
rawRoot,
|
|
1874
|
+
);
|
|
1875
|
+
await assertExactDirectoryEntries(
|
|
1876
|
+
attachmentsDirectory,
|
|
1877
|
+
[],
|
|
1878
|
+
"official export attachments directory",
|
|
1879
|
+
);
|
|
1880
|
+
const chatPath = join(directory, "chat.json");
|
|
1881
|
+
const chatDocument = await readOwnedJsonDocument(
|
|
1882
|
+
chatPath,
|
|
1883
|
+
rawRoot,
|
|
1884
|
+
MAX_CHAT_JSON_BYTES,
|
|
1885
|
+
false,
|
|
1886
|
+
signal,
|
|
1887
|
+
);
|
|
1888
|
+
const chat = parseBeeperExportConversation(chatDocument.value, accounts);
|
|
1889
|
+
if (
|
|
1890
|
+
chat.id !== listedChat.id
|
|
1891
|
+
|| chat.accountId !== listedChat.accountId
|
|
1892
|
+
|| chat.accountId !== selected.accountId
|
|
1893
|
+
) return fail("official export chat detail crossed its selected account boundary");
|
|
1894
|
+
const expectedMessageCount = stateMessageCounts.get(chat.id);
|
|
1895
|
+
if (expectedMessageCount === undefined) {
|
|
1896
|
+
return fail("official export chat state disappeared");
|
|
1897
|
+
}
|
|
1898
|
+
officialMessageCount += expectedMessageCount;
|
|
1899
|
+
if (!Number.isSafeInteger(officialMessageCount)) {
|
|
1900
|
+
return fail("official export message count overflowed");
|
|
1901
|
+
}
|
|
1902
|
+
try {
|
|
1903
|
+
await unlink(join(directory, "messages.markdown"));
|
|
1904
|
+
await unlink(join(directory, "messages.html"));
|
|
1905
|
+
} catch {
|
|
1906
|
+
return fail("redundant official export renderings could not be removed safely");
|
|
1907
|
+
}
|
|
1908
|
+
await assertExactDirectoryEntries(directory, [
|
|
1909
|
+
"attachments",
|
|
1910
|
+
"chat.json",
|
|
1911
|
+
"messages.json",
|
|
1912
|
+
], "sanitized official export chat directory");
|
|
1913
|
+
chats.push(Object.freeze({
|
|
1914
|
+
chat: Object.freeze({ id: chat.id, accountId: chat.accountId }),
|
|
1915
|
+
root: rawRoot,
|
|
1916
|
+
chatPath,
|
|
1917
|
+
chatSha256: chatDocument.sha256,
|
|
1918
|
+
messagesPath: join(directory, "messages.json"),
|
|
1919
|
+
expectedMessageCount,
|
|
1920
|
+
observedAt: manifest.completedAt,
|
|
1921
|
+
}));
|
|
1922
|
+
}
|
|
1923
|
+
if (officialMessageCount !== manifest.messageCount) {
|
|
1924
|
+
return fail("official export manifest message count did not match chat state");
|
|
1925
|
+
}
|
|
1926
|
+
throwIfAborted(signal);
|
|
1927
|
+
return Object.freeze({
|
|
1928
|
+
accountId: selected.accountId,
|
|
1929
|
+
completedAt: manifest.completedAt,
|
|
1930
|
+
chats: Object.freeze(chats),
|
|
1931
|
+
messageCount: officialMessageCount,
|
|
1932
|
+
});
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
async function prepassSelfAliases(
|
|
1936
|
+
chats: readonly ValidatedShardChat[],
|
|
1937
|
+
accounts: readonly BeeperAccountProjection[],
|
|
1938
|
+
accountsById: ReadonlyMap<string, BeeperAccountProjection>,
|
|
1939
|
+
maxMessagesJsonBytes: number,
|
|
1940
|
+
maxBundleRecords: number,
|
|
1941
|
+
maxBundleBytes: number,
|
|
1942
|
+
maxParticipantOccurrences: number,
|
|
1943
|
+
observedAtByAccount: ReadonlyMap<string, string>,
|
|
1944
|
+
signal: AbortSignal | undefined,
|
|
1945
|
+
heartbeat: () => void,
|
|
1946
|
+
onRefinementPass: ((pass: number) => Promise<void>) | undefined,
|
|
1947
|
+
): Promise<SelfAliasPrepass> {
|
|
1948
|
+
const evidenceSet = (
|
|
1949
|
+
collection: Map<string, Set<string>>,
|
|
1950
|
+
accountId: string,
|
|
1951
|
+
): Set<string> => {
|
|
1952
|
+
let values = collection.get(accountId);
|
|
1953
|
+
if (values === undefined) {
|
|
1954
|
+
values = new Set();
|
|
1955
|
+
collection.set(accountId, values);
|
|
1956
|
+
}
|
|
1957
|
+
return values;
|
|
1958
|
+
};
|
|
1959
|
+
type EvidencePass = Readonly<{
|
|
1960
|
+
aliasesByAccount: ReadonlyMap<string, ReadonlySet<string>>;
|
|
1961
|
+
coveredChats: readonly ValidatedShardChat[];
|
|
1962
|
+
evidenceLimitReached: boolean;
|
|
1963
|
+
identityConflict: boolean;
|
|
1964
|
+
messagesSha256ByPath: ReadonlyMap<string, string>;
|
|
1965
|
+
oversizedMessagesPaths: ReadonlySet<string>;
|
|
1966
|
+
participantOccurrenceLimitReached: boolean;
|
|
1967
|
+
}>;
|
|
1968
|
+
type BudgetPass = Readonly<{
|
|
1969
|
+
admittedChats: readonly ValidatedShardChat[];
|
|
1970
|
+
bundleByteLimitReached: boolean;
|
|
1971
|
+
bundleRecordLimitReached: boolean;
|
|
1972
|
+
provisionalRecordBytes: number;
|
|
1973
|
+
provisionalRecordCount: number;
|
|
1974
|
+
}>;
|
|
1975
|
+
const boundMessagesSha256ByPath = new Map<string, string>();
|
|
1976
|
+
const boundOversizedMessagesPaths = new Set<string>();
|
|
1977
|
+
|
|
1978
|
+
const collectEvidence = async (
|
|
1979
|
+
candidateChats: readonly ValidatedShardChat[],
|
|
1980
|
+
): Promise<EvidencePass> => {
|
|
1981
|
+
const aliasesByAccount = new Map<string, Set<string>>();
|
|
1982
|
+
const peerEvidenceByAccount = new Map<string, Set<string>>();
|
|
1983
|
+
const evidenceIdsByAccount = new Map<string, Set<string>>();
|
|
1984
|
+
const coveredChats: ValidatedShardChat[] = [];
|
|
1985
|
+
const messagesSha256ByPath = new Map<string, string>();
|
|
1986
|
+
const oversizedMessagesPaths = new Set<string>();
|
|
1987
|
+
let evidenceCount = 0;
|
|
1988
|
+
let participantOccurrenceCount = 0;
|
|
1989
|
+
for (const account of accounts) {
|
|
1990
|
+
if (account.user.isSelf === false) {
|
|
1991
|
+
return fail("Beeper account user contradicts its self identity anchor");
|
|
1992
|
+
}
|
|
1993
|
+
const coordinate = digest([
|
|
1994
|
+
"beeper-self-alias-v1",
|
|
1995
|
+
account.accountId,
|
|
1996
|
+
account.user.id,
|
|
1997
|
+
]);
|
|
1998
|
+
evidenceSet(aliasesByAccount, account.accountId).add(coordinate);
|
|
1999
|
+
evidenceSet(evidenceIdsByAccount, account.accountId).add(coordinate);
|
|
2000
|
+
evidenceCount += 1;
|
|
2001
|
+
participantOccurrenceCount += 1;
|
|
2002
|
+
}
|
|
2003
|
+
if (participantOccurrenceCount > maxParticipantOccurrences) {
|
|
2004
|
+
return fail("connected Beeper accounts exceed the participant work bound");
|
|
2005
|
+
}
|
|
2006
|
+
let evidenceLimitReached = evidenceCount > maxBundleRecords;
|
|
2007
|
+
let participantOccurrenceLimitReached = false;
|
|
2008
|
+
for (const validated of candidateChats) {
|
|
2009
|
+
if (evidenceLimitReached || participantOccurrenceLimitReached) break;
|
|
2010
|
+
heartbeat();
|
|
2011
|
+
throwIfAborted(signal);
|
|
2012
|
+
const chatDocument = await readOwnedJsonDocument(
|
|
2013
|
+
validated.chatPath,
|
|
2014
|
+
validated.root,
|
|
2015
|
+
MAX_CHAT_JSON_BYTES,
|
|
2016
|
+
false,
|
|
2017
|
+
signal,
|
|
2018
|
+
);
|
|
2019
|
+
if (chatDocument.sha256 !== validated.chatSha256) {
|
|
2020
|
+
return fail("official export chat changed before self-alias prepass");
|
|
2021
|
+
}
|
|
2022
|
+
const chat = parseBeeperExportConversation(chatDocument.value, accounts);
|
|
2023
|
+
if (
|
|
2024
|
+
chat.id !== validated.chat.id
|
|
2025
|
+
|| chat.accountId !== validated.chat.accountId
|
|
2026
|
+
) return fail("official export chat identity changed before self-alias prepass");
|
|
2027
|
+
const account = accountsById.get(chat.accountId);
|
|
2028
|
+
if (account === undefined) {
|
|
2029
|
+
return fail("official export chat references an unknown account");
|
|
2030
|
+
}
|
|
2031
|
+
const rosterOccurrences = chat.participants.items.length;
|
|
2032
|
+
if (rosterOccurrences > maxParticipantOccurrences - participantOccurrenceCount) {
|
|
2033
|
+
participantOccurrenceLimitReached = true;
|
|
2034
|
+
break;
|
|
2035
|
+
}
|
|
2036
|
+
const messagesSize = await ownedFileSize(validated.messagesPath, validated.root);
|
|
2037
|
+
const priorSha256 = boundMessagesSha256ByPath.get(validated.messagesPath);
|
|
2038
|
+
if (messagesSize > maxMessagesJsonBytes) {
|
|
2039
|
+
if (priorSha256 !== undefined) {
|
|
2040
|
+
return fail("official export messages changed between self-alias passes");
|
|
2041
|
+
}
|
|
2042
|
+
boundOversizedMessagesPaths.add(validated.messagesPath);
|
|
2043
|
+
oversizedMessagesPaths.add(validated.messagesPath);
|
|
2044
|
+
participantOccurrenceCount += rosterOccurrences;
|
|
2045
|
+
coveredChats.push(validated);
|
|
2046
|
+
continue;
|
|
2047
|
+
}
|
|
2048
|
+
if (boundOversizedMessagesPaths.has(validated.messagesPath)) {
|
|
2049
|
+
return fail("official export messages changed between self-alias passes");
|
|
2050
|
+
}
|
|
2051
|
+
const messagesDocument = await readOwnedJsonDocument(
|
|
2052
|
+
validated.messagesPath,
|
|
2053
|
+
validated.root,
|
|
2054
|
+
maxMessagesJsonBytes,
|
|
2055
|
+
false,
|
|
2056
|
+
signal,
|
|
2057
|
+
);
|
|
2058
|
+
if (priorSha256 !== undefined && priorSha256 !== messagesDocument.sha256) {
|
|
2059
|
+
return fail("official export messages changed between self-alias passes");
|
|
2060
|
+
}
|
|
2061
|
+
boundMessagesSha256ByPath.set(validated.messagesPath, messagesDocument.sha256);
|
|
2062
|
+
const messages = parseBeeperExportMessages(
|
|
2063
|
+
messagesDocument.value,
|
|
2064
|
+
chat.accountId,
|
|
2065
|
+
chat.id,
|
|
2066
|
+
MAX_EXPORT_MESSAGES_PER_CHAT,
|
|
2067
|
+
);
|
|
2068
|
+
if (messages.length !== validated.expectedMessageCount) {
|
|
2069
|
+
return fail("official export chat messages did not match completed state");
|
|
2070
|
+
}
|
|
2071
|
+
const reactionCount = messages.reduce(
|
|
2072
|
+
(count, message) => count + message.reactions.length,
|
|
2073
|
+
0,
|
|
2074
|
+
);
|
|
2075
|
+
const participantOccurrences = rosterOccurrences
|
|
2076
|
+
+ messages.length
|
|
2077
|
+
+ reactionCount
|
|
2078
|
+
+ (chat.type === "single" ? 1 : 0);
|
|
2079
|
+
if (
|
|
2080
|
+
!Number.isSafeInteger(reactionCount)
|
|
2081
|
+
|| !Number.isSafeInteger(participantOccurrences)
|
|
2082
|
+
) return fail("official export derived record count overflowed");
|
|
2083
|
+
if (participantOccurrences > maxParticipantOccurrences - participantOccurrenceCount) {
|
|
2084
|
+
participantOccurrenceLimitReached = true;
|
|
2085
|
+
break;
|
|
2086
|
+
}
|
|
2087
|
+
const positiveEvidence = new Set<string>();
|
|
2088
|
+
const peerEvidence = new Set<string>();
|
|
2089
|
+
for (const participant of chat.participants.items) {
|
|
2090
|
+
const coordinate = digest([
|
|
2091
|
+
"beeper-self-alias-v1",
|
|
2092
|
+
account.accountId,
|
|
2093
|
+
participant.id,
|
|
2094
|
+
]);
|
|
2095
|
+
if (participant.isSelf === true) positiveEvidence.add(coordinate);
|
|
2096
|
+
if (participant.isSelf === false) peerEvidence.add(coordinate);
|
|
2097
|
+
}
|
|
2098
|
+
for (const message of messages) {
|
|
2099
|
+
(message.isSender ? positiveEvidence : peerEvidence).add(digest([
|
|
2100
|
+
"beeper-self-alias-v1",
|
|
2101
|
+
account.accountId,
|
|
2102
|
+
message.senderId,
|
|
2103
|
+
]));
|
|
2104
|
+
}
|
|
2105
|
+
const accountEvidenceIds = evidenceSet(evidenceIdsByAccount, account.accountId);
|
|
2106
|
+
const candidateEvidence = new Set([...positiveEvidence, ...peerEvidence]);
|
|
2107
|
+
let newEvidence = 0;
|
|
2108
|
+
for (const sourceId of candidateEvidence) {
|
|
2109
|
+
if (!accountEvidenceIds.has(sourceId)) newEvidence += 1;
|
|
2110
|
+
}
|
|
2111
|
+
if (newEvidence > maxBundleRecords - evidenceCount) {
|
|
2112
|
+
evidenceLimitReached = true;
|
|
2113
|
+
break;
|
|
2114
|
+
}
|
|
2115
|
+
const aliases = evidenceSet(aliasesByAccount, account.accountId);
|
|
2116
|
+
const peers = evidenceSet(peerEvidenceByAccount, account.accountId);
|
|
2117
|
+
for (const sourceId of positiveEvidence) aliases.add(sourceId);
|
|
2118
|
+
for (const sourceId of peerEvidence) peers.add(sourceId);
|
|
2119
|
+
for (const sourceId of candidateEvidence) accountEvidenceIds.add(sourceId);
|
|
2120
|
+
evidenceCount += newEvidence;
|
|
2121
|
+
participantOccurrenceCount += participantOccurrences;
|
|
2122
|
+
messagesSha256ByPath.set(validated.messagesPath, messagesDocument.sha256);
|
|
2123
|
+
coveredChats.push(validated);
|
|
2124
|
+
}
|
|
2125
|
+
let identityConflict = false;
|
|
2126
|
+
for (const [accountId, peerEvidence] of peerEvidenceByAccount) {
|
|
2127
|
+
const aliases = aliasesByAccount.get(accountId);
|
|
2128
|
+
if (aliases === undefined) {
|
|
2129
|
+
return fail("Beeper account self-alias set disappeared");
|
|
2130
|
+
}
|
|
2131
|
+
if ([...peerEvidence].some((sourceId) => aliases.has(sourceId))) {
|
|
2132
|
+
identityConflict = true;
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
return Object.freeze({
|
|
2136
|
+
aliasesByAccount,
|
|
2137
|
+
coveredChats: Object.freeze(coveredChats),
|
|
2138
|
+
evidenceLimitReached,
|
|
2139
|
+
identityConflict,
|
|
2140
|
+
messagesSha256ByPath,
|
|
2141
|
+
oversizedMessagesPaths,
|
|
2142
|
+
participantOccurrenceLimitReached,
|
|
2143
|
+
});
|
|
2144
|
+
};
|
|
2145
|
+
|
|
2146
|
+
const planBudget = async (evidence: EvidencePass): Promise<BudgetPass> => {
|
|
2147
|
+
const participantFacts = new Map<string, ParticipantFact>();
|
|
2148
|
+
const selfParticipantByAccount = new Map<string, string>();
|
|
2149
|
+
let recordCount = accounts.length;
|
|
2150
|
+
let recordBytes = 0;
|
|
2151
|
+
for (const account of accounts) {
|
|
2152
|
+
const observedAt = observedAtByAccount.get(account.accountId);
|
|
2153
|
+
if (observedAt === undefined) {
|
|
2154
|
+
return fail("Beeper account observation time disappeared");
|
|
2155
|
+
}
|
|
2156
|
+
const self = planningParticipantFact(
|
|
2157
|
+
account,
|
|
2158
|
+
account.user,
|
|
2159
|
+
true,
|
|
2160
|
+
evidence.aliasesByAccount,
|
|
2161
|
+
);
|
|
2162
|
+
participantFacts.set(self.id, self);
|
|
2163
|
+
selfParticipantByAccount.set(account.accountId, self.id);
|
|
2164
|
+
recordCount += 1;
|
|
2165
|
+
const network = normalizeNetwork(account.network, account.bridge.type);
|
|
2166
|
+
recordBytes += bundleRecordBytes(accountRecord(
|
|
2167
|
+
account,
|
|
2168
|
+
network,
|
|
2169
|
+
observedAt,
|
|
2170
|
+
self.id,
|
|
2171
|
+
));
|
|
2172
|
+
recordBytes += bundleRecordBytes(participantRecord(
|
|
2173
|
+
self,
|
|
2174
|
+
account,
|
|
2175
|
+
network,
|
|
2176
|
+
observedAt,
|
|
2177
|
+
));
|
|
2178
|
+
}
|
|
2179
|
+
if (recordCount > maxBundleRecords || recordBytes > maxBundleBytes) {
|
|
2180
|
+
return fail("connected Beeper accounts exceed the bounded bundle foundation");
|
|
2181
|
+
}
|
|
2182
|
+
const admittedChats: ValidatedShardChat[] = [];
|
|
2183
|
+
for (const validated of evidence.coveredChats) {
|
|
2184
|
+
heartbeat();
|
|
2185
|
+
throwIfAborted(signal);
|
|
2186
|
+
const messagesSha256 = evidence.messagesSha256ByPath.get(validated.messagesPath);
|
|
2187
|
+
if (messagesSha256 === undefined) {
|
|
2188
|
+
if (!evidence.oversizedMessagesPaths.has(validated.messagesPath)) {
|
|
2189
|
+
return fail("official export message proof disappeared during bundle admission");
|
|
2190
|
+
}
|
|
2191
|
+
if (
|
|
2192
|
+
await ownedFileSize(validated.messagesPath, validated.root)
|
|
2193
|
+
<= maxMessagesJsonBytes
|
|
2194
|
+
) return fail("official export messages changed during bundle admission");
|
|
2195
|
+
admittedChats.push(validated);
|
|
2196
|
+
continue;
|
|
2197
|
+
}
|
|
2198
|
+
const chatDocument = await readOwnedJsonDocument(
|
|
2199
|
+
validated.chatPath,
|
|
2200
|
+
validated.root,
|
|
2201
|
+
MAX_CHAT_JSON_BYTES,
|
|
2202
|
+
false,
|
|
2203
|
+
signal,
|
|
2204
|
+
);
|
|
2205
|
+
if (chatDocument.sha256 !== validated.chatSha256) {
|
|
2206
|
+
return fail("official export chat changed during bundle admission");
|
|
2207
|
+
}
|
|
2208
|
+
const chat = parseBeeperExportConversation(chatDocument.value, accounts);
|
|
2209
|
+
if (
|
|
2210
|
+
chat.id !== validated.chat.id
|
|
2211
|
+
|| chat.accountId !== validated.chat.accountId
|
|
2212
|
+
) return fail("official export chat identity changed during bundle admission");
|
|
2213
|
+
const account = accountsById.get(chat.accountId);
|
|
2214
|
+
if (account === undefined) {
|
|
2215
|
+
return fail("official export chat references an unknown account");
|
|
2216
|
+
}
|
|
2217
|
+
const messagesDocument = await readOwnedJsonDocument(
|
|
2218
|
+
validated.messagesPath,
|
|
2219
|
+
validated.root,
|
|
2220
|
+
maxMessagesJsonBytes,
|
|
2221
|
+
false,
|
|
2222
|
+
signal,
|
|
2223
|
+
);
|
|
2224
|
+
if (messagesDocument.sha256 !== messagesSha256) {
|
|
2225
|
+
return fail("official export messages changed during bundle admission");
|
|
2226
|
+
}
|
|
2227
|
+
const messages = parseBeeperExportMessages(
|
|
2228
|
+
messagesDocument.value,
|
|
2229
|
+
chat.accountId,
|
|
2230
|
+
chat.id,
|
|
2231
|
+
MAX_EXPORT_MESSAGES_PER_CHAT,
|
|
2232
|
+
);
|
|
2233
|
+
if (messages.length !== validated.expectedMessageCount) {
|
|
2234
|
+
return fail("official export chat messages did not match completed state");
|
|
2235
|
+
}
|
|
2236
|
+
const selfParticipantId = selfParticipantByAccount.get(account.accountId);
|
|
2237
|
+
const selfParticipant = selfParticipantId === undefined
|
|
2238
|
+
? undefined
|
|
2239
|
+
: participantFacts.get(selfParticipantId);
|
|
2240
|
+
if (selfParticipantId === undefined || selfParticipant === undefined) {
|
|
2241
|
+
return fail("Beeper account self participant disappeared");
|
|
2242
|
+
}
|
|
2243
|
+
const scanParticipantFacts = new Map<string, ParticipantFact>([
|
|
2244
|
+
[selfParticipantId, selfParticipant],
|
|
2245
|
+
]);
|
|
2246
|
+
const participantIds = new Set<string>();
|
|
2247
|
+
const addPlanningFact = (fact: ParticipantFact): void => {
|
|
2248
|
+
scanParticipantFacts.set(
|
|
2249
|
+
fact.id,
|
|
2250
|
+
mergePlanningParticipantFact(scanParticipantFacts.get(fact.id), fact),
|
|
2251
|
+
);
|
|
2252
|
+
participantIds.add(fact.id);
|
|
2253
|
+
};
|
|
2254
|
+
for (const participant of chat.participants.items) {
|
|
2255
|
+
addPlanningFact(planningParticipantFact(
|
|
2256
|
+
account,
|
|
2257
|
+
participant,
|
|
2258
|
+
participant.isSelf,
|
|
2259
|
+
evidence.aliasesByAccount,
|
|
2260
|
+
));
|
|
2261
|
+
}
|
|
2262
|
+
if (chat.type === "single") participantIds.add(selfParticipantId);
|
|
2263
|
+
const messageIds = new Set(messages.map((message) => message.id));
|
|
2264
|
+
for (const message of messages) {
|
|
2265
|
+
addPlanningFact(planningParticipantFact(account, {
|
|
2266
|
+
id: message.senderId,
|
|
2267
|
+
fullName: message.senderName,
|
|
2268
|
+
phoneNumber: null,
|
|
2269
|
+
email: null,
|
|
2270
|
+
username: null,
|
|
2271
|
+
}, message.isSender, evidence.aliasesByAccount));
|
|
2272
|
+
for (const reaction of message.reactions) {
|
|
2273
|
+
addPlanningFact(planningParticipantFact(account, {
|
|
2274
|
+
id: reaction.participantId,
|
|
2275
|
+
fullName: null,
|
|
2276
|
+
phoneNumber: null,
|
|
2277
|
+
email: null,
|
|
2278
|
+
username: null,
|
|
2279
|
+
}, null, evidence.aliasesByAccount));
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2282
|
+
const reactionCount = messages.reduce(
|
|
2283
|
+
(count, message) => count + message.reactions.length,
|
|
2284
|
+
0,
|
|
2285
|
+
);
|
|
2286
|
+
const tombstoneCount = messages.reduce(
|
|
2287
|
+
(count, message) => count + (message.isDeleted || message.isHidden ? 1 : 0),
|
|
2288
|
+
0,
|
|
2289
|
+
);
|
|
2290
|
+
const reactionProviderIdNonUniqueGroups = messages.reduce(
|
|
2291
|
+
(count, message) => count + new Set(
|
|
2292
|
+
message.reactions
|
|
2293
|
+
.filter((reaction) => reaction.providerIdNonUnique)
|
|
2294
|
+
.map((reaction) => reaction.id),
|
|
2295
|
+
).size,
|
|
2296
|
+
0,
|
|
2297
|
+
);
|
|
2298
|
+
const range = messageTimestampRange(messages);
|
|
2299
|
+
const roster = [...participantIds].map((participantId) => {
|
|
2300
|
+
const participant = scanParticipantFacts.get(participantId);
|
|
2301
|
+
if (participant === undefined) {
|
|
2302
|
+
return fail("Beeper conversation participant disappeared");
|
|
2303
|
+
}
|
|
2304
|
+
return participant;
|
|
2305
|
+
});
|
|
2306
|
+
const directRosterComplete = chat.type !== "single"
|
|
2307
|
+
|| (
|
|
2308
|
+
roster.length === 2
|
|
2309
|
+
&& roster.filter((participant) => participant.isSelf === true).length === 1
|
|
2310
|
+
&& roster.filter((participant) => participant.isSelf !== true).length === 1
|
|
2311
|
+
);
|
|
2312
|
+
const scan: ConversationScan = Object.freeze({
|
|
2313
|
+
chat: Object.freeze({
|
|
2314
|
+
id: chat.id,
|
|
2315
|
+
accountId: chat.accountId,
|
|
2316
|
+
lastActivity: chat.lastActivity,
|
|
2317
|
+
title: chat.title,
|
|
2318
|
+
type: chat.type,
|
|
2319
|
+
}),
|
|
2320
|
+
root: validated.root,
|
|
2321
|
+
messagesPath: validated.messagesPath,
|
|
2322
|
+
messagesSha256: messagesDocument.sha256,
|
|
2323
|
+
observedAt: validated.observedAt,
|
|
2324
|
+
participantIds: Object.freeze([...participantIds].sort()),
|
|
2325
|
+
participantFactChanges: Object.freeze([]),
|
|
2326
|
+
participantsComplete: !chat.participants.hasMore
|
|
2327
|
+
&& chat.participants.items.length === chat.participants.total
|
|
2328
|
+
&& directRosterComplete,
|
|
2329
|
+
startedAt: range.first,
|
|
2330
|
+
lastMessageAt: range.last,
|
|
2331
|
+
messageCount: messages.length,
|
|
2332
|
+
reactionCount,
|
|
2333
|
+
reactionProviderIdNonUniqueGroups,
|
|
2334
|
+
tombstoneCount,
|
|
2335
|
+
nonParticipantRecordBytes: 0,
|
|
2336
|
+
});
|
|
2337
|
+
const network = normalizeNetwork(account.network, account.bridge.type);
|
|
2338
|
+
let nonParticipantBytes = bundleRecordBytes(conversationRecord(
|
|
2339
|
+
scan,
|
|
2340
|
+
account,
|
|
2341
|
+
network,
|
|
2342
|
+
validated.observedAt,
|
|
2343
|
+
));
|
|
2344
|
+
for (const message of messages) {
|
|
2345
|
+
nonParticipantBytes += bundleRecordBytes(messageRecord(
|
|
2346
|
+
message,
|
|
2347
|
+
scan,
|
|
2348
|
+
messageIds,
|
|
2349
|
+
account,
|
|
2350
|
+
evidence.aliasesByAccount,
|
|
2351
|
+
network,
|
|
2352
|
+
validated.observedAt,
|
|
2353
|
+
));
|
|
2354
|
+
for (const reaction of message.reactions) {
|
|
2355
|
+
nonParticipantBytes += bundleRecordBytes(reactionRecord(
|
|
2356
|
+
reaction,
|
|
2357
|
+
message,
|
|
2358
|
+
scan,
|
|
2359
|
+
account,
|
|
2360
|
+
evidence.aliasesByAccount,
|
|
2361
|
+
network,
|
|
2362
|
+
validated.observedAt,
|
|
2363
|
+
));
|
|
2364
|
+
}
|
|
2365
|
+
const tombstone = tombstoneRecord(
|
|
2366
|
+
message,
|
|
2367
|
+
scan,
|
|
2368
|
+
account,
|
|
2369
|
+
network,
|
|
2370
|
+
validated.observedAt,
|
|
2371
|
+
);
|
|
2372
|
+
if (tombstone !== null) nonParticipantBytes += bundleRecordBytes(tombstone);
|
|
2373
|
+
}
|
|
2374
|
+
const stagedParticipantFacts = new Map<string, ParticipantFact>();
|
|
2375
|
+
let addedParticipants = 0;
|
|
2376
|
+
let participantByteDelta = 0;
|
|
2377
|
+
for (const participantId of participantIds) {
|
|
2378
|
+
const incoming = scanParticipantFacts.get(participantId);
|
|
2379
|
+
if (incoming === undefined) {
|
|
2380
|
+
return fail("Beeper conversation participant disappeared");
|
|
2381
|
+
}
|
|
2382
|
+
const current = participantFacts.get(participantId);
|
|
2383
|
+
const merged = mergePlanningParticipantFact(current, incoming);
|
|
2384
|
+
stagedParticipantFacts.set(participantId, merged);
|
|
2385
|
+
if (current === undefined) addedParticipants += 1;
|
|
2386
|
+
const mergedBytes = bundleRecordBytes(participantRecord(
|
|
2387
|
+
merged,
|
|
2388
|
+
account,
|
|
2389
|
+
network,
|
|
2390
|
+
validated.observedAt,
|
|
2391
|
+
));
|
|
2392
|
+
const currentBytes = current === undefined
|
|
2393
|
+
? 0
|
|
2394
|
+
: bundleRecordBytes(participantRecord(
|
|
2395
|
+
current,
|
|
2396
|
+
account,
|
|
2397
|
+
network,
|
|
2398
|
+
validated.observedAt,
|
|
2399
|
+
));
|
|
2400
|
+
participantByteDelta += mergedBytes - currentBytes;
|
|
2401
|
+
}
|
|
2402
|
+
const candidateRecords = 1
|
|
2403
|
+
+ messages.length
|
|
2404
|
+
+ reactionCount
|
|
2405
|
+
+ tombstoneCount
|
|
2406
|
+
+ addedParticipants;
|
|
2407
|
+
const candidateBytes = nonParticipantBytes + participantByteDelta;
|
|
2408
|
+
if (
|
|
2409
|
+
!Number.isSafeInteger(candidateRecords)
|
|
2410
|
+
|| !Number.isSafeInteger(candidateBytes)
|
|
2411
|
+
|| candidateBytes < 0
|
|
2412
|
+
) return fail("official export derived bundle budget overflowed");
|
|
2413
|
+
if (candidateRecords > maxBundleRecords - recordCount) {
|
|
2414
|
+
return Object.freeze({
|
|
2415
|
+
admittedChats: Object.freeze(admittedChats),
|
|
2416
|
+
bundleByteLimitReached: false,
|
|
2417
|
+
bundleRecordLimitReached: true,
|
|
2418
|
+
provisionalRecordBytes: recordBytes,
|
|
2419
|
+
provisionalRecordCount: recordCount,
|
|
2420
|
+
});
|
|
2421
|
+
}
|
|
2422
|
+
if (candidateBytes > maxBundleBytes - recordBytes) {
|
|
2423
|
+
return Object.freeze({
|
|
2424
|
+
admittedChats: Object.freeze(admittedChats),
|
|
2425
|
+
bundleByteLimitReached: true,
|
|
2426
|
+
bundleRecordLimitReached: false,
|
|
2427
|
+
provisionalRecordBytes: recordBytes,
|
|
2428
|
+
provisionalRecordCount: recordCount,
|
|
2429
|
+
});
|
|
2430
|
+
}
|
|
2431
|
+
admittedChats.push(validated);
|
|
2432
|
+
recordCount += candidateRecords;
|
|
2433
|
+
recordBytes += candidateBytes;
|
|
2434
|
+
for (const [participantId, fact] of stagedParticipantFacts) {
|
|
2435
|
+
participantFacts.set(participantId, fact);
|
|
2436
|
+
}
|
|
2437
|
+
}
|
|
2438
|
+
return Object.freeze({
|
|
2439
|
+
admittedChats: Object.freeze(admittedChats),
|
|
2440
|
+
bundleByteLimitReached: false,
|
|
2441
|
+
bundleRecordLimitReached: false,
|
|
2442
|
+
provisionalRecordBytes: recordBytes,
|
|
2443
|
+
provisionalRecordCount: recordCount,
|
|
2444
|
+
});
|
|
2445
|
+
};
|
|
2446
|
+
|
|
2447
|
+
let candidateChats = chats;
|
|
2448
|
+
let bundleByteLimitReached = false;
|
|
2449
|
+
let bundleRecordLimitReached = false;
|
|
2450
|
+
for (let pass = 0; pass < 16; pass += 1) {
|
|
2451
|
+
const evidence = await collectEvidence(candidateChats);
|
|
2452
|
+
const budget = await planBudget(evidence);
|
|
2453
|
+
bundleByteLimitReached ||= budget.bundleByteLimitReached;
|
|
2454
|
+
bundleRecordLimitReached ||= budget.bundleRecordLimitReached;
|
|
2455
|
+
if (budget.admittedChats.length === evidence.coveredChats.length) {
|
|
2456
|
+
if (evidence.identityConflict) {
|
|
2457
|
+
return fail("official export has peer evidence for an account self alias");
|
|
2458
|
+
}
|
|
2459
|
+
return Object.freeze({
|
|
2460
|
+
aliasesByAccount: evidence.aliasesByAccount,
|
|
2461
|
+
coveredChats: evidence.coveredChats,
|
|
2462
|
+
bundleByteLimitReached,
|
|
2463
|
+
bundleRecordLimitReached,
|
|
2464
|
+
evidenceLimitReached: evidence.evidenceLimitReached,
|
|
2465
|
+
messagesSha256ByPath: evidence.messagesSha256ByPath,
|
|
2466
|
+
participantOccurrenceLimitReached:
|
|
2467
|
+
evidence.participantOccurrenceLimitReached,
|
|
2468
|
+
provisionalRecordBytes: budget.provisionalRecordBytes,
|
|
2469
|
+
provisionalRecordCount: budget.provisionalRecordCount,
|
|
2470
|
+
});
|
|
2471
|
+
}
|
|
2472
|
+
await onRefinementPass?.(pass + 1);
|
|
2473
|
+
candidateChats = budget.admittedChats;
|
|
2474
|
+
}
|
|
2475
|
+
return fail("bounded chat prefix did not stabilize");
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
function canonicalParticipantSourceId(
|
|
2479
|
+
account: BeeperAccountProjection,
|
|
2480
|
+
sourceId: string,
|
|
2481
|
+
aliasesByAccount: ReadonlyMap<string, ReadonlySet<string>>,
|
|
2482
|
+
): string {
|
|
2483
|
+
const aliases = aliasesByAccount.get(account.accountId);
|
|
2484
|
+
if (aliases === undefined) return fail("Beeper account self-alias set disappeared");
|
|
2485
|
+
const coordinate = digest(["beeper-self-alias-v1", account.accountId, sourceId]);
|
|
2486
|
+
return aliases.has(coordinate) ? account.user.id : sourceId;
|
|
2487
|
+
}
|
|
2488
|
+
|
|
2489
|
+
function upsertParticipant(
|
|
2490
|
+
facts: Map<string, ParticipantFact>,
|
|
2491
|
+
account: BeeperAccountProjection,
|
|
2492
|
+
user: Pick<BeeperUserProjection, "id" | "fullName" | "phoneNumber" | "email" | "username">,
|
|
2493
|
+
self: boolean | null,
|
|
2494
|
+
aliasesByAccount: ReadonlyMap<string, ReadonlySet<string>>,
|
|
2495
|
+
): ParticipantFact {
|
|
2496
|
+
const sourceId = canonicalParticipantSourceId(account, user.id, aliasesByAccount);
|
|
2497
|
+
const id = localId("participant", account.accountId, sourceId);
|
|
2498
|
+
const current = facts.get(id);
|
|
2499
|
+
const handle = user.phoneNumber ?? user.email ?? user.username;
|
|
2500
|
+
if (current !== undefined) {
|
|
2501
|
+
if (self !== null && current.isSelf !== null && current.isSelf !== self) {
|
|
2502
|
+
return fail("one Beeper participant has conflicting self-direction evidence");
|
|
2503
|
+
}
|
|
2504
|
+
const updated = Object.freeze({
|
|
2505
|
+
...current,
|
|
2506
|
+
displayName: current.displayName ?? user.fullName,
|
|
2507
|
+
handle: current.handle ?? handle,
|
|
2508
|
+
isSelf: current.isSelf ?? self,
|
|
2509
|
+
});
|
|
2510
|
+
if (
|
|
2511
|
+
updated.displayName === current.displayName
|
|
2512
|
+
&& updated.handle === current.handle
|
|
2513
|
+
&& updated.isSelf === current.isSelf
|
|
2514
|
+
) return current;
|
|
2515
|
+
facts.set(id, updated);
|
|
2516
|
+
return updated;
|
|
2517
|
+
}
|
|
2518
|
+
const created: ParticipantFact = Object.freeze({
|
|
2519
|
+
id,
|
|
2520
|
+
accountId: localId("account", account.accountId),
|
|
2521
|
+
sourceId,
|
|
2522
|
+
providerId: providerId("participant", account.accountId, sourceId),
|
|
2523
|
+
displayName: user.fullName,
|
|
2524
|
+
handle,
|
|
2525
|
+
isSelf: self,
|
|
2526
|
+
});
|
|
2527
|
+
facts.set(id, created);
|
|
2528
|
+
return created;
|
|
2529
|
+
}
|
|
2530
|
+
|
|
2531
|
+
function mergeParticipantFact(
|
|
2532
|
+
current: ParticipantFact | undefined,
|
|
2533
|
+
incoming: ParticipantFact,
|
|
2534
|
+
): ParticipantFact {
|
|
2535
|
+
if (current === undefined) return incoming;
|
|
2536
|
+
if (
|
|
2537
|
+
current.id !== incoming.id
|
|
2538
|
+
|| current.accountId !== incoming.accountId
|
|
2539
|
+
|| current.sourceId !== incoming.sourceId
|
|
2540
|
+
|| current.providerId !== incoming.providerId
|
|
2541
|
+
) return fail("one Beeper participant has conflicting source coordinates");
|
|
2542
|
+
if (
|
|
2543
|
+
current.isSelf !== null
|
|
2544
|
+
&& incoming.isSelf !== null
|
|
2545
|
+
&& current.isSelf !== incoming.isSelf
|
|
2546
|
+
) return fail("one Beeper participant has conflicting self-direction evidence");
|
|
2547
|
+
const merged = Object.freeze({
|
|
2548
|
+
...current,
|
|
2549
|
+
displayName: current.displayName ?? incoming.displayName,
|
|
2550
|
+
handle: current.handle ?? incoming.handle,
|
|
2551
|
+
isSelf: current.isSelf ?? incoming.isSelf,
|
|
2552
|
+
});
|
|
2553
|
+
return merged.displayName === current.displayName
|
|
2554
|
+
&& merged.handle === current.handle
|
|
2555
|
+
&& merged.isSelf === current.isSelf
|
|
2556
|
+
? current
|
|
2557
|
+
: merged;
|
|
2558
|
+
}
|
|
2559
|
+
|
|
2560
|
+
function planningParticipantFact(
|
|
2561
|
+
account: BeeperAccountProjection,
|
|
2562
|
+
user: Pick<BeeperUserProjection, "id" | "fullName" | "phoneNumber" | "email" | "username">,
|
|
2563
|
+
self: boolean | null,
|
|
2564
|
+
aliasesByAccount: ReadonlyMap<string, ReadonlySet<string>>,
|
|
2565
|
+
): ParticipantFact {
|
|
2566
|
+
const sourceId = canonicalParticipantSourceId(account, user.id, aliasesByAccount);
|
|
2567
|
+
return Object.freeze({
|
|
2568
|
+
id: localId("participant", account.accountId, sourceId),
|
|
2569
|
+
accountId: localId("account", account.accountId),
|
|
2570
|
+
sourceId,
|
|
2571
|
+
providerId: providerId("participant", account.accountId, sourceId),
|
|
2572
|
+
displayName: user.fullName,
|
|
2573
|
+
handle: user.phoneNumber ?? user.email ?? user.username,
|
|
2574
|
+
isSelf: self,
|
|
2575
|
+
});
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
function mergePlanningParticipantFact(
|
|
2579
|
+
current: ParticipantFact | undefined,
|
|
2580
|
+
incoming: ParticipantFact,
|
|
2581
|
+
): ParticipantFact {
|
|
2582
|
+
if (current === undefined) return incoming;
|
|
2583
|
+
if (
|
|
2584
|
+
current.id !== incoming.id
|
|
2585
|
+
|| current.accountId !== incoming.accountId
|
|
2586
|
+
|| current.sourceId !== incoming.sourceId
|
|
2587
|
+
|| current.providerId !== incoming.providerId
|
|
2588
|
+
) return fail("one Beeper participant has conflicting source coordinates");
|
|
2589
|
+
return Object.freeze({
|
|
2590
|
+
...current,
|
|
2591
|
+
displayName: current.displayName ?? incoming.displayName,
|
|
2592
|
+
handle: current.handle ?? incoming.handle,
|
|
2593
|
+
isSelf: current.isSelf !== null
|
|
2594
|
+
&& incoming.isSelf !== null
|
|
2595
|
+
&& current.isSelf !== incoming.isSelf
|
|
2596
|
+
? false
|
|
2597
|
+
: current.isSelf ?? incoming.isSelf,
|
|
2598
|
+
});
|
|
2599
|
+
}
|
|
2600
|
+
|
|
2601
|
+
function messageTimestampRange(
|
|
2602
|
+
messages: readonly BeeperMessageProjection[],
|
|
2603
|
+
): { readonly first: string | null; readonly last: string | null } {
|
|
2604
|
+
let first: string | null = null;
|
|
2605
|
+
let last: string | null = null;
|
|
2606
|
+
for (const message of messages) {
|
|
2607
|
+
if (first === null || message.timestamp < first) first = message.timestamp;
|
|
2608
|
+
if (last === null || message.timestamp > last) last = message.timestamp;
|
|
2609
|
+
}
|
|
2610
|
+
return Object.freeze({ first, last });
|
|
2611
|
+
}
|
|
2612
|
+
|
|
2613
|
+
function parseOfficialState(
|
|
2614
|
+
value: unknown,
|
|
2615
|
+
chatIds: readonly string[],
|
|
2616
|
+
createdAt: string,
|
|
2617
|
+
): ReadonlyMap<string, number> {
|
|
2618
|
+
const source = record(value, "official export state");
|
|
2619
|
+
exactKeys(source, [
|
|
2620
|
+
"chats",
|
|
2621
|
+
"completedChatIDs",
|
|
2622
|
+
"createdAt",
|
|
2623
|
+
"exportVersion",
|
|
2624
|
+
], [], "official export state");
|
|
2625
|
+
if (source.exportVersion !== 1 || timestamp(source.createdAt, "official export state createdAt") !== createdAt) {
|
|
2626
|
+
fail("official export state does not match the completed manifest");
|
|
2627
|
+
}
|
|
2628
|
+
const completed = array(
|
|
2629
|
+
source.completedChatIDs,
|
|
2630
|
+
"official export completedChatIDs",
|
|
2631
|
+
MAX_EXPORT_CHATS,
|
|
2632
|
+
).map((item) => string(item, "official export completed chat ID", 2_048));
|
|
2633
|
+
const completedSet = new Set(completed);
|
|
2634
|
+
if (completedSet.size !== completed.length || completedSet.size !== chatIds.length) {
|
|
2635
|
+
fail("official export completed chat inventory is inconsistent");
|
|
2636
|
+
}
|
|
2637
|
+
const states = record(source.chats, "official export chat states");
|
|
2638
|
+
if (Object.keys(states).length !== chatIds.length) {
|
|
2639
|
+
fail("official export chat state inventory is inconsistent");
|
|
2640
|
+
}
|
|
2641
|
+
const messageCounts = new Map<string, number>();
|
|
2642
|
+
for (const chatId of chatIds) {
|
|
2643
|
+
if (!completedSet.has(chatId)) {
|
|
2644
|
+
fail("official export omitted a completed chat marker");
|
|
2645
|
+
}
|
|
2646
|
+
const state = record(states[chatId], "official export chat state");
|
|
2647
|
+
exactKeys(state, [
|
|
2648
|
+
"attachmentCount",
|
|
2649
|
+
"complete",
|
|
2650
|
+
"cursor",
|
|
2651
|
+
"messageCount",
|
|
2652
|
+
"startedAt",
|
|
2653
|
+
"updatedAt",
|
|
2654
|
+
], [], "official export chat state");
|
|
2655
|
+
if (
|
|
2656
|
+
state.complete !== true
|
|
2657
|
+
|| state.cursor !== null
|
|
2658
|
+
|| integer(state.attachmentCount, "official export chat attachmentCount", Number.MAX_SAFE_INTEGER) !== 0
|
|
2659
|
+
) fail("official export chat did not complete without attachments");
|
|
2660
|
+
const messageCount = integer(
|
|
2661
|
+
state.messageCount,
|
|
2662
|
+
"official export chat messageCount",
|
|
2663
|
+
MAX_EXPORT_MESSAGES_PER_CHAT,
|
|
2664
|
+
);
|
|
2665
|
+
timestamp(state.startedAt, "official export chat startedAt");
|
|
2666
|
+
timestamp(state.updatedAt, "official export chat updatedAt");
|
|
2667
|
+
messageCounts.set(chatId, messageCount);
|
|
2668
|
+
}
|
|
2669
|
+
return messageCounts;
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2672
|
+
function accountRecord(
|
|
2673
|
+
account: BeeperAccountProjection,
|
|
2674
|
+
network: string,
|
|
2675
|
+
observedAt: string,
|
|
2676
|
+
selfParticipantId: string,
|
|
2677
|
+
): BeeperMessageLikeMeAccount {
|
|
2678
|
+
const id = localId("account", account.accountId);
|
|
2679
|
+
const connectedAccountProviderId = providerId("account", account.accountId);
|
|
2680
|
+
return Object.freeze({
|
|
2681
|
+
schemaVersion: 1,
|
|
2682
|
+
kind: "account",
|
|
2683
|
+
id,
|
|
2684
|
+
accountId: id,
|
|
2685
|
+
network,
|
|
2686
|
+
provenance: Object.freeze({
|
|
2687
|
+
providerId: connectedAccountProviderId,
|
|
2688
|
+
providerRevision: null,
|
|
2689
|
+
observedAt,
|
|
2690
|
+
connectedAccountProviderId,
|
|
2691
|
+
}),
|
|
2692
|
+
displayName: account.user.fullName ?? account.network,
|
|
2693
|
+
handle: preferredHandle(account.user),
|
|
2694
|
+
selfParticipantId,
|
|
2695
|
+
});
|
|
2696
|
+
}
|
|
2697
|
+
|
|
2698
|
+
function participantRecord(
|
|
2699
|
+
fact: ParticipantFact,
|
|
2700
|
+
account: BeeperAccountProjection,
|
|
2701
|
+
network: string,
|
|
2702
|
+
observedAt: string,
|
|
2703
|
+
): BeeperMessageLikeMeParticipant {
|
|
2704
|
+
return Object.freeze({
|
|
2705
|
+
schemaVersion: 1,
|
|
2706
|
+
kind: "participant",
|
|
2707
|
+
id: fact.id,
|
|
2708
|
+
accountId: fact.accountId,
|
|
2709
|
+
network,
|
|
2710
|
+
provenance: Object.freeze({
|
|
2711
|
+
providerId: fact.providerId,
|
|
2712
|
+
providerRevision: null,
|
|
2713
|
+
observedAt,
|
|
2714
|
+
connectedAccountProviderId: providerId("account", account.accountId),
|
|
2715
|
+
}),
|
|
2716
|
+
displayName: fact.displayName,
|
|
2717
|
+
handle: fact.handle,
|
|
2718
|
+
isSelf: fact.isSelf === true,
|
|
2719
|
+
});
|
|
2720
|
+
}
|
|
2721
|
+
|
|
2722
|
+
function conversationRecord(
|
|
2723
|
+
scan: ConversationScan,
|
|
2724
|
+
account: BeeperAccountProjection,
|
|
2725
|
+
network: string,
|
|
2726
|
+
observedAt: string,
|
|
2727
|
+
): BeeperMessageLikeMeConversation {
|
|
2728
|
+
const id = localId("conversation", account.accountId, scan.chat.id);
|
|
2729
|
+
return Object.freeze({
|
|
2730
|
+
schemaVersion: 1,
|
|
2731
|
+
kind: "conversation",
|
|
2732
|
+
id,
|
|
2733
|
+
accountId: localId("account", account.accountId),
|
|
2734
|
+
network,
|
|
2735
|
+
provenance: Object.freeze({
|
|
2736
|
+
providerId: providerId("conversation", account.accountId, scan.chat.id),
|
|
2737
|
+
providerRevision: scan.chat.lastActivity,
|
|
2738
|
+
observedAt,
|
|
2739
|
+
connectedAccountProviderId: providerId("account", account.accountId),
|
|
2740
|
+
}),
|
|
2741
|
+
type: scan.chat.type === "single" ? "direct" : "group",
|
|
2742
|
+
title: scan.chat.title,
|
|
2743
|
+
participantIds: scan.participantIds,
|
|
2744
|
+
participantsComplete: scan.participantsComplete,
|
|
2745
|
+
startedAt: scan.startedAt,
|
|
2746
|
+
lastMessageAt: scan.lastMessageAt,
|
|
2747
|
+
});
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
function messageRecord(
|
|
2751
|
+
message: BeeperMessageProjection,
|
|
2752
|
+
scan: ConversationScan,
|
|
2753
|
+
messageIds: ReadonlySet<string>,
|
|
2754
|
+
account: BeeperAccountProjection,
|
|
2755
|
+
aliasesByAccount: ReadonlyMap<string, ReadonlySet<string>>,
|
|
2756
|
+
network: string,
|
|
2757
|
+
observedAt: string,
|
|
2758
|
+
): BeeperMessageLikeMeMessage {
|
|
2759
|
+
const id = localId("message", account.accountId, scan.chat.id, message.id);
|
|
2760
|
+
const providerMessageId = providerId(
|
|
2761
|
+
"message",
|
|
2762
|
+
account.accountId,
|
|
2763
|
+
scan.chat.id,
|
|
2764
|
+
message.id,
|
|
2765
|
+
);
|
|
2766
|
+
const deletionState: BeeperMessageLikeMeMessage["deletion"] =
|
|
2767
|
+
message.isDeleted || message.isHidden
|
|
2768
|
+
? Object.freeze({
|
|
2769
|
+
state: message.isDeleted && message.isHidden
|
|
2770
|
+
? "revoked-and-deleted-for-me"
|
|
2771
|
+
: message.isDeleted
|
|
2772
|
+
? "revoked"
|
|
2773
|
+
: "deleted-for-me",
|
|
2774
|
+
observedAt,
|
|
2775
|
+
providerRevision: message.editedTimestamp ?? message.sortKey,
|
|
2776
|
+
})
|
|
2777
|
+
: null;
|
|
2778
|
+
const replyProviderId = message.linkedMessageId === null
|
|
2779
|
+
? null
|
|
2780
|
+
: providerId(
|
|
2781
|
+
"message",
|
|
2782
|
+
account.accountId,
|
|
2783
|
+
scan.chat.id,
|
|
2784
|
+
message.linkedMessageId,
|
|
2785
|
+
);
|
|
2786
|
+
return Object.freeze({
|
|
2787
|
+
schemaVersion: 1,
|
|
2788
|
+
kind: "message",
|
|
2789
|
+
id,
|
|
2790
|
+
accountId: localId("account", account.accountId),
|
|
2791
|
+
network,
|
|
2792
|
+
provenance: Object.freeze({
|
|
2793
|
+
providerId: providerMessageId,
|
|
2794
|
+
providerRevision: message.editedTimestamp ?? message.sortKey,
|
|
2795
|
+
observedAt,
|
|
2796
|
+
connectedAccountProviderId: providerId("account", account.accountId),
|
|
2797
|
+
}),
|
|
2798
|
+
conversationId: localId("conversation", account.accountId, scan.chat.id),
|
|
2799
|
+
senderParticipantId: localId(
|
|
2800
|
+
"participant",
|
|
2801
|
+
account.accountId,
|
|
2802
|
+
canonicalParticipantSourceId(account, message.senderId, aliasesByAccount),
|
|
2803
|
+
),
|
|
2804
|
+
direction: message.isSender === null
|
|
2805
|
+
? "unknown"
|
|
2806
|
+
: message.isSender ? "outgoing" : "incoming",
|
|
2807
|
+
sentAt: message.timestamp,
|
|
2808
|
+
sortKey: message.sortKey,
|
|
2809
|
+
body: deletionState === null ? message.text : null,
|
|
2810
|
+
bodyTruncated: false,
|
|
2811
|
+
replyTo: message.linkedMessageId === null || replyProviderId === null
|
|
2812
|
+
? null
|
|
2813
|
+
: Object.freeze({
|
|
2814
|
+
messageId: messageIds.has(message.linkedMessageId)
|
|
2815
|
+
? localId(
|
|
2816
|
+
"message",
|
|
2817
|
+
account.accountId,
|
|
2818
|
+
scan.chat.id,
|
|
2819
|
+
message.linkedMessageId,
|
|
2820
|
+
)
|
|
2821
|
+
: null,
|
|
2822
|
+
providerId: replyProviderId,
|
|
2823
|
+
}),
|
|
2824
|
+
edit: message.editedTimestamp === null
|
|
2825
|
+
? null
|
|
2826
|
+
: Object.freeze({
|
|
2827
|
+
kind: "in-place" as const,
|
|
2828
|
+
editedAt: message.editedTimestamp,
|
|
2829
|
+
providerRevision: message.editedTimestamp,
|
|
2830
|
+
}),
|
|
2831
|
+
deletion: deletionState,
|
|
2832
|
+
attachments: Object.freeze(message.attachments.map(attachment)),
|
|
2833
|
+
});
|
|
2834
|
+
}
|
|
2835
|
+
|
|
2836
|
+
function reactionRecord(
|
|
2837
|
+
reaction: BeeperReactionProjection,
|
|
2838
|
+
message: BeeperMessageProjection,
|
|
2839
|
+
scan: ConversationScan,
|
|
2840
|
+
account: BeeperAccountProjection,
|
|
2841
|
+
aliasesByAccount: ReadonlyMap<string, ReadonlySet<string>>,
|
|
2842
|
+
network: string,
|
|
2843
|
+
observedAt: string,
|
|
2844
|
+
): BeeperMessageLikeMeReaction {
|
|
2845
|
+
const participantSourceId = canonicalParticipantSourceId(
|
|
2846
|
+
account,
|
|
2847
|
+
reaction.participantId,
|
|
2848
|
+
aliasesByAccount,
|
|
2849
|
+
);
|
|
2850
|
+
const id = localId(
|
|
2851
|
+
"reaction",
|
|
2852
|
+
account.accountId,
|
|
2853
|
+
scan.chat.id,
|
|
2854
|
+
message.id,
|
|
2855
|
+
reaction.id,
|
|
2856
|
+
reaction.participantId,
|
|
2857
|
+
reaction.reactionKey,
|
|
2858
|
+
);
|
|
2859
|
+
return Object.freeze({
|
|
2860
|
+
schemaVersion: 1,
|
|
2861
|
+
kind: "reaction",
|
|
2862
|
+
id,
|
|
2863
|
+
accountId: localId("account", account.accountId),
|
|
2864
|
+
network,
|
|
2865
|
+
provenance: Object.freeze({
|
|
2866
|
+
providerId: providerId(
|
|
2867
|
+
"reaction",
|
|
2868
|
+
account.accountId,
|
|
2869
|
+
scan.chat.id,
|
|
2870
|
+
message.id,
|
|
2871
|
+
reaction.id,
|
|
2872
|
+
reaction.participantId,
|
|
2873
|
+
reaction.reactionKey,
|
|
2874
|
+
),
|
|
2875
|
+
providerRevision: null,
|
|
2876
|
+
observedAt,
|
|
2877
|
+
connectedAccountProviderId: providerId("account", account.accountId),
|
|
2878
|
+
}),
|
|
2879
|
+
messageId: localId("message", account.accountId, scan.chat.id, message.id),
|
|
2880
|
+
messageProviderId: providerId(
|
|
2881
|
+
"message",
|
|
2882
|
+
account.accountId,
|
|
2883
|
+
scan.chat.id,
|
|
2884
|
+
message.id,
|
|
2885
|
+
),
|
|
2886
|
+
participantId: localId("participant", account.accountId, participantSourceId),
|
|
2887
|
+
body: reactionBody(reaction.reactionKey),
|
|
2888
|
+
reactedAt: null,
|
|
2889
|
+
state: "active",
|
|
2890
|
+
});
|
|
2891
|
+
}
|
|
2892
|
+
|
|
2893
|
+
function tombstoneRecord(
|
|
2894
|
+
message: BeeperMessageProjection,
|
|
2895
|
+
scan: ConversationScan,
|
|
2896
|
+
account: BeeperAccountProjection,
|
|
2897
|
+
network: string,
|
|
2898
|
+
observedAt: string,
|
|
2899
|
+
): BeeperMessageLikeMeTombstone | null {
|
|
2900
|
+
if (!message.isDeleted && !message.isHidden) return null;
|
|
2901
|
+
const messageId = localId("message", account.accountId, scan.chat.id, message.id);
|
|
2902
|
+
const providerMessageId = providerId(
|
|
2903
|
+
"message",
|
|
2904
|
+
account.accountId,
|
|
2905
|
+
scan.chat.id,
|
|
2906
|
+
message.id,
|
|
2907
|
+
);
|
|
2908
|
+
return Object.freeze({
|
|
2909
|
+
schemaVersion: 1,
|
|
2910
|
+
kind: "tombstone",
|
|
2911
|
+
id: localId("tombstone", account.accountId, scan.chat.id, message.id),
|
|
2912
|
+
accountId: localId("account", account.accountId),
|
|
2913
|
+
network,
|
|
2914
|
+
provenance: Object.freeze({
|
|
2915
|
+
providerId: providerId("tombstone", account.accountId, scan.chat.id, message.id),
|
|
2916
|
+
providerRevision: message.editedTimestamp ?? message.sortKey,
|
|
2917
|
+
observedAt,
|
|
2918
|
+
connectedAccountProviderId: providerId("account", account.accountId),
|
|
2919
|
+
}),
|
|
2920
|
+
entityKind: "message",
|
|
2921
|
+
entityId: messageId,
|
|
2922
|
+
entityProviderId: providerMessageId,
|
|
2923
|
+
deletedAt: observedAt,
|
|
2924
|
+
scope: message.isDeleted && message.isHidden
|
|
2925
|
+
? "unknown"
|
|
2926
|
+
: message.isDeleted
|
|
2927
|
+
? "remote"
|
|
2928
|
+
: "local",
|
|
2929
|
+
providerRevision: message.editedTimestamp ?? message.sortKey,
|
|
2930
|
+
});
|
|
2931
|
+
}
|
|
2932
|
+
|
|
2933
|
+
async function assertExactDirectoryEntries(
|
|
2934
|
+
path: string,
|
|
2935
|
+
expected: readonly string[],
|
|
2936
|
+
label: string,
|
|
2937
|
+
): Promise<void> {
|
|
2938
|
+
const actual = (await readdir(path)).sort();
|
|
2939
|
+
const wanted = [...expected].sort();
|
|
2940
|
+
if (
|
|
2941
|
+
actual.length !== wanted.length
|
|
2942
|
+
|| actual.some((entry, index) => entry !== wanted[index])
|
|
2943
|
+
) fail(`${label} contained an unexpected file layout`);
|
|
2944
|
+
}
|
|
2945
|
+
|
|
2946
|
+
async function removePrivateOwnedDirectory(
|
|
2947
|
+
path: string,
|
|
2948
|
+
root: string,
|
|
2949
|
+
expected: PrivateDirectoryIdentity,
|
|
2950
|
+
): Promise<void> {
|
|
2951
|
+
if (path !== root && !path.startsWith(`${root}${sep}`)) {
|
|
2952
|
+
return fail("private export cleanup escaped its owned root");
|
|
2953
|
+
}
|
|
2954
|
+
try {
|
|
2955
|
+
removePrivateDirectoryTree(path, Object.freeze({
|
|
2956
|
+
device: String(expected.device),
|
|
2957
|
+
inode: String(expected.inode),
|
|
2958
|
+
}));
|
|
2959
|
+
} catch {
|
|
2960
|
+
return fail("private export directory could not be removed from quarantine safely");
|
|
2961
|
+
}
|
|
2962
|
+
}
|
|
2963
|
+
|
|
2964
|
+
/**
|
|
2965
|
+
* Creates a single-use source for the private bundle sink. The pinned official
|
|
2966
|
+
* CLI paginates one operation-private account shard at a time. Ghostget validates
|
|
2967
|
+
* every completed shard, reports only ordinal progress, and projects all shards
|
|
2968
|
+
* in one deterministic global conversation order.
|
|
2969
|
+
*/
|
|
2970
|
+
export function createBeeperMessageLikeMeSource(
|
|
2971
|
+
request: BeeperMessageLikeMeSourceRequest,
|
|
2972
|
+
): BeeperMessageLikeMeExportSource {
|
|
2973
|
+
const auth = requireAuth(request.auth);
|
|
2974
|
+
const limits = parseLimits(request.limits);
|
|
2975
|
+
let consumed = false;
|
|
2976
|
+
let completion: unknown;
|
|
2977
|
+
let disposeWorking: (() => Promise<void>) | undefined;
|
|
2978
|
+
let disposed = false;
|
|
2979
|
+
let disposalInFlight: Promise<void> | undefined;
|
|
2980
|
+
let progressHeartbeat: ReturnType<typeof setInterval> | undefined;
|
|
2981
|
+
let progressHeartbeatFailed = false;
|
|
2982
|
+
const stopProgressHeartbeat = (): void => {
|
|
2983
|
+
if (progressHeartbeat !== undefined) clearInterval(progressHeartbeat);
|
|
2984
|
+
progressHeartbeat = undefined;
|
|
2985
|
+
};
|
|
2986
|
+
const assertProgressHeartbeat = (): void => {
|
|
2987
|
+
if (progressHeartbeatFailed) return fail("export progress reporting failed");
|
|
2988
|
+
};
|
|
2989
|
+
|
|
2990
|
+
const records = (async function* (): AsyncGenerator<BeeperMessageLikeMeRecord> {
|
|
2991
|
+
if (consumed) return fail("record stream is single-use");
|
|
2992
|
+
consumed = true;
|
|
2993
|
+
throwIfAborted(request.signal);
|
|
2994
|
+
request.onProgress?.(Object.freeze({ phase: "preparing" }));
|
|
2995
|
+
const configDirectory = await validateBeeperCliStore(auth.path);
|
|
2996
|
+
const environment = request.environment ?? process.env;
|
|
2997
|
+
const binarySource = request.dependencies?.binaryPath
|
|
2998
|
+
?? await resolvePinnedBeeperCliBinary(environment);
|
|
2999
|
+
const maxBundleRecords = request.dependencies?.maxBundleRecords === undefined
|
|
3000
|
+
? BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS
|
|
3001
|
+
: positiveInteger(
|
|
3002
|
+
request.dependencies.maxBundleRecords,
|
|
3003
|
+
"test maxBundleRecords",
|
|
3004
|
+
BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS,
|
|
3005
|
+
);
|
|
3006
|
+
const maxBundleBytes = request.dependencies?.maxBundleBytes === undefined
|
|
3007
|
+
? BEEPER_MESSAGE_LIKE_ME_MAX_TOTAL_BYTES
|
|
3008
|
+
: positiveInteger(
|
|
3009
|
+
request.dependencies.maxBundleBytes,
|
|
3010
|
+
"test maxBundleBytes",
|
|
3011
|
+
BEEPER_MESSAGE_LIKE_ME_MAX_TOTAL_BYTES,
|
|
3012
|
+
);
|
|
3013
|
+
const maxMessagesJsonBytes = request.dependencies?.maxMessagesJsonBytes === undefined
|
|
3014
|
+
? MAX_MESSAGES_JSON_BYTES
|
|
3015
|
+
: positiveInteger(
|
|
3016
|
+
request.dependencies.maxMessagesJsonBytes,
|
|
3017
|
+
"test maxMessagesJsonBytes",
|
|
3018
|
+
MAX_MESSAGES_JSON_BYTES,
|
|
3019
|
+
);
|
|
3020
|
+
const maxParticipantOccurrences =
|
|
3021
|
+
request.dependencies?.maxParticipantOccurrences === undefined
|
|
3022
|
+
? MAX_PARTICIPANT_OCCURRENCES
|
|
3023
|
+
: positiveInteger(
|
|
3024
|
+
request.dependencies.maxParticipantOccurrences,
|
|
3025
|
+
"test maxParticipantOccurrences",
|
|
3026
|
+
MAX_PARTICIPANT_OCCURRENCES,
|
|
3027
|
+
);
|
|
3028
|
+
if (!isAbsolute(binarySource)) return fail("Beeper CLI binary path must be absolute");
|
|
3029
|
+
const customCreateWorking = request.dependencies?.createWorkingDirectory;
|
|
3030
|
+
const customRemoveWorking = request.dependencies?.removeWorkingDirectory;
|
|
3031
|
+
if ((customCreateWorking === undefined) !== (customRemoveWorking === undefined)) {
|
|
3032
|
+
return fail("test working-directory create and remove seams must be supplied together");
|
|
3033
|
+
}
|
|
3034
|
+
const createWorking = customCreateWorking
|
|
3035
|
+
?? (() => mkdtemp(join(tmpdir(), "wrench-beeper-message-like-me-")));
|
|
3036
|
+
const working = await createWorking();
|
|
3037
|
+
let canonicalWorking: string;
|
|
3038
|
+
try {
|
|
3039
|
+
if (!isAbsolute(working)) return fail("working directory must be absolute");
|
|
3040
|
+
await chmod(working, PRIVATE_DIRECTORY_MODE);
|
|
3041
|
+
canonicalWorking = await assertPrivateOwnedDirectory(await realpath(working));
|
|
3042
|
+
} catch (error) {
|
|
3043
|
+
if (customCreateWorking === undefined && isAbsolute(working)) {
|
|
3044
|
+
try {
|
|
3045
|
+
await rmdir(working);
|
|
3046
|
+
} catch (cleanupError) {
|
|
3047
|
+
throw new AggregateError(
|
|
3048
|
+
[error, cleanupError],
|
|
3049
|
+
"Beeper export working-directory setup and cleanup both failed",
|
|
3050
|
+
);
|
|
3051
|
+
}
|
|
3052
|
+
}
|
|
3053
|
+
throw error;
|
|
3054
|
+
}
|
|
3055
|
+
const workingMetadata = await lstat(canonicalWorking);
|
|
3056
|
+
const workingIdentity = Object.freeze({
|
|
3057
|
+
device: workingMetadata.dev,
|
|
3058
|
+
inode: workingMetadata.ino,
|
|
3059
|
+
});
|
|
3060
|
+
let workingLease: BeeperMessageLikeMeDirectoryLease | undefined;
|
|
3061
|
+
if (customCreateWorking === undefined) {
|
|
3062
|
+
try {
|
|
3063
|
+
workingLease = await createBeeperMessageLikeMeDirectoryLease({
|
|
3064
|
+
role: "raw-working",
|
|
3065
|
+
path: canonicalWorking,
|
|
3066
|
+
recoverAfterMs:
|
|
3067
|
+
Date.now() + limits.timeoutMs + RAW_WORKING_RECOVERY_GRACE_MS,
|
|
3068
|
+
environment,
|
|
3069
|
+
});
|
|
3070
|
+
} catch (error) {
|
|
3071
|
+
try {
|
|
3072
|
+
await removePrivateOwnedDirectory(
|
|
3073
|
+
canonicalWorking,
|
|
3074
|
+
canonicalWorking,
|
|
3075
|
+
workingIdentity,
|
|
3076
|
+
);
|
|
3077
|
+
} catch (cleanupError) {
|
|
3078
|
+
throw new AggregateError(
|
|
3079
|
+
[error, cleanupError],
|
|
3080
|
+
"Beeper export recovery setup and empty-directory cleanup both failed",
|
|
3081
|
+
);
|
|
3082
|
+
}
|
|
3083
|
+
throw error;
|
|
3084
|
+
}
|
|
3085
|
+
}
|
|
3086
|
+
disposeWorking = async () => {
|
|
3087
|
+
if (customRemoveWorking === undefined) {
|
|
3088
|
+
await removePrivateOwnedDirectory(
|
|
3089
|
+
canonicalWorking,
|
|
3090
|
+
canonicalWorking,
|
|
3091
|
+
workingIdentity,
|
|
3092
|
+
);
|
|
3093
|
+
if (workingLease !== undefined) {
|
|
3094
|
+
releaseBeeperMessageLikeMeDirectoryLease(workingLease);
|
|
3095
|
+
}
|
|
3096
|
+
return;
|
|
3097
|
+
}
|
|
3098
|
+
const current = await lstat(canonicalWorking);
|
|
3099
|
+
if (
|
|
3100
|
+
!current.isDirectory()
|
|
3101
|
+
|| current.isSymbolicLink()
|
|
3102
|
+
|| current.uid !== process.getuid?.()
|
|
3103
|
+
|| (current.mode & 0o777) !== PRIVATE_DIRECTORY_MODE
|
|
3104
|
+
|| current.dev !== workingIdentity.device
|
|
3105
|
+
|| current.ino !== workingIdentity.inode
|
|
3106
|
+
|| await realpath(canonicalWorking) !== canonicalWorking
|
|
3107
|
+
) return fail("private export working directory changed before cleanup");
|
|
3108
|
+
await customRemoveWorking(canonicalWorking);
|
|
3109
|
+
try {
|
|
3110
|
+
await lstat(canonicalWorking);
|
|
3111
|
+
} catch (error) {
|
|
3112
|
+
if (isErrno(error, "ENOENT")) return;
|
|
3113
|
+
throw error;
|
|
3114
|
+
}
|
|
3115
|
+
return fail("private export working directory survived cleanup");
|
|
3116
|
+
};
|
|
3117
|
+
const cacheDirectory = resolve(canonicalWorking, "cli-payload-cache");
|
|
3118
|
+
const shardsDirectory = resolve(canonicalWorking, "account-shards");
|
|
3119
|
+
const selectorsDirectory = resolve(canonicalWorking, "account-selectors");
|
|
3120
|
+
await mkdir(cacheDirectory, { mode: PRIVATE_DIRECTORY_MODE });
|
|
3121
|
+
await chmod(cacheDirectory, PRIVATE_DIRECTORY_MODE);
|
|
3122
|
+
await mkdir(shardsDirectory, { mode: PRIVATE_DIRECTORY_MODE });
|
|
3123
|
+
await chmod(shardsDirectory, PRIVATE_DIRECTORY_MODE);
|
|
3124
|
+
await mkdir(selectorsDirectory, { mode: PRIVATE_DIRECTORY_MODE });
|
|
3125
|
+
await chmod(selectorsDirectory, PRIVATE_DIRECTORY_MODE);
|
|
3126
|
+
await assertPrivateOwnedDirectory(cacheDirectory, canonicalWorking);
|
|
3127
|
+
await assertPrivateOwnedDirectory(shardsDirectory, canonicalWorking);
|
|
3128
|
+
await assertPrivateOwnedDirectory(selectorsDirectory, canonicalWorking);
|
|
3129
|
+
const binary = request.dependencies?.binaryPath !== undefined
|
|
3130
|
+
? binarySource
|
|
3131
|
+
: await materializePinnedBeeperCliBinary(
|
|
3132
|
+
binarySource,
|
|
3133
|
+
canonicalWorking,
|
|
3134
|
+
beeperCliArtifactForRuntime().executableSha256,
|
|
3135
|
+
);
|
|
3136
|
+
{
|
|
3137
|
+
const run = request.dependencies?.runCli ?? runExportCli;
|
|
3138
|
+
const deadlineMs = Date.now() + limits.timeoutMs;
|
|
3139
|
+
const storeSnapshot = await readCliStoreSnapshot(configDirectory);
|
|
3140
|
+
const inventoryStore = await createOperationPrivateBeeperStore(
|
|
3141
|
+
resolve(selectorsDirectory, "inventory"),
|
|
3142
|
+
storeSnapshot,
|
|
3143
|
+
);
|
|
3144
|
+
const baseEnvironment = environmentForExport(inventoryStore.path, cacheDirectory);
|
|
3145
|
+
const targetProof = await proveDesktopTargetRealm(
|
|
3146
|
+
binary,
|
|
3147
|
+
baseEnvironment,
|
|
3148
|
+
storeSnapshot.targetBaseUrl,
|
|
3149
|
+
deadlineMs,
|
|
3150
|
+
run,
|
|
3151
|
+
workingLease,
|
|
3152
|
+
request.signal,
|
|
3153
|
+
);
|
|
3154
|
+
const accounts = await enumerateAccounts(
|
|
3155
|
+
binary,
|
|
3156
|
+
baseEnvironment,
|
|
3157
|
+
deadlineMs,
|
|
3158
|
+
run,
|
|
3159
|
+
workingLease,
|
|
3160
|
+
request.signal,
|
|
3161
|
+
(elapsedSeconds) => request.onProgress?.(Object.freeze({
|
|
3162
|
+
phase: "accounts-progress",
|
|
3163
|
+
stage: "discovering",
|
|
3164
|
+
elapsedSeconds,
|
|
3165
|
+
})),
|
|
3166
|
+
);
|
|
3167
|
+
if (
|
|
3168
|
+
beeperSubjectFromAccountsAndTarget(
|
|
3169
|
+
accounts,
|
|
3170
|
+
storeSnapshot.targetBaseUrl,
|
|
3171
|
+
targetProof.bundleId,
|
|
3172
|
+
targetProof.version,
|
|
3173
|
+
)
|
|
3174
|
+
!== auth.subject
|
|
3175
|
+
) {
|
|
3176
|
+
return fail("official export account did not match the bound auth realm");
|
|
3177
|
+
}
|
|
3178
|
+
const baselineRealmDigest = outputRealmDigest(accounts);
|
|
3179
|
+
const orderedAccounts = [...accounts].sort((left, right) => {
|
|
3180
|
+
const leftKey = digest([left.accountId]);
|
|
3181
|
+
const rightKey = digest([right.accountId]);
|
|
3182
|
+
return leftKey.localeCompare(rightKey) || left.accountId.localeCompare(right.accountId);
|
|
3183
|
+
});
|
|
3184
|
+
for (const account of orderedAccounts) {
|
|
3185
|
+
assertUniqueOfficialAccountSelector(account, accounts);
|
|
3186
|
+
}
|
|
3187
|
+
request.onProgress?.(Object.freeze({
|
|
3188
|
+
phase: "accounts-discovered",
|
|
3189
|
+
accounts: orderedAccounts.length,
|
|
3190
|
+
}));
|
|
3191
|
+
|
|
3192
|
+
const shards: ValidatedAccountShard[] = [];
|
|
3193
|
+
const accountObservedAt = new Map<string, string>();
|
|
3194
|
+
const effectiveChatLimit = limits.limitChats ?? MAX_EXPORT_CHATS;
|
|
3195
|
+
const effectiveMessageLimit = limits.limitMessages
|
|
3196
|
+
?? MAX_EXPORT_MESSAGES_PER_CHAT;
|
|
3197
|
+
let remainingChats = effectiveChatLimit;
|
|
3198
|
+
let cumulativeChats = 0;
|
|
3199
|
+
let cumulativeMessages = 0;
|
|
3200
|
+
for (const [index, account] of orderedAccounts.entries()) {
|
|
3201
|
+
throwIfAborted(request.signal);
|
|
3202
|
+
const ordinal = index + 1;
|
|
3203
|
+
if (remainingChats === 0) {
|
|
3204
|
+
request.onProgress?.(Object.freeze({
|
|
3205
|
+
phase: "account-skipped",
|
|
3206
|
+
account: ordinal,
|
|
3207
|
+
accounts: orderedAccounts.length,
|
|
3208
|
+
reason: "chat-limit-reached",
|
|
3209
|
+
}));
|
|
3210
|
+
continue;
|
|
3211
|
+
}
|
|
3212
|
+
const ordinalSegment = `account-${String(ordinal).padStart(3, "0")}`;
|
|
3213
|
+
const selectorRoot = resolve(selectorsDirectory, ordinalSegment);
|
|
3214
|
+
const shardRoot = resolve(shardsDirectory, ordinalSegment);
|
|
3215
|
+
const selectorStore = await createOperationPrivateBeeperStore(
|
|
3216
|
+
selectorRoot,
|
|
3217
|
+
storeSnapshot,
|
|
3218
|
+
account.accountId,
|
|
3219
|
+
);
|
|
3220
|
+
await mkdir(shardRoot, { mode: PRIVATE_DIRECTORY_MODE });
|
|
3221
|
+
await chmod(shardRoot, PRIVATE_DIRECTORY_MODE);
|
|
3222
|
+
await assertPrivateOwnedDirectory(shardRoot, canonicalWorking);
|
|
3223
|
+
request.onProgress?.(Object.freeze({
|
|
3224
|
+
phase: "account-started",
|
|
3225
|
+
account: ordinal,
|
|
3226
|
+
accounts: orderedAccounts.length,
|
|
3227
|
+
}));
|
|
3228
|
+
const timeoutMs = remainingTimeoutMs(deadlineMs);
|
|
3229
|
+
const arguments_ = planBeeperMessageLikeMeExportCommand({
|
|
3230
|
+
outputDirectory: shardRoot,
|
|
3231
|
+
limitChats: remainingChats,
|
|
3232
|
+
limitMessages: effectiveMessageLimit,
|
|
3233
|
+
maxParticipants: limits.maxParticipants,
|
|
3234
|
+
}, timeoutMs);
|
|
3235
|
+
const result = await run({
|
|
3236
|
+
binary,
|
|
3237
|
+
arguments: arguments_,
|
|
3238
|
+
environment: environmentForExport(selectorStore.path, cacheDirectory),
|
|
3239
|
+
timeoutMs,
|
|
3240
|
+
maxOutputBytes: MAX_STDOUT_BYTES,
|
|
3241
|
+
maxStderrBytes: MAX_STDERR_BYTES,
|
|
3242
|
+
workingRoot: canonicalWorking,
|
|
3243
|
+
maxWorkingBytes: MAX_RAW_WORKING_BYTES,
|
|
3244
|
+
...(workingLease === undefined ? {} : { directoryLease: workingLease }),
|
|
3245
|
+
onHeartbeat: (elapsedSeconds) => request.onProgress?.(Object.freeze({
|
|
3246
|
+
phase: "account-progress",
|
|
3247
|
+
account: ordinal,
|
|
3248
|
+
accounts: orderedAccounts.length,
|
|
3249
|
+
elapsedSeconds,
|
|
3250
|
+
})),
|
|
3251
|
+
...(request.signal === undefined ? {} : { signal: request.signal }),
|
|
3252
|
+
});
|
|
3253
|
+
throwIfAborted(request.signal);
|
|
3254
|
+
if (result.exitCode !== 0 || result.stderr.trim().length !== 0) {
|
|
3255
|
+
return fail("official read-only export failed");
|
|
3256
|
+
}
|
|
3257
|
+
request.onProgress?.(Object.freeze({
|
|
3258
|
+
phase: "account-validating",
|
|
3259
|
+
account: ordinal,
|
|
3260
|
+
accounts: orderedAccounts.length,
|
|
3261
|
+
elapsedSeconds: 0,
|
|
3262
|
+
}));
|
|
3263
|
+
const shard = await withProgressHeartbeat(
|
|
3264
|
+
() => validateAccountShard(
|
|
3265
|
+
shardRoot,
|
|
3266
|
+
canonicalWorking,
|
|
3267
|
+
account,
|
|
3268
|
+
baselineRealmDigest,
|
|
3269
|
+
request.signal,
|
|
3270
|
+
),
|
|
3271
|
+
request.onProgress === undefined
|
|
3272
|
+
? undefined
|
|
3273
|
+
: (elapsedSeconds) => request.onProgress?.(Object.freeze({
|
|
3274
|
+
phase: "account-validating",
|
|
3275
|
+
account: ordinal,
|
|
3276
|
+
accounts: orderedAccounts.length,
|
|
3277
|
+
elapsedSeconds,
|
|
3278
|
+
})),
|
|
3279
|
+
);
|
|
3280
|
+
await removePrivateOwnedDirectory(
|
|
3281
|
+
selectorRoot,
|
|
3282
|
+
canonicalWorking,
|
|
3283
|
+
selectorStore.identity,
|
|
3284
|
+
);
|
|
3285
|
+
shards.push(shard);
|
|
3286
|
+
accountObservedAt.set(account.accountId, shard.completedAt);
|
|
3287
|
+
if (shard.chats.length > remainingChats) {
|
|
3288
|
+
return fail("official per-account export exceeded its allocated global chat budget");
|
|
3289
|
+
}
|
|
3290
|
+
cumulativeChats += shard.chats.length;
|
|
3291
|
+
cumulativeMessages += shard.messageCount;
|
|
3292
|
+
if (
|
|
3293
|
+
!Number.isSafeInteger(cumulativeChats)
|
|
3294
|
+
|| !Number.isSafeInteger(cumulativeMessages)
|
|
3295
|
+
) return fail("official export aggregate counts overflowed");
|
|
3296
|
+
remainingChats -= shard.chats.length;
|
|
3297
|
+
request.onProgress?.(Object.freeze({
|
|
3298
|
+
phase: "account-completed",
|
|
3299
|
+
account: ordinal,
|
|
3300
|
+
accounts: orderedAccounts.length,
|
|
3301
|
+
chats: cumulativeChats,
|
|
3302
|
+
messages: cumulativeMessages,
|
|
3303
|
+
}));
|
|
3304
|
+
}
|
|
3305
|
+
request.onProgress?.(Object.freeze({
|
|
3306
|
+
phase: "accounts-verifying",
|
|
3307
|
+
accounts: orderedAccounts.length,
|
|
3308
|
+
}));
|
|
3309
|
+
const finalTargetProof = await proveDesktopTargetRealm(
|
|
3310
|
+
binary,
|
|
3311
|
+
baseEnvironment,
|
|
3312
|
+
storeSnapshot.targetBaseUrl,
|
|
3313
|
+
deadlineMs,
|
|
3314
|
+
run,
|
|
3315
|
+
workingLease,
|
|
3316
|
+
request.signal,
|
|
3317
|
+
);
|
|
3318
|
+
if (
|
|
3319
|
+
finalTargetProof.bundleId !== targetProof.bundleId
|
|
3320
|
+
|| finalTargetProof.version !== targetProof.version
|
|
3321
|
+
) {
|
|
3322
|
+
return fail("official Desktop target realm changed during export");
|
|
3323
|
+
}
|
|
3324
|
+
const finalAccounts = await enumerateAccounts(
|
|
3325
|
+
binary,
|
|
3326
|
+
baseEnvironment,
|
|
3327
|
+
deadlineMs,
|
|
3328
|
+
run,
|
|
3329
|
+
workingLease,
|
|
3330
|
+
request.signal,
|
|
3331
|
+
(elapsedSeconds) => request.onProgress?.(Object.freeze({
|
|
3332
|
+
phase: "accounts-progress",
|
|
3333
|
+
stage: "verifying",
|
|
3334
|
+
elapsedSeconds,
|
|
3335
|
+
})),
|
|
3336
|
+
);
|
|
3337
|
+
if (
|
|
3338
|
+
beeperSubjectFromAccountsAndTarget(
|
|
3339
|
+
finalAccounts,
|
|
3340
|
+
storeSnapshot.targetBaseUrl,
|
|
3341
|
+
finalTargetProof.bundleId,
|
|
3342
|
+
finalTargetProof.version,
|
|
3343
|
+
)
|
|
3344
|
+
!== auth.subject
|
|
3345
|
+
) {
|
|
3346
|
+
return fail("official export account did not match the bound auth realm");
|
|
3347
|
+
}
|
|
3348
|
+
assertOutputRealm(finalAccounts, baselineRealmDigest);
|
|
3349
|
+
await removePrivateOwnedDirectory(
|
|
3350
|
+
resolve(selectorsDirectory, "inventory"),
|
|
3351
|
+
canonicalWorking,
|
|
3352
|
+
inventoryStore.identity,
|
|
3353
|
+
);
|
|
3354
|
+
const finalObservedAt = new Date().toISOString();
|
|
3355
|
+
for (const account of accounts) {
|
|
3356
|
+
accountObservedAt.set(
|
|
3357
|
+
account.accountId,
|
|
3358
|
+
accountObservedAt.get(account.accountId) ?? finalObservedAt,
|
|
3359
|
+
);
|
|
3360
|
+
}
|
|
3361
|
+
const observedAtForAccount = (accountId: string): string => {
|
|
3362
|
+
const value = accountObservedAt.get(accountId);
|
|
3363
|
+
if (value === undefined) return fail("Beeper account observation time disappeared");
|
|
3364
|
+
return value;
|
|
3365
|
+
};
|
|
3366
|
+
const validatedChats = shards.flatMap((shard) => shard.chats);
|
|
3367
|
+
request.onProgress?.(Object.freeze({
|
|
3368
|
+
phase: "conversion-started",
|
|
3369
|
+
accounts: orderedAccounts.length,
|
|
3370
|
+
chats: cumulativeChats,
|
|
3371
|
+
messages: cumulativeMessages,
|
|
3372
|
+
}));
|
|
3373
|
+
if (request.onProgress !== undefined) {
|
|
3374
|
+
const conversionStartedAt = Date.now();
|
|
3375
|
+
progressHeartbeat = setInterval(() => {
|
|
3376
|
+
try {
|
|
3377
|
+
request.onProgress?.(Object.freeze({
|
|
3378
|
+
phase: "conversion-progress",
|
|
3379
|
+
elapsedSeconds: Math.max(
|
|
3380
|
+
1,
|
|
3381
|
+
Math.floor((Date.now() - conversionStartedAt) / 1_000),
|
|
3382
|
+
),
|
|
3383
|
+
}));
|
|
3384
|
+
} catch {
|
|
3385
|
+
progressHeartbeatFailed = true;
|
|
3386
|
+
}
|
|
3387
|
+
}, ACCOUNT_HEARTBEAT_INTERVAL_MS);
|
|
3388
|
+
}
|
|
3389
|
+
const accountsById = new Map(accounts.map((account) => [account.accountId, account]));
|
|
3390
|
+
const allListedChatEntries = [...validatedChats].sort((left, right) =>
|
|
3391
|
+
localId("conversation", left.chat.accountId, left.chat.id).localeCompare(
|
|
3392
|
+
localId("conversation", right.chat.accountId, right.chat.id),
|
|
3393
|
+
));
|
|
3394
|
+
const selfAliasPrepass = await prepassSelfAliases(
|
|
3395
|
+
allListedChatEntries,
|
|
3396
|
+
accounts,
|
|
3397
|
+
accountsById,
|
|
3398
|
+
maxMessagesJsonBytes,
|
|
3399
|
+
maxBundleRecords,
|
|
3400
|
+
maxBundleBytes,
|
|
3401
|
+
maxParticipantOccurrences,
|
|
3402
|
+
accountObservedAt,
|
|
3403
|
+
request.signal,
|
|
3404
|
+
assertProgressHeartbeat,
|
|
3405
|
+
request.dependencies?.onSelfAliasRefinementPass,
|
|
3406
|
+
);
|
|
3407
|
+
const aliasesByAccount = selfAliasPrepass.aliasesByAccount;
|
|
3408
|
+
|
|
3409
|
+
const participantFacts = new Map<string, ParticipantFact>();
|
|
3410
|
+
const selfParticipantByAccount = new Map<string, string>();
|
|
3411
|
+
for (const account of accounts) {
|
|
3412
|
+
const self = upsertParticipant(
|
|
3413
|
+
participantFacts,
|
|
3414
|
+
account,
|
|
3415
|
+
account.user,
|
|
3416
|
+
true,
|
|
3417
|
+
aliasesByAccount,
|
|
3418
|
+
);
|
|
3419
|
+
selfParticipantByAccount.set(account.accountId, self.id);
|
|
3420
|
+
}
|
|
3421
|
+
|
|
3422
|
+
const scans: ConversationScan[] = [];
|
|
3423
|
+
let observedFrom: string | null = null;
|
|
3424
|
+
let observedThrough: string | null = null;
|
|
3425
|
+
let participantRosterIncomplete = false;
|
|
3426
|
+
let messageLimitReached = false;
|
|
3427
|
+
let oversizedChatSkipped = false;
|
|
3428
|
+
let scannedRecordCount = accounts.length + selfParticipantByAccount.size;
|
|
3429
|
+
const scannedParticipantIds = new Set(selfParticipantByAccount.values());
|
|
3430
|
+
let scanRecordBudgetExhausted = false;
|
|
3431
|
+
const listedChatEntries = selfAliasPrepass.coveredChats;
|
|
3432
|
+
for (const {
|
|
3433
|
+
chat: validatedChat,
|
|
3434
|
+
root: shardRoot,
|
|
3435
|
+
chatPath,
|
|
3436
|
+
chatSha256,
|
|
3437
|
+
messagesPath,
|
|
3438
|
+
expectedMessageCount,
|
|
3439
|
+
observedAt,
|
|
3440
|
+
} of listedChatEntries) {
|
|
3441
|
+
assertProgressHeartbeat();
|
|
3442
|
+
throwIfAborted(request.signal);
|
|
3443
|
+
const chatDocument = await readOwnedJsonDocument(
|
|
3444
|
+
chatPath,
|
|
3445
|
+
shardRoot,
|
|
3446
|
+
MAX_CHAT_JSON_BYTES,
|
|
3447
|
+
false,
|
|
3448
|
+
request.signal,
|
|
3449
|
+
);
|
|
3450
|
+
if (chatDocument.sha256 !== chatSha256) {
|
|
3451
|
+
return fail("official export chat changed between validated passes");
|
|
3452
|
+
}
|
|
3453
|
+
const chat = parseBeeperExportConversation(chatDocument.value, accounts);
|
|
3454
|
+
if (
|
|
3455
|
+
chat.id !== validatedChat.id
|
|
3456
|
+
|| chat.accountId !== validatedChat.accountId
|
|
3457
|
+
) return fail("official export chat identity changed between validated passes");
|
|
3458
|
+
const account = accountsById.get(chat.accountId);
|
|
3459
|
+
if (account === undefined) return fail("official export chat references an unknown account");
|
|
3460
|
+
if (
|
|
3461
|
+
expectedMessageCount >= effectiveMessageLimit
|
|
3462
|
+
) messageLimitReached = true;
|
|
3463
|
+
const prepassMessagesSha256 = selfAliasPrepass.messagesSha256ByPath.get(messagesPath);
|
|
3464
|
+
if (await ownedFileSize(messagesPath, shardRoot) > maxMessagesJsonBytes) {
|
|
3465
|
+
if (prepassMessagesSha256 !== undefined) {
|
|
3466
|
+
return fail("official export messages changed after self-alias prepass");
|
|
3467
|
+
}
|
|
3468
|
+
oversizedChatSkipped = true;
|
|
3469
|
+
continue;
|
|
3470
|
+
}
|
|
3471
|
+
if (prepassMessagesSha256 === undefined) {
|
|
3472
|
+
return fail("official export self-alias message proof disappeared");
|
|
3473
|
+
}
|
|
3474
|
+
const messagesDocument = await readOwnedJsonDocument(
|
|
3475
|
+
messagesPath,
|
|
3476
|
+
shardRoot,
|
|
3477
|
+
maxMessagesJsonBytes,
|
|
3478
|
+
false,
|
|
3479
|
+
request.signal,
|
|
3480
|
+
);
|
|
3481
|
+
if (messagesDocument.sha256 !== prepassMessagesSha256) {
|
|
3482
|
+
return fail("official export messages changed after self-alias prepass");
|
|
3483
|
+
}
|
|
3484
|
+
const messages = parseBeeperExportMessages(
|
|
3485
|
+
messagesDocument.value,
|
|
3486
|
+
chat.accountId,
|
|
3487
|
+
chat.id,
|
|
3488
|
+
MAX_EXPORT_MESSAGES_PER_CHAT,
|
|
3489
|
+
);
|
|
3490
|
+
if (messages.length !== expectedMessageCount) {
|
|
3491
|
+
return fail("official export chat messages did not match completed state");
|
|
3492
|
+
}
|
|
3493
|
+
if (
|
|
3494
|
+
scanRecordBudgetExhausted
|
|
3495
|
+
|| expectedMessageCount + 1 > maxBundleRecords - scannedRecordCount
|
|
3496
|
+
) {
|
|
3497
|
+
scanRecordBudgetExhausted = true;
|
|
3498
|
+
continue;
|
|
3499
|
+
}
|
|
3500
|
+
const selfParticipantId = selfParticipantByAccount.get(account.accountId);
|
|
3501
|
+
if (selfParticipantId === undefined) {
|
|
3502
|
+
return fail("Beeper account self participant disappeared");
|
|
3503
|
+
}
|
|
3504
|
+
const selfParticipant = participantFacts.get(selfParticipantId);
|
|
3505
|
+
if (selfParticipant === undefined) {
|
|
3506
|
+
return fail("Beeper account self participant disappeared");
|
|
3507
|
+
}
|
|
3508
|
+
const scanParticipantFacts = new Map<string, ParticipantFact>([
|
|
3509
|
+
[selfParticipantId, selfParticipant],
|
|
3510
|
+
]);
|
|
3511
|
+
const participantIds = new Set<string>();
|
|
3512
|
+
for (const participant of chat.participants.items) {
|
|
3513
|
+
participantIds.add(upsertParticipant(
|
|
3514
|
+
scanParticipantFacts,
|
|
3515
|
+
account,
|
|
3516
|
+
participant,
|
|
3517
|
+
participant.isSelf,
|
|
3518
|
+
aliasesByAccount,
|
|
3519
|
+
).id);
|
|
3520
|
+
}
|
|
3521
|
+
if (chat.type === "single") {
|
|
3522
|
+
participantIds.add(selfParticipantId);
|
|
3523
|
+
}
|
|
3524
|
+
const messageIds = new Set(messages.map((message) => message.id));
|
|
3525
|
+
const reactionCount = messages.reduce(
|
|
3526
|
+
(count, message) => count + message.reactions.length,
|
|
3527
|
+
0,
|
|
3528
|
+
);
|
|
3529
|
+
const reactionProviderIdNonUniqueGroups = messages.reduce(
|
|
3530
|
+
(count, message) => count + new Set(
|
|
3531
|
+
message.reactions
|
|
3532
|
+
.filter((reaction) => reaction.providerIdNonUnique)
|
|
3533
|
+
.map((reaction) => reaction.id),
|
|
3534
|
+
).size,
|
|
3535
|
+
0,
|
|
3536
|
+
);
|
|
3537
|
+
const tombstoneCount = messages.reduce(
|
|
3538
|
+
(count, message) => count + (message.isDeleted || message.isHidden ? 1 : 0),
|
|
3539
|
+
0,
|
|
3540
|
+
);
|
|
3541
|
+
if (
|
|
3542
|
+
!Number.isSafeInteger(reactionCount)
|
|
3543
|
+
|| !Number.isSafeInteger(tombstoneCount)
|
|
3544
|
+
) return fail("official export derived record count overflowed");
|
|
3545
|
+
for (const message of messages) {
|
|
3546
|
+
participantIds.add(upsertParticipant(scanParticipantFacts, account, {
|
|
3547
|
+
id: message.senderId,
|
|
3548
|
+
fullName: message.senderName,
|
|
3549
|
+
phoneNumber: null,
|
|
3550
|
+
email: null,
|
|
3551
|
+
username: null,
|
|
3552
|
+
}, message.isSender, aliasesByAccount).id);
|
|
3553
|
+
for (const reaction of message.reactions) {
|
|
3554
|
+
participantIds.add(upsertParticipant(scanParticipantFacts, account, {
|
|
3555
|
+
id: reaction.participantId,
|
|
3556
|
+
fullName: null,
|
|
3557
|
+
phoneNumber: null,
|
|
3558
|
+
email: null,
|
|
3559
|
+
username: null,
|
|
3560
|
+
}, null, aliasesByAccount).id);
|
|
3561
|
+
}
|
|
3562
|
+
}
|
|
3563
|
+
const newlySeenParticipantIds = [...participantIds].filter(
|
|
3564
|
+
(participantId) => !scannedParticipantIds.has(participantId),
|
|
3565
|
+
);
|
|
3566
|
+
const scanRecordCount = 1
|
|
3567
|
+
+ messages.length
|
|
3568
|
+
+ reactionCount
|
|
3569
|
+
+ tombstoneCount
|
|
3570
|
+
+ newlySeenParticipantIds.length;
|
|
3571
|
+
if (scanRecordCount > maxBundleRecords - scannedRecordCount) {
|
|
3572
|
+
scanRecordBudgetExhausted = true;
|
|
3573
|
+
continue;
|
|
3574
|
+
}
|
|
3575
|
+
scannedRecordCount += scanRecordCount;
|
|
3576
|
+
for (const participantId of newlySeenParticipantIds) {
|
|
3577
|
+
scannedParticipantIds.add(participantId);
|
|
3578
|
+
}
|
|
3579
|
+
const range = messageTimestampRange(messages);
|
|
3580
|
+
if (range.first !== null && (observedFrom === null || range.first < observedFrom)) {
|
|
3581
|
+
observedFrom = range.first;
|
|
3582
|
+
}
|
|
3583
|
+
if (range.last !== null && (observedThrough === null || range.last > observedThrough)) {
|
|
3584
|
+
observedThrough = range.last;
|
|
3585
|
+
}
|
|
3586
|
+
const roster = [...participantIds].map((participantId) => {
|
|
3587
|
+
const participant = scanParticipantFacts.get(participantId);
|
|
3588
|
+
if (participant === undefined) {
|
|
3589
|
+
return fail("Beeper conversation participant disappeared");
|
|
3590
|
+
}
|
|
3591
|
+
return participant;
|
|
3592
|
+
});
|
|
3593
|
+
const directRosterComplete = chat.type !== "single"
|
|
3594
|
+
|| (
|
|
3595
|
+
roster.length === 2
|
|
3596
|
+
&& roster.filter((participant) => participant.isSelf === true).length === 1
|
|
3597
|
+
&& roster.filter((participant) => participant.isSelf !== true).length === 1
|
|
3598
|
+
);
|
|
3599
|
+
const participantsComplete = !chat.participants.hasMore
|
|
3600
|
+
&& chat.participants.items.length === chat.participants.total
|
|
3601
|
+
&& directRosterComplete;
|
|
3602
|
+
if (!participantsComplete) participantRosterIncomplete = true;
|
|
3603
|
+
const scanDraft: ConversationScan = Object.freeze({
|
|
3604
|
+
chat: Object.freeze({
|
|
3605
|
+
id: chat.id,
|
|
3606
|
+
accountId: chat.accountId,
|
|
3607
|
+
lastActivity: chat.lastActivity,
|
|
3608
|
+
title: chat.title,
|
|
3609
|
+
type: chat.type,
|
|
3610
|
+
}),
|
|
3611
|
+
root: shardRoot,
|
|
3612
|
+
messagesPath,
|
|
3613
|
+
messagesSha256: messagesDocument.sha256,
|
|
3614
|
+
observedAt,
|
|
3615
|
+
participantIds: Object.freeze([...participantIds].sort()),
|
|
3616
|
+
participantFactChanges: Object.freeze([...participantIds]
|
|
3617
|
+
.sort()
|
|
3618
|
+
.map((participantId) => {
|
|
3619
|
+
const fact = scanParticipantFacts.get(participantId);
|
|
3620
|
+
if (fact === undefined) {
|
|
3621
|
+
return fail("Beeper conversation participant disappeared");
|
|
3622
|
+
}
|
|
3623
|
+
return fact;
|
|
3624
|
+
})),
|
|
3625
|
+
participantsComplete,
|
|
3626
|
+
startedAt: range.first,
|
|
3627
|
+
lastMessageAt: range.last,
|
|
3628
|
+
messageCount: messages.length,
|
|
3629
|
+
reactionCount,
|
|
3630
|
+
reactionProviderIdNonUniqueGroups,
|
|
3631
|
+
tombstoneCount,
|
|
3632
|
+
nonParticipantRecordBytes: 0,
|
|
3633
|
+
});
|
|
3634
|
+
const network = normalizeNetwork(account.network, account.bridge.type);
|
|
3635
|
+
let nonParticipantRecordBytes = bundleRecordBytes(
|
|
3636
|
+
conversationRecord(scanDraft, account, network, observedAt),
|
|
3637
|
+
);
|
|
3638
|
+
for (const message of messages) {
|
|
3639
|
+
nonParticipantRecordBytes += bundleRecordBytes(
|
|
3640
|
+
messageRecord(
|
|
3641
|
+
message,
|
|
3642
|
+
scanDraft,
|
|
3643
|
+
messageIds,
|
|
3644
|
+
account,
|
|
3645
|
+
aliasesByAccount,
|
|
3646
|
+
network,
|
|
3647
|
+
observedAt,
|
|
3648
|
+
),
|
|
3649
|
+
);
|
|
3650
|
+
for (const reaction of message.reactions) {
|
|
3651
|
+
nonParticipantRecordBytes += bundleRecordBytes(reactionRecord(
|
|
3652
|
+
reaction,
|
|
3653
|
+
message,
|
|
3654
|
+
scanDraft,
|
|
3655
|
+
account,
|
|
3656
|
+
aliasesByAccount,
|
|
3657
|
+
network,
|
|
3658
|
+
observedAt,
|
|
3659
|
+
));
|
|
3660
|
+
}
|
|
3661
|
+
const tombstone = tombstoneRecord(
|
|
3662
|
+
message,
|
|
3663
|
+
scanDraft,
|
|
3664
|
+
account,
|
|
3665
|
+
network,
|
|
3666
|
+
observedAt,
|
|
3667
|
+
);
|
|
3668
|
+
if (tombstone !== null) {
|
|
3669
|
+
nonParticipantRecordBytes += bundleRecordBytes(tombstone);
|
|
3670
|
+
}
|
|
3671
|
+
}
|
|
3672
|
+
if (!Number.isSafeInteger(nonParticipantRecordBytes)) {
|
|
3673
|
+
return fail("official export derived record bytes overflowed");
|
|
3674
|
+
}
|
|
3675
|
+
scans.push(Object.freeze({
|
|
3676
|
+
...scanDraft,
|
|
3677
|
+
nonParticipantRecordBytes,
|
|
3678
|
+
}));
|
|
3679
|
+
}
|
|
3680
|
+
const orderedScans = [...scans].sort((left, right) =>
|
|
3681
|
+
localId("conversation", left.chat.accountId, left.chat.id).localeCompare(
|
|
3682
|
+
localId("conversation", right.chat.accountId, right.chat.id),
|
|
3683
|
+
));
|
|
3684
|
+
const selectedParticipantFacts = new Map(participantFacts);
|
|
3685
|
+
const selectedParticipantIds = new Set(selectedParticipantFacts.keys());
|
|
3686
|
+
const selectedScans: ConversationScan[] = [];
|
|
3687
|
+
let selectedRecordCount = accounts.length + selectedParticipantIds.size;
|
|
3688
|
+
let selectedRecordBytes = 0;
|
|
3689
|
+
for (const account of accounts) {
|
|
3690
|
+
const selfParticipantId = selfParticipantByAccount.get(account.accountId);
|
|
3691
|
+
if (selfParticipantId === undefined) {
|
|
3692
|
+
return fail("Beeper account self participant disappeared");
|
|
3693
|
+
}
|
|
3694
|
+
const network = normalizeNetwork(account.network, account.bridge.type);
|
|
3695
|
+
selectedRecordBytes += bundleRecordBytes(accountRecord(
|
|
3696
|
+
account,
|
|
3697
|
+
network,
|
|
3698
|
+
observedAtForAccount(account.accountId),
|
|
3699
|
+
selfParticipantId,
|
|
3700
|
+
));
|
|
3701
|
+
const selfFact = participantFacts.get(selfParticipantId);
|
|
3702
|
+
if (selfFact === undefined) return fail("Beeper account self participant disappeared");
|
|
3703
|
+
selectedRecordBytes += bundleRecordBytes(participantRecord(
|
|
3704
|
+
selfFact,
|
|
3705
|
+
account,
|
|
3706
|
+
network,
|
|
3707
|
+
observedAtForAccount(account.accountId),
|
|
3708
|
+
));
|
|
3709
|
+
}
|
|
3710
|
+
let bundleRecordLimitReached = selfAliasPrepass.bundleRecordLimitReached;
|
|
3711
|
+
let bundleByteLimitReached = selfAliasPrepass.bundleByteLimitReached;
|
|
3712
|
+
for (const scan of orderedScans) {
|
|
3713
|
+
const account = accountsById.get(scan.chat.accountId);
|
|
3714
|
+
if (account === undefined) {
|
|
3715
|
+
return fail("selected participant source identity disappeared");
|
|
3716
|
+
}
|
|
3717
|
+
const expectedParticipantAccountId = localId("account", account.accountId);
|
|
3718
|
+
const stagedParticipantFacts = new Map<string, ParticipantFact>();
|
|
3719
|
+
for (const incoming of scan.participantFactChanges) {
|
|
3720
|
+
if (incoming.accountId !== expectedParticipantAccountId) {
|
|
3721
|
+
return fail("selected participant source identity changed accounts");
|
|
3722
|
+
}
|
|
3723
|
+
const current = stagedParticipantFacts.get(incoming.id)
|
|
3724
|
+
?? selectedParticipantFacts.get(incoming.id);
|
|
3725
|
+
stagedParticipantFacts.set(
|
|
3726
|
+
incoming.id,
|
|
3727
|
+
mergeParticipantFact(current, incoming),
|
|
3728
|
+
);
|
|
3729
|
+
}
|
|
3730
|
+
let addedParticipants = 0;
|
|
3731
|
+
let participantByteDelta = 0;
|
|
3732
|
+
for (const [participantId, fact] of stagedParticipantFacts) {
|
|
3733
|
+
const current = selectedParticipantFacts.get(participantId);
|
|
3734
|
+
if (current === undefined) addedParticipants += 1;
|
|
3735
|
+
const factBytes = bundleRecordBytes(participantRecord(
|
|
3736
|
+
fact,
|
|
3737
|
+
account,
|
|
3738
|
+
normalizeNetwork(account.network, account.bridge.type),
|
|
3739
|
+
observedAtForAccount(account.accountId),
|
|
3740
|
+
));
|
|
3741
|
+
const currentBytes = current === undefined
|
|
3742
|
+
? 0
|
|
3743
|
+
: bundleRecordBytes(participantRecord(
|
|
3744
|
+
current,
|
|
3745
|
+
account,
|
|
3746
|
+
normalizeNetwork(account.network, account.bridge.type),
|
|
3747
|
+
observedAtForAccount(account.accountId),
|
|
3748
|
+
));
|
|
3749
|
+
participantByteDelta += factBytes - currentBytes;
|
|
3750
|
+
}
|
|
3751
|
+
const scanRecords = 1
|
|
3752
|
+
+ scan.messageCount
|
|
3753
|
+
+ scan.reactionCount
|
|
3754
|
+
+ scan.tombstoneCount
|
|
3755
|
+
+ addedParticipants;
|
|
3756
|
+
const scanBytes = scan.nonParticipantRecordBytes + participantByteDelta;
|
|
3757
|
+
if (!Number.isSafeInteger(scanBytes) || scanBytes < 0) {
|
|
3758
|
+
return fail("official export derived record bytes overflowed");
|
|
3759
|
+
}
|
|
3760
|
+
const exceedsRecords = scanRecords > maxBundleRecords - selectedRecordCount;
|
|
3761
|
+
const exceedsBytes = scanBytes > maxBundleBytes - selectedRecordBytes;
|
|
3762
|
+
if (exceedsRecords || exceedsBytes) {
|
|
3763
|
+
bundleRecordLimitReached ||= exceedsRecords;
|
|
3764
|
+
bundleByteLimitReached ||= exceedsBytes;
|
|
3765
|
+
break;
|
|
3766
|
+
}
|
|
3767
|
+
selectedScans.push(scan);
|
|
3768
|
+
selectedRecordCount += scanRecords;
|
|
3769
|
+
selectedRecordBytes += scanBytes;
|
|
3770
|
+
for (const [participantId, fact] of stagedParticipantFacts) {
|
|
3771
|
+
selectedParticipantFacts.set(participantId, fact);
|
|
3772
|
+
}
|
|
3773
|
+
for (const participantId of scan.participantIds) {
|
|
3774
|
+
selectedParticipantIds.add(participantId);
|
|
3775
|
+
}
|
|
3776
|
+
}
|
|
3777
|
+
if (
|
|
3778
|
+
scanRecordBudgetExhausted
|
|
3779
|
+
|| selectedScans.length !== orderedScans.length
|
|
3780
|
+
) return fail("bounded chat prefix exceeded its conservative admission proof");
|
|
3781
|
+
if (
|
|
3782
|
+
selectedRecordCount > selfAliasPrepass.provisionalRecordCount
|
|
3783
|
+
|| selectedRecordBytes > selfAliasPrepass.provisionalRecordBytes
|
|
3784
|
+
) return fail("exact bundle exceeded its provisional admission proof");
|
|
3785
|
+
observedFrom = null;
|
|
3786
|
+
observedThrough = null;
|
|
3787
|
+
for (const scan of selectedScans) {
|
|
3788
|
+
if (
|
|
3789
|
+
scan.startedAt !== null
|
|
3790
|
+
&& (observedFrom === null || scan.startedAt < observedFrom)
|
|
3791
|
+
) observedFrom = scan.startedAt;
|
|
3792
|
+
if (
|
|
3793
|
+
scan.lastMessageAt !== null
|
|
3794
|
+
&& (observedThrough === null || scan.lastMessageAt > observedThrough)
|
|
3795
|
+
) observedThrough = scan.lastMessageAt;
|
|
3796
|
+
}
|
|
3797
|
+
participantRosterIncomplete = selectedScans.some(
|
|
3798
|
+
(scan) => !scan.participantsComplete,
|
|
3799
|
+
);
|
|
3800
|
+
const validatedReactionProviderIdNonUnique = selectedScans.some(
|
|
3801
|
+
(scan) => scan.reactionProviderIdNonUniqueGroups > 0,
|
|
3802
|
+
);
|
|
3803
|
+
|
|
3804
|
+
for (const account of [...accounts].sort((left, right) =>
|
|
3805
|
+
left.accountId.localeCompare(right.accountId))) {
|
|
3806
|
+
const selfParticipantId = selfParticipantByAccount.get(account.accountId);
|
|
3807
|
+
if (selfParticipantId === undefined) return fail("Beeper account has no self participant");
|
|
3808
|
+
const record = accountRecord(
|
|
3809
|
+
account,
|
|
3810
|
+
normalizeNetwork(account.network, account.bridge.type),
|
|
3811
|
+
observedAtForAccount(account.accountId),
|
|
3812
|
+
selfParticipantId,
|
|
3813
|
+
);
|
|
3814
|
+
request.onRecordCoordinate?.(record, Object.freeze({
|
|
3815
|
+
kind: "account",
|
|
3816
|
+
accountId: account.accountId,
|
|
3817
|
+
}));
|
|
3818
|
+
yield record;
|
|
3819
|
+
}
|
|
3820
|
+
for (const fact of [...selectedParticipantFacts.values()]
|
|
3821
|
+
.filter((candidate) => selectedParticipantIds.has(candidate.id))
|
|
3822
|
+
.sort((left, right) => left.id.localeCompare(right.id))) {
|
|
3823
|
+
const account = accounts.find((candidate) =>
|
|
3824
|
+
localId("account", candidate.accountId) === fact.accountId);
|
|
3825
|
+
if (account === undefined) return fail("participant account disappeared");
|
|
3826
|
+
const record = participantRecord(
|
|
3827
|
+
fact,
|
|
3828
|
+
account,
|
|
3829
|
+
normalizeNetwork(account.network, account.bridge.type),
|
|
3830
|
+
observedAtForAccount(account.accountId),
|
|
3831
|
+
);
|
|
3832
|
+
request.onRecordCoordinate?.(record, Object.freeze({
|
|
3833
|
+
kind: "participant",
|
|
3834
|
+
accountId: account.accountId,
|
|
3835
|
+
participantId: fact.sourceId,
|
|
3836
|
+
}));
|
|
3837
|
+
yield record;
|
|
3838
|
+
}
|
|
3839
|
+
let emittedReactionProviderIdNonUnique = false;
|
|
3840
|
+
for (const scan of selectedScans) {
|
|
3841
|
+
assertProgressHeartbeat();
|
|
3842
|
+
const account = accountsById.get(scan.chat.accountId);
|
|
3843
|
+
if (account === undefined) return fail("conversation account disappeared");
|
|
3844
|
+
const network = normalizeNetwork(account.network, account.bridge.type);
|
|
3845
|
+
const conversation = conversationRecord(
|
|
3846
|
+
scan,
|
|
3847
|
+
account,
|
|
3848
|
+
network,
|
|
3849
|
+
scan.observedAt,
|
|
3850
|
+
);
|
|
3851
|
+
request.onRecordCoordinate?.(conversation, Object.freeze({
|
|
3852
|
+
kind: "conversation",
|
|
3853
|
+
accountId: account.accountId,
|
|
3854
|
+
conversationId: scan.chat.id,
|
|
3855
|
+
}));
|
|
3856
|
+
yield conversation;
|
|
3857
|
+
const messagesDocument = await readOwnedJsonDocument(
|
|
3858
|
+
scan.messagesPath,
|
|
3859
|
+
scan.root,
|
|
3860
|
+
maxMessagesJsonBytes,
|
|
3861
|
+
false,
|
|
3862
|
+
request.signal,
|
|
3863
|
+
);
|
|
3864
|
+
if (messagesDocument.sha256 !== scan.messagesSha256) {
|
|
3865
|
+
return fail("official export messages changed between validated passes");
|
|
3866
|
+
}
|
|
3867
|
+
const messages = parseBeeperExportMessages(
|
|
3868
|
+
messagesDocument.value,
|
|
3869
|
+
scan.chat.accountId,
|
|
3870
|
+
scan.chat.id,
|
|
3871
|
+
MAX_EXPORT_MESSAGES_PER_CHAT,
|
|
3872
|
+
);
|
|
3873
|
+
if (messages.length !== scan.messageCount) {
|
|
3874
|
+
return fail("official export message count changed between validated passes");
|
|
3875
|
+
}
|
|
3876
|
+
const reactionCount = messages.reduce(
|
|
3877
|
+
(count, message) => count + message.reactions.length,
|
|
3878
|
+
0,
|
|
3879
|
+
);
|
|
3880
|
+
const reactionProviderIdNonUniqueGroups = messages.reduce(
|
|
3881
|
+
(count, message) => count + new Set(
|
|
3882
|
+
message.reactions
|
|
3883
|
+
.filter((reaction) => reaction.providerIdNonUnique)
|
|
3884
|
+
.map((reaction) => reaction.id),
|
|
3885
|
+
).size,
|
|
3886
|
+
0,
|
|
3887
|
+
);
|
|
3888
|
+
if (
|
|
3889
|
+
reactionCount !== scan.reactionCount
|
|
3890
|
+
|| reactionProviderIdNonUniqueGroups
|
|
3891
|
+
!== scan.reactionProviderIdNonUniqueGroups
|
|
3892
|
+
) {
|
|
3893
|
+
return fail("official export reaction identity changed between validated passes");
|
|
3894
|
+
}
|
|
3895
|
+
emittedReactionProviderIdNonUnique ||= reactionProviderIdNonUniqueGroups > 0;
|
|
3896
|
+
const messageIds = new Set(messages.map((message) => message.id));
|
|
3897
|
+
for (const message of messages) {
|
|
3898
|
+
const record = messageRecord(
|
|
3899
|
+
message,
|
|
3900
|
+
scan,
|
|
3901
|
+
messageIds,
|
|
3902
|
+
account,
|
|
3903
|
+
aliasesByAccount,
|
|
3904
|
+
network,
|
|
3905
|
+
scan.observedAt,
|
|
3906
|
+
);
|
|
3907
|
+
request.onRecordCoordinate?.(record, Object.freeze({
|
|
3908
|
+
kind: "message",
|
|
3909
|
+
accountId: account.accountId,
|
|
3910
|
+
conversationId: scan.chat.id,
|
|
3911
|
+
messageId: message.id,
|
|
3912
|
+
}));
|
|
3913
|
+
yield record;
|
|
3914
|
+
for (const reaction of message.reactions) {
|
|
3915
|
+
yield reactionRecord(
|
|
3916
|
+
reaction,
|
|
3917
|
+
message,
|
|
3918
|
+
scan,
|
|
3919
|
+
account,
|
|
3920
|
+
aliasesByAccount,
|
|
3921
|
+
network,
|
|
3922
|
+
scan.observedAt,
|
|
3923
|
+
);
|
|
3924
|
+
}
|
|
3925
|
+
const tombstone = tombstoneRecord(
|
|
3926
|
+
message,
|
|
3927
|
+
scan,
|
|
3928
|
+
account,
|
|
3929
|
+
network,
|
|
3930
|
+
scan.observedAt,
|
|
3931
|
+
);
|
|
3932
|
+
if (tombstone !== null) yield tombstone;
|
|
3933
|
+
}
|
|
3934
|
+
}
|
|
3935
|
+
if (
|
|
3936
|
+
emittedReactionProviderIdNonUnique
|
|
3937
|
+
!== validatedReactionProviderIdNonUnique
|
|
3938
|
+
) return fail("official export reaction identity changed between validated passes");
|
|
3939
|
+
|
|
3940
|
+
const warnings = new Set([
|
|
3941
|
+
"attachments-metadata-only",
|
|
3942
|
+
"remote-history-not-claimed",
|
|
3943
|
+
"connected-account-backfill-coverage-unknown",
|
|
3944
|
+
"sequential-account-snapshot",
|
|
3945
|
+
]);
|
|
3946
|
+
const chatLimitReached = cumulativeChats >= effectiveChatLimit;
|
|
3947
|
+
const hardChatLimitReached = chatLimitReached && limits.limitChats === null;
|
|
3948
|
+
const hardMessageLimitReached = messageLimitReached
|
|
3949
|
+
&& limits.limitMessages === null;
|
|
3950
|
+
const hardSourceLimitReached = hardChatLimitReached
|
|
3951
|
+
|| hardMessageLimitReached;
|
|
3952
|
+
if (chatLimitReached) warnings.add("chat-limit-reached");
|
|
3953
|
+
if (messageLimitReached) warnings.add("message-limit-reached");
|
|
3954
|
+
if (participantRosterIncomplete) warnings.add("participant-roster-incomplete");
|
|
3955
|
+
if (oversizedChatSkipped) warnings.add("oversized-chat-skipped");
|
|
3956
|
+
if (bundleRecordLimitReached) warnings.add("bundle-record-limit-reached");
|
|
3957
|
+
if (bundleByteLimitReached) warnings.add("bundle-byte-limit-reached");
|
|
3958
|
+
if (selfAliasPrepass.evidenceLimitReached) {
|
|
3959
|
+
warnings.add("self-alias-evidence-limit-reached");
|
|
3960
|
+
}
|
|
3961
|
+
if (selfAliasPrepass.participantOccurrenceLimitReached) {
|
|
3962
|
+
warnings.add("participant-occurrence-limit-reached");
|
|
3963
|
+
}
|
|
3964
|
+
if (emittedReactionProviderIdNonUnique) {
|
|
3965
|
+
warnings.add("reaction-provider-id-non-unique");
|
|
3966
|
+
}
|
|
3967
|
+
const truncated = chatLimitReached
|
|
3968
|
+
|| messageLimitReached
|
|
3969
|
+
|| oversizedChatSkipped
|
|
3970
|
+
|| bundleRecordLimitReached
|
|
3971
|
+
|| bundleByteLimitReached
|
|
3972
|
+
|| selfAliasPrepass.evidenceLimitReached
|
|
3973
|
+
|| selfAliasPrepass.participantOccurrenceLimitReached;
|
|
3974
|
+
stopProgressHeartbeat();
|
|
3975
|
+
assertProgressHeartbeat();
|
|
3976
|
+
completion = Object.freeze({
|
|
3977
|
+
completeness: Object.freeze({
|
|
3978
|
+
kind: truncated ? "truncated" : "bounded-local",
|
|
3979
|
+
reason: bundleRecordLimitReached
|
|
3980
|
+
? "bundle-record-limit"
|
|
3981
|
+
: bundleByteLimitReached
|
|
3982
|
+
? "bundle-byte-limit"
|
|
3983
|
+
: selfAliasPrepass.evidenceLimitReached
|
|
3984
|
+
? "self-alias-evidence-limit"
|
|
3985
|
+
: selfAliasPrepass.participantOccurrenceLimitReached
|
|
3986
|
+
? "participant-occurrence-limit"
|
|
3987
|
+
: oversizedChatSkipped
|
|
3988
|
+
? "oversized-chat"
|
|
3989
|
+
: hardSourceLimitReached
|
|
3990
|
+
? "source-hard-limit"
|
|
3991
|
+
: truncated
|
|
3992
|
+
? "explicit-source-limit"
|
|
3993
|
+
: "desktop-local-sequential-export",
|
|
3994
|
+
observedFrom,
|
|
3995
|
+
observedThrough,
|
|
3996
|
+
}),
|
|
3997
|
+
warnings: Object.freeze([...warnings].sort()),
|
|
3998
|
+
});
|
|
3999
|
+
}
|
|
4000
|
+
})();
|
|
4001
|
+
|
|
4002
|
+
return Object.freeze({
|
|
4003
|
+
descriptor: Object.freeze({
|
|
4004
|
+
source: Object.freeze({ id: "beeper-local", version: "1.1.0" }),
|
|
4005
|
+
provider: Object.freeze({ id: "beeper", version: BEEPER_CLI_PIN.version }),
|
|
4006
|
+
}),
|
|
4007
|
+
records,
|
|
4008
|
+
completion: async () => {
|
|
4009
|
+
if (completion === undefined) return fail("record stream did not complete");
|
|
4010
|
+
return completion;
|
|
4011
|
+
},
|
|
4012
|
+
dispose: async (_published: boolean) => {
|
|
4013
|
+
stopProgressHeartbeat();
|
|
4014
|
+
if (disposed) return;
|
|
4015
|
+
if (disposalInFlight !== undefined) return disposalInFlight;
|
|
4016
|
+
const dispose = disposeWorking;
|
|
4017
|
+
if (dispose === undefined) {
|
|
4018
|
+
disposed = true;
|
|
4019
|
+
return;
|
|
4020
|
+
}
|
|
4021
|
+
const attempt = (async () => {
|
|
4022
|
+
await dispose();
|
|
4023
|
+
disposeWorking = undefined;
|
|
4024
|
+
disposed = true;
|
|
4025
|
+
})();
|
|
4026
|
+
disposalInFlight = attempt;
|
|
4027
|
+
try {
|
|
4028
|
+
await attempt;
|
|
4029
|
+
} finally {
|
|
4030
|
+
if (!disposed) disposalInFlight = undefined;
|
|
4031
|
+
}
|
|
4032
|
+
},
|
|
4033
|
+
});
|
|
4034
|
+
}
|