@hraness/ghostget 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +458 -0
- package/LICENSE +21 -0
- package/README.md +1359 -0
- package/bunfig.toml +2 -0
- package/dist/apple-photos-client.js +704 -0
- package/dist/beeper-client.js +4602 -0
- package/dist/client.js +1605 -0
- package/dist/index-26yq8q16.js +31 -0
- package/dist/index-9t13b4k7.js +508 -0
- package/dist/index-axwm9b3v.js +5 -0
- package/dist/index-gwk7rbyj.js +30 -0
- package/dist/index.js +701 -0
- package/dist/messaging.js +1130 -0
- package/dist/omni-client.js +1406 -0
- package/dist/whatsapp-client.js +355 -0
- package/docs/imessage-direct-provider.md +106 -0
- package/docs/rental-listings.md +94 -0
- package/package.json +600 -0
- package/skills/ghostget/SKILL.md +251 -0
- package/skills/ghostget/agents/openai.yaml +4 -0
- package/skills/ghostget/references/adapter-contract.md +181 -0
- package/skills/ghostget/references/apple-photos.md +115 -0
- package/skills/ghostget/references/article-drafts.md +235 -0
- package/skills/ghostget/references/code-owned-provider-scaffold.md +134 -0
- package/skills/ghostget/references/cross-posting-video.md +139 -0
- package/skills/ghostget/references/cross-posting.md +82 -0
- package/skills/ghostget/references/derivation.md +390 -0
- package/skills/ghostget/references/hraness-social-profile-stats.json +269 -0
- package/skills/ghostget/references/install.md +66 -0
- package/skills/ghostget/references/linkedin-adapter.md +442 -0
- package/skills/ghostget/references/messaging.md +168 -0
- package/skills/ghostget/references/meta-comet-contract.md +153 -0
- package/skills/ghostget/references/platform-patterns.md +122 -0
- package/skills/ghostget/references/provider-plugins.md +496 -0
- package/skills/ghostget/references/reddit-flair.md +78 -0
- package/skills/ghostget/references/safety-and-state.md +214 -0
- package/skills/ghostget/references/settlement-and-duplicate-cleanup.md +79 -0
- package/skills/ghostget/references/social-platform-routing.md +66 -0
- package/skills/ghostget/references/social-profile-stats.md +195 -0
- package/skills/ghostget/references/social-video-platform-routing.md +89 -0
- package/skills/ghostget/references/whatsapp-adapter.md +101 -0
- package/skills/ghostget/references/x-adapter.md +332 -0
- package/skills/ghostget/references/x-ai-disclosure.md +71 -0
- package/src/apple-photos-cli.ts +639 -0
- package/src/apple-photos-client-types.ts +168 -0
- package/src/apple-photos-client.ts +201 -0
- package/src/apple-photos-contact-evidence.ts +838 -0
- package/src/apple-photos-local-source.ts +1131 -0
- package/src/args.ts +2053 -0
- package/src/article-draft-document.ts +417 -0
- package/src/article-draft-embeds.ts +123 -0
- package/src/article-draft-images.ts +214 -0
- package/src/assets/adapter-template/wrench-adapter.json +40 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.json +1500 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v1.0.0.json +129 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v1.1.0.json +207 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v2.0.0.json +1463 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v2.1.0.json +1463 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v2.2.0.json +1475 -0
- package/src/assets/adapters/beeper/wrench-web-adapter.v2.3.0.json +1488 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.json +684 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.0.0.json +581 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.1.0.json +581 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.2.0.json +581 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.3.0.json +615 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.4.0.json +684 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.5.0.json +684 -0
- package/src/assets/adapters/bluesky/wrench-web-adapter.v1.6.0.json +684 -0
- package/src/assets/adapters/clasificados/wrench-web-adapter.json +54 -0
- package/src/assets/adapters/facebook/wrench-web-adapter.json +750 -0
- package/src/assets/adapters/facebook/wrench-web-adapter.v1.0.0.json +706 -0
- package/src/assets/adapters/facebook/wrench-web-adapter.v1.1.0.json +703 -0
- package/src/assets/adapters/facebook/wrench-web-adapter.v1.2.0.json +750 -0
- package/src/assets/adapters/facebook-group/wrench-web-adapter.json +682 -0
- package/src/assets/adapters/facebook-group/wrench-web-adapter.v1.0.0.json +583 -0
- package/src/assets/adapters/facebook-group/wrench-web-adapter.v1.1.0.json +682 -0
- package/src/assets/adapters/facebook-marketplace/wrench-web-adapter.json +437 -0
- package/src/assets/adapters/facebook-marketplace/wrench-web-adapter.v1.0.0.json +383 -0
- package/src/assets/adapters/facebook-marketplace/wrench-web-adapter.v1.1.0.json +437 -0
- package/src/assets/adapters/facebook-marketplace/wrench-web-adapter.v1.1.1.json +437 -0
- package/src/assets/adapters/facebook-page/wrench-web-adapter.json +880 -0
- package/src/assets/adapters/facebook-page/wrench-web-adapter.v1.0.0.json +740 -0
- package/src/assets/adapters/facebook-page/wrench-web-adapter.v1.1.0.json +880 -0
- package/src/assets/adapters/github/wrench-web-adapter.json +69 -0
- package/src/assets/adapters/github/wrench-web-adapter.v1.0.0.json +42 -0
- package/src/assets/adapters/github/wrench-web-adapter.v1.1.0.json +69 -0
- package/src/assets/adapters/gmail/wrench-adapter.json +161 -0
- package/src/assets/adapters/gmail/wrench-adapter.v1.2.0.json +157 -0
- package/src/assets/adapters/hacker-news/wrench-web-adapter.json +315 -0
- package/src/assets/adapters/hacker-news/wrench-web-adapter.v1.0.0.json +315 -0
- package/src/assets/adapters/imessage/wrench-web-adapter.json +236 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.json +719 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.0.0.json +627 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.1.0.json +609 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.2.0.json +640 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.3.0.json +671 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.4.0.json +710 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.5.0.json +710 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.6.0.json +719 -0
- package/src/assets/adapters/instagram/wrench-web-adapter.v1.7.0.json +719 -0
- package/src/assets/adapters/linkedin/wrench-adapter.json +442 -0
- package/src/assets/adapters/linkedin/wrench-adapter.v0.4.0.json +124 -0
- package/src/assets/adapters/linkedin/wrench-adapter.v1.0.0.json +411 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.json +914 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.0.0.json +604 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.1.0.json +604 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.10.0.json +809 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.11.0.json +809 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.12.0.json +820 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.13.0.json +819 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.14.0.json +819 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.15.0.json +819 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.16.0.json +874 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.17.0.json +874 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.18.0.json +874 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.19.0.json +874 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.2.0.json +731 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.3.0.json +768 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.4.0.json +808 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.5.0.json +808 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.6.0.json +808 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.7.0.json +825 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.8.0.json +792 -0
- package/src/assets/adapters/linkedin/wrench-web-adapter.v1.9.0.json +809 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.json +855 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.0.0.json +577 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.1.0.json +604 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.10.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.11.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.12.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.2.0.json +653 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.3.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.4.0.json +713 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.5.0.json +712 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.6.0.json +713 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.7.0.json +713 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.8.0.json +713 -0
- package/src/assets/adapters/reddit/wrench-web-adapter.v1.9.0.json +713 -0
- package/src/assets/adapters/substack/wrench-web-adapter.json +936 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.0.0.json +817 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.1.0.json +812 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.2.0.json +812 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.3.0.json +902 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.4.0.json +936 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.5.0.json +936 -0
- package/src/assets/adapters/substack/wrench-web-adapter.v1.6.0.json +936 -0
- package/src/assets/adapters/threads/wrench-web-adapter.json +692 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.0.0.json +629 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.1.0.json +623 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.2.0.json +622 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.3.0.json +622 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.4.0.json +622 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.5.0.json +699 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.6.0.json +693 -0
- package/src/assets/adapters/threads/wrench-web-adapter.v1.7.0.json +692 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.json +762 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.v1.0.0.json +681 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.v1.1.0.json +708 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.v1.2.0.json +762 -0
- package/src/assets/adapters/tiktok/wrench-web-adapter.v1.3.0.json +762 -0
- package/src/assets/adapters/twitch/wrench-web-adapter.json +42 -0
- package/src/assets/adapters/twitch/wrench-web-adapter.v1.0.0.json +42 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.json +400 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.v1.0.0.json +355 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.v1.1.0.json +386 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.v1.2.0.json +386 -0
- package/src/assets/adapters/whatsapp/wrench-web-adapter.v1.3.0.json +400 -0
- package/src/assets/adapters/x/wrench-adapter.json +749 -0
- package/src/assets/adapters/x/wrench-adapter.v1.0.0.json +700 -0
- package/src/assets/adapters/x/wrench-adapter.v1.1.0.json +707 -0
- package/src/assets/adapters/x/wrench-web-adapter.json +712 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.1.0.json +579 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.10.0.json +685 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.11.0.json +712 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.12.0.json +712 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.13.0.json +712 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.2.0.json +578 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.3.0.json +610 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.4.0.json +617 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.5.0.json +634 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.6.0.json +632 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.7.0.json +649 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.8.0.json +649 -0
- package/src/assets/adapters/x/wrench-web-adapter.v1.9.0.json +683 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.json +561 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.v1.0.0.json +465 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.v1.1.0.json +519 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.v1.2.0.json +561 -0
- package/src/assets/adapters/youtube/wrench-web-adapter.v1.3.0.json +561 -0
- package/src/assets/code-owned-provider-template/plugin.test.ts.template +54 -0
- package/src/assets/code-owned-provider-template/plugin.ts.template +102 -0
- package/src/assets/code-owned-provider-template/runtime.internal.test.ts.template +44 -0
- package/src/assets/code-owned-provider-template/runtime.ts.template +59 -0
- package/src/assets/code-owned-provider-template/wrench-adapter.json.template +33 -0
- package/src/assets/fonts/nebula-sans/LICENSE.txt +96 -0
- package/src/assets/fonts/nebula-sans/NebulaSans-Book.woff2 +0 -0
- package/src/assets/fonts/nebula-sans/PROVENANCE.md +14 -0
- package/src/assets.d.ts +4 -0
- package/src/auth.ts +1157 -0
- package/src/beeper-client-types.ts +169 -0
- package/src/beeper-client.ts +250 -0
- package/src/beeper-contact-interactions-cli.ts +180 -0
- package/src/beeper-contact-interactions.ts +1612 -0
- package/src/beeper-message-bundle-v1.ts +257 -0
- package/src/beeper-message-like-me-cli.ts +84 -0
- package/src/beeper-message-like-me-export.ts +2274 -0
- package/src/beeper-message-like-me-recovery.ts +931 -0
- package/src/beeper-message-like-me-source.ts +4034 -0
- package/src/browser-admission.ts +928 -0
- package/src/browser-snapshots.ts +260 -0
- package/src/browser.ts +3873 -0
- package/src/canonical-json.ts +60 -0
- package/src/catalog-cli.ts +638 -0
- package/src/cli.ts +208 -0
- package/src/client-types.ts +301 -0
- package/src/client.ts +2726 -0
- package/src/confirmed-write-failure.ts +44 -0
- package/src/confirmed-write-model.ts +72 -0
- package/src/confirmed-write-platform.ts +1055 -0
- package/src/confirmed-write-program.ts +224 -0
- package/src/confirmed-write-runtime.ts +16 -0
- package/src/cursor-token.ts +228 -0
- package/src/derivation-dnr-readiness.ts +367 -0
- package/src/derivation-file-chooser.ts +384 -0
- package/src/derivation-network-boundary.ts +835 -0
- package/src/derivation-network-guard.ts +782 -0
- package/src/derivation-network-proxy-helper.ts +440 -0
- package/src/derivation-network-proxy.ts +385 -0
- package/src/derive-command-helper.ts +693 -0
- package/src/derive-fixtures.ts +343 -0
- package/src/derive-review.ts +980 -0
- package/src/derive.ts +4754 -0
- package/src/fixtures/beeper-message-like-me-v1/accounts.ndjson +2 -0
- package/src/fixtures/beeper-message-like-me-v1/conversations.ndjson +1 -0
- package/src/fixtures/beeper-message-like-me-v1/manifest.json +1 -0
- package/src/fixtures/beeper-message-like-me-v1/messages.ndjson +2 -0
- package/src/fixtures/beeper-message-like-me-v1/participants.ndjson +3 -0
- package/src/fixtures/beeper-message-like-me-v1/reactions.ndjson +1 -0
- package/src/fixtures/beeper-message-like-me-v1/tombstones.ndjson +1 -0
- package/src/ghostget.ts +3672 -0
- package/src/gmail-capture.ts +1244 -0
- package/src/har-internal.ts +1242 -0
- package/src/har.ts +743 -0
- package/src/index.ts +218 -0
- package/src/invocation-read-platform.ts +55 -0
- package/src/invocation-read-program.ts +93 -0
- package/src/invocation-read-runtime.ts +11 -0
- package/src/linked-device-lifecycle-admission.ts +450 -0
- package/src/linked-device-lifecycle-journal.ts +1560 -0
- package/src/linked-device-lifecycle-runtime.ts +1602 -0
- package/src/local-cli-admission.ts +135 -0
- package/src/local-cli-contracts.ts +271 -0
- package/src/local-cli-execution.ts +98 -0
- package/src/local-cli-surface-contract.ts +1510 -0
- package/src/local-cli-tool-identity.ts +336 -0
- package/src/media/archive.ts +2325 -0
- package/src/media/args.ts +303 -0
- package/src/media/cli.ts +287 -0
- package/src/media/doctor.ts +475 -0
- package/src/media/ffmpeg.ts +1330 -0
- package/src/media/http-capture.ts +711 -0
- package/src/media/http-probe.ts +593 -0
- package/src/media/http.ts +387 -0
- package/src/media/index.ts +10 -0
- package/src/media/local-transcription.ts +613 -0
- package/src/media/lock.ts +321 -0
- package/src/media/manifest.ts +1712 -0
- package/src/media/metadata.ts +906 -0
- package/src/media/process.ts +723 -0
- package/src/media/revision.ts +166 -0
- package/src/media/runtime-closure.ts +1063 -0
- package/src/media/source-router.ts +187 -0
- package/src/media/transcriber-config.ts +1122 -0
- package/src/media/transcript-persistence-model.ts +15 -0
- package/src/media/transcript-persistence-platform.ts +89 -0
- package/src/media/transcript-persistence-program.ts +39 -0
- package/src/media/transcript-persistence-runtime.ts +24 -0
- package/src/media/transcript.ts +616 -0
- package/src/media/utf8-order.ts +30 -0
- package/src/media/whisper-cpp.ts +464 -0
- package/src/media/whisper-language.ts +58 -0
- package/src/media/yt-dlp.ts +439 -0
- package/src/message-like-me-agentic-messaging.ts +882 -0
- package/src/messaging-action-store.ts +793 -0
- package/src/messaging-runtime.ts +2136 -0
- package/src/messaging-store.ts +539 -0
- package/src/messaging-types.ts +1663 -0
- package/src/messaging.ts +334 -0
- package/src/model.ts +2671 -0
- package/src/oauth-google.ts +888 -0
- package/src/omni-client-types.ts +306 -0
- package/src/omni-client.ts +2225 -0
- package/src/omni-limits.ts +34 -0
- package/src/omni-model.ts +1614 -0
- package/src/omni-request.ts +379 -0
- package/src/omni-runtime.ts +1632 -0
- package/src/operation-deadline.ts +189 -0
- package/src/path-helper.ts +1656 -0
- package/src/pinned-https.ts +194 -0
- package/src/plan-assets.ts +832 -0
- package/src/platform-catalog.ts +1832 -0
- package/src/plugins/beeper-linked-device/plugin.ts +310 -0
- package/src/plugins/bluesky-web/plugin.ts +178 -0
- package/src/plugins/clasificados-web/plugin.ts +66 -0
- package/src/plugins/github-web/plugin.ts +80 -0
- package/src/plugins/gmail-official/plugin.ts +110 -0
- package/src/plugins/hacker-news-web/plugin.ts +52 -0
- package/src/plugins/imessage-direct/plugin.ts +195 -0
- package/src/plugins/imessage-direct/vendor/0001-fix-keep-AppleScript-send-payloads-out-of-child-argv.patch +1430 -0
- package/src/plugins/imessage-direct/vendor/0002-feat-rpc-add-exact-chat-lookup.patch +225 -0
- package/src/plugins/imessage-direct/vendor/provenance.json +80 -0
- package/src/plugins/linkedin-official/plugin.ts +84 -0
- package/src/plugins/linkedin-web/plugin.ts +412 -0
- package/src/plugins/meta-web/plugin.ts +317 -0
- package/src/plugins/reddit-web/flair.ts +492 -0
- package/src/plugins/reddit-web/plugin.ts +477 -0
- package/src/plugins/substack-web/plugin.ts +124 -0
- package/src/plugins/tiktok-web/plugin.ts +67 -0
- package/src/plugins/twitch-web/plugin.ts +54 -0
- package/src/plugins/whatsapp-linked-device/plugin.ts +105 -0
- package/src/plugins/x-official/plugin.ts +64 -0
- package/src/plugins/x-web/plugin.ts +488 -0
- package/src/plugins/youtube-web/plugin.ts +101 -0
- package/src/portable-run-recovery.ts +466 -0
- package/src/process-identity.ts +501 -0
- package/src/profile-clone-helper.ts +104 -0
- package/src/provider-catalog-views.ts +51 -0
- package/src/provider-context.ts +69 -0
- package/src/provider-contract-definitions-x.ts +45 -0
- package/src/provider-contract-definitions.ts +948 -0
- package/src/provider-contract-input-gmail.ts +176 -0
- package/src/provider-contract-input-linkedin.ts +77 -0
- package/src/provider-contract-input-x.ts +304 -0
- package/src/provider-contract-planning.ts +23 -0
- package/src/provider-contract-semantic-identity.ts +54 -0
- package/src/provider-contracts.ts +251 -0
- package/src/provider-http.ts +723 -0
- package/src/provider-plugin-auth.ts +36 -0
- package/src/provider-plugin-builtins.ts +229 -0
- package/src/provider-plugin-cleanup-barrier.ts +229 -0
- package/src/provider-plugin-cleanup-execution.ts +79 -0
- package/src/provider-plugin-cleanup-resource.ts +470 -0
- package/src/provider-plugin-contract-identity.ts +707 -0
- package/src/provider-plugin-host.ts +1972 -0
- package/src/provider-plugin-identifiers.ts +60 -0
- package/src/provider-plugin-invocation-lease.ts +889 -0
- package/src/provider-plugin-lifecycle-kernel.ts +566 -0
- package/src/provider-plugin-lifecycle.ts +1455 -0
- package/src/provider-plugin-module-analysis.ts +336 -0
- package/src/provider-plugin-package.ts +2530 -0
- package/src/provider-plugin-portable-authority.ts +106 -0
- package/src/provider-plugin-portable-catalog.ts +94 -0
- package/src/provider-plugin-portable-identity.ts +315 -0
- package/src/provider-plugin-portable-runtime.ts +2057 -0
- package/src/provider-plugin-protocol.ts +1625 -0
- package/src/provider-plugin-registry.ts +3342 -0
- package/src/provider-plugin-store.ts +2155 -0
- package/src/provider-plugin-test-fixture.ts +5 -0
- package/src/provider-plugin.ts +4907 -0
- package/src/provider-plugins.generated.ts +43 -0
- package/src/provider-plugins.ts +8 -0
- package/src/provider-subject.ts +41 -0
- package/src/provider.ts +315 -0
- package/src/providers/beeper-local-runtime.ts +5426 -0
- package/src/providers/beeper-local.ts +2739 -0
- package/src/providers/beeper-messaging.ts +462 -0
- package/src/providers/beeper-omni.ts +1159 -0
- package/src/providers/bluesky-web-runtime.ts +2895 -0
- package/src/providers/bluesky-web.ts +1614 -0
- package/src/providers/clasificados-web-runtime.ts +255 -0
- package/src/providers/clasificados-web.ts +503 -0
- package/src/providers/contact-projection.ts +301 -0
- package/src/providers/github-read-model.ts +94 -0
- package/src/providers/github-read-platform.ts +165 -0
- package/src/providers/github-read-program.ts +97 -0
- package/src/providers/github-web-runtime.ts +320 -0
- package/src/providers/github-web.ts +403 -0
- package/src/providers/gmail-api.ts +2834 -0
- package/src/providers/gmail-omni.ts +830 -0
- package/src/providers/gmail.ts +1130 -0
- package/src/providers/hacker-news-web-runtime.ts +220 -0
- package/src/providers/hacker-news-web.ts +809 -0
- package/src/providers/imessage-direct-install.ts +354 -0
- package/src/providers/imessage-direct-messaging.ts +532 -0
- package/src/providers/imessage-direct-omni.ts +400 -0
- package/src/providers/imessage-direct-runtime.ts +1498 -0
- package/src/providers/imessage-direct.ts +351 -0
- package/src/providers/instagram-video-foundations.ts +913 -0
- package/src/providers/instagram-web-profile-browser.ts +600 -0
- package/src/providers/iso-bmff.ts +338 -0
- package/src/providers/linkedin-company-platform.ts +42 -0
- package/src/providers/linkedin-company-program.ts +88 -0
- package/src/providers/linkedin-contact-failure.ts +84 -0
- package/src/providers/linkedin-contact-platform.ts +45 -0
- package/src/providers/linkedin-contact-program.ts +91 -0
- package/src/providers/linkedin-profile-activity-failure.ts +76 -0
- package/src/providers/linkedin-profile-activity-platform.ts +41 -0
- package/src/providers/linkedin-profile-activity-program.ts +67 -0
- package/src/providers/linkedin-read-failure.ts +52 -0
- package/src/providers/linkedin-self-platform.ts +53 -0
- package/src/providers/linkedin-self-program.ts +122 -0
- package/src/providers/linkedin-web-article-browser.ts +794 -0
- package/src/providers/linkedin-web-bootstrap.ts +146 -0
- package/src/providers/linkedin-web-contact.ts +830 -0
- package/src/providers/linkedin-web-feed-browser.ts +832 -0
- package/src/providers/linkedin-web-feed.ts +864 -0
- package/src/providers/linkedin-web-post-browser.ts +653 -0
- package/src/providers/linkedin-web-profile-browser.ts +683 -0
- package/src/providers/linkedin-web-runtime.ts +2388 -0
- package/src/providers/linkedin-web.ts +3305 -0
- package/src/providers/linkedin.ts +1885 -0
- package/src/providers/meta-bootstrap.ts +1043 -0
- package/src/providers/meta-facebook-group.ts +690 -0
- package/src/providers/meta-marketplace-relay.ts +747 -0
- package/src/providers/meta-omni.ts +373 -0
- package/src/providers/meta-relay-bundle.ts +1049 -0
- package/src/providers/meta-web-descriptors.ts +1740 -0
- package/src/providers/meta-web-runtime.ts +3388 -0
- package/src/providers/meta-web.ts +2931 -0
- package/src/providers/read-failure.ts +139 -0
- package/src/providers/reddit-omni.ts +462 -0
- package/src/providers/reddit-web-runtime.ts +1929 -0
- package/src/providers/reddit-web.ts +1611 -0
- package/src/providers/rental-listings.ts +460 -0
- package/src/providers/substack-omni.ts +406 -0
- package/src/providers/substack-video-mp4.ts +33 -0
- package/src/providers/substack-web-runtime.ts +2812 -0
- package/src/providers/substack-web.ts +1502 -0
- package/src/providers/tiktok-video-mp4.ts +33 -0
- package/src/providers/tiktok-web-runtime.ts +770 -0
- package/src/providers/tiktok-web.ts +2786 -0
- package/src/providers/twitch-web-runtime.ts +226 -0
- package/src/providers/twitch-web.ts +274 -0
- package/src/providers/whatsapp-account-identity.ts +55 -0
- package/src/providers/whatsapp-contact-projection-helper.ts +874 -0
- package/src/providers/whatsapp-contact-projection-protocol.ts +403 -0
- package/src/providers/whatsapp-interaction-projection-helper.ts +1452 -0
- package/src/providers/whatsapp-interaction-projection-protocol.ts +462 -0
- package/src/providers/whatsapp-message-export-projection-protocol.ts +532 -0
- package/src/providers/whatsapp-messaging.ts +132 -0
- package/src/providers/whatsapp-omni.ts +586 -0
- package/src/providers/whatsapp-web-runtime.ts +3108 -0
- package/src/providers/whatsapp-web.ts +1037 -0
- package/src/providers/x-image-provenance.ts +523 -0
- package/src/providers/x-made-with-ai.ts +106 -0
- package/src/providers/x-transaction-id.ts +413 -0
- package/src/providers/x-web-runtime.ts +3485 -0
- package/src/providers/x-web.ts +2332 -0
- package/src/providers/x.ts +1889 -0
- package/src/providers/youtube-web-runtime.ts +1400 -0
- package/src/providers/youtube-web.ts +1221 -0
- package/src/read-admission-runtime.ts +91 -0
- package/src/read-client.ts +521 -0
- package/src/read-effect-platform.ts +7 -0
- package/src/read-effect-runtime.ts +14 -0
- package/src/read-effect.ts +52 -0
- package/src/read-projection-admission.ts +866 -0
- package/src/read-projections.ts +3151 -0
- package/src/recovery.ts +1492 -0
- package/src/reviewed-template.ts +146 -0
- package/src/run-journal.ts +1272 -0
- package/src/runtime.ts +5116 -0
- package/src/scripts/generate-provider-plugin-catalog.ts +121 -0
- package/src/scripts/install-whatsapp-protocol.sh +232 -0
- package/src/scripts/recover-install-stages.ts +1491 -0
- package/src/scripts/resolve-state-home.ts +30 -0
- package/src/scripts/scaffold-web-provider.ts +973 -0
- package/src/scripts/sync-bundled-adapters.ts +695 -0
- package/src/session-secrets.ts +1183 -0
- package/src/state-helper.bunfig.toml +2 -0
- package/src/state-helper.ts +2914 -0
- package/src/storage.ts +3842 -0
- package/src/transport-policy.ts +2 -0
- package/src/usage.ts +182 -0
- package/src/version.ts +2 -0
- package/src/web-session-authentication-policy.ts +136 -0
- package/src/web-session-cleanup-admission.ts +2344 -0
- package/src/web-session-cleanup-join.ts +99 -0
- package/src/web-session-client.ts +996 -0
- package/src/web-session-contract-definitions.ts +820 -0
- package/src/web-session-contract-planning.ts +42 -0
- package/src/web-session-contracts.ts +228 -0
- package/src/web-session-cookies.ts +500 -0
- package/src/web-session-execution.ts +385 -0
- package/src/web-session-http.ts +373 -0
- package/src/web-session-read-errors.ts +19 -0
- package/src/web-session-read-runtime.ts +124 -0
- package/src/web-session-recovery.ts +818 -0
- package/src/web-session-sites.ts +58 -0
- package/src/web-session-template.ts +994 -0
- package/src/web-session.ts +93 -0
- package/src/whatsapp-client-binding.ts +21 -0
- package/src/whatsapp-client-types.ts +75 -0
- package/src/whatsapp-client.ts +395 -0
- package/src/whatsapp-export-coordinate.ts +17 -0
- package/src/whatsapp-message-bundle-v2.ts +195 -0
- package/src/whatsapp-message-like-me-cli.ts +195 -0
- package/src/whatsapp-message-like-me-export.ts +1072 -0
- package/src/whatsapp-message-like-me-source.ts +1012 -0
- package/tsconfig.json +67 -0
|
@@ -0,0 +1,3151 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createCipheriv,
|
|
3
|
+
createDecipheriv,
|
|
4
|
+
createHash,
|
|
5
|
+
createHmac,
|
|
6
|
+
randomBytes,
|
|
7
|
+
randomUUID,
|
|
8
|
+
timingSafeEqual,
|
|
9
|
+
} from "node:crypto";
|
|
10
|
+
import { lstatSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
|
|
13
|
+
import { canonicalJson } from "./canonical-json";
|
|
14
|
+
import {
|
|
15
|
+
parseLocalCliToolIdentityV1,
|
|
16
|
+
type LocalCliToolIdentityV1,
|
|
17
|
+
} from "./local-cli-tool-identity";
|
|
18
|
+
import {
|
|
19
|
+
READ_PROJECTION_TRANSITION_SETTLEMENT_WAIT_MS,
|
|
20
|
+
withReadProjectionAuthAdmission,
|
|
21
|
+
withSettledReadProjectionAuthAdmission,
|
|
22
|
+
} from "./read-projection-admission";
|
|
23
|
+
import {
|
|
24
|
+
MAX_PRIVATE_STATE_BATCH_FILE_BYTES,
|
|
25
|
+
createPrivateJsonIfAbsent,
|
|
26
|
+
ensurePrivateStateDirectory,
|
|
27
|
+
listPrivateStateDirectory,
|
|
28
|
+
readPrivateStateChildFilesBatched,
|
|
29
|
+
readPrivateStateFileBytesIfPresent,
|
|
30
|
+
readPrivateStateFileIfPresent,
|
|
31
|
+
readRegularFile,
|
|
32
|
+
removePrivateStateDirectoryTree,
|
|
33
|
+
removePrivateStateFile,
|
|
34
|
+
snapshotPrivateStateDirectory,
|
|
35
|
+
ghostgetStateHome,
|
|
36
|
+
writePrivateJsonIfUnchanged,
|
|
37
|
+
type PrivateDirectoryIdentity,
|
|
38
|
+
} from "./storage";
|
|
39
|
+
|
|
40
|
+
export {
|
|
41
|
+
ReadProjectionAdmissionContentionError,
|
|
42
|
+
READ_PROJECTION_SHORT_SETTLEMENT_WAIT_MS,
|
|
43
|
+
READ_PROJECTION_TRANSITION_SETTLEMENT_WAIT_MS,
|
|
44
|
+
acquireReadProjectionAuthAdmission,
|
|
45
|
+
ensureReadProjectionAuthIncarnation,
|
|
46
|
+
projectionAuthIdentityHash,
|
|
47
|
+
removeReadProjectionAuthIncarnation,
|
|
48
|
+
rotateReadProjectionAuthIncarnation,
|
|
49
|
+
withReadProjectionAuthAdmission,
|
|
50
|
+
withSettledReadProjectionAuthAdmission,
|
|
51
|
+
type ReadProjectionAdmissionContentionReason,
|
|
52
|
+
type ReadProjectionAdmissionOwner,
|
|
53
|
+
type ReadProjectionAdmissionSettlementOptions,
|
|
54
|
+
type ReadProjectionAuthAdmission,
|
|
55
|
+
} from "./read-projection-admission";
|
|
56
|
+
|
|
57
|
+
type Environment = Readonly<Record<string, string | undefined>>;
|
|
58
|
+
type JsonRecord = Record<string, unknown>;
|
|
59
|
+
|
|
60
|
+
const STORE_DIRECTORY = "read-projections";
|
|
61
|
+
const OMNI_STORE_DIRECTORY = "omni-read-projections";
|
|
62
|
+
const CONTROL_DIRECTORY = "read-projection-control";
|
|
63
|
+
const KEY_FILE = ".projection-encryption-key";
|
|
64
|
+
const STORE_KEY_MARKER_FILE = "store-key.json";
|
|
65
|
+
const HEAD_FILE = "head.json";
|
|
66
|
+
const KEY_FILE_MAX_BYTES = 512;
|
|
67
|
+
const STORE_KEY_MARKER_MAX_BYTES = 512;
|
|
68
|
+
const INITIALIZATION_SETTLE_ATTEMPTS = 100;
|
|
69
|
+
const INITIALIZATION_SETTLE_WAIT_MS = 10;
|
|
70
|
+
const HEAD_MAX_BYTES = 16 * 1024;
|
|
71
|
+
const MANIFEST_MAX_BYTES = 64 * 1024;
|
|
72
|
+
const MAX_ATTRIBUTABLE_CORRUPT_FILE_BYTES = 4 * 1024 * 1024;
|
|
73
|
+
const PLAINTEXT_CHUNK_BYTES = 1_200_000;
|
|
74
|
+
const MAX_PLAINTEXT_BYTES = 16 * 1024 * 1024;
|
|
75
|
+
const MAX_CHUNKS = 16;
|
|
76
|
+
const MAX_QUERY_DIRECTORY_ENTRIES = 2 * MAX_CHUNKS + 16;
|
|
77
|
+
const MAX_REALM_QUERY_DIRECTORIES = 32;
|
|
78
|
+
const MAX_REALM_STORAGE_BYTES = 128 * 1024 * 1024;
|
|
79
|
+
const MAX_JSON_DEPTH = 64;
|
|
80
|
+
const MAX_JSON_NODES = 110_000;
|
|
81
|
+
const MAX_FRESH_FOR_MS = 365 * 24 * 60 * 60 * 1_000;
|
|
82
|
+
const immutableManifestNamePattern = /^manifest--([a-f0-9]{32})\.json$/u;
|
|
83
|
+
const immutableChunkNamePattern = /^chunk--([a-f0-9]{32})--([0-9]{3})\.json$/u;
|
|
84
|
+
const queryDirectoryNamePattern = /^[a-f0-9]{64}$/u;
|
|
85
|
+
const authIdPattern = /^[a-z][a-z0-9-]{0,47}$/u;
|
|
86
|
+
const stateHelperArtifactNamePatterns = Object.freeze([
|
|
87
|
+
/^\.io-write-[1-9][0-9]{0,9}-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.tmp$/u,
|
|
88
|
+
/^\.io-mutation-[a-f0-9]{64}-(?:waiting|candidate|held)-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.lock$/u,
|
|
89
|
+
/^\.io-mutation-stage-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}-[1-9][0-9]{0,9}\.tmp$/u,
|
|
90
|
+
/^\.io-remove-file-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.quarantine$/u,
|
|
91
|
+
/^\.io-remove(?:-tree)?-[1-9][0-9]{0,9}-[1-9][0-9]{0,15}-[0-9a-f]{12}4[0-9a-f]{3}[89ab][0-9a-f]{15}\.quarantine$/u,
|
|
92
|
+
]);
|
|
93
|
+
|
|
94
|
+
type ProjectionKey = {
|
|
95
|
+
readonly id: string;
|
|
96
|
+
readonly value: Buffer;
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
type ProjectionStoreKeyMarkerV1 = {
|
|
100
|
+
readonly schemaVersion: 1;
|
|
101
|
+
readonly keyId: string;
|
|
102
|
+
readonly authentication: string;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
export type ReadProjectionQueryIdentity = {
|
|
106
|
+
readonly adapter: {
|
|
107
|
+
readonly id: string;
|
|
108
|
+
readonly version: string;
|
|
109
|
+
readonly hash: string;
|
|
110
|
+
};
|
|
111
|
+
readonly operation: string;
|
|
112
|
+
readonly input: unknown;
|
|
113
|
+
readonly inputHash: string;
|
|
114
|
+
readonly auth: {
|
|
115
|
+
readonly id: string;
|
|
116
|
+
readonly kind: string;
|
|
117
|
+
readonly hash: string;
|
|
118
|
+
readonly subject: string;
|
|
119
|
+
};
|
|
120
|
+
readonly contract: {
|
|
121
|
+
readonly transport:
|
|
122
|
+
| "browser"
|
|
123
|
+
| "portable-provider-plugin"
|
|
124
|
+
| "provider-api"
|
|
125
|
+
| "reviewed-template-api"
|
|
126
|
+
| "web-session-api";
|
|
127
|
+
readonly hash: string;
|
|
128
|
+
} | {
|
|
129
|
+
readonly transport: "local-cli";
|
|
130
|
+
readonly hash: string;
|
|
131
|
+
readonly tool: LocalCliToolIdentityV1;
|
|
132
|
+
};
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
export type ReadProjectionQuery = {
|
|
136
|
+
/** Keyed digest safe to use as an opaque local cache identifier. */
|
|
137
|
+
readonly key: string;
|
|
138
|
+
/** Keyed auth-locator coordinate used only for private state custody. */
|
|
139
|
+
readonly realmKey: string;
|
|
140
|
+
readonly identity: ReadProjectionQueryIdentity;
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
export type OmniProjectionQuery = ReadProjectionQuery & {
|
|
144
|
+
/** Runtime storage-class tag. Exact-query values deliberately have no tag. */
|
|
145
|
+
readonly storageClass: "omni-v1";
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
type ProjectionStorageClass = "exact-v1" | "omni-v1";
|
|
149
|
+
|
|
150
|
+
export type OmniProjectionCurrent = {
|
|
151
|
+
readonly key: string;
|
|
152
|
+
/** Immutable encrypted revision named by the authoritative head. */
|
|
153
|
+
readonly storageRevisionId: string;
|
|
154
|
+
readonly output: unknown;
|
|
155
|
+
readonly dataRevision: string;
|
|
156
|
+
readonly createdAt: string;
|
|
157
|
+
readonly dataChangedAt: string;
|
|
158
|
+
readonly validatedAt: string;
|
|
159
|
+
readonly runId: string;
|
|
160
|
+
readonly startedAt: string;
|
|
161
|
+
readonly finishedAt: string;
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
export type OmniProjectionPublication = ReadProjectionPublication & {
|
|
165
|
+
readonly storageRevisionId: string;
|
|
166
|
+
readonly runId: string;
|
|
167
|
+
readonly startedAt: string;
|
|
168
|
+
readonly finishedAt: string;
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
export type OmniProjectionReductionResult = {
|
|
172
|
+
readonly publication: OmniProjectionPublication;
|
|
173
|
+
readonly current: OmniProjectionCurrent;
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
export type ReadProjectionCacheResult =
|
|
177
|
+
| {
|
|
178
|
+
readonly status: "miss";
|
|
179
|
+
readonly key: string;
|
|
180
|
+
}
|
|
181
|
+
| {
|
|
182
|
+
readonly status: "hit";
|
|
183
|
+
readonly source: "cache";
|
|
184
|
+
readonly key: string;
|
|
185
|
+
readonly output: unknown;
|
|
186
|
+
readonly dataRevision: string;
|
|
187
|
+
readonly createdAt: string;
|
|
188
|
+
readonly dataChangedAt: string;
|
|
189
|
+
readonly validatedAt: string;
|
|
190
|
+
readonly runId: string;
|
|
191
|
+
readonly ageMs: number;
|
|
192
|
+
readonly freshness: {
|
|
193
|
+
readonly state: "fresh" | "stale" | "unclassified";
|
|
194
|
+
readonly freshForMs: number | null;
|
|
195
|
+
};
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
export type OmniProjectionCacheResult =
|
|
199
|
+
| Extract<ReadProjectionCacheResult, { readonly status: "miss" }>
|
|
200
|
+
| (Extract<ReadProjectionCacheResult, { readonly status: "hit" }> & {
|
|
201
|
+
readonly storageRevisionId: string;
|
|
202
|
+
readonly startedAt: string;
|
|
203
|
+
readonly finishedAt: string;
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
/** Internal exact-head observation used to bind a derivative to exact bytes. */
|
|
207
|
+
export type ReadProjectionMaterializationSnapshot = OmniProjectionCacheResult;
|
|
208
|
+
|
|
209
|
+
/** Exact immutable head that must still own the auth admission at reduction. */
|
|
210
|
+
export type ReadProjectionExactHeadFence = {
|
|
211
|
+
readonly query: ReadProjectionQuery;
|
|
212
|
+
readonly storageRevisionId: string;
|
|
213
|
+
readonly dataRevision: string;
|
|
214
|
+
readonly runId: string;
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
export type ReadProjectionPublication = {
|
|
218
|
+
readonly key: string;
|
|
219
|
+
readonly dataRevision: string;
|
|
220
|
+
readonly validatedAt: string;
|
|
221
|
+
readonly dataChangedAt: string;
|
|
222
|
+
readonly disposition: "created" | "changed" | "unchanged" | "superseded";
|
|
223
|
+
readonly currentDataRevision?: string;
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
type ProjectionPayloadV1 = {
|
|
227
|
+
readonly schemaVersion: 1;
|
|
228
|
+
readonly query: ReadProjectionQueryIdentity;
|
|
229
|
+
readonly output: unknown;
|
|
230
|
+
readonly dataRevision: string;
|
|
231
|
+
readonly createdAt: string;
|
|
232
|
+
readonly dataChangedAt: string;
|
|
233
|
+
readonly validatedAt: string;
|
|
234
|
+
readonly runId: string;
|
|
235
|
+
readonly startedAt: string;
|
|
236
|
+
readonly finishedAt: string;
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
export type ReadProjectionHeadPublication = {
|
|
240
|
+
readonly dataRevision: string;
|
|
241
|
+
readonly createdAt: string;
|
|
242
|
+
readonly dataChangedAt: string;
|
|
243
|
+
readonly validatedAt: string;
|
|
244
|
+
readonly runId: string;
|
|
245
|
+
readonly startedAt: string;
|
|
246
|
+
readonly finishedAt: string;
|
|
247
|
+
};
|
|
248
|
+
|
|
249
|
+
type ProjectionHeadV1 = {
|
|
250
|
+
readonly schemaVersion: 1;
|
|
251
|
+
readonly keyId: string;
|
|
252
|
+
readonly revisionId: string;
|
|
253
|
+
readonly manifestFile: string;
|
|
254
|
+
readonly publication: ReadProjectionHeadPublication;
|
|
255
|
+
readonly authentication: string;
|
|
256
|
+
};
|
|
257
|
+
|
|
258
|
+
type ProjectionManifestV1 = {
|
|
259
|
+
readonly schemaVersion: 1;
|
|
260
|
+
readonly keyId: string;
|
|
261
|
+
readonly revisionId: string;
|
|
262
|
+
readonly plaintextBytes: number;
|
|
263
|
+
readonly chunkFiles: readonly string[];
|
|
264
|
+
readonly chunkHashes: readonly string[];
|
|
265
|
+
readonly authentication: string;
|
|
266
|
+
};
|
|
267
|
+
|
|
268
|
+
type EncryptedProjectionChunkV1 = {
|
|
269
|
+
readonly schemaVersion: 1;
|
|
270
|
+
readonly encryption: "aes-256-gcm";
|
|
271
|
+
readonly keyId: string;
|
|
272
|
+
readonly revisionId: string;
|
|
273
|
+
readonly index: number;
|
|
274
|
+
readonly count: number;
|
|
275
|
+
readonly iv: string;
|
|
276
|
+
readonly ciphertext: string;
|
|
277
|
+
readonly tag: string;
|
|
278
|
+
};
|
|
279
|
+
|
|
280
|
+
type LoadedProjection = {
|
|
281
|
+
readonly head: ProjectionHeadV1;
|
|
282
|
+
readonly headContentSha256: string;
|
|
283
|
+
readonly payload: ProjectionPayloadV1;
|
|
284
|
+
};
|
|
285
|
+
|
|
286
|
+
type ProjectionHeadSnapshot = {
|
|
287
|
+
readonly head: ProjectionHeadV1;
|
|
288
|
+
readonly contentSha256: string;
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
type ReadProjectionCorruptionEvidence = {
|
|
292
|
+
readonly storageClass: ProjectionStorageClass;
|
|
293
|
+
readonly queryKey: string;
|
|
294
|
+
readonly realmKey: string;
|
|
295
|
+
readonly headContentSha256: string;
|
|
296
|
+
readonly headPublication: ReadProjectionHeadPublication | null;
|
|
297
|
+
};
|
|
298
|
+
|
|
299
|
+
const corruptionConstructorToken = Symbol("read-projection-corruption");
|
|
300
|
+
const corruptionEvidence = new WeakMap<
|
|
301
|
+
ReadProjectionCorruptionError,
|
|
302
|
+
ReadProjectionCorruptionEvidence
|
|
303
|
+
>();
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Nominal evidence that one exact query's authenticated on-disk projection is
|
|
307
|
+
* corrupt. Control-state failures, contention, caller errors, and ordinary I/O
|
|
308
|
+
* failures deliberately retain their original error types.
|
|
309
|
+
*/
|
|
310
|
+
export class ReadProjectionCorruptionError extends Error {
|
|
311
|
+
readonly #nominal = true;
|
|
312
|
+
|
|
313
|
+
constructor(
|
|
314
|
+
message: string,
|
|
315
|
+
evidence: ReadProjectionCorruptionEvidence,
|
|
316
|
+
options: { readonly cause?: unknown },
|
|
317
|
+
token: typeof corruptionConstructorToken,
|
|
318
|
+
) {
|
|
319
|
+
if (token !== corruptionConstructorToken) {
|
|
320
|
+
throw new TypeError("read projection corruption evidence is internal");
|
|
321
|
+
}
|
|
322
|
+
super(message, options);
|
|
323
|
+
this.name = "ReadProjectionCorruptionError";
|
|
324
|
+
corruptionEvidence.set(this, evidence);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
get authenticatedOnDiskQueryCorruption(): true {
|
|
328
|
+
return this.#nominal;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
get queryKey(): string {
|
|
332
|
+
return corruptionEvidence.get(this)!.queryKey;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
get storageClass(): ProjectionStorageClass {
|
|
336
|
+
return corruptionEvidence.get(this)!.storageClass;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
get realmKey(): string {
|
|
340
|
+
return corruptionEvidence.get(this)!.realmKey;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
get headContentSha256(): string {
|
|
344
|
+
return corruptionEvidence.get(this)!.headContentSha256;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
get headPublication(): ReadProjectionHeadPublication | null {
|
|
348
|
+
return corruptionEvidence.get(this)!.headPublication;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
export function isReadProjectionCorruptionError(
|
|
353
|
+
error: unknown,
|
|
354
|
+
): error is ReadProjectionCorruptionError {
|
|
355
|
+
return error instanceof ReadProjectionCorruptionError
|
|
356
|
+
&& corruptionEvidence.has(error);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* The replacement head is already authoritative when this error is raised.
|
|
361
|
+
* Only reclamation of unreachable prior state failed.
|
|
362
|
+
*/
|
|
363
|
+
export class ReadProjectionDurableRepairError extends Error {
|
|
364
|
+
readonly durableRepair = true;
|
|
365
|
+
readonly headIsAuthoritative = true;
|
|
366
|
+
readonly queryKey: string;
|
|
367
|
+
readonly publication: ReadProjectionPublication;
|
|
368
|
+
|
|
369
|
+
constructor(
|
|
370
|
+
queryKey: string,
|
|
371
|
+
publicationValue: ReadProjectionPublication,
|
|
372
|
+
cause: unknown,
|
|
373
|
+
) {
|
|
374
|
+
super(
|
|
375
|
+
"read projection repair is durable but post-publication reclamation failed",
|
|
376
|
+
{ cause },
|
|
377
|
+
);
|
|
378
|
+
this.name = "ReadProjectionDurableRepairError";
|
|
379
|
+
this.queryKey = queryKey;
|
|
380
|
+
this.publication = publicationValue;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
function record(value: unknown, label: string): JsonRecord {
|
|
385
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
386
|
+
throw new Error(`${label} must be an object`);
|
|
387
|
+
}
|
|
388
|
+
const prototype: unknown = Object.getPrototypeOf(value) as unknown;
|
|
389
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
390
|
+
throw new Error(`${label} has an unsupported prototype`);
|
|
391
|
+
}
|
|
392
|
+
const descriptors = Object.getOwnPropertyDescriptors(value);
|
|
393
|
+
if (Reflect.ownKeys(descriptors).some((key) => typeof key !== "string")) {
|
|
394
|
+
throw new Error(`${label} has unsupported symbol fields`);
|
|
395
|
+
}
|
|
396
|
+
const result: JsonRecord = {};
|
|
397
|
+
for (const [key, descriptor] of Object.entries(descriptors)) {
|
|
398
|
+
if (!descriptor.enumerable || !("value" in descriptor)) {
|
|
399
|
+
throw new Error(`${label} has unsupported accessor fields`);
|
|
400
|
+
}
|
|
401
|
+
Object.defineProperty(result, key, {
|
|
402
|
+
value: descriptor.value,
|
|
403
|
+
enumerable: true,
|
|
404
|
+
configurable: false,
|
|
405
|
+
writable: false,
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
return result;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
function exactKeys(value: JsonRecord, expected: readonly string[], label: string): void {
|
|
412
|
+
const actual = Object.keys(value).sort();
|
|
413
|
+
const wanted = [...expected].sort();
|
|
414
|
+
if (
|
|
415
|
+
actual.length !== wanted.length
|
|
416
|
+
|| actual.some((key, index) => key !== wanted[index])
|
|
417
|
+
) {
|
|
418
|
+
throw new Error(`${label} has unsupported fields`);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function hexDigest(value: unknown, label: string): string {
|
|
423
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/u.test(value)) {
|
|
424
|
+
throw new Error(`${label} is malformed`);
|
|
425
|
+
}
|
|
426
|
+
return value;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function safeString(value: unknown, label: string, maximum: number): string {
|
|
430
|
+
if (
|
|
431
|
+
typeof value !== "string"
|
|
432
|
+
|| value.length < 1
|
|
433
|
+
|| value.length > maximum
|
|
434
|
+
|| value.trim() !== value
|
|
435
|
+
|| [...value].some((character) => {
|
|
436
|
+
const codePoint = character.codePointAt(0);
|
|
437
|
+
return codePoint !== undefined && (codePoint <= 0x1f || codePoint === 0x7f);
|
|
438
|
+
})
|
|
439
|
+
) throw new Error(`${label} is malformed`);
|
|
440
|
+
return value;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function authId(value: unknown): string {
|
|
444
|
+
if (typeof value !== "string" || !authIdPattern.test(value)) {
|
|
445
|
+
throw new Error("read projection auth ID must be lowercase kebab-case");
|
|
446
|
+
}
|
|
447
|
+
return value;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function safeInteger(
|
|
451
|
+
value: unknown,
|
|
452
|
+
label: string,
|
|
453
|
+
minimum: number,
|
|
454
|
+
maximum: number,
|
|
455
|
+
): number {
|
|
456
|
+
if (!Number.isSafeInteger(value) || typeof value !== "number" || value < minimum || value > maximum) {
|
|
457
|
+
throw new Error(`${label} is malformed`);
|
|
458
|
+
}
|
|
459
|
+
return value;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function timestamp(value: unknown, label: string): string {
|
|
463
|
+
const text = safeString(value, label, 64);
|
|
464
|
+
const date = new Date(text);
|
|
465
|
+
if (!Number.isFinite(date.getTime()) || date.toISOString() !== text) {
|
|
466
|
+
throw new Error(`${label} is malformed`);
|
|
467
|
+
}
|
|
468
|
+
return text;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function runId(value: unknown): string {
|
|
472
|
+
const text = safeString(value, "projection run ID", 64);
|
|
473
|
+
if (!/^[a-f0-9]{8}-[a-f0-9]{4}-[1-8][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/iu.test(text)) {
|
|
474
|
+
throw new Error("projection run ID is malformed");
|
|
475
|
+
}
|
|
476
|
+
return text.toLowerCase();
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
function boundedJson(value: unknown, label: string): unknown {
|
|
480
|
+
let nodes = 0;
|
|
481
|
+
const ancestors = new WeakSet<object>();
|
|
482
|
+
const visit = (candidate: unknown, depth: number): unknown => {
|
|
483
|
+
nodes += 1;
|
|
484
|
+
if (nodes > MAX_JSON_NODES || depth > MAX_JSON_DEPTH) {
|
|
485
|
+
throw new Error(`${label} exceeds its structural bound`);
|
|
486
|
+
}
|
|
487
|
+
if (
|
|
488
|
+
candidate === null
|
|
489
|
+
|| typeof candidate === "boolean"
|
|
490
|
+
|| typeof candidate === "string"
|
|
491
|
+
) return candidate;
|
|
492
|
+
if (typeof candidate === "number" && Number.isFinite(candidate)) return candidate;
|
|
493
|
+
if (typeof candidate !== "object") {
|
|
494
|
+
throw new Error(`${label} must contain only JSON data`);
|
|
495
|
+
}
|
|
496
|
+
if (ancestors.has(candidate)) throw new Error(`${label} must not be circular`);
|
|
497
|
+
ancestors.add(candidate);
|
|
498
|
+
try {
|
|
499
|
+
if (Array.isArray(candidate)) {
|
|
500
|
+
if (Object.getPrototypeOf(candidate) !== Array.prototype) {
|
|
501
|
+
throw new Error(`${label} arrays must use the standard prototype`);
|
|
502
|
+
}
|
|
503
|
+
const descriptors = Object.getOwnPropertyDescriptors(candidate) as unknown as Readonly<Record<string, PropertyDescriptor>>;
|
|
504
|
+
if (Reflect.ownKeys(descriptors).some((key) => typeof key !== "string")) {
|
|
505
|
+
throw new Error(`${label} arrays have unsupported symbol fields`);
|
|
506
|
+
}
|
|
507
|
+
const lengthDescriptor = descriptors.length;
|
|
508
|
+
if (
|
|
509
|
+
lengthDescriptor === undefined
|
|
510
|
+
|| !("value" in lengthDescriptor)
|
|
511
|
+
|| typeof lengthDescriptor.value !== "number"
|
|
512
|
+
|| !Number.isSafeInteger(lengthDescriptor.value)
|
|
513
|
+
|| lengthDescriptor.value < 0
|
|
514
|
+
) throw new Error(`${label} arrays are malformed`);
|
|
515
|
+
const length = lengthDescriptor.value;
|
|
516
|
+
const keys = Object.keys(descriptors).filter((key) => key !== "length");
|
|
517
|
+
if (
|
|
518
|
+
keys.length !== length
|
|
519
|
+
|| keys.some((key, index) => key !== String(index))
|
|
520
|
+
) throw new Error(`${label} arrays must be dense data arrays`);
|
|
521
|
+
const cloned: unknown[] = [];
|
|
522
|
+
for (let index = 0; index < length; index += 1) {
|
|
523
|
+
const descriptor = descriptors[String(index)];
|
|
524
|
+
if (
|
|
525
|
+
descriptor === undefined
|
|
526
|
+
|| !descriptor.enumerable
|
|
527
|
+
|| !("value" in descriptor)
|
|
528
|
+
) throw new Error(`${label} arrays must contain only data elements`);
|
|
529
|
+
cloned.push(visit(descriptor.value, depth + 1));
|
|
530
|
+
}
|
|
531
|
+
return cloned;
|
|
532
|
+
}
|
|
533
|
+
const data = record(candidate, label);
|
|
534
|
+
const cloned: JsonRecord = {};
|
|
535
|
+
for (const [key, item] of Object.entries(data)
|
|
536
|
+
.sort(([left], [right]) => left.localeCompare(right))) {
|
|
537
|
+
Object.defineProperty(cloned, key, {
|
|
538
|
+
value: visit(item, depth + 1),
|
|
539
|
+
enumerable: true,
|
|
540
|
+
configurable: false,
|
|
541
|
+
writable: false,
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
return cloned;
|
|
545
|
+
} finally {
|
|
546
|
+
ancestors.delete(candidate);
|
|
547
|
+
}
|
|
548
|
+
};
|
|
549
|
+
const cloned = visit(value, 0);
|
|
550
|
+
if (Buffer.byteLength(canonicalJson(cloned), "utf8") > MAX_PLAINTEXT_BYTES) {
|
|
551
|
+
throw new Error(`${label} exceeds its byte bound`);
|
|
552
|
+
}
|
|
553
|
+
return cloned;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
function deeplyFreezeJson(value: unknown): unknown {
|
|
557
|
+
if (typeof value !== "object" || value === null || Object.isFrozen(value)) {
|
|
558
|
+
return value;
|
|
559
|
+
}
|
|
560
|
+
for (const descriptor of Object.values(Object.getOwnPropertyDescriptors(value))) {
|
|
561
|
+
if ("value" in descriptor) deeplyFreezeJson(descriptor.value);
|
|
562
|
+
}
|
|
563
|
+
return Object.freeze(value);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function immutableBoundedJson(value: unknown, label: string): unknown {
|
|
567
|
+
return deeplyFreezeJson(boundedJson(value, label));
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
function hasThenableProtocol(value: unknown): boolean {
|
|
571
|
+
if (
|
|
572
|
+
(typeof value !== "object" || value === null)
|
|
573
|
+
&& typeof value !== "function"
|
|
574
|
+
) return false;
|
|
575
|
+
const visited = new WeakSet<object>();
|
|
576
|
+
let current: object | null = value;
|
|
577
|
+
while (current !== null) {
|
|
578
|
+
if (visited.has(current)) {
|
|
579
|
+
throw new Error("omni projection reducer result has a cyclic prototype chain");
|
|
580
|
+
}
|
|
581
|
+
visited.add(current);
|
|
582
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, "then");
|
|
583
|
+
if (descriptor !== undefined) {
|
|
584
|
+
return !("value" in descriptor) || typeof descriptor.value === "function";
|
|
585
|
+
}
|
|
586
|
+
current = Object.getPrototypeOf(current) as object | null;
|
|
587
|
+
}
|
|
588
|
+
return false;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
function hashBytes(value: string | Buffer): string {
|
|
592
|
+
return createHash("sha256").update(value).digest("hex");
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
function hmac(key: Buffer, domain: string, value: string | Buffer): string {
|
|
596
|
+
return createHmac("sha256", key)
|
|
597
|
+
.update(domain, "utf8")
|
|
598
|
+
.update("\0", "utf8")
|
|
599
|
+
.update(value)
|
|
600
|
+
.digest("hex");
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
function authenticated(
|
|
604
|
+
left: string,
|
|
605
|
+
right: string,
|
|
606
|
+
): boolean {
|
|
607
|
+
if (!/^[a-f0-9]{64}$/u.test(left) || !/^[a-f0-9]{64}$/u.test(right)) return false;
|
|
608
|
+
return timingSafeEqual(Buffer.from(left, "hex"), Buffer.from(right, "hex"));
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
function storeDirectory(environment: Environment): string {
|
|
612
|
+
return join(ghostgetStateHome(environment), STORE_DIRECTORY);
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
function omniStoreDirectory(environment: Environment): string {
|
|
616
|
+
return join(ghostgetStateHome(environment), OMNI_STORE_DIRECTORY);
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
function storeDirectoryForClass(
|
|
620
|
+
storageClass: ProjectionStorageClass,
|
|
621
|
+
environment: Environment,
|
|
622
|
+
): string {
|
|
623
|
+
return storageClass === "exact-v1"
|
|
624
|
+
? storeDirectory(environment)
|
|
625
|
+
: omniStoreDirectory(environment);
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
function keyPath(environment: Environment): string {
|
|
629
|
+
return join(ghostgetStateHome(environment), KEY_FILE);
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
function controlDirectory(environment: Environment): string {
|
|
633
|
+
return join(ghostgetStateHome(environment), CONTROL_DIRECTORY);
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function storeKeyMarkerPath(environment: Environment): string {
|
|
637
|
+
return join(controlDirectory(environment), STORE_KEY_MARKER_FILE);
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
function realmDirectory(
|
|
641
|
+
realmKey: string,
|
|
642
|
+
environment: Environment,
|
|
643
|
+
storageClass: ProjectionStorageClass = "exact-v1",
|
|
644
|
+
): string {
|
|
645
|
+
return join(
|
|
646
|
+
storeDirectoryForClass(storageClass, environment),
|
|
647
|
+
hexDigest(realmKey, "projection realm key"),
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
function queryDirectory(query: ReadProjectionQuery, environment: Environment): string {
|
|
652
|
+
return join(
|
|
653
|
+
realmDirectory(
|
|
654
|
+
query.realmKey,
|
|
655
|
+
environment,
|
|
656
|
+
projectionStorageClass(query),
|
|
657
|
+
),
|
|
658
|
+
query.key,
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
function headPath(query: ReadProjectionQuery, environment: Environment): string {
|
|
663
|
+
return join(queryDirectory(query, environment), HEAD_FILE);
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
function projectionKeyId(key: Buffer): string {
|
|
667
|
+
return hmac(key, "wrench-read-projection-key-id-v1", key);
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
function projectionStoreKeyMarkerBody(keyId: string): Readonly<{
|
|
671
|
+
schemaVersion: 1;
|
|
672
|
+
keyId: string;
|
|
673
|
+
}> {
|
|
674
|
+
return Object.freeze({ schemaVersion: 1, keyId });
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
function projectionStoreKeyMarker(key: ProjectionKey): ProjectionStoreKeyMarkerV1 {
|
|
678
|
+
const body = projectionStoreKeyMarkerBody(key.id);
|
|
679
|
+
return Object.freeze({
|
|
680
|
+
...body,
|
|
681
|
+
authentication: hmac(
|
|
682
|
+
key.value,
|
|
683
|
+
"wrench-read-projection-store-key-marker-v1",
|
|
684
|
+
canonicalJson(body),
|
|
685
|
+
),
|
|
686
|
+
});
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
function parseProjectionStoreKeyMarker(
|
|
690
|
+
text: string,
|
|
691
|
+
key: ProjectionKey,
|
|
692
|
+
): ProjectionStoreKeyMarkerV1 {
|
|
693
|
+
let marker: ProjectionStoreKeyMarkerV1;
|
|
694
|
+
try {
|
|
695
|
+
const value = record(
|
|
696
|
+
JSON.parse(text) as unknown,
|
|
697
|
+
"read projection store key marker",
|
|
698
|
+
);
|
|
699
|
+
exactKeys(
|
|
700
|
+
value,
|
|
701
|
+
["schemaVersion", "keyId", "authentication"],
|
|
702
|
+
"read projection store key marker",
|
|
703
|
+
);
|
|
704
|
+
if (value.schemaVersion !== 1) {
|
|
705
|
+
throw new Error("unsupported read projection store key marker");
|
|
706
|
+
}
|
|
707
|
+
marker = Object.freeze({
|
|
708
|
+
schemaVersion: 1,
|
|
709
|
+
keyId: hexDigest(value.keyId, "read projection store key marker key ID"),
|
|
710
|
+
authentication: hexDigest(
|
|
711
|
+
value.authentication,
|
|
712
|
+
"read projection store key marker authentication",
|
|
713
|
+
),
|
|
714
|
+
});
|
|
715
|
+
if (text !== `${canonicalJson(marker)}\n`) {
|
|
716
|
+
throw new Error("read projection store key marker is not canonical");
|
|
717
|
+
}
|
|
718
|
+
} catch {
|
|
719
|
+
throw new Error("read projection store key marker is malformed");
|
|
720
|
+
}
|
|
721
|
+
const expectedAuthentication = hmac(
|
|
722
|
+
key.value,
|
|
723
|
+
"wrench-read-projection-store-key-marker-v1",
|
|
724
|
+
canonicalJson(projectionStoreKeyMarkerBody(marker.keyId)),
|
|
725
|
+
);
|
|
726
|
+
if (
|
|
727
|
+
marker.keyId !== key.id
|
|
728
|
+
|| !authenticated(marker.authentication, expectedAuthentication)
|
|
729
|
+
) {
|
|
730
|
+
throw new Error(
|
|
731
|
+
"read projection store key marker does not match the projection encryption key",
|
|
732
|
+
);
|
|
733
|
+
}
|
|
734
|
+
return marker;
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
function newProjectionKeyRecord(): Readonly<{
|
|
738
|
+
schemaVersion: 1;
|
|
739
|
+
keyId: string;
|
|
740
|
+
key: string;
|
|
741
|
+
}> {
|
|
742
|
+
const key = randomBytes(32);
|
|
743
|
+
return Object.freeze({
|
|
744
|
+
schemaVersion: 1,
|
|
745
|
+
keyId: projectionKeyId(key),
|
|
746
|
+
key: key.toString("hex"),
|
|
747
|
+
});
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
function parseProjectionKey(text: string): ProjectionKey {
|
|
751
|
+
try {
|
|
752
|
+
const value = record(JSON.parse(text) as unknown, "projection encryption key");
|
|
753
|
+
exactKeys(value, ["schemaVersion", "keyId", "key"], "projection encryption key");
|
|
754
|
+
if (value.schemaVersion !== 1) throw new Error("unsupported projection encryption key");
|
|
755
|
+
const id = hexDigest(value.keyId, "projection encryption key ID");
|
|
756
|
+
const keyHex = hexDigest(value.key, "projection encryption key bytes");
|
|
757
|
+
const key = Buffer.from(keyHex, "hex");
|
|
758
|
+
if (projectionKeyId(key) !== id) throw new Error("projection encryption key is not identity-bound");
|
|
759
|
+
return Object.freeze({ id, value: key });
|
|
760
|
+
} catch {
|
|
761
|
+
throw new Error("projection encryption key is malformed");
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
function encryptedStoreHasState(environment: Environment): boolean {
|
|
766
|
+
const directory = storeDirectory(environment);
|
|
767
|
+
const identity = ensureProjectionStateDirectory(directory, environment);
|
|
768
|
+
const exactHasState = listPrivateStateDirectory(
|
|
769
|
+
directory,
|
|
770
|
+
environment,
|
|
771
|
+
identity,
|
|
772
|
+
{ recoverOrphanedMutationClaims: true },
|
|
773
|
+
).length > 0;
|
|
774
|
+
if (exactHasState) return true;
|
|
775
|
+
try {
|
|
776
|
+
lstatSync(omniStoreDirectory(environment));
|
|
777
|
+
} catch (error) {
|
|
778
|
+
if (
|
|
779
|
+
typeof error === "object"
|
|
780
|
+
&& error !== null
|
|
781
|
+
&& "code" in error
|
|
782
|
+
&& error.code === "ENOENT"
|
|
783
|
+
) return false;
|
|
784
|
+
throw error;
|
|
785
|
+
}
|
|
786
|
+
return snapshotPrivateStateDirectory(
|
|
787
|
+
omniStoreDirectory(environment),
|
|
788
|
+
environment,
|
|
789
|
+
undefined,
|
|
790
|
+
{ recoverOrphanedMutationClaims: true },
|
|
791
|
+
).entries.length > 0;
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
function readProjectionKeyIfPresent(environment: Environment): ProjectionKey | null {
|
|
795
|
+
try {
|
|
796
|
+
return parseProjectionKey(
|
|
797
|
+
readRegularFile(keyPath(environment), KEY_FILE_MAX_BYTES, "projection encryption key").trim(),
|
|
798
|
+
);
|
|
799
|
+
} catch (error) {
|
|
800
|
+
if (error instanceof Error && error.message === "projection encryption key is malformed") {
|
|
801
|
+
throw error;
|
|
802
|
+
}
|
|
803
|
+
return null;
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
function readProjectionStoreKeyMarkerIfPresent(
|
|
808
|
+
environment: Environment,
|
|
809
|
+
): string | null {
|
|
810
|
+
for (let attempt = 0; attempt < INITIALIZATION_SETTLE_ATTEMPTS; attempt += 1) {
|
|
811
|
+
try {
|
|
812
|
+
return readPrivateStateFileIfPresent(
|
|
813
|
+
storeKeyMarkerPath(environment),
|
|
814
|
+
STORE_KEY_MARKER_MAX_BYTES,
|
|
815
|
+
"read projection store key marker",
|
|
816
|
+
environment,
|
|
817
|
+
);
|
|
818
|
+
} catch (error) {
|
|
819
|
+
if (!isConcurrentDirectoryCreation(error)) throw error;
|
|
820
|
+
waitForProjectionInitialization();
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
throw new Error("read projection store marker lookup did not settle");
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
function waitForProjectionInitialization(): void {
|
|
827
|
+
Atomics.wait(
|
|
828
|
+
new Int32Array(new SharedArrayBuffer(4)),
|
|
829
|
+
0,
|
|
830
|
+
0,
|
|
831
|
+
INITIALIZATION_SETTLE_WAIT_MS,
|
|
832
|
+
);
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
function isActiveStateMutation(error: unknown): boolean {
|
|
836
|
+
return error instanceof Error
|
|
837
|
+
&& error.message.includes("state file mutation is already active");
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
function isConcurrentDirectoryCreation(error: unknown): boolean {
|
|
841
|
+
return error instanceof Error
|
|
842
|
+
&& (
|
|
843
|
+
error.message.includes("state directory appeared where absence was required")
|
|
844
|
+
|| error.message.includes("state directory appeared while being created")
|
|
845
|
+
|| error.message.includes("state directory appeared where absence was expected")
|
|
846
|
+
);
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
function ensureProjectionStateDirectory(
|
|
850
|
+
path: string,
|
|
851
|
+
environment: Environment,
|
|
852
|
+
): PrivateDirectoryIdentity {
|
|
853
|
+
for (let attempt = 0; attempt < INITIALIZATION_SETTLE_ATTEMPTS; attempt += 1) {
|
|
854
|
+
try {
|
|
855
|
+
return ensurePrivateStateDirectory(path, environment);
|
|
856
|
+
} catch (error) {
|
|
857
|
+
if (!isConcurrentDirectoryCreation(error)) throw error;
|
|
858
|
+
waitForProjectionInitialization();
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
throw new Error("read projection state-directory initialization did not settle");
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
function ensureProjectionStoreKeyMarker(
|
|
865
|
+
key: ProjectionKey,
|
|
866
|
+
environment: Environment,
|
|
867
|
+
create: boolean,
|
|
868
|
+
): boolean {
|
|
869
|
+
for (let attempt = 0; attempt < INITIALIZATION_SETTLE_ATTEMPTS; attempt += 1) {
|
|
870
|
+
const content = readProjectionStoreKeyMarkerIfPresent(environment);
|
|
871
|
+
if (content !== null) {
|
|
872
|
+
parseProjectionStoreKeyMarker(content, key);
|
|
873
|
+
return true;
|
|
874
|
+
}
|
|
875
|
+
if (encryptedStoreHasState(environment)) {
|
|
876
|
+
throw new Error(
|
|
877
|
+
"read projection store is missing its key ownership marker",
|
|
878
|
+
);
|
|
879
|
+
}
|
|
880
|
+
if (!create) return false;
|
|
881
|
+
|
|
882
|
+
const controlIdentity = ensureProjectionStateDirectory(
|
|
883
|
+
controlDirectory(environment),
|
|
884
|
+
environment,
|
|
885
|
+
);
|
|
886
|
+
try {
|
|
887
|
+
createPrivateJsonIfAbsent(
|
|
888
|
+
storeKeyMarkerPath(environment),
|
|
889
|
+
projectionStoreKeyMarker(key),
|
|
890
|
+
{
|
|
891
|
+
environment,
|
|
892
|
+
expectedStateParent: controlIdentity,
|
|
893
|
+
},
|
|
894
|
+
);
|
|
895
|
+
} catch (error) {
|
|
896
|
+
if (!isActiveStateMutation(error)) throw error;
|
|
897
|
+
waitForProjectionInitialization();
|
|
898
|
+
continue;
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
const settled = readPrivateStateFileIfPresent(
|
|
902
|
+
storeKeyMarkerPath(environment),
|
|
903
|
+
STORE_KEY_MARKER_MAX_BYTES,
|
|
904
|
+
"read projection store key marker",
|
|
905
|
+
environment,
|
|
906
|
+
[controlIdentity],
|
|
907
|
+
);
|
|
908
|
+
if (settled !== null) {
|
|
909
|
+
parseProjectionStoreKeyMarker(settled, key);
|
|
910
|
+
return true;
|
|
911
|
+
}
|
|
912
|
+
waitForProjectionInitialization();
|
|
913
|
+
}
|
|
914
|
+
throw new Error("read projection store key marker did not settle");
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
function projectionKey(environment: Environment, create: boolean): ProjectionKey | null {
|
|
918
|
+
let key = readProjectionKeyIfPresent(environment);
|
|
919
|
+
if (key === null) {
|
|
920
|
+
const markerExists = readProjectionStoreKeyMarkerIfPresent(environment) !== null;
|
|
921
|
+
const storeHasState = markerExists
|
|
922
|
+
? false
|
|
923
|
+
: encryptedStoreHasState(environment);
|
|
924
|
+
if (markerExists || storeHasState) {
|
|
925
|
+
key = readProjectionKeyIfPresent(environment);
|
|
926
|
+
if (key === null) {
|
|
927
|
+
throw new Error(
|
|
928
|
+
"projection encryption key is unavailable while encrypted read projections exist",
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
if (key === null && !create) return null;
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
if (key === null) {
|
|
936
|
+
for (let attempt = 0; attempt < INITIALIZATION_SETTLE_ATTEMPTS; attempt += 1) {
|
|
937
|
+
try {
|
|
938
|
+
createPrivateJsonIfAbsent(keyPath(environment), newProjectionKeyRecord(), {
|
|
939
|
+
environment,
|
|
940
|
+
privateParent: true,
|
|
941
|
+
});
|
|
942
|
+
} catch (error) {
|
|
943
|
+
if (!isActiveStateMutation(error)) throw error;
|
|
944
|
+
}
|
|
945
|
+
key = readProjectionKeyIfPresent(environment);
|
|
946
|
+
if (key !== null) break;
|
|
947
|
+
waitForProjectionInitialization();
|
|
948
|
+
}
|
|
949
|
+
if (key === null) throw new Error("projection encryption key is unavailable");
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
return ensureProjectionStoreKeyMarker(key, environment, create)
|
|
953
|
+
? key
|
|
954
|
+
: null;
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
function parseIdentity(value: unknown): ReadProjectionQueryIdentity {
|
|
958
|
+
const identity = record(value, "read projection query identity");
|
|
959
|
+
exactKeys(identity, ["adapter", "operation", "input", "inputHash", "auth", "contract"], "read projection query identity");
|
|
960
|
+
const adapter = record(identity.adapter, "read projection adapter");
|
|
961
|
+
exactKeys(adapter, ["id", "version", "hash"], "read projection adapter");
|
|
962
|
+
const auth = record(identity.auth, "read projection auth");
|
|
963
|
+
exactKeys(auth, ["id", "kind", "hash", "subject"], "read projection auth");
|
|
964
|
+
const contract = record(identity.contract, "read projection contract");
|
|
965
|
+
exactKeys(
|
|
966
|
+
contract,
|
|
967
|
+
contract.transport === "local-cli"
|
|
968
|
+
? ["transport", "hash", "tool"]
|
|
969
|
+
: ["transport", "hash"],
|
|
970
|
+
"read projection contract",
|
|
971
|
+
);
|
|
972
|
+
const input = boundedJson(identity.input, "read projection input");
|
|
973
|
+
const inputHash = hexDigest(identity.inputHash, "read projection input hash");
|
|
974
|
+
if (hashBytes(canonicalJson(input)) !== inputHash) {
|
|
975
|
+
throw new Error("read projection input is not hash-bound");
|
|
976
|
+
}
|
|
977
|
+
const transport = contract.transport;
|
|
978
|
+
if (
|
|
979
|
+
transport !== "browser"
|
|
980
|
+
&& transport !== "portable-provider-plugin"
|
|
981
|
+
&& transport !== "provider-api"
|
|
982
|
+
&& transport !== "reviewed-template-api"
|
|
983
|
+
&& transport !== "web-session-api"
|
|
984
|
+
&& transport !== "local-cli"
|
|
985
|
+
) throw new Error("read projection contract transport is malformed");
|
|
986
|
+
const contractHash = hexDigest(contract.hash, "read projection contract hash");
|
|
987
|
+
const parsedContract = transport === "local-cli"
|
|
988
|
+
? Object.freeze({
|
|
989
|
+
transport,
|
|
990
|
+
hash: contractHash,
|
|
991
|
+
tool: parseLocalCliToolIdentityV1(contract.tool),
|
|
992
|
+
})
|
|
993
|
+
: Object.freeze({ transport, hash: contractHash });
|
|
994
|
+
return Object.freeze({
|
|
995
|
+
adapter: Object.freeze({
|
|
996
|
+
id: safeString(adapter.id, "read projection adapter ID", 64),
|
|
997
|
+
version: safeString(adapter.version, "read projection adapter version", 64),
|
|
998
|
+
hash: hexDigest(adapter.hash, "read projection adapter hash"),
|
|
999
|
+
}),
|
|
1000
|
+
operation: safeString(identity.operation, "read projection operation", 128),
|
|
1001
|
+
input,
|
|
1002
|
+
inputHash,
|
|
1003
|
+
auth: Object.freeze({
|
|
1004
|
+
id: authId(auth.id),
|
|
1005
|
+
kind: safeString(auth.kind, "read projection auth kind", 64),
|
|
1006
|
+
hash: hexDigest(auth.hash, "read projection auth hash"),
|
|
1007
|
+
subject: safeString(auth.subject, "read projection auth subject", 512),
|
|
1008
|
+
}),
|
|
1009
|
+
contract: parsedContract,
|
|
1010
|
+
});
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
function queryForIdentity(
|
|
1014
|
+
identityValue: ReadProjectionQueryIdentity,
|
|
1015
|
+
environment: Environment,
|
|
1016
|
+
projectionKeyValue?: ProjectionKey,
|
|
1017
|
+
): ReadProjectionQuery {
|
|
1018
|
+
const identity = parseIdentity(identityValue);
|
|
1019
|
+
const key = projectionKeyValue ?? projectionKey(environment, true);
|
|
1020
|
+
if (key === null) throw new Error("projection encryption key is unavailable");
|
|
1021
|
+
return Object.freeze({
|
|
1022
|
+
key: hmac(key.value, "wrench-read-projection-query-v1", canonicalJson(identity)),
|
|
1023
|
+
realmKey: hmac(key.value, "wrench-read-projection-realm-v1", identity.auth.id),
|
|
1024
|
+
identity,
|
|
1025
|
+
});
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
function omniQueryForIdentity(
|
|
1029
|
+
identityValue: ReadProjectionQueryIdentity,
|
|
1030
|
+
environment: Environment,
|
|
1031
|
+
projectionKeyValue?: ProjectionKey,
|
|
1032
|
+
): OmniProjectionQuery {
|
|
1033
|
+
const identity = parseIdentity(identityValue);
|
|
1034
|
+
const key = projectionKeyValue ?? projectionKey(environment, true);
|
|
1035
|
+
if (key === null) throw new Error("projection encryption key is unavailable");
|
|
1036
|
+
return Object.freeze({
|
|
1037
|
+
storageClass: "omni-v1" as const,
|
|
1038
|
+
key: hmac(key.value, "wrench-omni-projection-query-v1", canonicalJson(identity)),
|
|
1039
|
+
realmKey: hmac(key.value, "wrench-omni-projection-realm-v1", identity.auth.id),
|
|
1040
|
+
identity,
|
|
1041
|
+
});
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
export function createReadProjectionQuery(
|
|
1045
|
+
identity: ReadProjectionQueryIdentity,
|
|
1046
|
+
environment: Environment = process.env,
|
|
1047
|
+
): ReadProjectionQuery {
|
|
1048
|
+
return queryForIdentity(identity, environment);
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
export function createOmniProjectionQuery(
|
|
1052
|
+
identity: ReadProjectionQueryIdentity,
|
|
1053
|
+
environment: Environment = process.env,
|
|
1054
|
+
): OmniProjectionQuery {
|
|
1055
|
+
return omniQueryForIdentity(identity, environment);
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
function parseQueryValue(value: unknown): ReadProjectionQuery {
|
|
1059
|
+
const query = record(value, "read projection query");
|
|
1060
|
+
exactKeys(
|
|
1061
|
+
query,
|
|
1062
|
+
["key", "realmKey", "identity"],
|
|
1063
|
+
"read projection query",
|
|
1064
|
+
);
|
|
1065
|
+
return Object.freeze({
|
|
1066
|
+
key: hexDigest(query.key, "read projection query key"),
|
|
1067
|
+
realmKey: hexDigest(query.realmKey, "read projection realm key"),
|
|
1068
|
+
identity: parseIdentity(query.identity),
|
|
1069
|
+
});
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
function parseOmniQueryValue(value: unknown): OmniProjectionQuery {
|
|
1073
|
+
const query = record(value, "omni projection query");
|
|
1074
|
+
exactKeys(
|
|
1075
|
+
query,
|
|
1076
|
+
["storageClass", "key", "realmKey", "identity"],
|
|
1077
|
+
"omni projection query",
|
|
1078
|
+
);
|
|
1079
|
+
if (query.storageClass !== "omni-v1") {
|
|
1080
|
+
throw new Error("omni projection storage class is malformed");
|
|
1081
|
+
}
|
|
1082
|
+
return Object.freeze({
|
|
1083
|
+
storageClass: "omni-v1",
|
|
1084
|
+
key: hexDigest(query.key, "omni projection query key"),
|
|
1085
|
+
realmKey: hexDigest(query.realmKey, "omni projection realm key"),
|
|
1086
|
+
identity: parseIdentity(query.identity),
|
|
1087
|
+
});
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
function projectionStorageClass(
|
|
1091
|
+
query: ReadProjectionQuery,
|
|
1092
|
+
): ProjectionStorageClass {
|
|
1093
|
+
return Object.hasOwn(query, "storageClass")
|
|
1094
|
+
? (query as OmniProjectionQuery).storageClass
|
|
1095
|
+
: "exact-v1";
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
function validateQuery(query: ReadProjectionQuery, environment: Environment): {
|
|
1099
|
+
readonly query: ReadProjectionQuery;
|
|
1100
|
+
readonly key: ProjectionKey;
|
|
1101
|
+
} {
|
|
1102
|
+
const parsed = parseQueryValue(query);
|
|
1103
|
+
const key = projectionKey(environment, true);
|
|
1104
|
+
if (key === null) throw new Error("projection encryption key is unavailable");
|
|
1105
|
+
const expected = queryForIdentity(parsed.identity, environment, key);
|
|
1106
|
+
if (parsed.key !== expected.key || parsed.realmKey !== expected.realmKey) {
|
|
1107
|
+
throw new Error("read projection query is not bound to its current private key");
|
|
1108
|
+
}
|
|
1109
|
+
return { query: expected, key };
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
function validateOmniQuery(
|
|
1113
|
+
query: OmniProjectionQuery,
|
|
1114
|
+
environment: Environment,
|
|
1115
|
+
): {
|
|
1116
|
+
readonly query: OmniProjectionQuery;
|
|
1117
|
+
readonly key: ProjectionKey;
|
|
1118
|
+
} {
|
|
1119
|
+
const parsed = parseOmniQueryValue(query);
|
|
1120
|
+
const key = projectionKey(environment, true);
|
|
1121
|
+
if (key === null) throw new Error("projection encryption key is unavailable");
|
|
1122
|
+
const expected = omniQueryForIdentity(parsed.identity, environment, key);
|
|
1123
|
+
if (parsed.key !== expected.key || parsed.realmKey !== expected.realmKey) {
|
|
1124
|
+
throw new Error("omni projection query is not bound to its current private key");
|
|
1125
|
+
}
|
|
1126
|
+
return { query: expected, key };
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
function parseHeadPublication(value: unknown): ReadProjectionHeadPublication {
|
|
1130
|
+
const publication = record(value, "read projection head publication");
|
|
1131
|
+
exactKeys(
|
|
1132
|
+
publication,
|
|
1133
|
+
[
|
|
1134
|
+
"dataRevision",
|
|
1135
|
+
"createdAt",
|
|
1136
|
+
"dataChangedAt",
|
|
1137
|
+
"validatedAt",
|
|
1138
|
+
"runId",
|
|
1139
|
+
"startedAt",
|
|
1140
|
+
"finishedAt",
|
|
1141
|
+
],
|
|
1142
|
+
"read projection head publication",
|
|
1143
|
+
);
|
|
1144
|
+
const createdAt = timestamp(
|
|
1145
|
+
publication.createdAt,
|
|
1146
|
+
"read projection head creation time",
|
|
1147
|
+
);
|
|
1148
|
+
const dataChangedAt = timestamp(
|
|
1149
|
+
publication.dataChangedAt,
|
|
1150
|
+
"read projection head data-change time",
|
|
1151
|
+
);
|
|
1152
|
+
const validatedAt = timestamp(
|
|
1153
|
+
publication.validatedAt,
|
|
1154
|
+
"read projection head validation time",
|
|
1155
|
+
);
|
|
1156
|
+
const startedAt = timestamp(
|
|
1157
|
+
publication.startedAt,
|
|
1158
|
+
"read projection head start time",
|
|
1159
|
+
);
|
|
1160
|
+
const finishedAt = timestamp(
|
|
1161
|
+
publication.finishedAt,
|
|
1162
|
+
"read projection head finish time",
|
|
1163
|
+
);
|
|
1164
|
+
if (
|
|
1165
|
+
createdAt > dataChangedAt
|
|
1166
|
+
|| dataChangedAt > validatedAt
|
|
1167
|
+
|| startedAt > finishedAt
|
|
1168
|
+
|| validatedAt !== finishedAt
|
|
1169
|
+
) throw new Error("read projection head timestamps are inconsistent");
|
|
1170
|
+
return Object.freeze({
|
|
1171
|
+
dataRevision: hexDigest(
|
|
1172
|
+
publication.dataRevision,
|
|
1173
|
+
"read projection head data revision",
|
|
1174
|
+
),
|
|
1175
|
+
createdAt,
|
|
1176
|
+
dataChangedAt,
|
|
1177
|
+
validatedAt,
|
|
1178
|
+
runId: runId(publication.runId),
|
|
1179
|
+
startedAt,
|
|
1180
|
+
finishedAt,
|
|
1181
|
+
});
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
function headPublication(
|
|
1185
|
+
payload: Pick<
|
|
1186
|
+
ProjectionPayloadV1,
|
|
1187
|
+
| "dataRevision"
|
|
1188
|
+
| "createdAt"
|
|
1189
|
+
| "dataChangedAt"
|
|
1190
|
+
| "validatedAt"
|
|
1191
|
+
| "runId"
|
|
1192
|
+
| "startedAt"
|
|
1193
|
+
| "finishedAt"
|
|
1194
|
+
>,
|
|
1195
|
+
): ReadProjectionHeadPublication {
|
|
1196
|
+
return Object.freeze({
|
|
1197
|
+
dataRevision: payload.dataRevision,
|
|
1198
|
+
createdAt: payload.createdAt,
|
|
1199
|
+
dataChangedAt: payload.dataChangedAt,
|
|
1200
|
+
validatedAt: payload.validatedAt,
|
|
1201
|
+
runId: payload.runId,
|
|
1202
|
+
startedAt: payload.startedAt,
|
|
1203
|
+
finishedAt: payload.finishedAt,
|
|
1204
|
+
});
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
function headBody(head: Omit<ProjectionHeadV1, "authentication">): string {
|
|
1208
|
+
return canonicalJson(head);
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
function headAuthentication(
|
|
1212
|
+
key: ProjectionKey,
|
|
1213
|
+
query: ReadProjectionQuery,
|
|
1214
|
+
body: Omit<ProjectionHeadV1, "authentication">,
|
|
1215
|
+
): string {
|
|
1216
|
+
return hmac(
|
|
1217
|
+
key.value,
|
|
1218
|
+
`wrench-read-projection-head-v1\0${query.realmKey}\0${query.key}`,
|
|
1219
|
+
headBody(body),
|
|
1220
|
+
);
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
function parseHead(
|
|
1224
|
+
value: unknown,
|
|
1225
|
+
query: ReadProjectionQuery,
|
|
1226
|
+
key: ProjectionKey,
|
|
1227
|
+
): ProjectionHeadV1 {
|
|
1228
|
+
const head = record(value, "read projection head");
|
|
1229
|
+
exactKeys(
|
|
1230
|
+
head,
|
|
1231
|
+
[
|
|
1232
|
+
"schemaVersion",
|
|
1233
|
+
"keyId",
|
|
1234
|
+
"revisionId",
|
|
1235
|
+
"manifestFile",
|
|
1236
|
+
"publication",
|
|
1237
|
+
"authentication",
|
|
1238
|
+
],
|
|
1239
|
+
"read projection head",
|
|
1240
|
+
);
|
|
1241
|
+
if (head.schemaVersion !== 1) throw new Error("unsupported read projection head schema");
|
|
1242
|
+
const revisionId = safeString(head.revisionId, "read projection revision ID", 64);
|
|
1243
|
+
if (!/^[a-f0-9]{32}$/u.test(revisionId)) throw new Error("read projection revision ID is malformed");
|
|
1244
|
+
const manifestFile = safeString(head.manifestFile, "read projection manifest filename", 128);
|
|
1245
|
+
if (manifestFile !== `manifest--${revisionId}.json`) {
|
|
1246
|
+
throw new Error("read projection head names the wrong manifest");
|
|
1247
|
+
}
|
|
1248
|
+
const body = {
|
|
1249
|
+
schemaVersion: 1 as const,
|
|
1250
|
+
keyId: hexDigest(head.keyId, "read projection head key ID"),
|
|
1251
|
+
revisionId,
|
|
1252
|
+
manifestFile,
|
|
1253
|
+
publication: parseHeadPublication(head.publication),
|
|
1254
|
+
};
|
|
1255
|
+
if (body.keyId !== key.id) throw new Error("read projection head uses another encryption key");
|
|
1256
|
+
const authentication = hexDigest(head.authentication, "read projection head authentication");
|
|
1257
|
+
if (!authenticated(authentication, headAuthentication(key, query, body))) {
|
|
1258
|
+
throw new Error("read projection head failed authentication");
|
|
1259
|
+
}
|
|
1260
|
+
return Object.freeze({ ...body, authentication });
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
function manifestAuthentication(
|
|
1264
|
+
key: ProjectionKey,
|
|
1265
|
+
query: ReadProjectionQuery,
|
|
1266
|
+
body: Omit<ProjectionManifestV1, "authentication">,
|
|
1267
|
+
): string {
|
|
1268
|
+
return hmac(
|
|
1269
|
+
key.value,
|
|
1270
|
+
`wrench-read-projection-manifest-v1\0${query.realmKey}\0${query.key}`,
|
|
1271
|
+
canonicalJson(body),
|
|
1272
|
+
);
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
function parseManifest(
|
|
1276
|
+
value: unknown,
|
|
1277
|
+
query: ReadProjectionQuery,
|
|
1278
|
+
key: ProjectionKey,
|
|
1279
|
+
revisionId: string,
|
|
1280
|
+
): ProjectionManifestV1 {
|
|
1281
|
+
const manifest = record(value, "read projection manifest");
|
|
1282
|
+
exactKeys(
|
|
1283
|
+
manifest,
|
|
1284
|
+
["schemaVersion", "keyId", "revisionId", "plaintextBytes", "chunkFiles", "chunkHashes", "authentication"],
|
|
1285
|
+
"read projection manifest",
|
|
1286
|
+
);
|
|
1287
|
+
if (manifest.schemaVersion !== 1) throw new Error("unsupported read projection manifest schema");
|
|
1288
|
+
const rawChunkFiles = manifest.chunkFiles;
|
|
1289
|
+
const rawChunkHashes = manifest.chunkHashes;
|
|
1290
|
+
if (!Array.isArray(rawChunkFiles) || !Array.isArray(rawChunkHashes)) {
|
|
1291
|
+
throw new Error("read projection manifest has malformed chunks");
|
|
1292
|
+
}
|
|
1293
|
+
const count = rawChunkFiles.length;
|
|
1294
|
+
if (count < 1 || count > MAX_CHUNKS || rawChunkHashes.length !== count) {
|
|
1295
|
+
throw new Error("read projection manifest has malformed chunks");
|
|
1296
|
+
}
|
|
1297
|
+
const chunkFiles = rawChunkFiles.map((file, index) => {
|
|
1298
|
+
const name = safeString(file, `read projection chunk filename ${index}`, 128);
|
|
1299
|
+
const expected = `chunk--${revisionId}--${String(index).padStart(3, "0")}.json`;
|
|
1300
|
+
if (name !== expected) throw new Error("read projection manifest names the wrong chunk");
|
|
1301
|
+
return name;
|
|
1302
|
+
});
|
|
1303
|
+
const chunkHashes = rawChunkHashes.map((hash, index) =>
|
|
1304
|
+
hexDigest(hash, `read projection chunk hash ${index}`));
|
|
1305
|
+
const body = {
|
|
1306
|
+
schemaVersion: 1 as const,
|
|
1307
|
+
keyId: hexDigest(manifest.keyId, "read projection manifest key ID"),
|
|
1308
|
+
revisionId: safeString(manifest.revisionId, "read projection manifest revision ID", 64),
|
|
1309
|
+
plaintextBytes: safeInteger(manifest.plaintextBytes, "read projection plaintext bytes", 1, MAX_PLAINTEXT_BYTES),
|
|
1310
|
+
chunkFiles: Object.freeze(chunkFiles),
|
|
1311
|
+
chunkHashes: Object.freeze(chunkHashes),
|
|
1312
|
+
};
|
|
1313
|
+
if (body.keyId !== key.id || body.revisionId !== revisionId) {
|
|
1314
|
+
throw new Error("read projection manifest identity does not match its head");
|
|
1315
|
+
}
|
|
1316
|
+
const authentication = hexDigest(manifest.authentication, "read projection manifest authentication");
|
|
1317
|
+
if (!authenticated(authentication, manifestAuthentication(key, query, body))) {
|
|
1318
|
+
throw new Error("read projection manifest failed authentication");
|
|
1319
|
+
}
|
|
1320
|
+
return Object.freeze({ ...body, authentication });
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
function boundedBase64(value: unknown, label: string, maximumBytes: number): Buffer {
|
|
1324
|
+
if (
|
|
1325
|
+
typeof value !== "string"
|
|
1326
|
+
|| value.length > Math.ceil(maximumBytes / 3) * 4 + 4
|
|
1327
|
+
|| !/^[A-Za-z0-9+/]*={0,2}$/u.test(value)
|
|
1328
|
+
) throw new Error(`${label} is malformed`);
|
|
1329
|
+
const bytes = Buffer.from(value, "base64");
|
|
1330
|
+
if (bytes.byteLength > maximumBytes || bytes.toString("base64") !== value) {
|
|
1331
|
+
throw new Error(`${label} is malformed`);
|
|
1332
|
+
}
|
|
1333
|
+
return bytes;
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
function chunkAdditionalData(
|
|
1337
|
+
key: ProjectionKey,
|
|
1338
|
+
query: ReadProjectionQuery,
|
|
1339
|
+
revisionId: string,
|
|
1340
|
+
index: number,
|
|
1341
|
+
count: number,
|
|
1342
|
+
): Buffer {
|
|
1343
|
+
return Buffer.from(
|
|
1344
|
+
`wrench-read-projection-chunk-v1\0${key.id}\0${query.realmKey}\0${query.key}\0${revisionId}\0${index}\0${count}`,
|
|
1345
|
+
"utf8",
|
|
1346
|
+
);
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
function parseChunk(
|
|
1350
|
+
value: unknown,
|
|
1351
|
+
key: ProjectionKey,
|
|
1352
|
+
revisionId: string,
|
|
1353
|
+
index: number,
|
|
1354
|
+
count: number,
|
|
1355
|
+
): EncryptedProjectionChunkV1 {
|
|
1356
|
+
const chunk = record(value, "encrypted read projection chunk");
|
|
1357
|
+
exactKeys(
|
|
1358
|
+
chunk,
|
|
1359
|
+
["schemaVersion", "encryption", "keyId", "revisionId", "index", "count", "iv", "ciphertext", "tag"],
|
|
1360
|
+
"encrypted read projection chunk",
|
|
1361
|
+
);
|
|
1362
|
+
if (chunk.schemaVersion !== 1 || chunk.encryption !== "aes-256-gcm") {
|
|
1363
|
+
throw new Error("unsupported read projection chunk schema");
|
|
1364
|
+
}
|
|
1365
|
+
const parsed: EncryptedProjectionChunkV1 = Object.freeze({
|
|
1366
|
+
schemaVersion: 1,
|
|
1367
|
+
encryption: "aes-256-gcm",
|
|
1368
|
+
keyId: hexDigest(chunk.keyId, "read projection chunk key ID"),
|
|
1369
|
+
revisionId: safeString(chunk.revisionId, "read projection chunk revision ID", 64),
|
|
1370
|
+
index: safeInteger(chunk.index, "read projection chunk index", 0, MAX_CHUNKS - 1),
|
|
1371
|
+
count: safeInteger(chunk.count, "read projection chunk count", 1, MAX_CHUNKS),
|
|
1372
|
+
iv: boundedBase64(chunk.iv, "read projection chunk IV", 12).toString("base64"),
|
|
1373
|
+
ciphertext: boundedBase64(chunk.ciphertext, "read projection chunk ciphertext", PLAINTEXT_CHUNK_BYTES).toString("base64"),
|
|
1374
|
+
tag: boundedBase64(chunk.tag, "read projection chunk tag", 16).toString("base64"),
|
|
1375
|
+
});
|
|
1376
|
+
if (
|
|
1377
|
+
parsed.keyId !== key.id
|
|
1378
|
+
|| parsed.revisionId !== revisionId
|
|
1379
|
+
|| parsed.index !== index
|
|
1380
|
+
|| parsed.count !== count
|
|
1381
|
+
) throw new Error("read projection chunk identity is inconsistent");
|
|
1382
|
+
return parsed;
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
function parsePayload(
|
|
1386
|
+
value: unknown,
|
|
1387
|
+
query: ReadProjectionQuery,
|
|
1388
|
+
dataRevisionForOutput: (output: unknown) => string,
|
|
1389
|
+
): ProjectionPayloadV1 {
|
|
1390
|
+
const payload = record(value, "read projection payload");
|
|
1391
|
+
exactKeys(
|
|
1392
|
+
payload,
|
|
1393
|
+
["schemaVersion", "query", "output", "dataRevision", "createdAt", "dataChangedAt", "validatedAt", "runId", "startedAt", "finishedAt"],
|
|
1394
|
+
"read projection payload",
|
|
1395
|
+
);
|
|
1396
|
+
if (payload.schemaVersion !== 1) throw new Error("unsupported read projection payload schema");
|
|
1397
|
+
const identity = parseIdentity(payload.query);
|
|
1398
|
+
if (canonicalJson(identity) !== canonicalJson(query.identity)) {
|
|
1399
|
+
throw new Error("read projection payload names another query");
|
|
1400
|
+
}
|
|
1401
|
+
const output = boundedJson(payload.output, "read projection output");
|
|
1402
|
+
const dataRevision = hexDigest(payload.dataRevision, "read projection data revision");
|
|
1403
|
+
if (dataRevision !== dataRevisionForOutput(output)) {
|
|
1404
|
+
throw new Error("read projection output is not revision-bound");
|
|
1405
|
+
}
|
|
1406
|
+
const createdAt = timestamp(payload.createdAt, "read projection creation time");
|
|
1407
|
+
const dataChangedAt = timestamp(payload.dataChangedAt, "read projection data-change time");
|
|
1408
|
+
const validatedAt = timestamp(payload.validatedAt, "read projection validation time");
|
|
1409
|
+
const startedAt = timestamp(payload.startedAt, "read projection start time");
|
|
1410
|
+
const finishedAt = timestamp(payload.finishedAt, "read projection finish time");
|
|
1411
|
+
if (
|
|
1412
|
+
createdAt > dataChangedAt
|
|
1413
|
+
|| dataChangedAt > validatedAt
|
|
1414
|
+
|| startedAt > finishedAt
|
|
1415
|
+
|| validatedAt !== finishedAt
|
|
1416
|
+
) throw new Error("read projection timestamps are inconsistent");
|
|
1417
|
+
return Object.freeze({
|
|
1418
|
+
schemaVersion: 1,
|
|
1419
|
+
query: identity,
|
|
1420
|
+
output,
|
|
1421
|
+
dataRevision,
|
|
1422
|
+
createdAt,
|
|
1423
|
+
dataChangedAt,
|
|
1424
|
+
validatedAt,
|
|
1425
|
+
runId: runId(payload.runId),
|
|
1426
|
+
startedAt,
|
|
1427
|
+
finishedAt,
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
function parseJson(text: string, label: string): unknown {
|
|
1432
|
+
try {
|
|
1433
|
+
return JSON.parse(text) as unknown;
|
|
1434
|
+
} catch {
|
|
1435
|
+
throw new Error(`${label} is malformed`);
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
function encryptedChunk(
|
|
1440
|
+
plaintext: Buffer,
|
|
1441
|
+
query: ReadProjectionQuery,
|
|
1442
|
+
key: ProjectionKey,
|
|
1443
|
+
revisionId: string,
|
|
1444
|
+
index: number,
|
|
1445
|
+
count: number,
|
|
1446
|
+
): EncryptedProjectionChunkV1 {
|
|
1447
|
+
const iv = randomBytes(12);
|
|
1448
|
+
const cipher = createCipheriv("aes-256-gcm", key.value, iv);
|
|
1449
|
+
cipher.setAAD(chunkAdditionalData(key, query, revisionId, index, count));
|
|
1450
|
+
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
|
|
1451
|
+
return Object.freeze({
|
|
1452
|
+
schemaVersion: 1,
|
|
1453
|
+
encryption: "aes-256-gcm",
|
|
1454
|
+
keyId: key.id,
|
|
1455
|
+
revisionId,
|
|
1456
|
+
index,
|
|
1457
|
+
count,
|
|
1458
|
+
iv: iv.toString("base64"),
|
|
1459
|
+
ciphertext: ciphertext.toString("base64"),
|
|
1460
|
+
tag: cipher.getAuthTag().toString("base64"),
|
|
1461
|
+
});
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
function decryptChunk(
|
|
1465
|
+
chunk: EncryptedProjectionChunkV1,
|
|
1466
|
+
query: ReadProjectionQuery,
|
|
1467
|
+
key: ProjectionKey,
|
|
1468
|
+
): Buffer {
|
|
1469
|
+
try {
|
|
1470
|
+
const iv = Buffer.from(chunk.iv, "base64");
|
|
1471
|
+
const decipher = createDecipheriv("aes-256-gcm", key.value, iv);
|
|
1472
|
+
decipher.setAAD(
|
|
1473
|
+
chunkAdditionalData(key, query, chunk.revisionId, chunk.index, chunk.count),
|
|
1474
|
+
);
|
|
1475
|
+
decipher.setAuthTag(Buffer.from(chunk.tag, "base64"));
|
|
1476
|
+
return Buffer.concat([
|
|
1477
|
+
decipher.update(Buffer.from(chunk.ciphertext, "base64")),
|
|
1478
|
+
decipher.final(),
|
|
1479
|
+
]);
|
|
1480
|
+
} catch {
|
|
1481
|
+
throw new Error("read projection chunk failed authentication");
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
export type AuthenticatedPrivatePayloadV1 = {
|
|
1486
|
+
readonly schemaVersion: 1;
|
|
1487
|
+
readonly encryption: "aes-256-gcm";
|
|
1488
|
+
readonly keyId: string;
|
|
1489
|
+
readonly iv: string;
|
|
1490
|
+
readonly ciphertext: string;
|
|
1491
|
+
readonly tag: string;
|
|
1492
|
+
};
|
|
1493
|
+
|
|
1494
|
+
function privatePayloadAdditionalData(keyId: string, domain: string): Buffer {
|
|
1495
|
+
if (
|
|
1496
|
+
domain.length < 1
|
|
1497
|
+
|| Buffer.byteLength(domain, "utf8") > 256
|
|
1498
|
+
|| /[\0\r\n]/u.test(domain)
|
|
1499
|
+
) throw new Error("authenticated private payload domain is malformed");
|
|
1500
|
+
return Buffer.from(canonicalJson({
|
|
1501
|
+
schemaVersion: 1,
|
|
1502
|
+
format: "wrench.authenticated-private-payload-aad",
|
|
1503
|
+
keyId,
|
|
1504
|
+
domain,
|
|
1505
|
+
}), "utf8");
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
/** Internal state primitive sharing Ghostget's authenticated projection key. */
|
|
1509
|
+
export function sealAuthenticatedPrivatePayload(
|
|
1510
|
+
value: unknown,
|
|
1511
|
+
domain: string,
|
|
1512
|
+
environment: Environment = process.env,
|
|
1513
|
+
maximumPlaintextBytes = 2 * 1024 * 1024,
|
|
1514
|
+
): AuthenticatedPrivatePayloadV1 {
|
|
1515
|
+
const plaintext = Buffer.from(canonicalJson(value), "utf8");
|
|
1516
|
+
if (
|
|
1517
|
+
!Number.isSafeInteger(maximumPlaintextBytes)
|
|
1518
|
+
|| maximumPlaintextBytes < 1
|
|
1519
|
+
|| plaintext.byteLength > maximumPlaintextBytes
|
|
1520
|
+
) throw new Error("authenticated private payload exceeds its plaintext bound");
|
|
1521
|
+
const key = projectionKey(environment, true);
|
|
1522
|
+
if (key === null) throw new Error("projection encryption key is unavailable");
|
|
1523
|
+
const iv = randomBytes(12);
|
|
1524
|
+
const cipher = createCipheriv("aes-256-gcm", key.value, iv);
|
|
1525
|
+
cipher.setAAD(privatePayloadAdditionalData(key.id, domain));
|
|
1526
|
+
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
|
|
1527
|
+
return Object.freeze({
|
|
1528
|
+
schemaVersion: 1,
|
|
1529
|
+
encryption: "aes-256-gcm",
|
|
1530
|
+
keyId: key.id,
|
|
1531
|
+
iv: iv.toString("base64"),
|
|
1532
|
+
ciphertext: ciphertext.toString("base64"),
|
|
1533
|
+
tag: cipher.getAuthTag().toString("base64"),
|
|
1534
|
+
});
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
function privatePayloadBase64(
|
|
1538
|
+
value: unknown,
|
|
1539
|
+
label: string,
|
|
1540
|
+
maximumBytes: number,
|
|
1541
|
+
): Buffer {
|
|
1542
|
+
if (
|
|
1543
|
+
typeof value !== "string"
|
|
1544
|
+
|| value.length > Math.ceil(maximumBytes / 3) * 4 + 4
|
|
1545
|
+
|| !/^[A-Za-z0-9+/]*={0,2}$/u.test(value)
|
|
1546
|
+
) throw new Error(`${label} is malformed`);
|
|
1547
|
+
const decoded = Buffer.from(value, "base64");
|
|
1548
|
+
if (
|
|
1549
|
+
decoded.byteLength > maximumBytes
|
|
1550
|
+
|| decoded.toString("base64") !== value
|
|
1551
|
+
) throw new Error(`${label} is malformed`);
|
|
1552
|
+
return decoded;
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
/** Internal inverse of sealAuthenticatedPrivatePayload. */
|
|
1556
|
+
export function openAuthenticatedPrivatePayload(
|
|
1557
|
+
value: unknown,
|
|
1558
|
+
domain: string,
|
|
1559
|
+
environment: Environment = process.env,
|
|
1560
|
+
maximumPlaintextBytes = 2 * 1024 * 1024,
|
|
1561
|
+
): unknown {
|
|
1562
|
+
const source = record(value, "authenticated private payload");
|
|
1563
|
+
exactKeys(source, [
|
|
1564
|
+
"schemaVersion",
|
|
1565
|
+
"encryption",
|
|
1566
|
+
"keyId",
|
|
1567
|
+
"iv",
|
|
1568
|
+
"ciphertext",
|
|
1569
|
+
"tag",
|
|
1570
|
+
], "authenticated private payload");
|
|
1571
|
+
if (
|
|
1572
|
+
source.schemaVersion !== 1
|
|
1573
|
+
|| source.encryption !== "aes-256-gcm"
|
|
1574
|
+
|| !Number.isSafeInteger(maximumPlaintextBytes)
|
|
1575
|
+
|| maximumPlaintextBytes < 1
|
|
1576
|
+
) throw new Error("authenticated private payload is malformed");
|
|
1577
|
+
const keyId = hexDigest(source.keyId, "authenticated private payload key ID");
|
|
1578
|
+
const key = projectionKey(environment, false);
|
|
1579
|
+
if (key === null || key.id !== keyId) {
|
|
1580
|
+
throw new Error("authenticated private payload encryption key is unavailable");
|
|
1581
|
+
}
|
|
1582
|
+
const iv = privatePayloadBase64(source.iv, "authenticated private payload IV", 12);
|
|
1583
|
+
const tag = privatePayloadBase64(source.tag, "authenticated private payload tag", 16);
|
|
1584
|
+
const ciphertext = privatePayloadBase64(
|
|
1585
|
+
source.ciphertext,
|
|
1586
|
+
"authenticated private payload ciphertext",
|
|
1587
|
+
maximumPlaintextBytes + 16,
|
|
1588
|
+
);
|
|
1589
|
+
if (iv.byteLength !== 12 || tag.byteLength !== 16) {
|
|
1590
|
+
throw new Error("authenticated private payload is malformed");
|
|
1591
|
+
}
|
|
1592
|
+
let plaintext: Buffer;
|
|
1593
|
+
try {
|
|
1594
|
+
const decipher = createDecipheriv("aes-256-gcm", key.value, iv);
|
|
1595
|
+
decipher.setAAD(privatePayloadAdditionalData(key.id, domain));
|
|
1596
|
+
decipher.setAuthTag(tag);
|
|
1597
|
+
plaintext = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
|
|
1598
|
+
} catch {
|
|
1599
|
+
throw new Error("authenticated private payload failed authentication");
|
|
1600
|
+
}
|
|
1601
|
+
if (plaintext.byteLength > maximumPlaintextBytes) {
|
|
1602
|
+
throw new Error("authenticated private payload exceeds its plaintext bound");
|
|
1603
|
+
}
|
|
1604
|
+
let decoded: string;
|
|
1605
|
+
try {
|
|
1606
|
+
decoded = new TextDecoder("utf-8", { fatal: true }).decode(plaintext);
|
|
1607
|
+
} catch {
|
|
1608
|
+
throw new Error("authenticated private payload plaintext is malformed");
|
|
1609
|
+
}
|
|
1610
|
+
return parseJson(decoded, "authenticated private payload plaintext");
|
|
1611
|
+
}
|
|
1612
|
+
|
|
1613
|
+
function dataRevision(
|
|
1614
|
+
key: ProjectionKey,
|
|
1615
|
+
output: unknown,
|
|
1616
|
+
storageClass: ProjectionStorageClass = "exact-v1",
|
|
1617
|
+
): string {
|
|
1618
|
+
return hmac(
|
|
1619
|
+
key.value,
|
|
1620
|
+
storageClass === "exact-v1"
|
|
1621
|
+
? "wrench-read-projection-data-v1"
|
|
1622
|
+
: "wrench-omni-projection-data-v1",
|
|
1623
|
+
canonicalJson(output),
|
|
1624
|
+
);
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
function corruptionContext(
|
|
1628
|
+
query: ReadProjectionQuery,
|
|
1629
|
+
headContentSha256: string,
|
|
1630
|
+
headPublicationValue: ReadProjectionHeadPublication | null,
|
|
1631
|
+
): ReadProjectionCorruptionEvidence {
|
|
1632
|
+
return Object.freeze({
|
|
1633
|
+
storageClass: projectionStorageClass(query),
|
|
1634
|
+
queryKey: query.key,
|
|
1635
|
+
realmKey: query.realmKey,
|
|
1636
|
+
headContentSha256,
|
|
1637
|
+
headPublication: headPublicationValue,
|
|
1638
|
+
});
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
function corruption(
|
|
1642
|
+
evidence: ReadProjectionCorruptionEvidence,
|
|
1643
|
+
message: string,
|
|
1644
|
+
cause?: unknown,
|
|
1645
|
+
): ReadProjectionCorruptionError {
|
|
1646
|
+
if (isReadProjectionCorruptionError(cause)) return cause;
|
|
1647
|
+
return new ReadProjectionCorruptionError(
|
|
1648
|
+
message,
|
|
1649
|
+
evidence,
|
|
1650
|
+
{ cause },
|
|
1651
|
+
corruptionConstructorToken,
|
|
1652
|
+
);
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
function decodeProjectionArtifact(
|
|
1656
|
+
bytes: Buffer,
|
|
1657
|
+
evidence: ReadProjectionCorruptionEvidence,
|
|
1658
|
+
label: string,
|
|
1659
|
+
): string {
|
|
1660
|
+
try {
|
|
1661
|
+
return new TextDecoder("utf-8", {
|
|
1662
|
+
fatal: true,
|
|
1663
|
+
ignoreBOM: true,
|
|
1664
|
+
}).decode(bytes);
|
|
1665
|
+
} catch (error) {
|
|
1666
|
+
throw corruption(evidence, `${label} is not valid UTF-8`, error);
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
|
|
1670
|
+
function readHead(
|
|
1671
|
+
query: ReadProjectionQuery,
|
|
1672
|
+
key: ProjectionKey,
|
|
1673
|
+
environment: Environment,
|
|
1674
|
+
): ProjectionHeadSnapshot | null {
|
|
1675
|
+
const bytes = readPrivateStateFileBytesIfPresent(
|
|
1676
|
+
headPath(query, environment),
|
|
1677
|
+
MAX_ATTRIBUTABLE_CORRUPT_FILE_BYTES,
|
|
1678
|
+
"read projection head",
|
|
1679
|
+
environment,
|
|
1680
|
+
);
|
|
1681
|
+
if (bytes === null) return null;
|
|
1682
|
+
const contentSha256 = hashBytes(bytes);
|
|
1683
|
+
const evidence = corruptionContext(query, contentSha256, null);
|
|
1684
|
+
if (bytes.byteLength > HEAD_MAX_BYTES) {
|
|
1685
|
+
throw corruption(evidence, "read projection head exceeds its byte bound");
|
|
1686
|
+
}
|
|
1687
|
+
const text = decodeProjectionArtifact(
|
|
1688
|
+
bytes,
|
|
1689
|
+
evidence,
|
|
1690
|
+
"read projection head",
|
|
1691
|
+
);
|
|
1692
|
+
let head: ProjectionHeadV1;
|
|
1693
|
+
try {
|
|
1694
|
+
head = parseHead(parseJson(text, "read projection head"), query, key);
|
|
1695
|
+
} catch (error) {
|
|
1696
|
+
throw corruption(evidence, "read projection head is corrupt", error);
|
|
1697
|
+
}
|
|
1698
|
+
return Object.freeze({ head, contentSha256 });
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
function loadFromHead(
|
|
1702
|
+
query: ReadProjectionQuery,
|
|
1703
|
+
key: ProjectionKey,
|
|
1704
|
+
headSnapshot: ProjectionHeadSnapshot,
|
|
1705
|
+
environment: Environment,
|
|
1706
|
+
): LoadedProjection {
|
|
1707
|
+
const directory = queryDirectory(query, environment);
|
|
1708
|
+
const evidence = corruptionContext(
|
|
1709
|
+
query,
|
|
1710
|
+
headSnapshot.contentSha256,
|
|
1711
|
+
headSnapshot.head.publication,
|
|
1712
|
+
);
|
|
1713
|
+
const manifestBytes = readPrivateStateFileBytesIfPresent(
|
|
1714
|
+
join(directory, headSnapshot.head.manifestFile),
|
|
1715
|
+
MAX_ATTRIBUTABLE_CORRUPT_FILE_BYTES,
|
|
1716
|
+
"read projection manifest",
|
|
1717
|
+
environment,
|
|
1718
|
+
);
|
|
1719
|
+
if (manifestBytes === null) {
|
|
1720
|
+
throw corruption(evidence, "read projection head names a missing manifest");
|
|
1721
|
+
}
|
|
1722
|
+
if (manifestBytes.byteLength > MANIFEST_MAX_BYTES) {
|
|
1723
|
+
throw corruption(
|
|
1724
|
+
evidence,
|
|
1725
|
+
"read projection manifest exceeds its byte bound",
|
|
1726
|
+
);
|
|
1727
|
+
}
|
|
1728
|
+
const manifestText = decodeProjectionArtifact(
|
|
1729
|
+
manifestBytes,
|
|
1730
|
+
evidence,
|
|
1731
|
+
"read projection manifest",
|
|
1732
|
+
);
|
|
1733
|
+
let manifest: ProjectionManifestV1;
|
|
1734
|
+
try {
|
|
1735
|
+
manifest = parseManifest(
|
|
1736
|
+
parseJson(manifestText, "read projection manifest"),
|
|
1737
|
+
query,
|
|
1738
|
+
key,
|
|
1739
|
+
headSnapshot.head.revisionId,
|
|
1740
|
+
);
|
|
1741
|
+
} catch (error) {
|
|
1742
|
+
throw corruption(evidence, "read projection manifest is corrupt", error);
|
|
1743
|
+
}
|
|
1744
|
+
const plaintext: Buffer[] = [];
|
|
1745
|
+
for (const [index, file] of manifest.chunkFiles.entries()) {
|
|
1746
|
+
const bytes = readPrivateStateFileBytesIfPresent(
|
|
1747
|
+
join(directory, file),
|
|
1748
|
+
MAX_ATTRIBUTABLE_CORRUPT_FILE_BYTES,
|
|
1749
|
+
`read projection chunk ${index}`,
|
|
1750
|
+
environment,
|
|
1751
|
+
);
|
|
1752
|
+
if (bytes === null) {
|
|
1753
|
+
throw corruption(evidence, "read projection manifest names a missing chunk");
|
|
1754
|
+
}
|
|
1755
|
+
if (bytes.byteLength > MAX_PRIVATE_STATE_BATCH_FILE_BYTES) {
|
|
1756
|
+
throw corruption(
|
|
1757
|
+
evidence,
|
|
1758
|
+
`read projection chunk ${index} exceeds its byte bound`,
|
|
1759
|
+
);
|
|
1760
|
+
}
|
|
1761
|
+
if (hashBytes(bytes) !== manifest.chunkHashes[index]) {
|
|
1762
|
+
throw corruption(evidence, "read projection chunk does not match its manifest");
|
|
1763
|
+
}
|
|
1764
|
+
const text = decodeProjectionArtifact(
|
|
1765
|
+
bytes,
|
|
1766
|
+
evidence,
|
|
1767
|
+
`read projection chunk ${index}`,
|
|
1768
|
+
);
|
|
1769
|
+
try {
|
|
1770
|
+
const chunk = parseChunk(
|
|
1771
|
+
parseJson(text, `read projection chunk ${index}`),
|
|
1772
|
+
key,
|
|
1773
|
+
manifest.revisionId,
|
|
1774
|
+
index,
|
|
1775
|
+
manifest.chunkFiles.length,
|
|
1776
|
+
);
|
|
1777
|
+
plaintext.push(decryptChunk(chunk, query, key));
|
|
1778
|
+
} catch (error) {
|
|
1779
|
+
throw corruption(evidence, `read projection chunk ${index} is corrupt`, error);
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1782
|
+
const bytes = Buffer.concat(plaintext);
|
|
1783
|
+
if (bytes.byteLength !== manifest.plaintextBytes) {
|
|
1784
|
+
throw corruption(
|
|
1785
|
+
evidence,
|
|
1786
|
+
"read projection plaintext length does not match its manifest",
|
|
1787
|
+
);
|
|
1788
|
+
}
|
|
1789
|
+
let parsed: unknown;
|
|
1790
|
+
try {
|
|
1791
|
+
parsed = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)) as unknown;
|
|
1792
|
+
} catch (error) {
|
|
1793
|
+
throw corruption(evidence, "read projection plaintext is malformed", error);
|
|
1794
|
+
}
|
|
1795
|
+
let payload: ProjectionPayloadV1;
|
|
1796
|
+
try {
|
|
1797
|
+
payload = parsePayload(
|
|
1798
|
+
parsed,
|
|
1799
|
+
query,
|
|
1800
|
+
(output) => dataRevision(key, output, projectionStorageClass(query)),
|
|
1801
|
+
);
|
|
1802
|
+
} catch (error) {
|
|
1803
|
+
throw corruption(evidence, "read projection payload is corrupt", error);
|
|
1804
|
+
}
|
|
1805
|
+
if (
|
|
1806
|
+
canonicalJson(headPublication(payload))
|
|
1807
|
+
!== canonicalJson(headSnapshot.head.publication)
|
|
1808
|
+
) {
|
|
1809
|
+
throw corruption(
|
|
1810
|
+
evidence,
|
|
1811
|
+
"read projection payload does not match its authenticated head publication",
|
|
1812
|
+
);
|
|
1813
|
+
}
|
|
1814
|
+
return Object.freeze({
|
|
1815
|
+
head: headSnapshot.head,
|
|
1816
|
+
headContentSha256: headSnapshot.contentSha256,
|
|
1817
|
+
payload,
|
|
1818
|
+
});
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
function loadProjection(
|
|
1822
|
+
query: ReadProjectionQuery,
|
|
1823
|
+
key: ProjectionKey,
|
|
1824
|
+
environment: Environment,
|
|
1825
|
+
): LoadedProjection | null {
|
|
1826
|
+
let first = readHead(query, key, environment);
|
|
1827
|
+
if (first === null) return null;
|
|
1828
|
+
try {
|
|
1829
|
+
return loadFromHead(query, key, first, environment);
|
|
1830
|
+
} catch (error) {
|
|
1831
|
+
const second = readHead(query, key, environment);
|
|
1832
|
+
if (second === null) return null;
|
|
1833
|
+
if (second.contentSha256 === first.contentSha256) throw error;
|
|
1834
|
+
first = second;
|
|
1835
|
+
return loadFromHead(query, key, first, environment);
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
|
|
1839
|
+
function freshness(
|
|
1840
|
+
validatedAt: string,
|
|
1841
|
+
now: Date,
|
|
1842
|
+
freshForMs: number | undefined,
|
|
1843
|
+
): { readonly state: "fresh" | "stale" | "unclassified"; readonly freshForMs: number | null; readonly ageMs: number } {
|
|
1844
|
+
if (!Number.isFinite(now.getTime())) throw new Error("read projection observation time is invalid");
|
|
1845
|
+
const ageMs = Math.max(0, now.getTime() - new Date(validatedAt).getTime());
|
|
1846
|
+
if (freshForMs === undefined) {
|
|
1847
|
+
return { state: "unclassified", freshForMs: null, ageMs };
|
|
1848
|
+
}
|
|
1849
|
+
safeInteger(freshForMs, "read projection freshness window", 0, MAX_FRESH_FOR_MS);
|
|
1850
|
+
return {
|
|
1851
|
+
state: ageMs <= freshForMs ? "fresh" : "stale",
|
|
1852
|
+
freshForMs,
|
|
1853
|
+
ageMs,
|
|
1854
|
+
};
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
export function readReadProjection(
|
|
1858
|
+
queryValue: ReadProjectionQuery,
|
|
1859
|
+
options: {
|
|
1860
|
+
readonly environment?: Environment;
|
|
1861
|
+
readonly now?: Date;
|
|
1862
|
+
readonly freshForMs?: number;
|
|
1863
|
+
} = {},
|
|
1864
|
+
): ReadProjectionCacheResult {
|
|
1865
|
+
const environment = options.environment ?? process.env;
|
|
1866
|
+
const authId = parseQueryValue(queryValue).identity.auth.id;
|
|
1867
|
+
return withReadProjectionAuthAdmission(authId, environment, () => {
|
|
1868
|
+
const { query, key } = validateQuery(queryValue, environment);
|
|
1869
|
+
const loaded = loadProjection(query, key, environment);
|
|
1870
|
+
if (loaded === null) {
|
|
1871
|
+
return Object.freeze({ status: "miss" as const, key: query.key });
|
|
1872
|
+
}
|
|
1873
|
+
const observed = freshness(
|
|
1874
|
+
loaded.payload.validatedAt,
|
|
1875
|
+
options.now ?? new Date(),
|
|
1876
|
+
options.freshForMs,
|
|
1877
|
+
);
|
|
1878
|
+
return Object.freeze({
|
|
1879
|
+
status: "hit" as const,
|
|
1880
|
+
source: "cache" as const,
|
|
1881
|
+
key: query.key,
|
|
1882
|
+
output: loaded.payload.output,
|
|
1883
|
+
dataRevision: loaded.payload.dataRevision,
|
|
1884
|
+
createdAt: loaded.payload.createdAt,
|
|
1885
|
+
dataChangedAt: loaded.payload.dataChangedAt,
|
|
1886
|
+
validatedAt: loaded.payload.validatedAt,
|
|
1887
|
+
runId: loaded.payload.runId,
|
|
1888
|
+
ageMs: observed.ageMs,
|
|
1889
|
+
freshness: Object.freeze({
|
|
1890
|
+
state: observed.state,
|
|
1891
|
+
freshForMs: observed.freshForMs,
|
|
1892
|
+
}),
|
|
1893
|
+
});
|
|
1894
|
+
});
|
|
1895
|
+
}
|
|
1896
|
+
|
|
1897
|
+
export function readOmniProjection(
|
|
1898
|
+
queryValue: OmniProjectionQuery,
|
|
1899
|
+
options: {
|
|
1900
|
+
readonly environment?: Environment;
|
|
1901
|
+
readonly now?: Date;
|
|
1902
|
+
readonly freshForMs?: number;
|
|
1903
|
+
} = {},
|
|
1904
|
+
): OmniProjectionCacheResult {
|
|
1905
|
+
const environment = options.environment ?? process.env;
|
|
1906
|
+
const authId = parseOmniQueryValue(queryValue).identity.auth.id;
|
|
1907
|
+
return withReadProjectionAuthAdmission(authId, environment, () => {
|
|
1908
|
+
const { query, key } = validateOmniQuery(queryValue, environment);
|
|
1909
|
+
const loaded = loadProjection(query, key, environment);
|
|
1910
|
+
if (loaded === null) {
|
|
1911
|
+
return Object.freeze({ status: "miss" as const, key: query.key });
|
|
1912
|
+
}
|
|
1913
|
+
const observed = freshness(
|
|
1914
|
+
loaded.payload.validatedAt,
|
|
1915
|
+
options.now ?? new Date(),
|
|
1916
|
+
options.freshForMs,
|
|
1917
|
+
);
|
|
1918
|
+
return Object.freeze({
|
|
1919
|
+
status: "hit" as const,
|
|
1920
|
+
source: "cache" as const,
|
|
1921
|
+
key: query.key,
|
|
1922
|
+
storageRevisionId: loaded.head.revisionId,
|
|
1923
|
+
output: immutableBoundedJson(
|
|
1924
|
+
loaded.payload.output,
|
|
1925
|
+
"omni projection current output",
|
|
1926
|
+
),
|
|
1927
|
+
dataRevision: loaded.payload.dataRevision,
|
|
1928
|
+
createdAt: loaded.payload.createdAt,
|
|
1929
|
+
dataChangedAt: loaded.payload.dataChangedAt,
|
|
1930
|
+
validatedAt: loaded.payload.validatedAt,
|
|
1931
|
+
runId: loaded.payload.runId,
|
|
1932
|
+
startedAt: loaded.payload.startedAt,
|
|
1933
|
+
finishedAt: loaded.payload.finishedAt,
|
|
1934
|
+
ageMs: observed.ageMs,
|
|
1935
|
+
freshness: Object.freeze({
|
|
1936
|
+
state: observed.state,
|
|
1937
|
+
freshForMs: observed.freshForMs,
|
|
1938
|
+
}),
|
|
1939
|
+
});
|
|
1940
|
+
});
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
/**
|
|
1944
|
+
* Observe the authoritative exact head with the immutable storage revision and
|
|
1945
|
+
* execution interval required by a provider-owned materializer. The ordinary
|
|
1946
|
+
* exact cache API intentionally keeps its existing public shape.
|
|
1947
|
+
*/
|
|
1948
|
+
export function readReadProjectionForMaterialization(
|
|
1949
|
+
queryValue: ReadProjectionQuery,
|
|
1950
|
+
options: {
|
|
1951
|
+
readonly environment?: Environment;
|
|
1952
|
+
readonly now?: Date;
|
|
1953
|
+
readonly freshForMs?: number;
|
|
1954
|
+
} = {},
|
|
1955
|
+
): ReadProjectionMaterializationSnapshot {
|
|
1956
|
+
const environment = options.environment ?? process.env;
|
|
1957
|
+
const authId = parseQueryValue(queryValue).identity.auth.id;
|
|
1958
|
+
return withReadProjectionAuthAdmission(authId, environment, () => {
|
|
1959
|
+
const { query, key } = validateQuery(queryValue, environment);
|
|
1960
|
+
const loaded = loadProjection(query, key, environment);
|
|
1961
|
+
if (loaded === null) {
|
|
1962
|
+
return Object.freeze({ status: "miss" as const, key: query.key });
|
|
1963
|
+
}
|
|
1964
|
+
const observed = freshness(
|
|
1965
|
+
loaded.payload.validatedAt,
|
|
1966
|
+
options.now ?? new Date(),
|
|
1967
|
+
options.freshForMs,
|
|
1968
|
+
);
|
|
1969
|
+
return Object.freeze({
|
|
1970
|
+
status: "hit" as const,
|
|
1971
|
+
source: "cache" as const,
|
|
1972
|
+
key: query.key,
|
|
1973
|
+
storageRevisionId: loaded.head.revisionId,
|
|
1974
|
+
output: immutableBoundedJson(
|
|
1975
|
+
loaded.payload.output,
|
|
1976
|
+
"exact projection materialization output",
|
|
1977
|
+
),
|
|
1978
|
+
dataRevision: loaded.payload.dataRevision,
|
|
1979
|
+
createdAt: loaded.payload.createdAt,
|
|
1980
|
+
dataChangedAt: loaded.payload.dataChangedAt,
|
|
1981
|
+
validatedAt: loaded.payload.validatedAt,
|
|
1982
|
+
runId: loaded.payload.runId,
|
|
1983
|
+
startedAt: loaded.payload.startedAt,
|
|
1984
|
+
finishedAt: loaded.payload.finishedAt,
|
|
1985
|
+
ageMs: observed.ageMs,
|
|
1986
|
+
freshness: Object.freeze({
|
|
1987
|
+
state: observed.state,
|
|
1988
|
+
freshForMs: observed.freshForMs,
|
|
1989
|
+
}),
|
|
1990
|
+
});
|
|
1991
|
+
});
|
|
1992
|
+
}
|
|
1993
|
+
|
|
1994
|
+
function publicationOrder(payload: Pick<ProjectionPayloadV1, "startedAt" | "finishedAt" | "runId">): string {
|
|
1995
|
+
return `${payload.startedAt}\0${payload.finishedAt}\0${payload.runId}`;
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
function revisionFiles(revisionId: string, count: number): readonly string[] {
|
|
1999
|
+
return Object.freeze([
|
|
2000
|
+
`manifest--${revisionId}.json`,
|
|
2001
|
+
...Array.from(
|
|
2002
|
+
{ length: count },
|
|
2003
|
+
(_, index) => `chunk--${revisionId}--${String(index).padStart(3, "0")}.json`,
|
|
2004
|
+
),
|
|
2005
|
+
]);
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
function isImmutableRevisionFile(name: string): boolean {
|
|
2009
|
+
const manifest = immutableManifestNamePattern.exec(name);
|
|
2010
|
+
if (manifest !== null) return true;
|
|
2011
|
+
const chunk = immutableChunkNamePattern.exec(name);
|
|
2012
|
+
if (chunk === null || chunk[2] === undefined) return false;
|
|
2013
|
+
const index = Number(chunk[2]);
|
|
2014
|
+
return Number.isSafeInteger(index) && index >= 0 && index < MAX_CHUNKS;
|
|
2015
|
+
}
|
|
2016
|
+
|
|
2017
|
+
function removeRevision(
|
|
2018
|
+
query: ReadProjectionQuery,
|
|
2019
|
+
revisionId: string,
|
|
2020
|
+
count: number,
|
|
2021
|
+
environment: Environment,
|
|
2022
|
+
): void {
|
|
2023
|
+
const directory = queryDirectory(query, environment);
|
|
2024
|
+
for (const file of revisionFiles(revisionId, count)) {
|
|
2025
|
+
try {
|
|
2026
|
+
removePrivateStateFile(join(directory, file), environment);
|
|
2027
|
+
} catch {
|
|
2028
|
+
// The promoted head is authoritative. Unreachable immutable files are
|
|
2029
|
+
// inert and can be reclaimed later without changing cache truth.
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
function currentChunkCount(
|
|
2035
|
+
loaded: LoadedProjection,
|
|
2036
|
+
query: ReadProjectionQuery,
|
|
2037
|
+
key: ProjectionKey,
|
|
2038
|
+
environment: Environment,
|
|
2039
|
+
): number {
|
|
2040
|
+
const manifestText = readPrivateStateFileIfPresent(
|
|
2041
|
+
join(queryDirectory(query, environment), loaded.head.manifestFile),
|
|
2042
|
+
MANIFEST_MAX_BYTES,
|
|
2043
|
+
"read projection manifest",
|
|
2044
|
+
environment,
|
|
2045
|
+
);
|
|
2046
|
+
if (manifestText === null) throw new Error("read projection head names a missing manifest");
|
|
2047
|
+
return parseManifest(
|
|
2048
|
+
parseJson(manifestText, "read projection manifest"),
|
|
2049
|
+
query,
|
|
2050
|
+
key,
|
|
2051
|
+
loaded.head.revisionId,
|
|
2052
|
+
).chunkFiles.length;
|
|
2053
|
+
}
|
|
2054
|
+
|
|
2055
|
+
function reclaimNonHeadRevisionFiles(
|
|
2056
|
+
query: ReadProjectionQuery,
|
|
2057
|
+
current: LoadedProjection | null,
|
|
2058
|
+
currentCount: number,
|
|
2059
|
+
prospectiveRevisionEntries: number,
|
|
2060
|
+
environment: Environment,
|
|
2061
|
+
): void {
|
|
2062
|
+
const directory = queryDirectory(query, environment);
|
|
2063
|
+
const directoryIdentity = ensurePrivateStateDirectory(directory, environment);
|
|
2064
|
+
const expected = current === null
|
|
2065
|
+
? new Set<string>()
|
|
2066
|
+
: new Set([HEAD_FILE, ...revisionFiles(current.head.revisionId, currentCount)]);
|
|
2067
|
+
const initial = listPrivateStateDirectory(
|
|
2068
|
+
directory,
|
|
2069
|
+
environment,
|
|
2070
|
+
directoryIdentity,
|
|
2071
|
+
{ recoverOrphanedMutationClaims: true },
|
|
2072
|
+
);
|
|
2073
|
+
for (const entry of initial) {
|
|
2074
|
+
if (
|
|
2075
|
+
entry.kind === "file"
|
|
2076
|
+
&& !expected.has(entry.name)
|
|
2077
|
+
&& isImmutableRevisionFile(entry.name)
|
|
2078
|
+
) {
|
|
2079
|
+
removePrivateStateFile(join(directory, entry.name), environment);
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
|
|
2083
|
+
const settled = listPrivateStateDirectory(
|
|
2084
|
+
directory,
|
|
2085
|
+
environment,
|
|
2086
|
+
directoryIdentity,
|
|
2087
|
+
{ recoverOrphanedMutationClaims: true },
|
|
2088
|
+
);
|
|
2089
|
+
if (
|
|
2090
|
+
settled.length
|
|
2091
|
+
+ prospectiveRevisionEntries
|
|
2092
|
+
+ (current === null ? 1 : 0)
|
|
2093
|
+
> MAX_QUERY_DIRECTORY_ENTRIES
|
|
2094
|
+
) {
|
|
2095
|
+
throw new Error(
|
|
2096
|
+
"read projection query directory exceeds its prospective entry bound",
|
|
2097
|
+
);
|
|
2098
|
+
}
|
|
2099
|
+
if (
|
|
2100
|
+
settled.some((entry) =>
|
|
2101
|
+
entry.kind !== "file" || !expected.has(entry.name))
|
|
2102
|
+
) {
|
|
2103
|
+
throw new Error("read projection query directory has unsupported state");
|
|
2104
|
+
}
|
|
2105
|
+
if (
|
|
2106
|
+
current !== null
|
|
2107
|
+
&& [...expected].some((name) =>
|
|
2108
|
+
!settled.some((entry) => entry.name === name))
|
|
2109
|
+
) {
|
|
2110
|
+
throw corruption(
|
|
2111
|
+
corruptionContext(
|
|
2112
|
+
query,
|
|
2113
|
+
current.headContentSha256,
|
|
2114
|
+
current.head.publication,
|
|
2115
|
+
),
|
|
2116
|
+
"read projection current revision became incomplete",
|
|
2117
|
+
);
|
|
2118
|
+
}
|
|
2119
|
+
}
|
|
2120
|
+
|
|
2121
|
+
type RealmQueryUsage = {
|
|
2122
|
+
readonly name: string;
|
|
2123
|
+
readonly identity: PrivateDirectoryIdentity;
|
|
2124
|
+
readonly bytes: number;
|
|
2125
|
+
readonly state: "usable" | "invalid" | "busy";
|
|
2126
|
+
};
|
|
2127
|
+
|
|
2128
|
+
function readRealmQueryUsage(
|
|
2129
|
+
storeIdentity: PrivateDirectoryIdentity,
|
|
2130
|
+
realm: string,
|
|
2131
|
+
realmIdentity: PrivateDirectoryIdentity,
|
|
2132
|
+
name: string,
|
|
2133
|
+
identity: PrivateDirectoryIdentity,
|
|
2134
|
+
environment: Environment,
|
|
2135
|
+
targetFileMaximumBytes?: number,
|
|
2136
|
+
): RealmQueryUsage {
|
|
2137
|
+
const entries = listPrivateStateDirectory(
|
|
2138
|
+
join(realm, name),
|
|
2139
|
+
environment,
|
|
2140
|
+
identity,
|
|
2141
|
+
{ recoverOrphanedMutationClaims: true },
|
|
2142
|
+
);
|
|
2143
|
+
const busy = entries.some((entry) =>
|
|
2144
|
+
stateHelperArtifactNamePatterns.some((pattern) =>
|
|
2145
|
+
pattern.test(entry.name)));
|
|
2146
|
+
if (
|
|
2147
|
+
entries.length > MAX_QUERY_DIRECTORY_ENTRIES
|
|
2148
|
+
|| entries.some((entry) => entry.kind !== "file")
|
|
2149
|
+
) {
|
|
2150
|
+
return Object.freeze({
|
|
2151
|
+
name,
|
|
2152
|
+
identity,
|
|
2153
|
+
bytes: MAX_REALM_STORAGE_BYTES + 1,
|
|
2154
|
+
state: busy ? "busy" : "invalid",
|
|
2155
|
+
});
|
|
2156
|
+
}
|
|
2157
|
+
let bytes = 0;
|
|
2158
|
+
if (targetFileMaximumBytes !== undefined) {
|
|
2159
|
+
for (const entry of entries) {
|
|
2160
|
+
const content = readPrivateStateFileBytesIfPresent(
|
|
2161
|
+
join(realm, name, entry.name),
|
|
2162
|
+
targetFileMaximumBytes,
|
|
2163
|
+
`read projection repair quota file ${entry.name}`,
|
|
2164
|
+
environment,
|
|
2165
|
+
[storeIdentity, realmIdentity, identity],
|
|
2166
|
+
);
|
|
2167
|
+
if (content === null) {
|
|
2168
|
+
return Object.freeze({
|
|
2169
|
+
name,
|
|
2170
|
+
identity,
|
|
2171
|
+
bytes: MAX_REALM_STORAGE_BYTES + 1,
|
|
2172
|
+
state: busy ? "busy" : "invalid",
|
|
2173
|
+
});
|
|
2174
|
+
}
|
|
2175
|
+
bytes += content.byteLength;
|
|
2176
|
+
if (!Number.isSafeInteger(bytes) || bytes > MAX_REALM_STORAGE_BYTES) {
|
|
2177
|
+
return Object.freeze({
|
|
2178
|
+
name,
|
|
2179
|
+
identity,
|
|
2180
|
+
bytes: MAX_REALM_STORAGE_BYTES + 1,
|
|
2181
|
+
state: busy ? "busy" : "invalid",
|
|
2182
|
+
});
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
} else {
|
|
2186
|
+
const results = readPrivateStateChildFilesBatched(
|
|
2187
|
+
realm,
|
|
2188
|
+
entries.map((entry) => ({
|
|
2189
|
+
directoryName: name,
|
|
2190
|
+
directoryIdentity: identity,
|
|
2191
|
+
fileName: entry.name,
|
|
2192
|
+
})),
|
|
2193
|
+
{
|
|
2194
|
+
maximumBytesPerFile: MAX_PRIVATE_STATE_BATCH_FILE_BYTES,
|
|
2195
|
+
environment,
|
|
2196
|
+
expectedDirectoryIdentity: realmIdentity,
|
|
2197
|
+
},
|
|
2198
|
+
);
|
|
2199
|
+
for (const result of results) {
|
|
2200
|
+
if (result.status !== "present") {
|
|
2201
|
+
return Object.freeze({
|
|
2202
|
+
name,
|
|
2203
|
+
identity,
|
|
2204
|
+
bytes: MAX_REALM_STORAGE_BYTES + 1,
|
|
2205
|
+
state: busy ? "busy" : "invalid",
|
|
2206
|
+
});
|
|
2207
|
+
}
|
|
2208
|
+
bytes += Buffer.byteLength(result.content, "utf8");
|
|
2209
|
+
if (!Number.isSafeInteger(bytes) || bytes > MAX_REALM_STORAGE_BYTES) {
|
|
2210
|
+
return Object.freeze({
|
|
2211
|
+
name,
|
|
2212
|
+
identity,
|
|
2213
|
+
bytes: MAX_REALM_STORAGE_BYTES + 1,
|
|
2214
|
+
state: busy ? "busy" : "invalid",
|
|
2215
|
+
});
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
return Object.freeze({
|
|
2220
|
+
name,
|
|
2221
|
+
identity,
|
|
2222
|
+
bytes,
|
|
2223
|
+
state: busy ? "busy" : "usable",
|
|
2224
|
+
});
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
function realmUsage(
|
|
2228
|
+
query: ReadProjectionQuery,
|
|
2229
|
+
environment: Environment,
|
|
2230
|
+
targetFileMaximumBytes?: number,
|
|
2231
|
+
): {
|
|
2232
|
+
readonly realm: string;
|
|
2233
|
+
readonly realmIdentity: PrivateDirectoryIdentity;
|
|
2234
|
+
readonly queries: readonly RealmQueryUsage[];
|
|
2235
|
+
} {
|
|
2236
|
+
const storageClass = projectionStorageClass(query);
|
|
2237
|
+
const storeIdentity = ensurePrivateStateDirectory(
|
|
2238
|
+
storeDirectoryForClass(storageClass, environment),
|
|
2239
|
+
environment,
|
|
2240
|
+
);
|
|
2241
|
+
const realm = realmDirectory(query.realmKey, environment, storageClass);
|
|
2242
|
+
const realmIdentity = ensurePrivateStateDirectory(realm, environment);
|
|
2243
|
+
const snapshot = snapshotPrivateStateDirectory(
|
|
2244
|
+
realm,
|
|
2245
|
+
environment,
|
|
2246
|
+
realmIdentity,
|
|
2247
|
+
);
|
|
2248
|
+
if (snapshot.identity === null) {
|
|
2249
|
+
throw new Error("read projection realm disappeared during quota inspection");
|
|
2250
|
+
}
|
|
2251
|
+
const queries: RealmQueryUsage[] = [];
|
|
2252
|
+
for (const entry of snapshot.entries) {
|
|
2253
|
+
if (
|
|
2254
|
+
entry.kind !== "directory"
|
|
2255
|
+
|| entry.identity === undefined
|
|
2256
|
+
|| !queryDirectoryNamePattern.test(entry.name)
|
|
2257
|
+
) {
|
|
2258
|
+
throw new Error("read projection realm contains unsupported state");
|
|
2259
|
+
}
|
|
2260
|
+
queries.push(readRealmQueryUsage(
|
|
2261
|
+
storeIdentity,
|
|
2262
|
+
realm,
|
|
2263
|
+
snapshot.identity,
|
|
2264
|
+
entry.name,
|
|
2265
|
+
entry.identity,
|
|
2266
|
+
environment,
|
|
2267
|
+
entry.name === query.key ? targetFileMaximumBytes : undefined,
|
|
2268
|
+
));
|
|
2269
|
+
}
|
|
2270
|
+
return Object.freeze({
|
|
2271
|
+
realm,
|
|
2272
|
+
realmIdentity: snapshot.identity,
|
|
2273
|
+
queries: Object.freeze(queries),
|
|
2274
|
+
});
|
|
2275
|
+
}
|
|
2276
|
+
|
|
2277
|
+
function admitRealmStorage(
|
|
2278
|
+
query: ReadProjectionQuery,
|
|
2279
|
+
prospectiveBytes: number,
|
|
2280
|
+
environment: Environment,
|
|
2281
|
+
targetFileMaximumBytes?: number,
|
|
2282
|
+
): void {
|
|
2283
|
+
for (let attempt = 0; attempt <= MAX_REALM_QUERY_DIRECTORIES; attempt += 1) {
|
|
2284
|
+
const usage = realmUsage(query, environment, targetFileMaximumBytes);
|
|
2285
|
+
const target = usage.queries.find((entry) => entry.name === query.key);
|
|
2286
|
+
if (target?.state === "invalid") {
|
|
2287
|
+
throw new Error("read projection target query has invalid quota state");
|
|
2288
|
+
}
|
|
2289
|
+
const prospectiveCount = usage.queries.length + (target === undefined ? 1 : 0);
|
|
2290
|
+
const storedBytes = usage.queries.reduce(
|
|
2291
|
+
(total, entry) => total + entry.bytes,
|
|
2292
|
+
0,
|
|
2293
|
+
);
|
|
2294
|
+
if (
|
|
2295
|
+
prospectiveCount <= MAX_REALM_QUERY_DIRECTORIES
|
|
2296
|
+
&& storedBytes + prospectiveBytes <= MAX_REALM_STORAGE_BYTES
|
|
2297
|
+
) return;
|
|
2298
|
+
|
|
2299
|
+
const candidate = usage.queries
|
|
2300
|
+
.filter((entry) =>
|
|
2301
|
+
entry.name !== query.key && entry.state !== "busy")
|
|
2302
|
+
.sort((left, right) => {
|
|
2303
|
+
if (left.state !== right.state) {
|
|
2304
|
+
return left.state === "invalid" ? -1 : 1;
|
|
2305
|
+
}
|
|
2306
|
+
return left.name.localeCompare(right.name);
|
|
2307
|
+
})[0];
|
|
2308
|
+
if (candidate === undefined) {
|
|
2309
|
+
throw new Error("read projection realm exceeds its bounded storage quota");
|
|
2310
|
+
}
|
|
2311
|
+
if (!removePrivateStateDirectoryTree(
|
|
2312
|
+
join(usage.realm, candidate.name),
|
|
2313
|
+
environment,
|
|
2314
|
+
candidate.identity,
|
|
2315
|
+
usage.realmIdentity,
|
|
2316
|
+
)) {
|
|
2317
|
+
throw new Error("read projection realm reclamation did not settle");
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
throw new Error("read projection realm reclamation exceeded its retry bound");
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
type PreparedProjectionRevision = {
|
|
2324
|
+
readonly revisionId: string;
|
|
2325
|
+
readonly chunks: readonly Readonly<{
|
|
2326
|
+
file: string;
|
|
2327
|
+
value: EncryptedProjectionChunkV1;
|
|
2328
|
+
}>[];
|
|
2329
|
+
readonly manifestFile: string;
|
|
2330
|
+
readonly manifest: ProjectionManifestV1;
|
|
2331
|
+
readonly head: ProjectionHeadV1;
|
|
2332
|
+
readonly serializedArtifactBytes: number;
|
|
2333
|
+
};
|
|
2334
|
+
|
|
2335
|
+
function serializedJsonBytes(value: unknown): number {
|
|
2336
|
+
return Buffer.byteLength(`${canonicalJson(value)}\n`, "utf8");
|
|
2337
|
+
}
|
|
2338
|
+
|
|
2339
|
+
function prepareRevision(
|
|
2340
|
+
query: ReadProjectionQuery,
|
|
2341
|
+
key: ProjectionKey,
|
|
2342
|
+
payload: ProjectionPayloadV1,
|
|
2343
|
+
plaintext: Buffer,
|
|
2344
|
+
chunkCount: number,
|
|
2345
|
+
): PreparedProjectionRevision {
|
|
2346
|
+
const revisionId = randomUUID().replaceAll("-", "");
|
|
2347
|
+
const chunks: Array<Readonly<{
|
|
2348
|
+
file: string;
|
|
2349
|
+
value: EncryptedProjectionChunkV1;
|
|
2350
|
+
}>> = [];
|
|
2351
|
+
const chunkHashes: string[] = [];
|
|
2352
|
+
let serializedArtifactBytes = 0;
|
|
2353
|
+
for (let index = 0; index < chunkCount; index += 1) {
|
|
2354
|
+
const file = `chunk--${revisionId}--${String(index).padStart(3, "0")}.json`;
|
|
2355
|
+
const value = encryptedChunk(
|
|
2356
|
+
plaintext.subarray(
|
|
2357
|
+
index * PLAINTEXT_CHUNK_BYTES,
|
|
2358
|
+
Math.min(plaintext.byteLength, (index + 1) * PLAINTEXT_CHUNK_BYTES),
|
|
2359
|
+
),
|
|
2360
|
+
query,
|
|
2361
|
+
key,
|
|
2362
|
+
revisionId,
|
|
2363
|
+
index,
|
|
2364
|
+
chunkCount,
|
|
2365
|
+
);
|
|
2366
|
+
const serialized = `${canonicalJson(value)}\n`;
|
|
2367
|
+
chunks.push(Object.freeze({ file, value }));
|
|
2368
|
+
chunkHashes.push(hashBytes(serialized));
|
|
2369
|
+
serializedArtifactBytes += Buffer.byteLength(serialized, "utf8");
|
|
2370
|
+
}
|
|
2371
|
+
const manifestBody = {
|
|
2372
|
+
schemaVersion: 1 as const,
|
|
2373
|
+
keyId: key.id,
|
|
2374
|
+
revisionId,
|
|
2375
|
+
plaintextBytes: plaintext.byteLength,
|
|
2376
|
+
chunkFiles: Object.freeze(chunks.map((chunk) => chunk.file)),
|
|
2377
|
+
chunkHashes: Object.freeze(chunkHashes),
|
|
2378
|
+
};
|
|
2379
|
+
const manifest: ProjectionManifestV1 = Object.freeze({
|
|
2380
|
+
...manifestBody,
|
|
2381
|
+
authentication: manifestAuthentication(key, query, manifestBody),
|
|
2382
|
+
});
|
|
2383
|
+
const manifestFile = `manifest--${revisionId}.json`;
|
|
2384
|
+
serializedArtifactBytes += serializedJsonBytes(manifest);
|
|
2385
|
+
const headBodyValue = {
|
|
2386
|
+
schemaVersion: 1 as const,
|
|
2387
|
+
keyId: key.id,
|
|
2388
|
+
revisionId,
|
|
2389
|
+
manifestFile,
|
|
2390
|
+
publication: headPublication(payload),
|
|
2391
|
+
};
|
|
2392
|
+
const head: ProjectionHeadV1 = Object.freeze({
|
|
2393
|
+
...headBodyValue,
|
|
2394
|
+
authentication: headAuthentication(key, query, headBodyValue),
|
|
2395
|
+
});
|
|
2396
|
+
serializedArtifactBytes += serializedJsonBytes(head);
|
|
2397
|
+
return Object.freeze({
|
|
2398
|
+
revisionId,
|
|
2399
|
+
chunks: Object.freeze(chunks),
|
|
2400
|
+
manifestFile,
|
|
2401
|
+
manifest,
|
|
2402
|
+
head,
|
|
2403
|
+
serializedArtifactBytes,
|
|
2404
|
+
});
|
|
2405
|
+
}
|
|
2406
|
+
|
|
2407
|
+
function writeRevision(
|
|
2408
|
+
query: ReadProjectionQuery,
|
|
2409
|
+
key: ProjectionKey,
|
|
2410
|
+
payload: ProjectionPayloadV1,
|
|
2411
|
+
current: LoadedProjection | null,
|
|
2412
|
+
environment: Environment,
|
|
2413
|
+
mode: "normal" | "preserve-for-repair" = "normal",
|
|
2414
|
+
): { readonly head: ProjectionHeadV1; readonly chunkCount: number } {
|
|
2415
|
+
const directory = queryDirectory(query, environment);
|
|
2416
|
+
const plaintext = Buffer.from(canonicalJson(payload), "utf8");
|
|
2417
|
+
if (plaintext.byteLength < 1 || plaintext.byteLength > MAX_PLAINTEXT_BYTES) {
|
|
2418
|
+
throw new Error("read projection payload exceeds its byte bound");
|
|
2419
|
+
}
|
|
2420
|
+
const chunkCount = Math.ceil(plaintext.byteLength / PLAINTEXT_CHUNK_BYTES);
|
|
2421
|
+
if (chunkCount < 1 || chunkCount > MAX_CHUNKS) {
|
|
2422
|
+
throw new Error("read projection payload exceeds its chunk bound");
|
|
2423
|
+
}
|
|
2424
|
+
const revision = prepareRevision(
|
|
2425
|
+
query,
|
|
2426
|
+
key,
|
|
2427
|
+
payload,
|
|
2428
|
+
plaintext,
|
|
2429
|
+
chunkCount,
|
|
2430
|
+
);
|
|
2431
|
+
if (mode === "normal") {
|
|
2432
|
+
const oldChunkCount = current === null
|
|
2433
|
+
? 0
|
|
2434
|
+
: currentChunkCount(current, query, key, environment);
|
|
2435
|
+
reclaimNonHeadRevisionFiles(
|
|
2436
|
+
query,
|
|
2437
|
+
current,
|
|
2438
|
+
oldChunkCount,
|
|
2439
|
+
chunkCount + 1,
|
|
2440
|
+
environment,
|
|
2441
|
+
);
|
|
2442
|
+
} else {
|
|
2443
|
+
const directoryIdentity = ensurePrivateStateDirectory(directory, environment);
|
|
2444
|
+
const entries = listPrivateStateDirectory(
|
|
2445
|
+
directory,
|
|
2446
|
+
environment,
|
|
2447
|
+
directoryIdentity,
|
|
2448
|
+
{ recoverOrphanedMutationClaims: true },
|
|
2449
|
+
);
|
|
2450
|
+
if (entries.length + chunkCount + 1 > MAX_QUERY_DIRECTORY_ENTRIES) {
|
|
2451
|
+
throw new Error(
|
|
2452
|
+
"read projection repair exceeds its prospective entry bound",
|
|
2453
|
+
);
|
|
2454
|
+
}
|
|
2455
|
+
}
|
|
2456
|
+
admitRealmStorage(
|
|
2457
|
+
query,
|
|
2458
|
+
revision.serializedArtifactBytes,
|
|
2459
|
+
environment,
|
|
2460
|
+
mode === "preserve-for-repair"
|
|
2461
|
+
? MAX_ATTRIBUTABLE_CORRUPT_FILE_BYTES
|
|
2462
|
+
: undefined,
|
|
2463
|
+
);
|
|
2464
|
+
try {
|
|
2465
|
+
for (const chunk of revision.chunks) {
|
|
2466
|
+
if (!createPrivateJsonIfAbsent(
|
|
2467
|
+
join(directory, chunk.file),
|
|
2468
|
+
chunk.value,
|
|
2469
|
+
{ environment },
|
|
2470
|
+
).created) {
|
|
2471
|
+
throw new Error("read projection revision collided with existing chunk state");
|
|
2472
|
+
}
|
|
2473
|
+
}
|
|
2474
|
+
if (!createPrivateJsonIfAbsent(
|
|
2475
|
+
join(directory, revision.manifestFile),
|
|
2476
|
+
revision.manifest,
|
|
2477
|
+
{ environment },
|
|
2478
|
+
).created) {
|
|
2479
|
+
throw new Error("read projection revision collided with existing manifest state");
|
|
2480
|
+
}
|
|
2481
|
+
return Object.freeze({
|
|
2482
|
+
head: revision.head,
|
|
2483
|
+
chunkCount,
|
|
2484
|
+
});
|
|
2485
|
+
} catch (error) {
|
|
2486
|
+
removeRevision(query, revision.revisionId, chunkCount, environment);
|
|
2487
|
+
throw error;
|
|
2488
|
+
}
|
|
2489
|
+
}
|
|
2490
|
+
|
|
2491
|
+
function publication(
|
|
2492
|
+
query: ReadProjectionQuery,
|
|
2493
|
+
payload: Pick<
|
|
2494
|
+
ProjectionPayloadV1,
|
|
2495
|
+
"dataRevision" | "validatedAt" | "dataChangedAt"
|
|
2496
|
+
>,
|
|
2497
|
+
disposition: ReadProjectionPublication["disposition"],
|
|
2498
|
+
currentDataRevision?: string,
|
|
2499
|
+
): ReadProjectionPublication {
|
|
2500
|
+
return Object.freeze({
|
|
2501
|
+
key: query.key,
|
|
2502
|
+
dataRevision: payload.dataRevision,
|
|
2503
|
+
validatedAt: payload.validatedAt,
|
|
2504
|
+
dataChangedAt: payload.dataChangedAt,
|
|
2505
|
+
disposition,
|
|
2506
|
+
...(currentDataRevision === undefined ? {} : { currentDataRevision }),
|
|
2507
|
+
});
|
|
2508
|
+
}
|
|
2509
|
+
|
|
2510
|
+
export function publishReadProjection(
|
|
2511
|
+
queryValue: ReadProjectionQuery,
|
|
2512
|
+
outputValue: unknown,
|
|
2513
|
+
options: {
|
|
2514
|
+
readonly environment?: Environment;
|
|
2515
|
+
readonly runId: string;
|
|
2516
|
+
readonly startedAt: string;
|
|
2517
|
+
readonly finishedAt: string;
|
|
2518
|
+
},
|
|
2519
|
+
): ReadProjectionPublication {
|
|
2520
|
+
const environment = options.environment ?? process.env;
|
|
2521
|
+
const authId = parseQueryValue(queryValue).identity.auth.id;
|
|
2522
|
+
return withReadProjectionAuthAdmission(authId, environment, () => {
|
|
2523
|
+
const { query, key } = validateQuery(queryValue, environment);
|
|
2524
|
+
const output = boundedJson(outputValue, "read projection output");
|
|
2525
|
+
const startedAt = timestamp(options.startedAt, "read projection start time");
|
|
2526
|
+
const finishedAt = timestamp(options.finishedAt, "read projection finish time");
|
|
2527
|
+
if (startedAt > finishedAt) {
|
|
2528
|
+
throw new Error("read projection finished before it started");
|
|
2529
|
+
}
|
|
2530
|
+
const normalizedRunId = runId(options.runId);
|
|
2531
|
+
const nextRevision = dataRevision(key, output);
|
|
2532
|
+
|
|
2533
|
+
for (let attempt = 0; attempt < 8; attempt += 1) {
|
|
2534
|
+
const current = loadProjection(query, key, environment);
|
|
2535
|
+
const nextOrder = publicationOrder({
|
|
2536
|
+
startedAt,
|
|
2537
|
+
finishedAt,
|
|
2538
|
+
runId: normalizedRunId,
|
|
2539
|
+
});
|
|
2540
|
+
if (current !== null && nextOrder <= publicationOrder(current.payload)) {
|
|
2541
|
+
return publication(
|
|
2542
|
+
query,
|
|
2543
|
+
current.payload,
|
|
2544
|
+
"superseded",
|
|
2545
|
+
current.payload.dataRevision,
|
|
2546
|
+
);
|
|
2547
|
+
}
|
|
2548
|
+
const unchanged = current?.payload.dataRevision === nextRevision;
|
|
2549
|
+
const createdAt = current === null
|
|
2550
|
+
? finishedAt
|
|
2551
|
+
: current.payload.createdAt < finishedAt
|
|
2552
|
+
? current.payload.createdAt
|
|
2553
|
+
: finishedAt;
|
|
2554
|
+
const dataChangedAt = unchanged
|
|
2555
|
+
? current.payload.dataChangedAt < finishedAt
|
|
2556
|
+
? current.payload.dataChangedAt
|
|
2557
|
+
: finishedAt
|
|
2558
|
+
: finishedAt;
|
|
2559
|
+
const payload: ProjectionPayloadV1 = Object.freeze({
|
|
2560
|
+
schemaVersion: 1,
|
|
2561
|
+
query: query.identity,
|
|
2562
|
+
output,
|
|
2563
|
+
dataRevision: nextRevision,
|
|
2564
|
+
createdAt,
|
|
2565
|
+
dataChangedAt,
|
|
2566
|
+
validatedAt: finishedAt,
|
|
2567
|
+
runId: normalizedRunId,
|
|
2568
|
+
startedAt,
|
|
2569
|
+
finishedAt,
|
|
2570
|
+
});
|
|
2571
|
+
const revision = writeRevision(query, key, payload, current, environment);
|
|
2572
|
+
const promoted = current === null
|
|
2573
|
+
? createPrivateJsonIfAbsent(
|
|
2574
|
+
headPath(query, environment),
|
|
2575
|
+
revision.head,
|
|
2576
|
+
{ environment },
|
|
2577
|
+
).created
|
|
2578
|
+
: writePrivateJsonIfUnchanged(
|
|
2579
|
+
headPath(query, environment),
|
|
2580
|
+
revision.head,
|
|
2581
|
+
{ expectedCurrentContentSha256: current.headContentSha256 },
|
|
2582
|
+
);
|
|
2583
|
+
if (!promoted) {
|
|
2584
|
+
removeRevision(
|
|
2585
|
+
query,
|
|
2586
|
+
revision.head.revisionId,
|
|
2587
|
+
revision.chunkCount,
|
|
2588
|
+
environment,
|
|
2589
|
+
);
|
|
2590
|
+
continue;
|
|
2591
|
+
}
|
|
2592
|
+
if (current !== null) {
|
|
2593
|
+
const oldCount = currentChunkCount(current, query, key, environment);
|
|
2594
|
+
removeRevision(query, current.head.revisionId, oldCount, environment);
|
|
2595
|
+
}
|
|
2596
|
+
return publication(
|
|
2597
|
+
query,
|
|
2598
|
+
payload,
|
|
2599
|
+
current === null ? "created" : unchanged ? "unchanged" : "changed",
|
|
2600
|
+
);
|
|
2601
|
+
}
|
|
2602
|
+
throw new Error(
|
|
2603
|
+
"read projection publication could not settle after concurrent updates",
|
|
2604
|
+
);
|
|
2605
|
+
});
|
|
2606
|
+
}
|
|
2607
|
+
|
|
2608
|
+
function omniCurrent(
|
|
2609
|
+
query: OmniProjectionQuery,
|
|
2610
|
+
payload: ProjectionPayloadV1,
|
|
2611
|
+
storageRevisionId: string,
|
|
2612
|
+
): OmniProjectionCurrent {
|
|
2613
|
+
return Object.freeze({
|
|
2614
|
+
key: query.key,
|
|
2615
|
+
storageRevisionId,
|
|
2616
|
+
output: immutableBoundedJson(
|
|
2617
|
+
payload.output,
|
|
2618
|
+
"omni projection current output",
|
|
2619
|
+
),
|
|
2620
|
+
dataRevision: payload.dataRevision,
|
|
2621
|
+
createdAt: payload.createdAt,
|
|
2622
|
+
dataChangedAt: payload.dataChangedAt,
|
|
2623
|
+
validatedAt: payload.validatedAt,
|
|
2624
|
+
runId: payload.runId,
|
|
2625
|
+
startedAt: payload.startedAt,
|
|
2626
|
+
finishedAt: payload.finishedAt,
|
|
2627
|
+
});
|
|
2628
|
+
}
|
|
2629
|
+
|
|
2630
|
+
function omniPublication(
|
|
2631
|
+
query: OmniProjectionQuery,
|
|
2632
|
+
payload: ProjectionPayloadV1,
|
|
2633
|
+
storageRevisionId: string,
|
|
2634
|
+
disposition: OmniProjectionPublication["disposition"],
|
|
2635
|
+
): OmniProjectionPublication {
|
|
2636
|
+
return Object.freeze({
|
|
2637
|
+
...publication(query, payload, disposition),
|
|
2638
|
+
storageRevisionId,
|
|
2639
|
+
runId: payload.runId,
|
|
2640
|
+
startedAt: payload.startedAt,
|
|
2641
|
+
finishedAt: payload.finishedAt,
|
|
2642
|
+
});
|
|
2643
|
+
}
|
|
2644
|
+
|
|
2645
|
+
function omniMutationTime(
|
|
2646
|
+
now: Date,
|
|
2647
|
+
current: LoadedProjection | null,
|
|
2648
|
+
): string {
|
|
2649
|
+
if (!Number.isFinite(now.getTime())) {
|
|
2650
|
+
throw new Error("omni projection mutation time is invalid");
|
|
2651
|
+
}
|
|
2652
|
+
let milliseconds = now.getTime();
|
|
2653
|
+
if (current !== null) {
|
|
2654
|
+
milliseconds = Math.max(
|
|
2655
|
+
milliseconds,
|
|
2656
|
+
new Date(current.payload.validatedAt).getTime() + 1,
|
|
2657
|
+
);
|
|
2658
|
+
}
|
|
2659
|
+
try {
|
|
2660
|
+
return timestamp(
|
|
2661
|
+
new Date(milliseconds).toISOString(),
|
|
2662
|
+
"omni projection mutation time",
|
|
2663
|
+
);
|
|
2664
|
+
} catch (error) {
|
|
2665
|
+
throw new Error("omni projection mutation time cannot advance", { cause: error });
|
|
2666
|
+
}
|
|
2667
|
+
}
|
|
2668
|
+
|
|
2669
|
+
/**
|
|
2670
|
+
* Serialize one pure synchronous aggregate transition under the auth admission.
|
|
2671
|
+
* The reducer sees an immutable JSON snapshot. A candidate revision remains
|
|
2672
|
+
* unreachable until its head wins the compare-and-swap publication.
|
|
2673
|
+
*/
|
|
2674
|
+
export function reduceOmniProjection(
|
|
2675
|
+
queryValue: OmniProjectionQuery,
|
|
2676
|
+
reducer: (currentOutput: unknown) => unknown,
|
|
2677
|
+
options: {
|
|
2678
|
+
readonly environment?: Environment;
|
|
2679
|
+
readonly now?: Date;
|
|
2680
|
+
/** Runs inside the auth admission before any reducer state is observed. */
|
|
2681
|
+
readonly assertCurrent?: () => void;
|
|
2682
|
+
/** Exact head whose bytes the reducer interpreted. */
|
|
2683
|
+
readonly exactHead?: ReadProjectionExactHeadFence;
|
|
2684
|
+
} = {},
|
|
2685
|
+
): OmniProjectionReductionResult {
|
|
2686
|
+
if (typeof reducer !== "function") {
|
|
2687
|
+
throw new Error("omni projection reducer must be a function");
|
|
2688
|
+
}
|
|
2689
|
+
const environment = options.environment ?? process.env;
|
|
2690
|
+
const parsedOmniQuery = parseOmniQueryValue(queryValue);
|
|
2691
|
+
const authId = parsedOmniQuery.identity.auth.id;
|
|
2692
|
+
return withSettledReadProjectionAuthAdmission(
|
|
2693
|
+
authId,
|
|
2694
|
+
environment,
|
|
2695
|
+
() => {
|
|
2696
|
+
if (
|
|
2697
|
+
options.assertCurrent !== undefined
|
|
2698
|
+
&& typeof options.assertCurrent !== "function"
|
|
2699
|
+
) {
|
|
2700
|
+
throw new Error("omni projection current-authority guard is malformed");
|
|
2701
|
+
}
|
|
2702
|
+
options.assertCurrent?.();
|
|
2703
|
+
if (options.exactHead !== undefined) {
|
|
2704
|
+
const fence = record(options.exactHead, "omni projection exact-head fence");
|
|
2705
|
+
exactKeys(
|
|
2706
|
+
fence,
|
|
2707
|
+
["query", "storageRevisionId", "dataRevision", "runId"],
|
|
2708
|
+
"omni projection exact-head fence",
|
|
2709
|
+
);
|
|
2710
|
+
const { query: exactQuery, key: exactKey } = validateQuery(
|
|
2711
|
+
fence.query as ReadProjectionQuery,
|
|
2712
|
+
environment,
|
|
2713
|
+
);
|
|
2714
|
+
if (
|
|
2715
|
+
exactQuery.identity.auth.id !== parsedOmniQuery.identity.auth.id
|
|
2716
|
+
|| exactQuery.identity.auth.hash !== parsedOmniQuery.identity.auth.hash
|
|
2717
|
+
) {
|
|
2718
|
+
throw new Error(
|
|
2719
|
+
"omni projection exact-head fence belongs to another auth lifetime",
|
|
2720
|
+
);
|
|
2721
|
+
}
|
|
2722
|
+
const expectedStorageRevisionId = safeString(
|
|
2723
|
+
fence.storageRevisionId,
|
|
2724
|
+
"omni projection exact storage revision ID",
|
|
2725
|
+
64,
|
|
2726
|
+
);
|
|
2727
|
+
if (!/^[a-f0-9]{32}$/u.test(expectedStorageRevisionId)) {
|
|
2728
|
+
throw new Error("omni projection exact storage revision ID is malformed");
|
|
2729
|
+
}
|
|
2730
|
+
const expectedDataRevision = hexDigest(
|
|
2731
|
+
fence.dataRevision,
|
|
2732
|
+
"omni projection exact data revision",
|
|
2733
|
+
);
|
|
2734
|
+
const expectedRunId = runId(fence.runId);
|
|
2735
|
+
const exact = loadProjection(exactQuery, exactKey, environment);
|
|
2736
|
+
if (
|
|
2737
|
+
exact === null
|
|
2738
|
+
|| exact.head.revisionId !== expectedStorageRevisionId
|
|
2739
|
+
|| exact.payload.dataRevision !== expectedDataRevision
|
|
2740
|
+
|| exact.payload.runId !== expectedRunId
|
|
2741
|
+
) {
|
|
2742
|
+
throw new Error(
|
|
2743
|
+
"exact projection changed before normalized publication admission",
|
|
2744
|
+
);
|
|
2745
|
+
}
|
|
2746
|
+
}
|
|
2747
|
+
const { query, key } = validateOmniQuery(queryValue, environment);
|
|
2748
|
+
const current = loadProjection(query, key, environment);
|
|
2749
|
+
const mutationTime = omniMutationTime(options.now ?? new Date(), current);
|
|
2750
|
+
const currentOutput = current === null
|
|
2751
|
+
? null
|
|
2752
|
+
: immutableBoundedJson(
|
|
2753
|
+
current.payload.output,
|
|
2754
|
+
"omni projection reducer input",
|
|
2755
|
+
);
|
|
2756
|
+
const reduced = reducer(currentOutput);
|
|
2757
|
+
if (hasThenableProtocol(reduced)) {
|
|
2758
|
+
throw new Error(
|
|
2759
|
+
"omni projection reducers must be synchronous and must not return promises or thenables",
|
|
2760
|
+
);
|
|
2761
|
+
}
|
|
2762
|
+
const output = immutableBoundedJson(reduced, "omni projection output");
|
|
2763
|
+
const nextRevision = dataRevision(key, output, "omni-v1");
|
|
2764
|
+
if (current?.payload.dataRevision === nextRevision) {
|
|
2765
|
+
return Object.freeze({
|
|
2766
|
+
publication: omniPublication(
|
|
2767
|
+
query,
|
|
2768
|
+
current.payload,
|
|
2769
|
+
current.head.revisionId,
|
|
2770
|
+
"unchanged",
|
|
2771
|
+
),
|
|
2772
|
+
current: omniCurrent(
|
|
2773
|
+
query,
|
|
2774
|
+
current.payload,
|
|
2775
|
+
current.head.revisionId,
|
|
2776
|
+
),
|
|
2777
|
+
});
|
|
2778
|
+
}
|
|
2779
|
+
|
|
2780
|
+
const normalizedRunId = randomUUID();
|
|
2781
|
+
const payload: ProjectionPayloadV1 = Object.freeze({
|
|
2782
|
+
schemaVersion: 1,
|
|
2783
|
+
query: query.identity,
|
|
2784
|
+
output,
|
|
2785
|
+
dataRevision: nextRevision,
|
|
2786
|
+
createdAt: current?.payload.createdAt ?? mutationTime,
|
|
2787
|
+
dataChangedAt: mutationTime,
|
|
2788
|
+
validatedAt: mutationTime,
|
|
2789
|
+
runId: normalizedRunId,
|
|
2790
|
+
startedAt: mutationTime,
|
|
2791
|
+
finishedAt: mutationTime,
|
|
2792
|
+
});
|
|
2793
|
+
const oldChunkCount = current === null
|
|
2794
|
+
? 0
|
|
2795
|
+
: currentChunkCount(current, query, key, environment);
|
|
2796
|
+
const revision = writeRevision(query, key, payload, current, environment);
|
|
2797
|
+
const promoted = current === null
|
|
2798
|
+
? createPrivateJsonIfAbsent(
|
|
2799
|
+
headPath(query, environment),
|
|
2800
|
+
revision.head,
|
|
2801
|
+
{ environment },
|
|
2802
|
+
).created
|
|
2803
|
+
: writePrivateJsonIfUnchanged(
|
|
2804
|
+
headPath(query, environment),
|
|
2805
|
+
revision.head,
|
|
2806
|
+
{ expectedCurrentContentSha256: current.headContentSha256 },
|
|
2807
|
+
);
|
|
2808
|
+
if (!promoted) {
|
|
2809
|
+
removeRevision(
|
|
2810
|
+
query,
|
|
2811
|
+
revision.head.revisionId,
|
|
2812
|
+
revision.chunkCount,
|
|
2813
|
+
environment,
|
|
2814
|
+
);
|
|
2815
|
+
throw new Error(
|
|
2816
|
+
"omni projection publication lost its authenticated compare-and-swap",
|
|
2817
|
+
);
|
|
2818
|
+
}
|
|
2819
|
+
if (current !== null) {
|
|
2820
|
+
removeRevision(
|
|
2821
|
+
query,
|
|
2822
|
+
current.head.revisionId,
|
|
2823
|
+
oldChunkCount,
|
|
2824
|
+
environment,
|
|
2825
|
+
);
|
|
2826
|
+
}
|
|
2827
|
+
return Object.freeze({
|
|
2828
|
+
publication: omniPublication(
|
|
2829
|
+
query,
|
|
2830
|
+
payload,
|
|
2831
|
+
revision.head.revisionId,
|
|
2832
|
+
current === null ? "created" : "changed",
|
|
2833
|
+
),
|
|
2834
|
+
current: omniCurrent(query, payload, revision.head.revisionId),
|
|
2835
|
+
});
|
|
2836
|
+
},
|
|
2837
|
+
{ maximumWaitMs: READ_PROJECTION_TRANSITION_SETTLEMENT_WAIT_MS },
|
|
2838
|
+
);
|
|
2839
|
+
}
|
|
2840
|
+
|
|
2841
|
+
function rawHeadSnapshot(
|
|
2842
|
+
query: ReadProjectionQuery,
|
|
2843
|
+
environment: Environment,
|
|
2844
|
+
): { readonly contentSha256: string } | null {
|
|
2845
|
+
const content = readPrivateStateFileBytesIfPresent(
|
|
2846
|
+
headPath(query, environment),
|
|
2847
|
+
MAX_ATTRIBUTABLE_CORRUPT_FILE_BYTES,
|
|
2848
|
+
"read projection repair head",
|
|
2849
|
+
environment,
|
|
2850
|
+
);
|
|
2851
|
+
return content === null
|
|
2852
|
+
? null
|
|
2853
|
+
: Object.freeze({ contentSha256: hashBytes(content) });
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
function reclaimAfterDurableRepair(
|
|
2857
|
+
query: ReadProjectionQuery,
|
|
2858
|
+
revisionId: string,
|
|
2859
|
+
chunkCount: number,
|
|
2860
|
+
environment: Environment,
|
|
2861
|
+
): void {
|
|
2862
|
+
const directory = queryDirectory(query, environment);
|
|
2863
|
+
const directoryIdentity = ensurePrivateStateDirectory(directory, environment);
|
|
2864
|
+
const expected = new Set([HEAD_FILE, ...revisionFiles(revisionId, chunkCount)]);
|
|
2865
|
+
const entries = listPrivateStateDirectory(
|
|
2866
|
+
directory,
|
|
2867
|
+
environment,
|
|
2868
|
+
directoryIdentity,
|
|
2869
|
+
{ recoverOrphanedMutationClaims: true },
|
|
2870
|
+
);
|
|
2871
|
+
if (
|
|
2872
|
+
entries.some((entry) =>
|
|
2873
|
+
stateHelperArtifactNamePatterns.some((pattern) =>
|
|
2874
|
+
pattern.test(entry.name)))
|
|
2875
|
+
) {
|
|
2876
|
+
throw new Error(
|
|
2877
|
+
"read projection repaired durably but an in-flight state-helper artifact prevents reclamation",
|
|
2878
|
+
);
|
|
2879
|
+
}
|
|
2880
|
+
for (const entry of entries) {
|
|
2881
|
+
if (expected.has(entry.name)) continue;
|
|
2882
|
+
if (entry.kind === "file") {
|
|
2883
|
+
removePrivateStateFile(join(directory, entry.name), environment);
|
|
2884
|
+
continue;
|
|
2885
|
+
}
|
|
2886
|
+
if (entry.kind === "directory" && entry.identity !== undefined) {
|
|
2887
|
+
removePrivateStateDirectoryTree(
|
|
2888
|
+
join(directory, entry.name),
|
|
2889
|
+
environment,
|
|
2890
|
+
entry.identity,
|
|
2891
|
+
directoryIdentity,
|
|
2892
|
+
);
|
|
2893
|
+
continue;
|
|
2894
|
+
}
|
|
2895
|
+
throw new Error(
|
|
2896
|
+
"read projection repaired durably but inert corrupt state could not be reclaimed",
|
|
2897
|
+
);
|
|
2898
|
+
}
|
|
2899
|
+
const settled = listPrivateStateDirectory(
|
|
2900
|
+
directory,
|
|
2901
|
+
environment,
|
|
2902
|
+
directoryIdentity,
|
|
2903
|
+
{ recoverOrphanedMutationClaims: true },
|
|
2904
|
+
);
|
|
2905
|
+
if (
|
|
2906
|
+
settled.length !== expected.size
|
|
2907
|
+
|| settled.some((entry) =>
|
|
2908
|
+
entry.kind !== "file" || !expected.has(entry.name))
|
|
2909
|
+
) {
|
|
2910
|
+
throw new Error(
|
|
2911
|
+
"read projection repaired durably but old revision reclamation did not settle",
|
|
2912
|
+
);
|
|
2913
|
+
}
|
|
2914
|
+
}
|
|
2915
|
+
|
|
2916
|
+
/**
|
|
2917
|
+
* Replace a corrupt exact-query head only after a complete new immutable
|
|
2918
|
+
* revision is durable. If the cache recovered or changed meanwhile, ordinary
|
|
2919
|
+
* publication ordering remains authoritative.
|
|
2920
|
+
*/
|
|
2921
|
+
export function repairReadProjection(
|
|
2922
|
+
queryValue: ReadProjectionQuery,
|
|
2923
|
+
outputValue: unknown,
|
|
2924
|
+
options: {
|
|
2925
|
+
readonly environment?: Environment;
|
|
2926
|
+
readonly runId: string;
|
|
2927
|
+
readonly startedAt: string;
|
|
2928
|
+
readonly finishedAt: string;
|
|
2929
|
+
readonly corruption: ReadProjectionCorruptionError;
|
|
2930
|
+
readonly observedBeforeLive: boolean;
|
|
2931
|
+
},
|
|
2932
|
+
): ReadProjectionPublication {
|
|
2933
|
+
const environment = options.environment ?? process.env;
|
|
2934
|
+
const authId = parseQueryValue(queryValue).identity.auth.id;
|
|
2935
|
+
return withReadProjectionAuthAdmission(authId, environment, () => {
|
|
2936
|
+
const { query, key } = validateQuery(queryValue, environment);
|
|
2937
|
+
const output = boundedJson(outputValue, "read projection output");
|
|
2938
|
+
const startedAt = timestamp(options.startedAt, "read projection start time");
|
|
2939
|
+
const finishedAt = timestamp(options.finishedAt, "read projection finish time");
|
|
2940
|
+
if (startedAt > finishedAt) {
|
|
2941
|
+
throw new Error("read projection finished before it started");
|
|
2942
|
+
}
|
|
2943
|
+
const normalizedRunId = runId(options.runId);
|
|
2944
|
+
if (typeof options.observedBeforeLive !== "boolean") {
|
|
2945
|
+
throw new Error("read projection repair observation timing is malformed");
|
|
2946
|
+
}
|
|
2947
|
+
const observed = corruptionEvidence.get(options.corruption);
|
|
2948
|
+
if (
|
|
2949
|
+
observed === undefined
|
|
2950
|
+
|| observed.storageClass !== "exact-v1"
|
|
2951
|
+
|| observed.queryKey !== query.key
|
|
2952
|
+
|| observed.realmKey !== query.realmKey
|
|
2953
|
+
) {
|
|
2954
|
+
throw new Error(
|
|
2955
|
+
"read projection repair requires exact observed corruption evidence",
|
|
2956
|
+
);
|
|
2957
|
+
}
|
|
2958
|
+
const nextOrder = publicationOrder({
|
|
2959
|
+
startedAt,
|
|
2960
|
+
finishedAt,
|
|
2961
|
+
runId: normalizedRunId,
|
|
2962
|
+
});
|
|
2963
|
+
const nextRevision = dataRevision(key, output);
|
|
2964
|
+
|
|
2965
|
+
const publishAgainstCurrentState = (): ReadProjectionPublication => {
|
|
2966
|
+
try {
|
|
2967
|
+
return publishReadProjection(query, output, {
|
|
2968
|
+
environment,
|
|
2969
|
+
runId: normalizedRunId,
|
|
2970
|
+
startedAt,
|
|
2971
|
+
finishedAt,
|
|
2972
|
+
});
|
|
2973
|
+
} catch (error) {
|
|
2974
|
+
if (!isReadProjectionCorruptionError(error)) throw error;
|
|
2975
|
+
throw new Error(
|
|
2976
|
+
"read projection corrupt head changed after the repair observation",
|
|
2977
|
+
{ cause: error },
|
|
2978
|
+
);
|
|
2979
|
+
}
|
|
2980
|
+
};
|
|
2981
|
+
|
|
2982
|
+
for (let attempt = 0; attempt < 8; attempt += 1) {
|
|
2983
|
+
const currentRaw = rawHeadSnapshot(query, environment);
|
|
2984
|
+
if (
|
|
2985
|
+
currentRaw === null
|
|
2986
|
+
|| currentRaw.contentSha256 !== observed.headContentSha256
|
|
2987
|
+
) {
|
|
2988
|
+
return publishAgainstCurrentState();
|
|
2989
|
+
}
|
|
2990
|
+
|
|
2991
|
+
try {
|
|
2992
|
+
const recovered = loadProjection(query, key, environment);
|
|
2993
|
+
if (recovered === null) continue;
|
|
2994
|
+
return publishAgainstCurrentState();
|
|
2995
|
+
} catch (error) {
|
|
2996
|
+
if (!isReadProjectionCorruptionError(error)) throw error;
|
|
2997
|
+
if (error.headContentSha256 !== observed.headContentSha256) continue;
|
|
2998
|
+
}
|
|
2999
|
+
|
|
3000
|
+
if (observed.headPublication === null && !options.observedBeforeLive) {
|
|
3001
|
+
throw new Error(
|
|
3002
|
+
"unorderable read projection corruption was not observed before the live read",
|
|
3003
|
+
);
|
|
3004
|
+
}
|
|
3005
|
+
if (
|
|
3006
|
+
observed.headPublication !== null
|
|
3007
|
+
&& nextOrder <= publicationOrder(observed.headPublication)
|
|
3008
|
+
) {
|
|
3009
|
+
return publication(
|
|
3010
|
+
query,
|
|
3011
|
+
observed.headPublication,
|
|
3012
|
+
"superseded",
|
|
3013
|
+
observed.headPublication.dataRevision,
|
|
3014
|
+
);
|
|
3015
|
+
}
|
|
3016
|
+
const unchanged = observed.headPublication?.dataRevision === nextRevision;
|
|
3017
|
+
const createdAt = observed.headPublication === null
|
|
3018
|
+
? finishedAt
|
|
3019
|
+
: observed.headPublication.createdAt < finishedAt
|
|
3020
|
+
? observed.headPublication.createdAt
|
|
3021
|
+
: finishedAt;
|
|
3022
|
+
const dataChangedAt = unchanged && observed.headPublication !== null
|
|
3023
|
+
? observed.headPublication.dataChangedAt < finishedAt
|
|
3024
|
+
? observed.headPublication.dataChangedAt
|
|
3025
|
+
: finishedAt
|
|
3026
|
+
: finishedAt;
|
|
3027
|
+
const payload: ProjectionPayloadV1 = Object.freeze({
|
|
3028
|
+
schemaVersion: 1,
|
|
3029
|
+
query: query.identity,
|
|
3030
|
+
output,
|
|
3031
|
+
dataRevision: nextRevision,
|
|
3032
|
+
createdAt,
|
|
3033
|
+
dataChangedAt,
|
|
3034
|
+
validatedAt: finishedAt,
|
|
3035
|
+
runId: normalizedRunId,
|
|
3036
|
+
startedAt,
|
|
3037
|
+
finishedAt,
|
|
3038
|
+
});
|
|
3039
|
+
const revision = writeRevision(
|
|
3040
|
+
query,
|
|
3041
|
+
key,
|
|
3042
|
+
payload,
|
|
3043
|
+
null,
|
|
3044
|
+
environment,
|
|
3045
|
+
"preserve-for-repair",
|
|
3046
|
+
);
|
|
3047
|
+
const promoted = writePrivateJsonIfUnchanged(
|
|
3048
|
+
headPath(query, environment),
|
|
3049
|
+
revision.head,
|
|
3050
|
+
{
|
|
3051
|
+
expectedCurrentContentSha256: observed.headContentSha256,
|
|
3052
|
+
maximumExpectedCurrentBytes: MAX_ATTRIBUTABLE_CORRUPT_FILE_BYTES,
|
|
3053
|
+
},
|
|
3054
|
+
);
|
|
3055
|
+
if (!promoted) {
|
|
3056
|
+
removeRevision(
|
|
3057
|
+
query,
|
|
3058
|
+
revision.head.revisionId,
|
|
3059
|
+
revision.chunkCount,
|
|
3060
|
+
environment,
|
|
3061
|
+
);
|
|
3062
|
+
continue;
|
|
3063
|
+
}
|
|
3064
|
+
const published = publication(
|
|
3065
|
+
query,
|
|
3066
|
+
payload,
|
|
3067
|
+
observed.headPublication === null
|
|
3068
|
+
? "changed"
|
|
3069
|
+
: unchanged
|
|
3070
|
+
? "unchanged"
|
|
3071
|
+
: "changed",
|
|
3072
|
+
);
|
|
3073
|
+
try {
|
|
3074
|
+
reclaimAfterDurableRepair(
|
|
3075
|
+
query,
|
|
3076
|
+
revision.head.revisionId,
|
|
3077
|
+
revision.chunkCount,
|
|
3078
|
+
environment,
|
|
3079
|
+
);
|
|
3080
|
+
} catch (error) {
|
|
3081
|
+
throw new ReadProjectionDurableRepairError(
|
|
3082
|
+
query.key,
|
|
3083
|
+
published,
|
|
3084
|
+
error,
|
|
3085
|
+
);
|
|
3086
|
+
}
|
|
3087
|
+
return published;
|
|
3088
|
+
}
|
|
3089
|
+
throw new Error(
|
|
3090
|
+
"read projection repair could not settle after concurrent updates",
|
|
3091
|
+
);
|
|
3092
|
+
});
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
export function removeReadProjectionsForAuth(
|
|
3096
|
+
authIdValue: string,
|
|
3097
|
+
environment: Environment = process.env,
|
|
3098
|
+
): boolean {
|
|
3099
|
+
return withReadProjectionAuthAdmission(authIdValue, environment, () => {
|
|
3100
|
+
const key = projectionKey(environment, false);
|
|
3101
|
+
if (key === null) return false;
|
|
3102
|
+
const exactRealmKey = hmac(
|
|
3103
|
+
key.value,
|
|
3104
|
+
"wrench-read-projection-realm-v1",
|
|
3105
|
+
authIdValue,
|
|
3106
|
+
);
|
|
3107
|
+
const omniRealmKey = hmac(
|
|
3108
|
+
key.value,
|
|
3109
|
+
"wrench-omni-projection-realm-v1",
|
|
3110
|
+
authIdValue,
|
|
3111
|
+
);
|
|
3112
|
+
let removed = false;
|
|
3113
|
+
const failures: unknown[] = [];
|
|
3114
|
+
for (const [storageClass, realmKey] of [
|
|
3115
|
+
["exact-v1", exactRealmKey],
|
|
3116
|
+
["omni-v1", omniRealmKey],
|
|
3117
|
+
] as const) {
|
|
3118
|
+
try {
|
|
3119
|
+
removed = removePrivateStateDirectoryTree(
|
|
3120
|
+
realmDirectory(realmKey, environment, storageClass),
|
|
3121
|
+
environment,
|
|
3122
|
+
) || removed;
|
|
3123
|
+
} catch (error) {
|
|
3124
|
+
failures.push(error);
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
3127
|
+
if (failures.length === 1) throw failures[0];
|
|
3128
|
+
if (failures.length > 1) {
|
|
3129
|
+
throw new AggregateError(
|
|
3130
|
+
failures,
|
|
3131
|
+
"read projection auth cleanup failed for exact and omni stores",
|
|
3132
|
+
);
|
|
3133
|
+
}
|
|
3134
|
+
return removed;
|
|
3135
|
+
});
|
|
3136
|
+
}
|
|
3137
|
+
|
|
3138
|
+
/** Remove only one exact query revision tree, never sibling queries or a replacement realm. */
|
|
3139
|
+
export function removeReadProjection(
|
|
3140
|
+
queryValue: ReadProjectionQuery,
|
|
3141
|
+
environment: Environment = process.env,
|
|
3142
|
+
): boolean {
|
|
3143
|
+
const authId = parseQueryValue(queryValue).identity.auth.id;
|
|
3144
|
+
return withReadProjectionAuthAdmission(authId, environment, () => {
|
|
3145
|
+
const { query } = validateQuery(queryValue, environment);
|
|
3146
|
+
return removePrivateStateDirectoryTree(
|
|
3147
|
+
queryDirectory(query, environment),
|
|
3148
|
+
environment,
|
|
3149
|
+
);
|
|
3150
|
+
});
|
|
3151
|
+
}
|