@cosmicdrift/kumiko-framework 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -38
- package/package.json +21 -2
- package/src/__tests__/anonymous-access.integration.test.ts +185 -0
- package/src/__tests__/consumer-cli.integration.test.ts +172 -0
- package/src/__tests__/entity-permalink-open.integration.test.ts +94 -0
- package/src/__tests__/full-stack.integration.test.ts +1 -1
- package/src/__tests__/schema-cli-temporal-polyfill.test.ts +48 -0
- package/src/__tests__/schema-cli.integration.test.ts +50 -0
- package/src/__tests__/store-table.integration.test.ts +93 -0
- package/src/api/__tests__/api.test.ts +373 -0
- package/src/api/__tests__/auth-middleware-anonymous-access-boot.test.ts +40 -0
- package/src/api/__tests__/auth-routes-cookie.test.ts +17 -1
- package/src/api/__tests__/auth-routes-invalid-body-invite.test.ts +253 -0
- package/src/api/__tests__/auth-routes-mfa-preauth-confirm.test.ts +222 -0
- package/src/api/__tests__/auth-routes-mfa-preauth-enable-start.test.ts +249 -0
- package/src/api/__tests__/auth-routes-mfa-verify.test.ts +202 -0
- package/src/api/__tests__/auth-routes-trusted-proxy.test.ts +135 -0
- package/src/api/__tests__/batch.integration.test.ts +101 -11
- package/src/api/__tests__/csrf-constants-sync.test.ts +20 -0
- package/src/api/__tests__/dispatcher-live.integration.test.ts +74 -0
- package/src/api/__tests__/jwt.test.ts +200 -1
- package/src/api/__tests__/login-rate-limiter-sweep.test.ts +50 -0
- package/src/api/__tests__/pat-scope.test.ts +36 -0
- package/src/api/__tests__/pii-leak-guard.integration.test.ts +103 -0
- package/src/api/__tests__/redis-login-rate-limiter.integration.test.ts +138 -0
- package/src/api/__tests__/request-id-middleware.test.ts +51 -0
- package/src/api/__tests__/server-boot-guards.test.ts +71 -0
- package/src/api/__tests__/server-jwt-ttl.test.ts +58 -0
- package/src/api/__tests__/sse-broker.test.ts +57 -0
- package/src/api/__tests__/sse-route.test.ts +4 -0
- package/src/api/api-constants.ts +11 -0
- package/src/api/auth-middleware.ts +248 -40
- package/src/api/auth-routes.ts +576 -95
- package/src/api/index.ts +13 -4
- package/src/api/jwt.ts +170 -11
- package/src/api/pat-scope.ts +14 -0
- package/src/api/pii-leak-guard.ts +47 -0
- package/src/api/request-context.ts +3 -0
- package/src/api/request-id-middleware.ts +2 -0
- package/src/api/routes.ts +169 -2
- package/src/api/server.ts +112 -16
- package/src/api/sse-broker.ts +39 -0
- package/src/bun-db/__tests__/PATTERN.md +0 -1
- package/src/bun-db/__tests__/coerce-row-temporal.test.ts +41 -0
- package/src/bun-db/__tests__/select-many-retry.test.ts +79 -0
- package/src/bun-db/__tests__/write-brand.test.ts +21 -0
- package/src/bun-db/connection.ts +3 -3
- package/src/bun-db/index.ts +2 -0
- package/src/bun-db/query.ts +105 -32
- package/src/consumer-cli.ts +134 -0
- package/src/crypto/__tests__/blind-index.test.ts +130 -0
- package/src/crypto/__tests__/event-pii.test.ts +161 -0
- package/src/crypto/__tests__/kek-rotation.integration.test.ts +180 -0
- package/src/crypto/__tests__/kms-adapter-contract.ts +134 -0
- package/src/crypto/__tests__/kms-adapter.contract.test.ts +4 -0
- package/src/crypto/__tests__/pg-kms-adapter.integration.test.ts +117 -0
- package/src/crypto/__tests__/pii-field-encryption.test.ts +376 -0
- package/src/crypto/__tests__/request-kms-cache.test.ts +74 -0
- package/src/crypto/__tests__/subject-resolver.test.ts +108 -0
- package/src/crypto/blind-index.ts +118 -0
- package/src/crypto/event-pii.ts +70 -0
- package/src/crypto/in-memory-kms-adapter.ts +50 -0
- package/src/crypto/index.ts +67 -0
- package/src/crypto/kms-adapter.ts +2 -0
- package/src/crypto/pg-kms-adapter.ts +295 -0
- package/src/crypto/pii-field-encryption.ts +248 -0
- package/src/crypto/request-kms-cache.ts +38 -0
- package/src/crypto/subject-resolver.ts +91 -0
- package/src/db/__tests__/assert-no-unreachable-live-rows.integration.test.ts +191 -0
- package/src/db/__tests__/blind-index.integration.test.ts +248 -0
- package/src/db/__tests__/build-filter-where.test.ts +34 -0
- package/src/db/__tests__/collect-table-metas.test.ts +10 -10
- package/src/db/__tests__/config-seed.integration.test.ts +13 -5
- package/src/db/__tests__/dialect-instant.test.ts +1 -4
- package/src/db/__tests__/entity-field-encryption.test.ts +7 -7
- package/src/db/__tests__/entity-table-meta-source.test.ts +50 -0
- package/src/db/__tests__/event-store-executor-context.pii-roundtrip.test.ts +67 -0
- package/src/db/__tests__/event-store-executor-write-verbs.integration.test.ts +402 -0
- package/src/db/__tests__/event-store-executor.integration.test.ts +299 -29
- package/src/db/__tests__/feature-table-sources.test.ts +34 -0
- package/src/db/__tests__/implicit-projection-equivalence.integration.test.ts +118 -39
- package/src/db/__tests__/instant-to-driver-temporal.test.ts +21 -0
- package/src/db/__tests__/located-timestamp.test.ts +19 -0
- package/src/db/__tests__/migrate-generator.test.ts +21 -0
- package/src/db/__tests__/migrate-runner.test.ts +61 -0
- package/src/db/__tests__/number-field-fractional.integration.test.ts +58 -0
- package/src/db/__tests__/rebuild-marker.test.ts +13 -3
- package/src/db/__tests__/replay-migration-sql.test.ts +384 -0
- package/src/db/__tests__/schema-inspection.test.ts +7 -0
- package/src/db/__tests__/schema-migration.integration.test.ts +1 -1
- package/src/db/__tests__/table-builder-meta-lockstep.test.ts +39 -0
- package/src/db/__tests__/tenant-db-where-merge.test.ts +49 -3
- package/src/db/__tests__/tenant-db.integration.test.ts +6 -2
- package/src/db/api.ts +2 -2
- package/src/db/apply-entity-event.ts +18 -14
- package/src/db/blind-index-cleanup.ts +55 -0
- package/src/db/bun-provider.ts +2 -2
- package/src/db/collect-table-metas.ts +6 -7
- package/src/db/config-seed.ts +9 -9
- package/src/db/connection.ts +7 -12
- package/src/db/cursor.ts +1 -18
- package/src/db/dialect.ts +20 -26
- package/src/db/entity-field-encryption.ts +81 -24
- package/src/db/entity-table-meta-types.ts +2 -0
- package/src/db/entity-table-meta.ts +63 -90
- package/src/db/event-store-executor-context.ts +404 -0
- package/src/db/event-store-executor-read.ts +275 -0
- package/src/db/event-store-executor-write.ts +644 -0
- package/src/db/event-store-executor.ts +28 -1155
- package/src/db/feature-table-sources.ts +6 -5
- package/src/db/index.ts +20 -3
- package/src/db/located-timestamp.ts +4 -0
- package/src/db/migrate-generator.ts +39 -6
- package/src/db/migrate-runner.ts +107 -11
- package/src/db/pg-error.ts +9 -1
- package/src/db/postgres-provider.ts +2 -2
- package/src/db/queries/__tests__/event-store-idempotency-index.integration.test.ts +80 -0
- package/src/db/queries/backfill-pii.ts +277 -0
- package/src/db/queries/ddl.ts +45 -0
- package/src/db/queries/event-consumer.ts +35 -2
- package/src/db/queries/event-store.ts +126 -19
- package/src/db/queries/projection-rebuild.ts +22 -8
- package/src/db/queries/shadow-swap.ts +183 -0
- package/src/db/queries/test-stack.ts +4 -30
- package/src/db/query-api.ts +1 -0
- package/src/db/query.ts +1 -0
- package/src/db/rebuild-marker.ts +18 -2
- package/src/db/reference-data.ts +2 -3
- package/src/db/replay-migration-sql.ts +257 -0
- package/src/db/schema-inspection.ts +1 -1
- package/src/db/table-builder.ts +102 -71
- package/src/db/tenant-db.ts +15 -53
- package/src/engine/__tests__/boot-validator-action-wiring.test.ts +242 -0
- package/src/engine/__tests__/boot-validator-boot-check.test.ts +99 -0
- package/src/engine/__tests__/boot-validator-dashboard.test.ts +237 -0
- package/src/engine/__tests__/boot-validator-entity-list.test.ts +104 -0
- package/src/engine/__tests__/boot-validator-gdpr-storage.test.ts +7 -69
- package/src/engine/__tests__/boot-validator-i18n-keys.test.ts +26 -5
- package/src/engine/__tests__/boot-validator-located-timestamps.test.ts +3 -19
- package/src/engine/__tests__/boot-validator-pii-retention.test.ts +246 -4
- package/src/engine/__tests__/boot-validator.test.ts +219 -15
- package/src/engine/__tests__/build-app-schema.test.ts +82 -0
- package/src/engine/__tests__/build-config-feature-schema.test.ts +125 -0
- package/src/engine/__tests__/build-target.test.ts +3 -11
- package/src/engine/__tests__/codemod-pipeline.test.ts +152 -21
- package/src/engine/__tests__/config-helpers.test.ts +16 -0
- package/src/engine/__tests__/define-feature-entity-mapping.test.ts +6 -0
- package/src/engine/__tests__/define-roles.test.ts +21 -0
- package/src/engine/__tests__/engine.test.ts +163 -1
- package/src/engine/__tests__/entity-handlers.test.ts +80 -0
- package/src/engine/__tests__/event-migration-declarative.test.ts +65 -0
- package/src/engine/__tests__/event-type-map-augmentation.test.ts +24 -0
- package/src/engine/__tests__/extend-entity-projection.test.ts +123 -0
- package/src/engine/__tests__/factories-time.test.ts +2 -66
- package/src/engine/__tests__/feature-crud-shorthand.test.ts +28 -0
- package/src/engine/__tests__/feature-manifest.test.ts +31 -1
- package/src/engine/__tests__/field-access.test.ts +23 -1
- package/src/engine/__tests__/hook-phases.test.ts +5 -5
- package/src/engine/__tests__/membership-roles.test.ts +4 -10
- package/src/engine/__tests__/nav.test.ts +86 -1
- package/src/engine/__tests__/pipeline-engine.test.ts +9 -9
- package/src/engine/__tests__/pipeline-handler.integration.test.ts +18 -18
- package/src/engine/__tests__/pipeline-observability.integration.test.ts +2 -2
- package/src/engine/__tests__/pipeline-performance.integration.test.ts +3 -3
- package/src/engine/__tests__/pipeline-sub-pipelines.test.ts +9 -9
- package/src/engine/__tests__/post-query-hook.test.ts +7 -7
- package/src/engine/__tests__/registrar-object-form.test.ts +141 -0
- package/src/engine/__tests__/registry.test.ts +144 -0
- package/src/engine/__tests__/schema-builder.test.ts +18 -0
- package/src/engine/__tests__/screen.test.ts +147 -1
- package/src/engine/__tests__/soft-delete-cleanup.test.ts +3 -0
- package/src/engine/__tests__/store-table.test.ts +229 -0
- package/src/engine/__tests__/tier-resolver-extension.test.ts +19 -1
- package/src/engine/__tests__/{visual-tree-patterns.test.ts → tree-actions-patterns.test.ts} +7 -95
- package/src/engine/__tests__/validate-projection-allowlist.test.ts +15 -15
- package/src/engine/boot-validator/__tests__/config-deps.test.ts +92 -0
- package/src/engine/boot-validator/action-wiring.ts +149 -0
- package/src/engine/boot-validator/boot-check.ts +21 -0
- package/src/engine/boot-validator/config-deps.ts +40 -4
- package/src/engine/boot-validator/entity-handler.ts +20 -21
- package/src/engine/boot-validator/entity-list-screens.ts +88 -0
- package/src/engine/boot-validator/gdpr-storage.ts +0 -20
- package/src/engine/boot-validator/i18n-keys.ts +58 -4
- package/src/engine/boot-validator/index.ts +33 -12
- package/src/engine/boot-validator/nav.ts +125 -0
- package/src/engine/boot-validator/pii-retention.ts +95 -10
- package/src/engine/boot-validator/{screens-nav.ts → screens.ts} +214 -191
- package/src/engine/boot-validator/workspaces.ts +68 -0
- package/src/engine/build-app-schema.ts +16 -0
- package/src/engine/build-config-feature-schema.ts +44 -7
- package/src/engine/codemod/pipeline-codemod.ts +5 -5
- package/src/engine/config-helpers.ts +15 -0
- package/src/engine/constants.ts +32 -6
- package/src/engine/create-app.ts +11 -0
- package/src/engine/define-feature.ts +95 -947
- package/src/engine/define-handler.ts +28 -94
- package/src/engine/define-workflow.ts +1 -1
- package/src/engine/effective-features.ts +12 -2
- package/src/engine/entity-handlers.ts +76 -11
- package/src/engine/extensions/tenant-data.ts +19 -0
- package/src/engine/extensions/user-data.ts +29 -2
- package/src/engine/factories.ts +8 -49
- package/src/engine/feature-ast/__tests__/canonical-form.test.ts +26 -29
- package/src/engine/feature-ast/__tests__/fixtures/cross-file-registrar/feature.ts +9 -0
- package/src/engine/feature-ast/__tests__/fixtures/cross-file-registrar/screens.ts +4 -0
- package/src/engine/feature-ast/__tests__/parse-happy-path.test.ts +1 -2
- package/src/engine/feature-ast/__tests__/parse-real-features.test.ts +18 -8
- package/src/engine/feature-ast/__tests__/parse.test.ts +1640 -160
- package/src/engine/feature-ast/__tests__/patch.test.ts +206 -12
- package/src/engine/feature-ast/__tests__/patcher.test.ts +20 -23
- package/src/engine/feature-ast/__tests__/render-roundtrip.test.ts +380 -5
- package/src/engine/feature-ast/extractors/events.ts +322 -0
- package/src/engine/feature-ast/extractors/handlers.ts +234 -0
- package/src/engine/feature-ast/extractors/hooks.ts +243 -0
- package/src/engine/feature-ast/extractors/index.ts +34 -31
- package/src/engine/feature-ast/extractors/jobs-routes.ts +221 -0
- package/src/engine/feature-ast/extractors/projections-screens.ts +269 -0
- package/src/engine/feature-ast/extractors/round1.ts +3 -3
- package/src/engine/feature-ast/extractors/round5.ts +3 -3
- package/src/engine/feature-ast/extractors/round6.ts +2 -36
- package/src/engine/feature-ast/extractors/shared.ts +64 -6
- package/src/engine/feature-ast/index.ts +2 -4
- package/src/engine/feature-ast/parse.ts +130 -23
- package/src/engine/feature-ast/patch.ts +39 -50
- package/src/engine/feature-ast/patcher.ts +37 -38
- package/src/engine/feature-ast/patterns.ts +50 -62
- package/src/engine/feature-ast/render.ts +67 -44
- package/src/engine/feature-builder-state.ts +168 -0
- package/src/engine/feature-config-events-jobs.ts +403 -0
- package/src/engine/feature-entity-handlers.ts +208 -0
- package/src/engine/feature-manifest.ts +2 -1
- package/src/engine/feature-ui-extensions.ts +500 -0
- package/src/engine/field-access.ts +13 -2
- package/src/engine/field-helpers.ts +31 -0
- package/src/engine/handler-helpers.ts +26 -0
- package/src/engine/hook-helpers.ts +16 -0
- package/src/engine/index.ts +23 -7
- package/src/engine/membership-roles.ts +13 -0
- package/src/engine/object-form.ts +27 -0
- package/src/engine/ownership.ts +26 -79
- package/src/engine/pattern-library/__tests__/library.test.ts +7 -20
- package/src/engine/pattern-library/library.ts +44 -1152
- package/src/engine/pattern-library/mixed-schemas.ts +450 -0
- package/src/engine/pattern-library/opaque-schemas.ts +124 -0
- package/src/engine/pattern-library/shared-fields.ts +75 -0
- package/src/engine/pattern-library/static-schemas.ts +456 -0
- package/src/engine/pipeline.ts +6 -11
- package/src/engine/registry-facade.ts +362 -0
- package/src/engine/registry-ingest.ts +478 -0
- package/src/engine/registry-state.ts +388 -0
- package/src/engine/registry-validate.ts +642 -0
- package/src/engine/registry.ts +78 -1658
- package/src/engine/run-pipeline.ts +1 -1
- package/src/engine/schema-builder.ts +1 -0
- package/src/engine/screen-helpers.ts +54 -0
- package/src/engine/soft-delete-cleanup.ts +6 -2
- package/src/engine/steps/__tests__/duration-utils.test.ts +20 -0
- package/src/engine/steps/_duration-utils.ts +2 -0
- package/src/engine/steps/unsafe-projection-upsert.ts +1 -4
- package/src/engine/tier-resolver-extension.ts +3 -2
- package/src/engine/types/config.ts +2 -482
- package/src/engine/types/define-handler.ts +2 -0
- package/src/engine/types/entity-handlers.ts +2 -0
- package/src/engine/types/event-type-map.ts +1 -37
- package/src/engine/types/feature.ts +2 -972
- package/src/engine/types/fields.ts +2 -675
- package/src/engine/types/handlers.ts +2 -774
- package/src/engine/types/hooks.ts +2 -184
- package/src/engine/types/http-route.ts +1 -54
- package/src/engine/types/identifiers.ts +1 -47
- package/src/engine/types/index.ts +86 -40
- package/src/engine/types/nav.ts +2 -63
- package/src/engine/types/ownership.ts +2 -0
- package/src/engine/types/projection.ts +2 -138
- package/src/engine/types/relations.ts +1 -51
- package/src/engine/types/screen.ts +2 -574
- package/src/engine/types/step.ts +2 -334
- package/src/engine/types/target-ref.ts +1 -21
- package/src/engine/types/tree-node.ts +1 -132
- package/src/engine/types/workspace.ts +2 -49
- package/src/engine/validate-projection-allowlist.ts +6 -6
- package/src/entrypoint/__tests__/entrypoint-job-wiring.integration.test.ts +120 -1
- package/src/entrypoint/index.ts +49 -6
- package/src/errors/classes.ts +22 -1
- package/src/errors/field-issue.ts +0 -3
- package/src/errors/index.ts +1 -1
- package/src/errors/write-error-info.ts +10 -22
- package/src/es-ops/README.md +1 -1
- package/src/es-ops/__tests__/runner.integration.test.ts +74 -0
- package/src/es-ops/context.ts +4 -2
- package/src/es-ops/types.ts +8 -1
- package/src/event-store/__tests__/admin-api.integration.test.ts +27 -1
- package/src/event-store/__tests__/backfill-pii.integration.test.ts +279 -0
- package/src/event-store/__tests__/event-store.integration.test.ts +201 -0
- package/src/event-store/__tests__/row-to-stored-event.test.ts +2 -2
- package/src/event-store/__tests__/snapshot.integration.test.ts +86 -0
- package/src/event-store/__tests__/unscoped-stream-primitives.guard.test.ts +58 -0
- package/src/event-store/__tests__/upcaster.integration.test.ts +77 -45
- package/src/event-store/admin-api.ts +11 -4
- package/src/event-store/errors.ts +2 -35
- package/src/event-store/event-store.ts +64 -78
- package/src/event-store/events-schema.ts +11 -13
- package/src/event-store/index.ts +18 -3
- package/src/event-store/rebuild-dead-letter.ts +111 -0
- package/src/event-store/snapshot.ts +63 -40
- package/src/event-store/types.ts +2 -0
- package/src/files/__tests__/build-storage-key.test.ts +28 -0
- package/src/files/__tests__/file-ref-entity.test.ts +8 -0
- package/src/files/__tests__/files.integration.test.ts +24 -0
- package/src/files/__tests__/in-memory-provider.contract.test.ts +4 -0
- package/src/files/__tests__/local-provider.test.ts +31 -0
- package/src/files/__tests__/provider-resolver.test.ts +70 -0
- package/src/files/__tests__/write-stream.test.ts +13 -0
- package/src/files/file-handle.ts +2 -19
- package/src/files/file-ref-entity.ts +2 -2
- package/src/files/file-routes.ts +13 -0
- package/src/files/index.ts +1 -1
- package/src/files/local-provider.ts +27 -8
- package/src/files/provider-resolver.ts +31 -20
- package/src/files/types.ts +13 -55
- package/src/i18n/__tests__/required-surface-keys.test.ts +17 -0
- package/src/i18n/required-surface-keys.ts +120 -1
- package/src/jobs/__tests__/job-queue-depth.integration.test.ts +82 -0
- package/src/jobs/__tests__/jobs.integration.test.ts +329 -3
- package/src/jobs/job-runner.ts +82 -12
- package/src/logging/types.ts +1 -7
- package/src/migrations/pending-rebuilds.ts +1 -1
- package/src/observability/__tests__/observability.integration.test.ts +4 -1
- package/src/observability/__tests__/recording-tracer.test.ts +6 -4
- package/src/observability/index.ts +2 -0
- package/src/observability/noop-provider.ts +0 -9
- package/src/observability/recording-tracer.ts +0 -12
- package/src/observability/standard-metrics.ts +64 -2
- package/src/observability/types/index.ts +1 -29
- package/src/observability/types/metric.ts +1 -56
- package/src/observability/types/provider.ts +1 -32
- package/src/observability/types/span.ts +1 -64
- package/src/pipeline/__tests__/ctx-bridge.integration.test.ts +2 -2
- package/src/pipeline/__tests__/dispatcher.test.ts +366 -1
- package/src/pipeline/__tests__/event-consumer-state.integration.test.ts +31 -0
- package/src/pipeline/__tests__/event-dispatcher-delivery-max-attempts.test.ts +126 -0
- package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts +263 -0
- package/src/pipeline/__tests__/event-dispatcher.integration.test.ts +10 -0
- package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts +106 -0
- package/src/pipeline/__tests__/lifecycle-pipeline.test.ts +308 -77
- package/src/pipeline/__tests__/load-aggregate-query.integration.test.ts +16 -8
- package/src/pipeline/__tests__/post-query-hook.integration.test.ts +3 -3
- package/src/pipeline/__tests__/projection-rebuild.integration.test.ts +80 -2
- package/src/pipeline/__tests__/rebuild-poison-quarantine.integration.test.ts +274 -0
- package/src/pipeline/__tests__/try-append-event.integration.test.ts +166 -0
- package/src/pipeline/dispatch-batch.ts +187 -0
- package/src/pipeline/dispatch-query.ts +165 -0
- package/src/pipeline/dispatch-shared.ts +826 -0
- package/src/pipeline/dispatch-stream.ts +90 -0
- package/src/pipeline/dispatch-write.ts +451 -0
- package/src/pipeline/dispatcher-utils.ts +1 -1
- package/src/pipeline/dispatcher.ts +44 -1372
- package/src/pipeline/entity-cache.ts +2 -33
- package/src/pipeline/event-consumer-state.ts +30 -2
- package/src/pipeline/event-dispatcher-admin.ts +293 -0
- package/src/pipeline/event-dispatcher-delivery.ts +305 -0
- package/src/pipeline/event-dispatcher.ts +85 -542
- package/src/pipeline/index.ts +2 -0
- package/src/pipeline/msp-rebuild.ts +42 -3
- package/src/pipeline/multi-stream-apply-context.ts +4 -42
- package/src/pipeline/projection-rebuild.ts +105 -3
- package/src/pipeline/system-hooks.ts +144 -1
- package/src/rate-limit/__tests__/resolver.integration.test.ts +18 -0
- package/src/rate-limit/resolver.ts +16 -32
- package/src/schema-cli.ts +76 -16
- package/src/search/__tests__/meilisearch-adapter.integration.test.ts +207 -185
- package/src/search/__tests__/meilisearch-ids.test.ts +30 -0
- package/src/search/__tests__/reindex-entity.integration.test.ts +144 -0
- package/src/search/index.ts +6 -0
- package/src/search/meilisearch-adapter.ts +13 -12
- package/src/search/reindex-entity.ts +177 -0
- package/src/search/types.ts +1 -39
- package/src/secrets/__tests__/contains-secret.test.ts +34 -0
- package/src/secrets/__tests__/envelope-cipher.test.ts +59 -0
- package/src/secrets/__tests__/envelope.test.ts +1 -1
- package/src/secrets/dek-cache.ts +26 -5
- package/src/secrets/envelope-cipher.ts +53 -0
- package/src/secrets/envelope.ts +5 -3
- package/src/secrets/index.ts +13 -1
- package/src/secrets/stored-envelope.ts +46 -0
- package/src/secrets/types.ts +2 -162
- package/src/stack/__tests__/event-collector.test.ts +42 -0
- package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts +125 -0
- package/src/stack/db.ts +2 -1
- package/src/stack/push-entity-projection-tables.ts +4 -3
- package/src/stack/redis.ts +8 -0
- package/src/stack/request-helper.ts +38 -2
- package/src/stack/table-helpers.ts +6 -4
- package/src/stack/test-stack.ts +249 -131
- package/src/testing/__tests__/late-bound.test.ts +32 -0
- package/src/testing/__tests__/wait-for.test.ts +59 -0
- package/src/testing/boot-validator-fixture.ts +121 -0
- package/src/testing/e2e-generator.ts +8 -0
- package/src/testing/file-provider-contract.ts +104 -0
- package/src/testing/handler-context.ts +3 -1
- package/src/testing/index.ts +5 -0
- package/src/testing/late-bound.ts +5 -3
- package/src/testing/mutable-master-key-provider.ts +29 -3
- package/src/testing/wait-for.ts +3 -0
- package/src/testing/without-ambient-temporal.ts +14 -0
- package/src/time/__tests__/tz-dateline.test.ts +8 -10
- package/src/time/geo-tz.ts +1 -32
- package/src/time/index.ts +1 -0
- package/src/time/legacy-date.ts +18 -0
- package/src/time/polyfill.ts +21 -38
- package/src/time/tz-context.ts +44 -85
- package/src/ui-types/app-schema.ts +12 -0
- package/src/ui-types/index.ts +20 -7
- package/src/utils/__tests__/safe-json-temporal.test.ts +18 -0
- package/src/utils/safe-json.ts +13 -1
- package/src/__tests__/raw-table.integration.test.ts +0 -116
- package/src/bun-db/__tests__/bun-test-stack.ts +0 -6
- package/src/db/__tests__/encryption.test.ts +0 -39
- package/src/db/encryption.ts +0 -39
- package/src/db/row-helpers.ts +0 -4
- package/src/engine/__tests__/raw-table.test.ts +0 -150
- package/src/engine/__tests__/unmanaged-table.test.ts +0 -127
- package/src/engine/feature-ast/__tests__/visual-tree-parse.test.ts +0 -184
- package/src/engine/feature-ast/extractors/round4.ts +0 -1366
package/src/pipeline/index.ts
CHANGED
|
@@ -57,6 +57,8 @@ export {
|
|
|
57
57
|
} from "./projection-state";
|
|
58
58
|
export { runProjectionsForEvent } from "./projections-runner";
|
|
59
59
|
export {
|
|
60
|
+
ACCESS_INVALIDATION_CONSUMER_NAME,
|
|
61
|
+
createAccessInvalidationEventConsumer,
|
|
60
62
|
createSearchEventConsumer,
|
|
61
63
|
createSseBroadcastEventConsumer,
|
|
62
64
|
SEARCH_CONSUMER_NAME,
|
|
@@ -6,16 +6,22 @@ import {
|
|
|
6
6
|
updateConsumerRebuildCursor,
|
|
7
7
|
} from "../db/queries/event-consumer";
|
|
8
8
|
import {
|
|
9
|
+
assertLiveColumnsMatchMeta,
|
|
9
10
|
buildShadowTable,
|
|
10
11
|
ensureRebuildSchema,
|
|
11
12
|
rebuildMetaOrThrow,
|
|
12
13
|
swapShadowIntoLive,
|
|
13
14
|
} from "../db/queries/shadow-swap";
|
|
14
|
-
import { selectMany } from "../db/query";
|
|
15
|
+
import { runInSavepoint, selectMany } from "../db/query";
|
|
15
16
|
import type { Registry, TenantId } from "../engine/types";
|
|
16
17
|
import { InternalError } from "../errors";
|
|
17
18
|
import { eventsTable, type StoredEvent, upcastStoredEvent } from "../event-store";
|
|
18
19
|
import { loadAggregate, loadAggregateAsOf } from "../event-store/event-store";
|
|
20
|
+
import {
|
|
21
|
+
createRebuildDeadLetterTable,
|
|
22
|
+
recordRebuildDeadLetters,
|
|
23
|
+
type SkippedApply,
|
|
24
|
+
} from "../event-store/rebuild-dead-letter";
|
|
19
25
|
import { upcastStoredEvents } from "../event-store/upcaster";
|
|
20
26
|
import { emitProjectionRebuild } from "../observability/standard-metrics";
|
|
21
27
|
import type { Meter } from "../observability/types/metric";
|
|
@@ -123,16 +129,26 @@ export async function rebuildMultiStreamProjection(
|
|
|
123
129
|
|
|
124
130
|
const meta = rebuildMetaOrThrow(msp.table, mspName);
|
|
125
131
|
|
|
132
|
+
// Declared per-projection (MspErrorMode): the rebuild policy falls back to
|
|
133
|
+
// continuous when omitted — previously declared but never honored here.
|
|
134
|
+
const skipApplyErrors =
|
|
135
|
+
(msp.errorMode?.rebuild ?? msp.errorMode?.continuous)?.skipApplyErrors === true;
|
|
136
|
+
|
|
126
137
|
const startedAt = Date.now();
|
|
127
138
|
let eventsProcessed = 0;
|
|
128
139
|
let lastProcessedEventId = 0n;
|
|
140
|
+
const skipped: SkippedApply[] = [];
|
|
129
141
|
|
|
130
142
|
try {
|
|
131
143
|
await ensureRebuildSchema(db);
|
|
144
|
+
// Outside the rebuild tx, like the schema: idempotent DDL colliding
|
|
145
|
+
// inside the tx would roll the whole replay back.
|
|
146
|
+
if (skipApplyErrors) await createRebuildDeadLetterTable(db);
|
|
132
147
|
await db.begin(async (tx: DbTx) => {
|
|
133
148
|
await resetConsumerForMspRebuild(tx, mspName, SHARED_INSTANCE_SENTINEL);
|
|
134
149
|
await selectConsumerForUpdate(tx, mspName, SHARED_INSTANCE_SENTINEL);
|
|
135
150
|
|
|
151
|
+
await assertLiveColumnsMatchMeta(tx, meta, mspName);
|
|
136
152
|
await buildShadowTable(tx, meta);
|
|
137
153
|
|
|
138
154
|
const subscribedTypes = Object.keys(msp.apply);
|
|
@@ -178,13 +194,31 @@ export async function rebuildMultiStreamProjection(
|
|
|
178
194
|
});
|
|
179
195
|
const applyFn = msp.apply[row.type];
|
|
180
196
|
if (!applyFn) continue;
|
|
181
|
-
|
|
182
|
-
|
|
197
|
+
if (skipApplyErrors) {
|
|
198
|
+
// Driver-native savepoint: a throwing SQL statement would
|
|
199
|
+
// otherwise poison the rebuild tx (25P02) AND make the driver
|
|
200
|
+
// reject the whole begin() even after a caught error. Apply +
|
|
201
|
+
// ctx run on the savepoint-scoped handle.
|
|
202
|
+
try {
|
|
203
|
+
await runInSavepoint(tx, async (sp) => {
|
|
204
|
+
const rebuildCtx = createRebuildCtx(registry, sp as DbRunner, row.tenantId);
|
|
205
|
+
await applyFn(storedEvent, sp as DbRunner, rebuildCtx);
|
|
206
|
+
});
|
|
207
|
+
} catch (e) {
|
|
208
|
+
skipped.push({ event: storedEvent, error: e });
|
|
209
|
+
}
|
|
210
|
+
} else {
|
|
211
|
+
const rebuildCtx = createRebuildCtx(registry, tx, row.tenantId);
|
|
212
|
+
await applyFn(storedEvent, tx, rebuildCtx);
|
|
213
|
+
}
|
|
183
214
|
eventsProcessed++;
|
|
184
215
|
lastProcessedEventId = row.id;
|
|
185
216
|
}
|
|
186
217
|
}
|
|
187
218
|
|
|
219
|
+
if (skipped.length > 0) {
|
|
220
|
+
await recordRebuildDeadLetters(tx, mspName, skipped);
|
|
221
|
+
}
|
|
188
222
|
await updateConsumerRebuildCursor(
|
|
189
223
|
tx,
|
|
190
224
|
mspName,
|
|
@@ -210,8 +244,13 @@ export async function rebuildMultiStreamProjection(
|
|
|
210
244
|
const result: RebuildResult = {
|
|
211
245
|
projection: mspName,
|
|
212
246
|
eventsProcessed,
|
|
247
|
+
eventsSkipped: skipped.length,
|
|
213
248
|
lastProcessedEventId,
|
|
214
249
|
durationMs: Date.now() - startedAt,
|
|
250
|
+
// countColumnDrift is single-table (public.<t> vs kumiko_rebuild.<t>) and
|
|
251
|
+
// not wired into the multi-stream rebuild path — MSPs don't run the
|
|
252
|
+
// implicit-projection ghost-row guard either. #916 scope is single-table.
|
|
253
|
+
columnDriftCount: 0,
|
|
215
254
|
};
|
|
216
255
|
if (deps.meter) {
|
|
217
256
|
emitProjectionRebuild(
|
|
@@ -1,50 +1,12 @@
|
|
|
1
|
+
import type { MultiStreamApplyContext } from "@cosmicdrift/kumiko-types/multi-stream-apply-context-types";
|
|
1
2
|
import type { DbRunner } from "../db/connection";
|
|
2
|
-
import type {
|
|
3
|
-
|
|
4
|
-
AppendEventFn,
|
|
5
|
-
KumikoEventTypeMap,
|
|
6
|
-
Registry,
|
|
7
|
-
TenantId,
|
|
8
|
-
UnsafeAppendEventFn,
|
|
9
|
-
} from "../engine/types";
|
|
10
|
-
import { loadAggregate, loadAggregateAsOf, type StoredEvent } from "../event-store/event-store";
|
|
3
|
+
import type { AppendEventArgs, AppendEventFn, Registry, TenantId } from "../engine/types";
|
|
4
|
+
import { loadAggregate, loadAggregateAsOf } from "../event-store/event-store";
|
|
11
5
|
import { upcastStoredEvents } from "../event-store/upcaster";
|
|
12
6
|
import type { FileContext } from "../files/file-handle";
|
|
13
7
|
import { appendDomainEventCore } from "./append-event-core";
|
|
14
8
|
|
|
15
|
-
|
|
16
|
-
// when it needs to produce follow-up events (saga / process-manager
|
|
17
|
-
// pattern). Keeps the MSP feature-decoupled: applies don't reach into
|
|
18
|
-
// handler-bridge (no query/write/writeAs), they just read the aggregate
|
|
19
|
-
// stream and append new events — Marten's session scope for projections.
|
|
20
|
-
//
|
|
21
|
-
// TMap propagates the strict event-type-map (see HandlerContext). Default
|
|
22
|
-
// matches the global KumikoEventTypeMap; runtime-pluggable callers route
|
|
23
|
-
// through unsafeAppendEvent.
|
|
24
|
-
export type MultiStreamApplyContext<TMap extends object = KumikoEventTypeMap> = {
|
|
25
|
-
// Append a domain event onto an aggregate stream in the CURRENT tx.
|
|
26
|
-
// Schema-validated, archive-guarded, stream-version derived. Metadata
|
|
27
|
-
// inherits from the triggering event (correlationId) + requestContext
|
|
28
|
-
// (causationId is already set to the triggering event.id by the
|
|
29
|
-
// dispatcher wrap). Strict against KumikoEventTypeMap — same contract
|
|
30
|
-
// as HandlerContext.appendEvent (compile-time-validated payload).
|
|
31
|
-
readonly appendEvent: AppendEventFn<TMap>;
|
|
32
|
-
// Escape hatch for runtime-pluggable events without compile-time
|
|
33
|
-
// augmentation. Same runtime semantics; type-surface is `payload: unknown`.
|
|
34
|
-
readonly unsafeAppendEvent: UnsafeAppendEventFn;
|
|
35
|
-
// Read an aggregate stream — useful when a saga needs to inspect the
|
|
36
|
-
// current state of a different aggregate before deciding what to emit.
|
|
37
|
-
readonly loadAggregate: (
|
|
38
|
-
aggregateId: string,
|
|
39
|
-
options?: { readonly asOf?: Temporal.Instant },
|
|
40
|
-
) => Promise<readonly StoredEvent[]>;
|
|
41
|
-
// Binary storage handle factory, mirrors AppContext.files. Present when
|
|
42
|
-
// the app booted with `files.storageProvider`; undefined otherwise.
|
|
43
|
-
// Post-processing MSPs (resize, EXIF-strip, virus-scan) read bytes via
|
|
44
|
-
// `ctx.files.ref(payload.storageKey).read()` and write derivates via
|
|
45
|
-
// `.derive("thumb").write(...)` — binaries never ride through events.
|
|
46
|
-
readonly files?: FileContext;
|
|
47
|
-
};
|
|
9
|
+
export type { MultiStreamApplyContext } from "@cosmicdrift/kumiko-types/multi-stream-apply-context-types";
|
|
48
10
|
|
|
49
11
|
export type MultiStreamApplyContextDeps = {
|
|
50
12
|
readonly registry: Registry;
|
|
@@ -7,13 +7,17 @@ import {
|
|
|
7
7
|
selectEventsForProjectionRebuildBatch,
|
|
8
8
|
} from "../db/queries/projection-rebuild";
|
|
9
9
|
import {
|
|
10
|
+
assertLiveColumnsMatchMeta,
|
|
11
|
+
assertNoUnreachableLiveRows,
|
|
10
12
|
buildShadowTable,
|
|
13
|
+
type ColumnDriftResult,
|
|
14
|
+
countColumnDrift,
|
|
11
15
|
ensureRebuildSchema,
|
|
12
16
|
fenceLiveTable,
|
|
13
17
|
rebuildMetaOrThrow,
|
|
14
18
|
swapShadowIntoLive,
|
|
15
19
|
} from "../db/queries/shadow-swap";
|
|
16
|
-
import { coerceRow, extractTableInfo, selectMany } from "../db/query";
|
|
20
|
+
import { coerceRow, extractTableInfo, runInSavepoint, selectMany } from "../db/query";
|
|
17
21
|
import type { Registry, TenantId } from "../engine/types";
|
|
18
22
|
import {
|
|
19
23
|
eventsTable,
|
|
@@ -22,6 +26,11 @@ import {
|
|
|
22
26
|
upcastStoredEvent,
|
|
23
27
|
} from "../event-store";
|
|
24
28
|
import type { EventMetadata } from "../event-store/event-store";
|
|
29
|
+
import {
|
|
30
|
+
createRebuildDeadLetterTable,
|
|
31
|
+
recordRebuildDeadLetters,
|
|
32
|
+
type SkippedApply,
|
|
33
|
+
} from "../event-store/rebuild-dead-letter";
|
|
25
34
|
import { emitProjectionRebuild } from "../observability/standard-metrics";
|
|
26
35
|
import type { Meter } from "../observability/types/metric";
|
|
27
36
|
import { projectionStateTable } from "./projection-state";
|
|
@@ -119,9 +128,19 @@ function rowToStoredEvent(row: StoredEventRow): StoredEvent {
|
|
|
119
128
|
|
|
120
129
|
export type RebuildResult = {
|
|
121
130
|
readonly projection: string;
|
|
131
|
+
// Events consumed from the log — includes quarantined ones (the cursor
|
|
132
|
+
// advanced past them).
|
|
122
133
|
readonly eventsProcessed: number;
|
|
134
|
+
// Poison events quarantined into kumiko_rebuild_dead_letters this run.
|
|
135
|
+
// Always 0 unless quarantine mode was active (#760).
|
|
136
|
+
readonly eventsSkipped: number;
|
|
123
137
|
readonly lastProcessedEventId: bigint;
|
|
124
138
|
readonly durationMs: number;
|
|
139
|
+
// Live rows whose columns differ from the freshly-replayed shadow (bidx
|
|
140
|
+
// columns excluded — see countColumnDrift). Non-blocking: the swap
|
|
141
|
+
// proceeds regardless (#916). 0 for explicit projections (guard is
|
|
142
|
+
// implicit-only) or when no drift was found.
|
|
143
|
+
readonly columnDriftCount: number;
|
|
125
144
|
};
|
|
126
145
|
|
|
127
146
|
type RebuildDeps = {
|
|
@@ -145,6 +164,15 @@ type RebuildDeps = {
|
|
|
145
164
|
// on the live table; if a long-running writer holds it past this, the rebuild
|
|
146
165
|
// fails loud instead of hanging. Defaults to DEFAULT_FENCE_LOCK_TIMEOUT_MS.
|
|
147
166
|
readonly fenceLockTimeoutMs?: number;
|
|
167
|
+
// Apply-error policy for THIS run. Default strict: the first throwing
|
|
168
|
+
// apply aborts the rebuild (tx rollback, status "failed") — a poison
|
|
169
|
+
// event makes the projection permanently un-rebuildable (#760).
|
|
170
|
+
// skipApplyErrors: true confines every apply to a savepoint; a throwing
|
|
171
|
+
// apply is rolled back, recorded into kumiko_rebuild_dead_letters and
|
|
172
|
+
// skipped, and the rebuild completes without its effect. Opt-in for
|
|
173
|
+
// operators unblocking a pinned rebuild — the projection then knowingly
|
|
174
|
+
// misses the quarantined events until they are repaired and replayed.
|
|
175
|
+
readonly errorPolicy?: { readonly skipApplyErrors?: boolean };
|
|
148
176
|
// Test-only seam: fires once after the unlocked bulk drain and before the
|
|
149
177
|
// cutover fence. Lets a concurrency test inject a committed write into the
|
|
150
178
|
// replay window deterministically. The `__test_` prefix marks it test-only:
|
|
@@ -157,6 +185,27 @@ type RebuildDeps = {
|
|
|
157
185
|
readonly __test_onBuildShadowTable?: () => void;
|
|
158
186
|
};
|
|
159
187
|
|
|
188
|
+
// Ops-facing log line for countColumnDrift's result. No-op when clean. #916.
|
|
189
|
+
function warnOnColumnDrift(
|
|
190
|
+
projectionName: string,
|
|
191
|
+
tableName: string,
|
|
192
|
+
drift: ColumnDriftResult,
|
|
193
|
+
): void {
|
|
194
|
+
// skip: no drift found — nothing to warn about
|
|
195
|
+
if (drift.rowCount === 0) return;
|
|
196
|
+
const sampledIds = new Set(drift.sample.map((s) => s.slice(0, s.lastIndexOf("."))));
|
|
197
|
+
const countLabel =
|
|
198
|
+
sampledIds.size < drift.rowCount
|
|
199
|
+
? `${drift.rowCount} (sample truncated)`
|
|
200
|
+
: String(drift.rowCount);
|
|
201
|
+
console.warn(
|
|
202
|
+
`projection-rebuild "${projectionName}": ${countLabel} live row(s) in "${tableName}" have ` +
|
|
203
|
+
`column values that differ from the freshly-replayed shadow (sample: ${drift.sample.join(", ")}). ` +
|
|
204
|
+
"Not blocking — replay wins on swap. Likely cause: a legacy direct-write predating the row's " +
|
|
205
|
+
"event history (#494), or a sensitive custom field pending #972. Investigate if unexpected.",
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
|
|
160
209
|
export async function rebuildProjection(
|
|
161
210
|
projectionName: string,
|
|
162
211
|
deps: RebuildDeps,
|
|
@@ -174,7 +223,7 @@ export async function rebuildProjection(
|
|
|
174
223
|
const meta = rebuildMetaOrThrow(projection.table, projectionName);
|
|
175
224
|
|
|
176
225
|
const sources = Array.isArray(projection.source) ? projection.source : [projection.source];
|
|
177
|
-
const sourcesList = [...sources];
|
|
226
|
+
const sourcesList = [...sources, ...(projection.extraSources ?? [])];
|
|
178
227
|
const subscribedList = Object.keys(projection.apply);
|
|
179
228
|
// Upcasters run at read time: older stored payloads get walked through the
|
|
180
229
|
// registered r.eventMigration chain until their shape matches the current
|
|
@@ -184,7 +233,13 @@ export async function rebuildProjection(
|
|
|
184
233
|
const fenceLockTimeoutMs = deps.fenceLockTimeoutMs ?? DEFAULT_FENCE_LOCK_TIMEOUT_MS;
|
|
185
234
|
const startedAt = Date.now();
|
|
186
235
|
let eventsProcessed = 0;
|
|
236
|
+
let columnDriftCount = 0;
|
|
187
237
|
let lastProcessedEventId = 0n;
|
|
238
|
+
const skipApplyErrors = deps.errorPolicy?.skipApplyErrors === true;
|
|
239
|
+
// Quarantined events, collected in memory and written once after the
|
|
240
|
+
// replay settles — the #443 full-re-replay path would otherwise duplicate
|
|
241
|
+
// the dead-letter rows.
|
|
242
|
+
let skipped: SkippedApply[] = [];
|
|
188
243
|
|
|
189
244
|
// One chronological batch of events after lastProcessedEventId, applied into
|
|
190
245
|
// the shadow. Returns the batch size so the caller can detect the tail
|
|
@@ -208,7 +263,21 @@ export async function rebuildProjection(
|
|
|
208
263
|
// skip: apply-key validation ensures every subscribed type has a handler;
|
|
209
264
|
// defensive check against runtime-mutated registry
|
|
210
265
|
if (!applyFn) continue;
|
|
211
|
-
|
|
266
|
+
if (skipApplyErrors) {
|
|
267
|
+
// Driver-native savepoint: a throwing SQL statement would otherwise
|
|
268
|
+
// poison the rebuild tx (25P02) AND make the driver reject the whole
|
|
269
|
+
// begin() even after a caught error. The apply runs on the
|
|
270
|
+
// savepoint-scoped handle so its statements are confined.
|
|
271
|
+
try {
|
|
272
|
+
await runInSavepoint(tx, async (sp) => {
|
|
273
|
+
await applyFn(storedEvent, sp as DbTx, projection.table);
|
|
274
|
+
});
|
|
275
|
+
} catch (e) {
|
|
276
|
+
skipped.push({ event: storedEvent, error: e });
|
|
277
|
+
}
|
|
278
|
+
} else {
|
|
279
|
+
await applyFn(storedEvent, tx, projection.table);
|
|
280
|
+
}
|
|
212
281
|
eventsProcessed++;
|
|
213
282
|
lastProcessedEventId = row.id;
|
|
214
283
|
}
|
|
@@ -217,8 +286,12 @@ export async function rebuildProjection(
|
|
|
217
286
|
|
|
218
287
|
try {
|
|
219
288
|
await ensureRebuildSchema(db);
|
|
289
|
+
// Outside the rebuild tx, like the schema: idempotent DDL colliding
|
|
290
|
+
// inside the tx would roll the whole replay back.
|
|
291
|
+
if (skipApplyErrors) await createRebuildDeadLetterTable(db);
|
|
220
292
|
await db.begin(async (tx: DbTx) => {
|
|
221
293
|
await markProjectionRebuilding(tx, projectionName);
|
|
294
|
+
await assertLiveColumnsMatchMeta(tx, meta, projectionName);
|
|
222
295
|
await buildShadowTable(tx, meta);
|
|
223
296
|
deps.__test_onBuildShadowTable?.();
|
|
224
297
|
|
|
@@ -261,12 +334,39 @@ export async function rebuildProjection(
|
|
|
261
334
|
deps.__test_onBuildShadowTable?.();
|
|
262
335
|
lastProcessedEventId = 0n;
|
|
263
336
|
eventsProcessed = 0;
|
|
337
|
+
skipped = [];
|
|
264
338
|
while ((await drainBatch(tx)) === REBUILD_BATCH_SIZE) {
|
|
265
339
|
// re-replay the full log into the fresh shadow
|
|
266
340
|
}
|
|
341
|
+
// drainBatch increments eventsProcessed for every drained row
|
|
342
|
+
// regardless of quarantine (skipApplyErrors), so this must hold
|
|
343
|
+
// exactly under the fence — a mismatch means drainBatch silently
|
|
344
|
+
// dropped a row instead of counting it, which the rest of this
|
|
345
|
+
// function has no other way to detect.
|
|
346
|
+
if (eventsProcessed < Number(expectedEvents)) {
|
|
347
|
+
throw new Error(
|
|
348
|
+
`projection-rebuild "${projectionName}": fenced re-replay processed ` +
|
|
349
|
+
`${eventsProcessed} events but ${expectedEvents} were expected — drainBatch ` +
|
|
350
|
+
"silently dropped rows instead of counting them.",
|
|
351
|
+
);
|
|
352
|
+
}
|
|
267
353
|
}
|
|
268
354
|
}
|
|
269
355
|
|
|
356
|
+
if (skipped.length > 0) {
|
|
357
|
+
await recordRebuildDeadLetters(tx, projectionName, skipped);
|
|
358
|
+
}
|
|
359
|
+
// Guard the swap: abort if the live table holds a row no event can
|
|
360
|
+
// reconstruct (#498 ghost — direct-inserted without a .created event),
|
|
361
|
+
// which the swap would silently drop. Implicit projections only.
|
|
362
|
+
if (projection.isImplicit === true) {
|
|
363
|
+
await assertNoUnreachableLiveRows(tx, projectionName, meta.tableName, sourcesList);
|
|
364
|
+
// Non-blocking counterpart: reports column-level drift (bidx excluded)
|
|
365
|
+
// without aborting — see countColumnDrift for why this isn't fail-hard.
|
|
366
|
+
const drift = await countColumnDrift(tx, meta.tableName, meta);
|
|
367
|
+
columnDriftCount = drift.rowCount;
|
|
368
|
+
warnOnColumnDrift(projectionName, meta.tableName, drift);
|
|
369
|
+
}
|
|
270
370
|
await finalizeProjectionRebuild(tx, projectionName, lastProcessedEventId);
|
|
271
371
|
await swapShadowIntoLive(tx, meta.tableName);
|
|
272
372
|
});
|
|
@@ -292,8 +392,10 @@ export async function rebuildProjection(
|
|
|
292
392
|
const result: RebuildResult = {
|
|
293
393
|
projection: projectionName,
|
|
294
394
|
eventsProcessed,
|
|
395
|
+
eventsSkipped: skipped.length,
|
|
295
396
|
lastProcessedEventId,
|
|
296
397
|
durationMs: Date.now() - startedAt,
|
|
398
|
+
columnDriftCount,
|
|
297
399
|
};
|
|
298
400
|
if (deps.meter) {
|
|
299
401
|
emitProjectionRebuild(
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { SseBroker } from "../api/sse-broker";
|
|
2
2
|
import type { DbRow } from "../db/connection";
|
|
3
3
|
import { tenantChannel } from "../engine/constants";
|
|
4
|
-
import type { EntityId, Registry } from "../engine/types";
|
|
4
|
+
import type { EntityId, JobRunnerRef, Registry, SessionUser } from "../engine/types";
|
|
5
5
|
import type { SearchAdapter, SearchDocument } from "../search/types";
|
|
6
6
|
import type { EventConsumer } from "./event-dispatcher";
|
|
7
7
|
|
|
@@ -41,6 +41,13 @@ export function createSearchEventConsumer(
|
|
|
41
41
|
): EventConsumer {
|
|
42
42
|
return {
|
|
43
43
|
name: SEARCH_CONSUMER_NAME,
|
|
44
|
+
// ponytail: count-based maxAttempts, not real backoff — retries hammer
|
|
45
|
+
// the search adapter every pollIntervalMs during an outage instead of
|
|
46
|
+
// spacing out. 1200 attempts * 100ms default poll ~= 2min budget for a
|
|
47
|
+
// still-provisioning Meilisearch to come up at boot (was 10 = ~1s,
|
|
48
|
+
// killing the consumer near-instantly). Upgrade path: time-based
|
|
49
|
+
// dead-lettering in event-dispatcher-delivery.ts if 2min isn't enough.
|
|
50
|
+
errorPolicy: { maxAttempts: 1200 },
|
|
44
51
|
handler: async (event) => {
|
|
45
52
|
const entityName = event.aggregateType;
|
|
46
53
|
const verb = event.type.split(".").pop();
|
|
@@ -242,3 +249,139 @@ export function createSseBroadcastEventConsumer(sseBroker: SseBroker): EventCons
|
|
|
242
249
|
},
|
|
243
250
|
};
|
|
244
251
|
}
|
|
252
|
+
|
|
253
|
+
// --- Job-Trigger Consumer (async, via event-dispatcher) ---
|
|
254
|
+
//
|
|
255
|
+
// r.job's `trigger.on` historically only fired via the synchronous
|
|
256
|
+
// write-handler dispatch path (dispatch-write.ts's afterCommitHooks calling
|
|
257
|
+
// jobRunner.handleEvent). Events appended any other way — an
|
|
258
|
+
// r.multiStreamProjection's ctx.unsafeAppendEvent, or a raw
|
|
259
|
+
// event-store-executor write (e.g. `files`' fileRef.created) — never
|
|
260
|
+
// reached it, so a job could never trigger on an r.defineEvent-registered
|
|
261
|
+
// event (kumiko-framework#1505).
|
|
262
|
+
//
|
|
263
|
+
// This consumer closes that gap the same way search/SSE do: read every
|
|
264
|
+
// committed event off the shared cursor and re-check job triggers. It is
|
|
265
|
+
// scoped EXACTLY to the gap — the write/query-handler-QN skip below is
|
|
266
|
+
// defense-in-depth: no stored event's `type` is a handler QN today (entity
|
|
267
|
+
// events are "entity.verb"; ctx.appendEvent enforces defineEvent-only
|
|
268
|
+
// ownership), but if that ever changes, this consumer must not re-fire a
|
|
269
|
+
// trigger the synchronous dispatch-write.ts path already handled.
|
|
270
|
+
//
|
|
271
|
+
// Delivery is at-least-once (cursor semantics) where the synchronous path
|
|
272
|
+
// is effectively once — job handlers reached via an r.defineEvent trigger
|
|
273
|
+
// must be idempotent (same expectation r.multiStreamProjection applies
|
|
274
|
+
// already carry).
|
|
275
|
+
export const JOB_TRIGGER_CONSUMER_NAME = "system:consumer:job-trigger";
|
|
276
|
+
|
|
277
|
+
export function createJobTriggerEventConsumer(
|
|
278
|
+
jobRunner: JobRunnerRef,
|
|
279
|
+
registry: Registry,
|
|
280
|
+
): EventConsumer {
|
|
281
|
+
return {
|
|
282
|
+
name: JOB_TRIGGER_CONSUMER_NAME,
|
|
283
|
+
handler: async (event) => {
|
|
284
|
+
// skip: write/query-handler QN — already dispatched synchronously by
|
|
285
|
+
// dispatch-write.ts's afterCommitHooks. Re-firing here would
|
|
286
|
+
// double-enqueue every existing handler-triggered job.
|
|
287
|
+
if (registry.getWriteHandler(event.type) || registry.getQueryHandler(event.type)) return;
|
|
288
|
+
// skip: no r.defineEvent registered under this type — nothing this
|
|
289
|
+
// consumer is responsible for.
|
|
290
|
+
if (!registry.getEvent(event.type)) return;
|
|
291
|
+
|
|
292
|
+
const user: SessionUser = {
|
|
293
|
+
id: event.metadata.userId,
|
|
294
|
+
tenantId: event.tenantId,
|
|
295
|
+
roles: [],
|
|
296
|
+
};
|
|
297
|
+
await jobRunner.handleEvent(event.type, event.payload, user);
|
|
298
|
+
},
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// --- Access-Invalidation Consumer (async, via event-dispatcher) ---
|
|
303
|
+
//
|
|
304
|
+
// #1524 (Design), #1558 (channel + stream subscribe), #1559 (session-revoke
|
|
305
|
+
// emits an event). This consumer is the last leg: it watches for the two
|
|
306
|
+
// event types that can make an already-issued JWT stale mid-stream and
|
|
307
|
+
// pushes an invalidation through sseBroker.publishAccessInvalidation, which
|
|
308
|
+
// dispatch-stream.ts's subscribeAccessInvalidation listener turns into an
|
|
309
|
+
// AccessDeniedError thrown into the open stream.
|
|
310
|
+
//
|
|
311
|
+
// Event types are literal strings, not imports — this package (framework)
|
|
312
|
+
// cannot depend on bundled-features (sessions, tenant own these events),
|
|
313
|
+
// mirrors the es-ops-seed precedent (literal QNs, no subpath import).
|
|
314
|
+
//
|
|
315
|
+
// "sessions:event:session-revoked" — payload.userId direct (own aggregate,
|
|
316
|
+
// see bundled-features/sessions/session-revoked-event.ts). Fired for
|
|
317
|
+
// both self-service revoke and the privileged cross-tenant
|
|
318
|
+
// revoke-all-for-user (SYSTEM_TENANT_ID-anchored DSGVO Art.18 freeze).
|
|
319
|
+
// fetchPendingEvents has no tenant predicate, so the SYSTEM_TENANT_ID-
|
|
320
|
+
// anchored event reaches this consumer the same as any other — routing
|
|
321
|
+
// is purely on payload.userId, never on event.tenantId.
|
|
322
|
+
// "tenant-membership.updated" / "tenant-membership.deleted" — role change
|
|
323
|
+
// or member removal. userId isn't in payload.changes (update only
|
|
324
|
+
// carries the changed fields, e.g. { roles }) so it's read from
|
|
325
|
+
// payload.previous, the full pre-write entity snapshot written by
|
|
326
|
+
// createEventStoreExecutor. tenantMembershipEntity declares no
|
|
327
|
+
// encrypted/PII fields, so previous.userId is plaintext — no KMS
|
|
328
|
+
// decrypt step (see kumiko-framework#1560 PR discussion; the issue's
|
|
329
|
+
// original handoff assumed encryption that doesn't apply here).
|
|
330
|
+
//
|
|
331
|
+
// Over-invalidation (e.g. tenant-membership.updated fired by something
|
|
332
|
+
// other than a role change) is fail-safe: the stream closes, the client
|
|
333
|
+
// reconnects and re-authorizes. No opt-out/allowlist — #1524 chose
|
|
334
|
+
// global-by-default specifically so this can't be silently disabled per
|
|
335
|
+
// handler.
|
|
336
|
+
//
|
|
337
|
+
// Scope note: entityEventName also emits "tenant-membership.forgotten" (DSGVO
|
|
338
|
+
// erasure) and ".restored" — .restored re-grants access so ignoring it is
|
|
339
|
+
// correct, .forgotten is not currently produced by any user-data-rights
|
|
340
|
+
// pipeline for memberships but would be revocation-relevant if it ever is.
|
|
341
|
+
// Out of #1560's stated scope (session-revoke + role/membership change),
|
|
342
|
+
// tracked rather than handled speculatively here.
|
|
343
|
+
export const ACCESS_INVALIDATION_CONSUMER_NAME = "system:consumer:access-invalidation";
|
|
344
|
+
|
|
345
|
+
const SESSION_REVOKED_EVENT_TYPE = "sessions:event:session-revoked";
|
|
346
|
+
const TENANT_MEMBERSHIP_UPDATED_EVENT_TYPE = "tenant-membership.updated";
|
|
347
|
+
const TENANT_MEMBERSHIP_DELETED_EVENT_TYPE = "tenant-membership.deleted";
|
|
348
|
+
|
|
349
|
+
function readUserIdFromPreviousSnapshot(payload: Record<string, unknown>): string | undefined {
|
|
350
|
+
const previous = payload["previous"];
|
|
351
|
+
if (typeof previous !== "object" || previous === null) return undefined;
|
|
352
|
+
const userId = (previous as Record<string, unknown>)["userId"];
|
|
353
|
+
return typeof userId === "string" && userId.length > 0 ? userId : undefined;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
export function createAccessInvalidationEventConsumer(sseBroker: SseBroker): EventConsumer {
|
|
357
|
+
return {
|
|
358
|
+
name: ACCESS_INVALIDATION_CONSUMER_NAME,
|
|
359
|
+
// Per-instance, same reasoning as SSE broadcast: subscribeAccessInvalidation
|
|
360
|
+
// listeners live in this process's in-memory sseBroker only. A shared
|
|
361
|
+
// cursor would deliver the event to exactly one instance and leave
|
|
362
|
+
// every other instance's open streams for that user un-invalidated.
|
|
363
|
+
delivery: "per-instance",
|
|
364
|
+
handler: async (event) => {
|
|
365
|
+
if (event.type === SESSION_REVOKED_EVENT_TYPE) {
|
|
366
|
+
const userId = event.payload["userId"];
|
|
367
|
+
// skip: malformed session-revoked payload — fail open on this one
|
|
368
|
+
// event rather than dead-lettering the whole consumer (halt-on-
|
|
369
|
+
// poison would otherwise permanently stop access-invalidation for
|
|
370
|
+
// every user behind one bad row).
|
|
371
|
+
if (typeof userId !== "string" || userId.length === 0) return;
|
|
372
|
+
sseBroker.publishAccessInvalidation(userId);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
if (
|
|
376
|
+
event.type === TENANT_MEMBERSHIP_UPDATED_EVENT_TYPE ||
|
|
377
|
+
event.type === TENANT_MEMBERSHIP_DELETED_EVENT_TYPE
|
|
378
|
+
) {
|
|
379
|
+
const userId = readUserIdFromPreviousSnapshot(event.payload);
|
|
380
|
+
// skip: previous snapshot missing/malformed userId — same fail-open
|
|
381
|
+
// reasoning as above.
|
|
382
|
+
if (userId === undefined) return;
|
|
383
|
+
sseBroker.publishAccessInvalidation(userId);
|
|
384
|
+
}
|
|
385
|
+
},
|
|
386
|
+
};
|
|
387
|
+
}
|
|
@@ -169,6 +169,24 @@ describe("createRateLimitResolver — peek", () => {
|
|
|
169
169
|
});
|
|
170
170
|
});
|
|
171
171
|
|
|
172
|
+
describe("createRateLimitResolver — kumiko-framework#1525: no ambient Temporal global", () => {
|
|
173
|
+
test("check() and peek() compute resetAt without relying on globalThis.Temporal", async () => {
|
|
174
|
+
const config = { limit: 5, windowSeconds: 60 };
|
|
175
|
+
const savedGlobal = (globalThis as { Temporal?: unknown }).Temporal;
|
|
176
|
+
delete (globalThis as { Temporal?: unknown }).Temporal;
|
|
177
|
+
try {
|
|
178
|
+
const checked = await resolver.check("no-ambient:user", config);
|
|
179
|
+
expect(checked.resetAt).toBeDefined();
|
|
180
|
+
|
|
181
|
+
const peeked = await resolver.peek("no-ambient:user", config);
|
|
182
|
+
expect(peeked.resetAt).toBeDefined();
|
|
183
|
+
} finally {
|
|
184
|
+
if (savedGlobal === undefined) delete (globalThis as { Temporal?: unknown }).Temporal;
|
|
185
|
+
else (globalThis as { Temporal?: unknown }).Temporal = savedGlobal;
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
});
|
|
189
|
+
|
|
172
190
|
describe("createRateLimitResolver — enforce", () => {
|
|
173
191
|
test("enforce throws RateLimitError with the bucket details when blocked", async () => {
|
|
174
192
|
const config = { limit: 1, windowSeconds: 60 };
|
|
@@ -1,4 +1,13 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
RateLimitConfig,
|
|
3
|
+
RateLimitDecision,
|
|
4
|
+
RateLimitResolver,
|
|
5
|
+
} from "@cosmicdrift/kumiko-types/rate-limit-types";
|
|
1
6
|
import type Redis from "ioredis";
|
|
7
|
+
// Value-only import, aliased to avoid shadowing the ambient global
|
|
8
|
+
// `Temporal` TYPE that RateLimitDecision.resetAt resolves against (see
|
|
9
|
+
// event-store.ts for the same #1438 dual-package-hazard pattern).
|
|
10
|
+
import { Temporal as TemporalPolyfill } from "temporal-polyfill";
|
|
2
11
|
import { RateLimitError } from "../errors";
|
|
3
12
|
import { RedisKeys } from "../pipeline/redis-keys";
|
|
4
13
|
|
|
@@ -108,36 +117,11 @@ end
|
|
|
108
117
|
return { math.floor(tokens), retryAfterMs }
|
|
109
118
|
`;
|
|
110
119
|
|
|
111
|
-
export type
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
readonly windowSeconds: number;
|
|
117
|
-
readonly resetAt: Temporal.Instant;
|
|
118
|
-
};
|
|
119
|
-
|
|
120
|
-
export type RateLimitConfig = {
|
|
121
|
-
readonly limit: number;
|
|
122
|
-
readonly windowSeconds: number;
|
|
123
|
-
readonly cost?: number;
|
|
124
|
-
};
|
|
125
|
-
|
|
126
|
-
export type RateLimitResolver = {
|
|
127
|
-
// Atomic check + deduct. Returns the decision and current bucket state
|
|
128
|
-
// — caller decides whether to throw RateLimitError or proceed.
|
|
129
|
-
check(bucket: string, config: RateLimitConfig): Promise<RateLimitDecision>;
|
|
130
|
-
|
|
131
|
-
// Convenience: throws RateLimitError when blocked. Useful inside the
|
|
132
|
-
// dispatcher / middleware code-paths where the failure shape is fixed.
|
|
133
|
-
enforce(bucket: string, config: RateLimitConfig): Promise<RateLimitDecision>;
|
|
134
|
-
|
|
135
|
-
// Read-only inspection: returns the same shape as check() but never
|
|
136
|
-
// mutates the bucket — no token deduction, no refill-timestamp update.
|
|
137
|
-
// Use for ops/status queries (e.g. "kumiko rl status user:42") that
|
|
138
|
-
// must observe the bucket without disturbing it.
|
|
139
|
-
peek(bucket: string, config: Omit<RateLimitConfig, "cost">): Promise<RateLimitDecision>;
|
|
140
|
-
};
|
|
120
|
+
export type {
|
|
121
|
+
RateLimitConfig,
|
|
122
|
+
RateLimitDecision,
|
|
123
|
+
RateLimitResolver,
|
|
124
|
+
} from "@cosmicdrift/kumiko-types/rate-limit-types";
|
|
141
125
|
|
|
142
126
|
export type RateLimitResolverOptions = {
|
|
143
127
|
readonly redis: Redis;
|
|
@@ -209,7 +193,7 @@ export function createRateLimitResolver(opts: RateLimitResolverOptions): RateLim
|
|
|
209
193
|
);
|
|
210
194
|
|
|
211
195
|
const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
|
|
212
|
-
const resetAt =
|
|
196
|
+
const resetAt = TemporalPolyfill.Instant.fromEpochMilliseconds(nowMs + retryAfterMs);
|
|
213
197
|
|
|
214
198
|
return {
|
|
215
199
|
allowed: allowedFlag === 1,
|
|
@@ -249,7 +233,7 @@ export function createRateLimitResolver(opts: RateLimitResolverOptions): RateLim
|
|
|
249
233
|
);
|
|
250
234
|
|
|
251
235
|
const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
|
|
252
|
-
const resetAt =
|
|
236
|
+
const resetAt = TemporalPolyfill.Instant.fromEpochMilliseconds(nowMs + retryAfterMs);
|
|
253
237
|
|
|
254
238
|
return {
|
|
255
239
|
// peek doesn't deduct, so a "would-be" allowed flag is meaningful:
|