@cosmicdrift/kumiko-framework 0.413.1 → 0.414.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -2
- package/agents/api/framework-db.md +14 -3
- package/agents/api/framework-engine.md +63 -13
- package/agents/api/framework-event-store.md +14 -0
- package/agents/api/framework-ui-types.md +149 -21
- package/agents/api/renderer-web.md +93 -9
- package/agents/api/testing-helpers.md +3 -2
- package/agents/api/testing-stack.md +3 -0
- package/agents/api/types.md +205 -14
- package/agents/api.md +7 -7
- package/agents/boot-rules.md +63 -9
- package/agents/examples/api-docs.md +2 -6
- package/agents/examples/auth-mfa.md +57 -10
- package/agents/examples/chat-channels.md +2 -0
- package/agents/examples/currencies-global.md +1 -0
- package/agents/examples/env-composition.md +7 -11
- package/agents/examples/managed-pages.md +90 -1
- package/agents/examples/pwa-shell.md +210 -0
- package/agents/examples/rate-limiting.md +3 -3
- package/agents/examples/record-detail-layout.md +1537 -67
- package/agents/examples/workflow-engine.md +16 -12
- package/agents/examples.md +1 -0
- package/agents/features-index.md +4 -2
- package/agents/features.md +22 -7
- package/agents/widgets.md +19 -1
- package/dist/api/api-constants.d.ts +1 -0
- package/dist/api/api-constants.js +4 -0
- package/dist/api/auth-middleware.js +4 -3
- package/dist/api/auth-routes-shared.js +2 -1
- package/dist/api/entity-read-access.d.ts +1 -0
- package/dist/api/entity-read-access.js +18 -0
- package/dist/api/export/export-route.d.ts +6 -1
- package/dist/api/export/export-route.js +27 -1
- package/dist/api/export/export-rows.js +19 -3
- package/dist/api/routes.d.ts +2 -0
- package/dist/api/routes.js +9 -1
- package/dist/api/server-extra-routes.js +2 -1
- package/dist/api/server-feature-http-routes.d.ts +9 -0
- package/dist/api/server-feature-http-routes.js +109 -0
- package/dist/api/server-middleware.js +2 -19
- package/dist/api/server-pat-rate-limit-guard.d.ts +3 -0
- package/dist/api/server-pat-rate-limit-guard.js +19 -0
- package/dist/api/server-rate-limit-wiring.d.ts +10 -0
- package/dist/api/server-rate-limit-wiring.js +37 -0
- package/dist/api/server-routes.d.ts +4 -12
- package/dist/api/server-routes.js +12 -171
- package/dist/api/server-schema-route.d.ts +3 -0
- package/dist/api/server-schema-route.js +58 -0
- package/dist/api/server-wiring.js +2 -37
- package/dist/api/server.js +3 -1
- package/dist/compliance/override-schema.js +5 -5
- package/dist/compliance/sub-processors.js +10 -1
- package/dist/crypto/index.d.ts +1 -1
- package/dist/crypto/index.js +1 -1
- package/dist/crypto/pii-field-encryption.d.ts +2 -0
- package/dist/crypto/pii-field-encryption.js +16 -1
- package/dist/db/apply-entity-event.js +4 -0
- package/dist/db/bun-sql-connection-codes.d.ts +1 -0
- package/dist/db/bun-sql-connection-codes.js +7 -0
- package/dist/db/collect-table-metas.js +4 -1
- package/dist/db/config-seed.js +1 -1
- package/dist/db/derived-field-sql.js +15 -13
- package/dist/db/derived-related-sql.d.ts +2 -2
- package/dist/db/derived-related-sql.js +4 -2
- package/dist/db/dialect.d.ts +3 -0
- package/dist/db/dialect.js +1 -3
- package/dist/db/driver-row-coercion.js +2 -2
- package/dist/db/driver-value-preparation.js +1 -1
- package/dist/db/eagerload.js +15 -12
- package/dist/db/entity-table-meta.d.ts +2 -0
- package/dist/db/entity-table-meta.js +7 -0
- package/dist/db/event-store-executor-context.js +13 -8
- package/dist/db/event-store-executor-read-aggregate-currencies.js +2 -2
- package/dist/db/event-store-executor-read-aggregate.js +2 -2
- package/dist/db/event-store-executor-read-filter.d.ts +4 -0
- package/dist/db/event-store-executor-read-filter.js +28 -11
- package/dist/db/event-store-executor-read-search.d.ts +1 -0
- package/dist/db/event-store-executor-read-search.js +6 -6
- package/dist/db/event-store-executor-read-sort.js +3 -3
- package/dist/db/event-store-executor-read-where.js +14 -5
- package/dist/db/event-store-executor-read.js +7 -7
- package/dist/db/event-store-executor-write.js +50 -95
- package/dist/db/executor-managed-tables.js +4 -1
- package/dist/db/index.d.ts +4 -2
- package/dist/db/index.js +4 -2
- package/dist/db/known-table-names.d.ts +1 -0
- package/dist/db/known-table-names.js +7 -5
- package/dist/db/list-aggregate-labels.d.ts +1 -0
- package/dist/db/list-aggregate-labels.js +9 -1
- package/dist/db/migrate-generator.js +23 -4
- package/dist/db/migrate-rls-diff.d.ts +1 -0
- package/dist/db/migrate-rls-diff.js +15 -1
- package/dist/db/migrate-runner.js +2 -2
- package/dist/db/money.d.ts +1 -1
- package/dist/db/money.js +1 -4
- package/dist/db/parent-ref-clause.d.ts +6 -2
- package/dist/db/parent-ref-clause.js +76 -23
- package/dist/db/parent-ref-global-hosts.d.ts +20 -0
- package/dist/db/parent-ref-global-hosts.js +47 -0
- package/dist/db/production-schema-command-hint.d.ts +1 -0
- package/dist/db/production-schema-command-hint.js +3 -0
- package/dist/db/queries/backfill-pii.js +55 -29
- package/dist/db/queries/ddl.js +13 -2
- package/dist/db/queries/event-store-open-streams.d.ts +15 -0
- package/dist/db/queries/event-store-open-streams.js +33 -0
- package/dist/db/queries/event-store-payload-scrub.d.ts +5 -0
- package/dist/db/queries/event-store-payload-scrub.js +11 -0
- package/dist/db/queries/event-store.js +0 -1
- package/dist/db/queries/raw-sql.js +0 -1
- package/dist/db/queries/rebuild-rls-guard.d.ts +2 -1
- package/dist/db/queries/rebuild-rls-guard.js +24 -11
- package/dist/db/queries/rls-boot-catalog.d.ts +1 -0
- package/dist/db/queries/rls-boot-catalog.js +13 -8
- package/dist/db/queries/rls-shape.d.ts +2 -1
- package/dist/db/queries/rls-shape.js +10 -0
- package/dist/db/queries/session-advisory-lock.js +2 -1
- package/dist/db/queries/test-stack.js +7 -7
- package/dist/db/query.js +16 -11
- package/dist/db/rebuild-marker.js +11 -2
- package/dist/db/render-rls-ddl.d.ts +2 -0
- package/dist/db/render-rls-ddl.js +16 -11
- package/dist/db/replay-migration-sql.js +1 -1
- package/dist/db/row-level-security-boot-check.js +18 -5
- package/dist/db/row-level-security-connection.js +25 -12
- package/dist/db/row-level-security-scope.d.ts +2 -1
- package/dist/db/row-level-security-scope.js +9 -5
- package/dist/db/sql-identifier.d.ts +1 -0
- package/dist/db/sql-identifier.js +3 -0
- package/dist/db/system-tenant-read-db.d.ts +5 -0
- package/dist/db/system-tenant-read-db.js +28 -0
- package/dist/db/table-builder.d.ts +3 -1
- package/dist/db/table-builder.js +18 -8
- package/dist/db/table-info.d.ts +1 -2
- package/dist/db/table-introspection.d.ts +3 -0
- package/dist/db/table-introspection.js +6 -0
- package/dist/db/table-push.d.ts +1 -0
- package/dist/db/table-push.js +7 -1
- package/dist/db/tenant-db-runner.d.ts +3 -0
- package/dist/db/tenant-db-runner.js +19 -1
- package/dist/db/transaction-serialization.d.ts +1 -0
- package/dist/db/transaction-serialization.js +20 -3
- package/dist/db/where-clause.js +11 -2
- package/dist/db/write-ownership-failures.d.ts +25 -0
- package/dist/db/write-ownership-failures.js +82 -0
- package/dist/engine/boot-validator/boot-check.js +7 -1
- package/dist/engine/boot-validator/boot-phases-app.js +5 -5
- package/dist/engine/boot-validator/boot-phases.js +3 -2
- package/dist/engine/boot-validator/boot-rules.d.ts +293 -26
- package/dist/engine/boot-validator/boot-rules.js +4 -0
- package/dist/engine/boot-validator/bundled-lists-table.js +2 -2
- package/dist/engine/boot-validator/card-meta-overflow.js +1 -0
- package/dist/engine/boot-validator/config-deps.d.ts +0 -1
- package/dist/engine/boot-validator/config-deps.js +0 -28
- package/dist/engine/boot-validator/config-key-rules.d.ts +0 -15
- package/dist/engine/boot-validator/config-key-rules.js +0 -15
- package/dist/engine/boot-validator/dashboard-screen-target.d.ts +22 -0
- package/dist/engine/boot-validator/dashboard-screen-target.js +20 -0
- package/dist/engine/boot-validator/dashboard-stat-navigate.d.ts +1 -0
- package/dist/engine/boot-validator/dashboard-stat-navigate.js +25 -17
- package/dist/engine/boot-validator/definition-webhook-events-rules.d.ts +5 -0
- package/dist/engine/boot-validator/definition-webhook-events-rules.js +5 -0
- package/dist/engine/boot-validator/derived-sql-fields.js +1 -2
- package/dist/engine/boot-validator/entity-declaration-rules.d.ts +5 -0
- package/dist/engine/boot-validator/entity-declaration-rules.js +5 -0
- package/dist/engine/boot-validator/entity-list-screens.d.ts +1 -0
- package/dist/engine/boot-validator/entity-list-screens.js +33 -6
- package/dist/engine/boot-validator/entity-structure-rules.d.ts +5 -0
- package/dist/engine/boot-validator/entity-structure-rules.js +5 -0
- package/dist/engine/boot-validator/list-column-layout.js +10 -6
- package/dist/engine/boot-validator/list-row-guidance-rules.d.ts +67 -0
- package/dist/engine/boot-validator/list-row-guidance-rules.js +67 -0
- package/dist/engine/boot-validator/list-row-guidance.d.ts +20 -0
- package/dist/engine/boot-validator/list-row-guidance.js +129 -0
- package/dist/engine/boot-validator/list-sort-summary-rules.d.ts +42 -0
- package/dist/engine/boot-validator/list-sort-summary-rules.js +42 -0
- package/dist/engine/boot-validator/list-sort-summary.d.ts +5 -0
- package/dist/engine/boot-validator/list-sort-summary.js +149 -0
- package/dist/engine/boot-validator/list-table-options.js +15 -1
- package/dist/engine/boot-validator/list-table-rules.d.ts +10 -5
- package/dist/engine/boot-validator/list-table-rules.js +10 -5
- package/dist/engine/boot-validator/list-toolbar-options.d.ts +10 -0
- package/dist/engine/boot-validator/list-toolbar-options.js +31 -0
- package/dist/engine/boot-validator/named-filter-rules.d.ts +32 -2
- package/dist/engine/boot-validator/named-filter-rules.js +32 -2
- package/dist/engine/boot-validator/ownership.js +10 -1
- package/dist/engine/boot-validator/parent-ref.js +87 -50
- package/dist/engine/boot-validator/projection-list-screens.d.ts +1 -0
- package/dist/engine/boot-validator/projection-list-screens.js +31 -1
- package/dist/engine/boot-validator/query-output-columns.js +62 -9
- package/dist/engine/boot-validator/query-output-guidance.d.ts +7 -0
- package/dist/engine/boot-validator/query-output-guidance.js +39 -0
- package/dist/engine/boot-validator/query-refs.js +24 -1
- package/dist/engine/boot-validator/row-decoration.d.ts +11 -0
- package/dist/engine/boot-validator/row-decoration.js +16 -0
- package/dist/engine/boot-validator/screens-dashboard-links.d.ts +10 -0
- package/dist/engine/boot-validator/screens-dashboard-links.js +36 -0
- package/dist/engine/boot-validator/screens-dashboard-narrow.d.ts +4 -0
- package/dist/engine/boot-validator/screens-dashboard-narrow.js +211 -0
- package/dist/engine/boot-validator/screens-dashboard-rules.d.ts +84 -4
- package/dist/engine/boot-validator/screens-dashboard-rules.js +84 -4
- package/dist/engine/boot-validator/screens-dashboard.js +29 -7
- package/dist/engine/boot-validator/screens-kind-entity-list.js +32 -27
- package/dist/engine/boot-validator/screens-kind-projection-detail-guidance.d.ts +3 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail-guidance.js +46 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.d.ts +20 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.js +20 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail.js +2 -0
- package/dist/engine/boot-validator/screens-kind-projection-list.js +4 -0
- package/dist/engine/boot-validator/screens-kind-rules.d.ts +25 -0
- package/dist/engine/boot-validator/screens-kind-rules.js +25 -0
- package/dist/engine/boot-validator/screens-lists.d.ts +8 -2
- package/dist/engine/boot-validator/screens-lists.js +19 -3
- package/dist/engine/boot-validator/screens.js +1 -1
- package/dist/engine/boot-validator/sort-nulls.d.ts +2 -0
- package/dist/engine/boot-validator/sort-nulls.js +15 -0
- package/dist/engine/boot-validator/zod-shape.d.ts +2 -1
- package/dist/engine/boot-validator/zod-shape.js +19 -1
- package/dist/engine/build-app-schema.js +4 -0
- package/dist/engine/config-helpers.d.ts +1 -2
- package/dist/engine/config-helpers.js +6 -7
- package/dist/engine/define-feature.js +1 -0
- package/dist/engine/define-handler.d.ts +1 -0
- package/dist/engine/define-handler.js +4 -2
- package/dist/engine/definition-violations.js +2 -2
- package/dist/engine/entity-handlers.js +3 -8
- package/dist/engine/entity-list-runtime-options.js +9 -8
- package/dist/engine/factories.d.ts +2 -0
- package/dist/engine/feature-config-events-jobs.js +1 -0
- package/dist/engine/feature-entity-handlers.js +5 -2
- package/dist/engine/field-access.d.ts +1 -1
- package/dist/engine/field-access.js +7 -10
- package/dist/engine/find-entity.js +3 -3
- package/dist/engine/form-derivations.js +3 -3
- package/dist/engine/index.d.ts +3 -3
- package/dist/engine/index.js +2 -2
- package/dist/engine/named-filter-roles.d.ts +3 -0
- package/dist/engine/named-filter-roles.js +23 -0
- package/dist/engine/pipeline.js +0 -1
- package/dist/engine/project-app-schema-for-roles.js +17 -2
- package/dist/engine/qualified-name.js +0 -1
- package/dist/engine/registry-facade.js +1 -2
- package/dist/engine/registry-state.d.ts +1 -1
- package/dist/engine/registry-state.js +5 -2
- package/dist/engine/registry-validate.js +2 -1
- package/dist/engine/resolve-named-list-filters.js +5 -2
- package/dist/engine/schema-builder.js +6 -10
- package/dist/engine/screen-helpers.d.ts +12 -0
- package/dist/engine/screen-helpers.js +17 -1
- package/dist/engine/soft-delete-cleanup.js +1 -5
- package/dist/engine/state-machine.js +0 -1
- package/dist/engine/steps/_database-clock.d.ts +2 -0
- package/dist/engine/steps/_database-clock.js +6 -0
- package/dist/engine/steps/compute.js +1 -1
- package/dist/engine/steps/retry.js +5 -1
- package/dist/engine/steps/return.js +0 -1
- package/dist/engine/steps/wait-for-event.js +2 -1
- package/dist/engine/steps/wait.js +2 -1
- package/dist/engine/system-user.d.ts +2 -2
- package/dist/engine/system-user.js +2 -2
- package/dist/engine/types/handlers.d.ts +1 -1
- package/dist/engine/types/handlers.js +1 -1
- package/dist/engine/types/screen.d.ts +1 -1
- package/dist/engine/types/screen.js +1 -1
- package/dist/engine/webhook-contracts.d.ts +2 -3
- package/dist/engine/webhook-contracts.js +2 -25
- package/dist/engine/webhook-events.js +48 -29
- package/dist/engine/webhook-payload-strictness.d.ts +7 -0
- package/dist/engine/webhook-payload-strictness.js +53 -0
- package/dist/env/_zod-introspect.d.ts +1 -0
- package/dist/env/dry-run.d.ts +1 -1
- package/dist/env/dry-run.js +1 -1
- package/dist/env/field-meta.d.ts +4 -0
- package/dist/env/field-meta.js +67 -0
- package/dist/env/field-rules.d.ts +9 -0
- package/dist/env/field-rules.js +46 -0
- package/dist/env/index.d.ts +3 -3
- package/dist/env/index.js +16 -71
- package/dist/env/internals.d.ts +3 -0
- package/dist/env/internals.js +1 -2
- package/dist/env/required-env.d.ts +3 -1
- package/dist/env/required-env.js +5 -2
- package/dist/errors/serialize.js +2 -2
- package/dist/errors/write-error-info.js +0 -2
- package/dist/es-ops/operations-schema.js +18 -7
- package/dist/event-store/admin-api.js +3 -3
- package/dist/event-store/event-store.js +0 -3
- package/dist/event-store/index.d.ts +1 -0
- package/dist/event-store/index.js +1 -0
- package/dist/event-store/transfer.js +1 -1
- package/dist/files/assert-file-refs-belong-to-tenant.d.ts +2 -1
- package/dist/files/assert-file-refs-belong-to-tenant.js +15 -13
- package/dist/files/bind-pending-file-refs.d.ts +3 -1
- package/dist/files/bind-pending-file-refs.js +54 -27
- package/dist/files/copy-pending-file-refs-to-tenant.js +7 -3
- package/dist/files/create-file-ref.d.ts +4 -1
- package/dist/files/create-file-ref.js +14 -3
- package/dist/files/default-file-guard.js +4 -1
- package/dist/files/file-claim.d.ts +6 -0
- package/dist/files/file-claim.js +17 -0
- package/dist/files/file-field-slots.d.ts +2 -0
- package/dist/files/file-field-slots.js +28 -0
- package/dist/files/file-ref-entity.d.ts +3 -0
- package/dist/files/file-ref-entity.js +2 -0
- package/dist/files/file-ref-table.d.ts +3 -0
- package/dist/files/file-routes.d.ts +4 -1
- package/dist/files/file-routes.js +80 -16
- package/dist/files/in-memory-provider.d.ts +1 -1
- package/dist/files/in-memory-provider.js +2 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +1 -1
- package/dist/files/invalid-file-ref-error.d.ts +2 -0
- package/dist/files/invalid-file-ref-error.js +15 -0
- package/dist/files/local-provider.js +9 -2
- package/dist/files/provider-resolver.js +1 -1
- package/dist/files/rebind-file-ref.d.ts +3 -2
- package/dist/files/rebind-file-ref.js +24 -18
- package/dist/files/resolve-file-handles.d.ts +13 -0
- package/dist/files/resolve-file-handles.js +57 -0
- package/dist/files/storage-tracking.d.ts +1 -1
- package/dist/files/storage-tracking.js +5 -2
- package/dist/files/sweep-stale-pending-file-refs.js +10 -4
- package/dist/files/types.d.ts +4 -0
- package/dist/files/types.js +10 -0
- package/dist/files/upload-field-resolution.d.ts +1 -0
- package/dist/files/upload-field-resolution.js +16 -2
- package/dist/files/upload-ledger.js +1 -1
- package/dist/i18n/required-surface-guidance-keys.d.ts +3 -0
- package/dist/i18n/required-surface-guidance-keys.js +9 -0
- package/dist/i18n/required-surface-keys.js +19 -4
- package/dist/internal/boot-rule-table-expectations.d.ts +2 -0
- package/dist/internal/boot-rule-table-expectations.js +29 -0
- package/dist/internal/row-level-security.d.ts +1 -0
- package/dist/internal/row-level-security.js +1 -0
- package/dist/internal/testing.d.ts +1 -0
- package/dist/internal/testing.js +1 -0
- package/dist/jobs/index.d.ts +1 -0
- package/dist/jobs/index.js +1 -0
- package/dist/jobs/job-load-events.d.ts +10 -0
- package/dist/jobs/job-load-events.js +12 -0
- package/dist/jobs/job-runner-context.js +6 -158
- package/dist/jobs/job-runner-dispatch.d.ts +7 -0
- package/dist/jobs/job-runner-dispatch.js +57 -0
- package/dist/jobs/job-runner-event-triggers.js +18 -9
- package/dist/jobs/job-runner-execution.js +2 -2
- package/dist/jobs/job-runner-facets.d.ts +48 -0
- package/dist/jobs/job-runner-facets.js +156 -0
- package/dist/jobs/job-runner-gates.js +19 -11
- package/dist/jobs/job-runner-helpers.d.ts +5 -1
- package/dist/jobs/job-runner-helpers.js +18 -3
- package/dist/jobs/job-runner-scope.d.ts +1 -0
- package/dist/jobs/job-runner-scope.js +2 -0
- package/dist/jobs/job-runner-tenant-cap-lock.d.ts +2 -0
- package/dist/jobs/job-runner-tenant-cap-lock.js +30 -0
- package/dist/jobs/job-runner.js +31 -70
- package/dist/jobs/obliterate-job-queues.d.ts +6 -0
- package/dist/jobs/obliterate-job-queues.js +23 -0
- package/dist/lifecycle/lifecycle.js +1 -1
- package/dist/migrations/event-header-scrub.d.ts +11 -0
- package/dist/migrations/event-header-scrub.js +33 -0
- package/dist/migrations/event-payload-scrub.d.ts +1 -1
- package/dist/migrations/event-payload-scrub.js +36 -4
- package/dist/migrations/index.d.ts +1 -0
- package/dist/migrations/index.js +2 -0
- package/dist/migrations/kumiko-drift.d.ts +5 -2
- package/dist/migrations/kumiko-drift.js +51 -14
- package/dist/migrations/pending-rebuilds.js +4 -5
- package/dist/observability/sensitive-filter.js +0 -12
- package/dist/pipeline/aggregate-type-removal.js +17 -4
- package/dist/pipeline/context-services.d.ts +2 -0
- package/dist/pipeline/context-services.js +5 -2
- package/dist/pipeline/dispatch-batch.js +4 -3
- package/dist/pipeline/dispatch-context-helpers.d.ts +11 -0
- package/dist/pipeline/dispatch-context-helpers.js +58 -0
- package/dist/pipeline/dispatch-nested-write.d.ts +7 -0
- package/dist/pipeline/dispatch-nested-write.js +119 -0
- package/dist/pipeline/dispatch-query.js +10 -3
- package/dist/pipeline/dispatch-scope.js +4 -3
- package/dist/pipeline/dispatch-shared.d.ts +2 -6
- package/dist/pipeline/dispatch-shared.js +6 -38
- package/dist/pipeline/dispatch-write.d.ts +0 -1
- package/dist/pipeline/dispatch-write.js +3 -302
- package/dist/pipeline/dispatcher-utils.js +6 -0
- package/dist/pipeline/effective-system-scope.d.ts +3 -0
- package/dist/pipeline/effective-system-scope.js +22 -0
- package/dist/pipeline/event-dispatcher-delivery.d.ts +1 -0
- package/dist/pipeline/event-dispatcher-delivery.js +17 -9
- package/dist/pipeline/event-dispatcher.d.ts +4 -1
- package/dist/pipeline/event-dispatcher.js +6 -7
- package/dist/pipeline/handler-context-auth.d.ts +3 -2
- package/dist/pipeline/handler-context-auth.js +3 -4
- package/dist/pipeline/handler-context-db.js +11 -4
- package/dist/pipeline/handler-context-dispatch.js +2 -2
- package/dist/pipeline/handler-context-events.d.ts +8 -3
- package/dist/pipeline/handler-context-events.js +20 -65
- package/dist/pipeline/handler-context-member-resolution.js +16 -8
- package/dist/pipeline/handler-context-notify.d.ts +1 -0
- package/dist/pipeline/handler-context-notify.js +2 -0
- package/dist/pipeline/handler-context-phase-denials.js +3 -2
- package/dist/pipeline/handler-context-projection.d.ts +3 -0
- package/dist/pipeline/handler-context-projection.js +39 -0
- package/dist/pipeline/handler-context-scope.d.ts +1 -1
- package/dist/pipeline/handler-context-scope.js +14 -2
- package/dist/pipeline/index.d.ts +1 -1
- package/dist/pipeline/index.js +1 -1
- package/dist/pipeline/lifecycle-pipeline.d.ts +1 -0
- package/dist/pipeline/lifecycle-pipeline.js +7 -4
- package/dist/pipeline/member-reader.js +4 -2
- package/dist/pipeline/msp-rebuild.js +1 -1
- package/dist/pipeline/nested-write-parent-guard.d.ts +5 -0
- package/dist/pipeline/nested-write-parent-guard.js +81 -0
- package/dist/pipeline/projection-rebuild.d.ts +2 -0
- package/dist/pipeline/projection-rebuild.js +16 -7
- package/dist/pipeline/projection-state.js +4 -6
- package/dist/pipeline/system-hooks.js +36 -6
- package/dist/pipeline/write-result-masking.js +31 -3
- package/dist/pipeline/write-transition-guard.d.ts +7 -0
- package/dist/pipeline/write-transition-guard.js +95 -0
- package/dist/schema-cli/shared.d.ts +2 -0
- package/dist/schema-cli/shared.js +4 -2
- package/dist/schema-cli/validate-command.js +43 -28
- package/dist/search/meilisearch-adapter.js +14 -9
- package/dist/search/reindex-entity.d.ts +2 -0
- package/dist/search/reindex-entity.js +32 -4
- package/dist/search/reindex-tenants-search.js +20 -4
- package/dist/time/tz-context.js +1 -1
- package/dist/ui-types/card-column-roles.d.ts +4 -0
- package/dist/ui-types/card-column-roles.js +6 -3
- package/dist/ui-types/index.d.ts +2 -2
- package/dist/ui-types/index.js +1 -1
- package/dist/utils/ids.js +0 -1
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.js +1 -0
- package/dist/utils/return-path-limit.d.ts +1 -0
- package/dist/utils/return-path-limit.js +1 -0
- package/package.json +5 -3
- package/presets/biome-app.json +39 -21
- package/presets/biome-plugins/admin-api.grit +10 -0
- package/presets/biome-plugins/error-reasons.grit +53 -0
- package/presets/biome-plugins/es-literal-writes-tests.grit +18 -0
- package/presets/biome-plugins/es-literal-writes.grit +22 -0
- package/presets/biome-plugins/i18n-locale-terminology-de.grit +15 -0
- package/presets/biome-plugins/i18n-locale-terminology-romance.grit +10 -0
- package/presets/biome-plugins/no-create-test-dispatcher.grit +5 -0
- package/presets/biome-plugins/no-css-properties-type.grit +7 -0
- package/presets/biome-plugins/no-date-api.grit +9 -0
- package/presets/biome-plugins/no-direct-fetch.grit +16 -0
- package/presets/biome-plugins/no-direct-fs.grit +16 -0
- package/presets/biome-plugins/no-external-font-cdn-css.grit +9 -0
- package/presets/biome-plugins/no-external-font-cdn.grit +12 -0
- package/presets/biome-plugins/no-raw-hooks.grit +10 -0
- package/presets/biome-plugins/no-same-folder-barrel-import.grit +11 -0
- package/presets/biome-plugins/no-screen-container-classes.grit +7 -0
- package/presets/biome-plugins/no-skipped-or-retried-tests.grit +34 -0
- package/presets/biome-plugins/no-temporal-polyfill-require.grit +10 -0
- package/presets/biome-plugins/raw-html-in-web.grit +40 -0
- package/presets/biome-plugins/raw-ui-in-app.grit +142 -0
- package/presets/biome-plugins/restricted-event-store-symbols.grit +18 -0
- package/presets/biome-plugins/test-title.grit +20 -0
- package/presets/biome-plugins/unsafe-json-parse.grit +6 -0
- package/presets/biome-rules.json +465 -0
- package/src/changes.json +309 -0
- package/dist/engine/minor-units.d.ts +0 -2
- package/dist/engine/minor-units.js +0 -8
|
@@ -129,6 +129,10 @@ export function priorAttemptsOf(data) {
|
|
|
129
129
|
const value = data["_priorAttempts"];
|
|
130
130
|
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
|
|
131
131
|
}
|
|
132
|
+
// Only a run picked up from the queue can be moved to delayed; an inline boot-gate run has neither token nor moveToDelayed.
|
|
133
|
+
export function isQueuedRun(bullJob, token) {
|
|
134
|
+
return token !== undefined && bullJob.moveToDelayed !== undefined;
|
|
135
|
+
}
|
|
132
136
|
export function attemptNumberOf(bullJob) {
|
|
133
137
|
return bullJob.attemptsMade + 1 + priorAttemptsOf(bullJob.data);
|
|
134
138
|
}
|
|
@@ -202,19 +206,30 @@ export async function startQueueDepthPolling(queue, lane, meter) {
|
|
|
202
206
|
await pollQueueDepth();
|
|
203
207
|
return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
|
|
204
208
|
}
|
|
209
|
+
// A credential with a stray "%" is accepted by URL but not decodable; ioredis then uses it as written.
|
|
210
|
+
function decodeCredential(raw) {
|
|
211
|
+
try {
|
|
212
|
+
return decodeURIComponent(raw);
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
// skip: not percent-encoded, the raw value is the credential
|
|
216
|
+
return raw;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
205
219
|
export function parseRedisOpts(url) {
|
|
206
220
|
const parsed = new URL(url);
|
|
207
221
|
const result = {
|
|
208
|
-
|
|
222
|
+
// URL keeps the brackets of an IPv6 literal; ioredis and BullMQ need the bare address.
|
|
223
|
+
host: parsed.hostname.replace(/^\[(.*)\]$/, "$1"),
|
|
209
224
|
port: Number(parsed.port) || 6379,
|
|
210
225
|
};
|
|
211
226
|
if (parsed.pathname.length > 1) {
|
|
212
227
|
result.db = Number(parsed.pathname.slice(1));
|
|
213
228
|
}
|
|
214
229
|
if (parsed.username)
|
|
215
|
-
result.username =
|
|
230
|
+
result.username = decodeCredential(parsed.username);
|
|
216
231
|
if (parsed.password)
|
|
217
|
-
result.password =
|
|
232
|
+
result.password = decodeCredential(parsed.password);
|
|
218
233
|
if (parsed.protocol === "rediss:")
|
|
219
234
|
result.tls = {};
|
|
220
235
|
return result;
|
|
@@ -17,6 +17,7 @@ export type JobRunnerScope = {
|
|
|
17
17
|
readonly tracer: Tracer;
|
|
18
18
|
readonly errorLogger: ReturnType<typeof createFallbackLogger>;
|
|
19
19
|
readonly sequentialLock: DistributedLock | null;
|
|
20
|
+
readonly tenantCapLock: DistributedLock | null;
|
|
20
21
|
readonly getActiveTenantIds: (() => Promise<TenantId[]>) | undefined;
|
|
21
22
|
readonly escapeHatchReportWindow: EscapeHatchReportWindow;
|
|
22
23
|
readonly systemCronEscapeHatchDedup: EscapeHatchProcessDedup;
|
|
@@ -16,6 +16,8 @@ export function laneForJob(def) {
|
|
|
16
16
|
// One getJobs per queue reads the requested lists in a single Lua script, so a job
|
|
17
17
|
// moving from waiting to active between two separate reads can no longer vanish
|
|
18
18
|
// from the count; that gap let a third dispatch pass the cap in CI.
|
|
19
|
+
// Only this snapshot is atomic: dispatch and handleEvent hold the tenant-cap lock across the
|
|
20
|
+
// check and the following queue.add, otherwise concurrent dispatches could pass together.
|
|
19
21
|
// `activeOnly` + `self` serve the defer check, which runs inside the job being
|
|
20
22
|
// counted: it must not see waiting jobs or itself, and only counts jobs that
|
|
21
23
|
// became active before it. Without that order, workers that fetch a burst of
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
const CAP_LOCK_TTL_SEC = 10;
|
|
2
|
+
const CAP_LOCK_POLL_MS = 15;
|
|
3
|
+
const CAP_LOCK_MAX_WAIT_MS = 3_000;
|
|
4
|
+
const releaseNothing = async () => { };
|
|
5
|
+
// Count-then-add is two steps; holding this lock across both makes the "skip" cap exact for
|
|
6
|
+
// every dispatcher sharing the Redis. The lock only guards the short check+add window, so a
|
|
7
|
+
// holder that crashes frees it via the TTL. A lock that stays contended past the wait budget
|
|
8
|
+
// degrades to the soft cap instead of failing the dispatch.
|
|
9
|
+
export async function acquireTenantCapLock(scope, jobName, tenantId) {
|
|
10
|
+
const { tenantCapLock } = scope;
|
|
11
|
+
if (!tenantCapLock)
|
|
12
|
+
return releaseNothing;
|
|
13
|
+
const key = `${jobName}:${tenantId}`;
|
|
14
|
+
const deadline = Date.now() + CAP_LOCK_MAX_WAIT_MS;
|
|
15
|
+
for (;;) {
|
|
16
|
+
const token = await tenantCapLock.acquire(key, { ttlSeconds: CAP_LOCK_TTL_SEC });
|
|
17
|
+
if (token) {
|
|
18
|
+
return async () => {
|
|
19
|
+
await tenantCapLock.release(key, token);
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
if (Date.now() >= deadline) {
|
|
23
|
+
scope.errorLogger.warn("maxPerTenant lock stayed contended, enforcing the cap softly", {
|
|
24
|
+
job: jobName,
|
|
25
|
+
});
|
|
26
|
+
return releaseNothing;
|
|
27
|
+
}
|
|
28
|
+
await new Promise((resolve) => setTimeout(resolve, CAP_LOCK_POLL_MS));
|
|
29
|
+
}
|
|
30
|
+
}
|
package/dist/jobs/job-runner.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { DelayedError, Queue, Worker } from "bullmq";
|
|
2
2
|
import { Redis } from "ioredis";
|
|
3
|
-
import { requestContext } from "../api/request-context.js";
|
|
4
3
|
import { assertNoActiveRowLevelSecurityScope, runInSystemRowLevelSecurityScope, } from "../db/row-level-security-scope.js";
|
|
5
4
|
import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
|
|
6
5
|
import { bridgeStub } from "../engine/bridge-stub.js";
|
|
@@ -15,12 +14,14 @@ import { attachOutsideTransaction } from "../pipeline/attach-outside-transaction
|
|
|
15
14
|
import { createDistributedLock } from "../pipeline/distributed-lock.js";
|
|
16
15
|
import { RedisKeys } from "../pipeline/redis-keys.js";
|
|
17
16
|
import { isScheduledTrigger, scheduleRepeatOptions } from "./cron-repeat-options.js";
|
|
17
|
+
import { applyConcurrencyMode, buildEnqueueInputs } from "./job-runner-dispatch.js";
|
|
18
18
|
import { createJobEventTriggers } from "./job-runner-event-triggers.js";
|
|
19
19
|
import { applyHandlerDeferral, executeJobInSpan } from "./job-runner-execution.js";
|
|
20
20
|
import { claimSequentialSlot, deferOverTenantLimit, fanOutPerTenantJob, PER_TENANT_PREFIX, perTenantTargetOf, releaseSequentialSlot, resolveJobDefinitionOrFail, startJobRun, } from "./job-runner-gates.js";
|
|
21
|
-
import { BOOT_GATE_SLOW_WARN_MS, BOOT_REDIS_TIMEOUT_MS, buildRetryBullOpts, COMPLETED_JOB_RETENTION_AGE_SEC,
|
|
21
|
+
import { BOOT_GATE_SLOW_WARN_MS, BOOT_REDIS_TIMEOUT_MS, buildRetryBullOpts, COMPLETED_JOB_RETENTION_AGE_SEC, FAILED_JOB_RETENTION_AGE_SEC, maxRetryWindowMs, parseRedisOpts, STOP_QUEUE_READY_TIMEOUT_MS, stampGatedWriteOrigin, startQueueDepthPolling, timeoutReject, } from "./job-runner-helpers.js";
|
|
22
22
|
import { ACTIVE_TENANT_IDS_QUERY_NAME, bootJobIdForJobName, DEFAULT_QUEUE_NAME_PREFIX, markSchedulersSeen, ORPHAN_SCHEDULER_GRACE_MS, queueNameFor, removeOrphanJobSchedulers, schedulerIdForJobName, } from "./job-runner-schedulers.js";
|
|
23
|
-
import {
|
|
23
|
+
import { announceQueued, isOverPerTenantLimit, laneForJob, } from "./job-runner-scope.js";
|
|
24
|
+
import { acquireTenantCapLock } from "./job-runner-tenant-cap-lock.js";
|
|
24
25
|
import { resolveMaxPerTenant } from "./max-per-tenant.js";
|
|
25
26
|
export { serializeJobSubject } from "./job-runner-helpers.js";
|
|
26
27
|
export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.js";
|
|
@@ -133,13 +134,15 @@ export function createJobRunner(options) {
|
|
|
133
134
|
// Sequential coordination: BullMQ OSS has no `group`, so we serialise
|
|
134
135
|
// same-name jobs ourselves with a per-name Redis lock. Only built when at
|
|
135
136
|
// least one job actually requested it — keeps the no-sequential boot path
|
|
136
|
-
// free of the extra Redis client. Scoped under the consumer lane so two
|
|
137
|
+
// free of the extra Redis client (also built for maxPerTenant "skip" caps). Scoped under the consumer lane so two
|
|
137
138
|
// runners on different lanes cannot collide on the same lock-key for
|
|
138
139
|
// unrelated jobs.
|
|
139
140
|
const hasSequential = [...allJobs.values()].some((def) => def.concurrency === "sequential");
|
|
141
|
+
const hasSkipCap = [...allJobs.values()].some((def) => resolveMaxPerTenant(def)?.onLimit === "skip");
|
|
140
142
|
let lockRedis = null;
|
|
141
143
|
let sequentialLock = null;
|
|
142
|
-
|
|
144
|
+
let tenantCapLock = null;
|
|
145
|
+
if (hasSequential || hasSkipCap) {
|
|
143
146
|
lockRedis = new Redis({ ...redisOpts, keyPrefix });
|
|
144
147
|
// Without a listener, a post-close 'error' (e.g. a teardown-race
|
|
145
148
|
// "Connection is closed") is unhandled and crashes the process — in
|
|
@@ -149,6 +152,8 @@ export function createJobRunner(options) {
|
|
|
149
152
|
// Scoped by queueNamePrefix like the queues themselves, so runners
|
|
150
153
|
// isolated by prefix on a shared Redis don't contend for the same lock.
|
|
151
154
|
sequentialLock = createDistributedLock(lockRedis, `${RedisKeys.lock}seq:${queueNamePrefix}:${lockScope}:`);
|
|
155
|
+
// No lane in the prefix: api and worker runners dispatch the same job and must share the lock.
|
|
156
|
+
tenantCapLock = createDistributedLock(lockRedis, `${RedisKeys.lock}cap:${queueNamePrefix}:`);
|
|
152
157
|
}
|
|
153
158
|
// Two queue-clients — one per lane. Every runner holds both, regardless of
|
|
154
159
|
// its own consumerLane, so dispatch()/handleEvent() always write to the
|
|
@@ -204,6 +209,7 @@ export function createJobRunner(options) {
|
|
|
204
209
|
tracer,
|
|
205
210
|
errorLogger,
|
|
206
211
|
sequentialLock,
|
|
212
|
+
tenantCapLock,
|
|
207
213
|
getActiveTenantIds,
|
|
208
214
|
escapeHatchReportWindow,
|
|
209
215
|
systemCronEscapeHatchDedup,
|
|
@@ -536,75 +542,30 @@ export function createJobRunner(options) {
|
|
|
536
542
|
// dispatch). Fan-out children of perTenant jobs land here on their
|
|
537
543
|
// recursive queue.add and DO carry _tenantId.
|
|
538
544
|
const maxPerTenant = resolveMaxPerTenant(jobDef);
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
545
|
+
const capTenantId = maxPerTenant?.onLimit === "skip"
|
|
546
|
+
? payload?._tenantId // @cast-boundary dynamic-key
|
|
547
|
+
: undefined;
|
|
548
|
+
const releaseCapLock = capTenantId === undefined
|
|
549
|
+
? undefined
|
|
550
|
+
: await acquireTenantCapLock(scope, jobName, capTenantId);
|
|
551
|
+
try {
|
|
552
|
+
if (maxPerTenant !== undefined &&
|
|
553
|
+
capTenantId !== undefined &&
|
|
554
|
+
(await isOverPerTenantLimit(scope, jobName, capTenantId, maxPerTenant.limit))) {
|
|
543
555
|
return "skipped:max-per-tenant";
|
|
544
556
|
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
switch (concurrency) {
|
|
549
|
-
case "skip": {
|
|
550
|
-
const active = await targetQueue.getActive();
|
|
551
|
-
const waiting = await targetQueue.getWaiting();
|
|
552
|
-
const isRunning = [...active, ...waiting].some((j) => j.name === jobName);
|
|
553
|
-
if (isRunning) {
|
|
554
|
-
return "skipped";
|
|
555
|
-
}
|
|
556
|
-
break;
|
|
557
|
-
}
|
|
558
|
-
case "replace": {
|
|
559
|
-
const waiting = await targetQueue.getWaiting();
|
|
560
|
-
for (const j of waiting) {
|
|
561
|
-
if (j.name === jobName && j.id) {
|
|
562
|
-
await j.remove();
|
|
563
|
-
await announceDropped(scope, jobName, jobDef, j.id, j.data);
|
|
564
|
-
}
|
|
565
|
-
}
|
|
566
|
-
break;
|
|
557
|
+
const bullOpts = {};
|
|
558
|
+
if ((await applyConcurrencyMode(scope, targetQueue, jobName, jobDef, bullOpts)) === "skipped") {
|
|
559
|
+
return "skipped";
|
|
567
560
|
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
}
|
|
576
|
-
default:
|
|
577
|
-
break;
|
|
561
|
+
const data = buildEnqueueInputs(tracer, jobDef, payload, meta, bullOpts);
|
|
562
|
+
const job = await targetQueue.add(jobName, data, bullOpts);
|
|
563
|
+
await announceQueued(scope, jobName, jobDef, job, data);
|
|
564
|
+
return job.id ?? "unknown";
|
|
565
|
+
}
|
|
566
|
+
finally {
|
|
567
|
+
await releaseCapLock?.();
|
|
578
568
|
}
|
|
579
|
-
Object.assign(bullOpts, buildRetryBullOpts(jobDef));
|
|
580
|
-
if (jobDef.timeout)
|
|
581
|
-
bullOpts["timeout"] = jobDef.timeout;
|
|
582
|
-
if (meta?.priority !== undefined)
|
|
583
|
-
bullOpts["priority"] = meta.priority;
|
|
584
|
-
if (meta?.jobId !== undefined)
|
|
585
|
-
bullOpts["jobId"] = meta.jobId;
|
|
586
|
-
// Pack meta into job data with _ prefix
|
|
587
|
-
const data = { ...payload };
|
|
588
|
-
if (meta?.triggeredById !== undefined)
|
|
589
|
-
data["_triggeredById"] = meta.triggeredById;
|
|
590
|
-
if (meta?.payload !== undefined)
|
|
591
|
-
data["_payload"] = meta.payload;
|
|
592
|
-
// Carry the enqueuing span context into the worker so job.execute shows
|
|
593
|
-
// as a child of the caller.
|
|
594
|
-
const traceCtx = captureTraceContext(tracer);
|
|
595
|
-
if (traceCtx)
|
|
596
|
-
data[TRACE_CONTEXT_KEY] = traceCtx;
|
|
597
|
-
// Propagate correlation from the scheduling request into the job
|
|
598
|
-
// execution context. The worker re-enters requestContext.run with
|
|
599
|
-
// this value so ctx.events.append / executor writes during the job
|
|
600
|
-
// stamp the same correlation as the HTTP request that scheduled it.
|
|
601
|
-
const reqCtx = requestContext.get();
|
|
602
|
-
if (reqCtx?.correlationId)
|
|
603
|
-
data["_correlationId"] = reqCtx.correlationId;
|
|
604
|
-
stampGatedWriteOrigin(data);
|
|
605
|
-
const job = await targetQueue.add(jobName, data, bullOpts);
|
|
606
|
-
await announceQueued(scope, jobName, jobDef, job, data);
|
|
607
|
-
return job.id ?? "unknown";
|
|
608
569
|
},
|
|
609
570
|
...createJobEventTriggers({ scope, queues, tracer, allJobs }),
|
|
610
571
|
attachDispatcher(ref) {
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { Queue } from "bullmq";
|
|
2
|
+
import { parseRedisOpts } from "./job-runner-helpers.js";
|
|
3
|
+
import { queueNameFor } from "./job-runner-schedulers.js";
|
|
4
|
+
const JOB_RUNNER_LANES = ["api", "worker"];
|
|
5
|
+
// Deletes the lane queues of a disposable runner by name, so the cost is the queues' own size
|
|
6
|
+
// instead of a SCAN over the whole Redis. The runner must be stopped first.
|
|
7
|
+
export async function obliterateJobQueues(options) {
|
|
8
|
+
const connection = parseRedisOpts(options.redisUrl);
|
|
9
|
+
await Promise.all(JOB_RUNNER_LANES.map(async (lane) => {
|
|
10
|
+
// skipMetasUpdate: a fresh Queue otherwise writes its meta key again after obliterate removed it.
|
|
11
|
+
const queue = new Queue(queueNameFor(options.queueNamePrefix, lane), {
|
|
12
|
+
connection,
|
|
13
|
+
prefix: options.bullmqPrefix ?? "bull",
|
|
14
|
+
skipMetasUpdate: true,
|
|
15
|
+
});
|
|
16
|
+
try {
|
|
17
|
+
await queue.obliterate({ force: true });
|
|
18
|
+
}
|
|
19
|
+
finally {
|
|
20
|
+
await queue.close();
|
|
21
|
+
}
|
|
22
|
+
}));
|
|
23
|
+
}
|
|
@@ -92,7 +92,7 @@ export function createLifecycle(opts = {}) {
|
|
|
92
92
|
const timeoutMs = drainOpts.timeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
|
|
93
93
|
// Re-entrant drain: concurrent calls share the in-flight promise.
|
|
94
94
|
// Post-stop calls are a no-op so double-SIGTERM doesn't double-run hooks.
|
|
95
|
-
if (drainPromise)
|
|
95
|
+
if (drainPromise !== null)
|
|
96
96
|
return drainPromise;
|
|
97
97
|
// skip: already fully stopped (e.g. SIGTERM arrives after drain finished).
|
|
98
98
|
// Returning without action keeps the post-drain state pristine.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { DbRunner } from "../db/connection.ts";
|
|
2
|
+
export type ScrubEventHeaderOptions = {
|
|
3
|
+
readonly eventTypes: readonly string[];
|
|
4
|
+
readonly keys: readonly string[];
|
|
5
|
+
readonly reason: string;
|
|
6
|
+
readonly dryRun?: boolean;
|
|
7
|
+
};
|
|
8
|
+
export type ScrubEventHeaderResult = {
|
|
9
|
+
readonly removedKeys: number;
|
|
10
|
+
};
|
|
11
|
+
export declare function scrubEventHeaderKeys(db: DbRunner, options: ScrubEventHeaderOptions): Promise<ScrubEventHeaderResult>;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Removes chosen keys from the headers of stored events. For personal data that was written into
|
|
2
|
+
// event headers, which crypto-shredding cannot reach (only payload fields are encrypted).
|
|
3
|
+
import { removeEventHeaderKey } from "../db/queries/event-store-payload-scrub.js";
|
|
4
|
+
import { runInSystemRowLevelSecurityScope } from "../db/row-level-security-scope.js";
|
|
5
|
+
import { assertReason } from "../engine/escape-hatch-reason.js";
|
|
6
|
+
import { createFallbackLogger } from "../logging/utils.js";
|
|
7
|
+
// Headers are not part of any projection or snapshot, so no rebuild follows.
|
|
8
|
+
export function scrubEventHeaderKeys(db, options) {
|
|
9
|
+
assertReason("scrubEventHeaderKeys", options.reason);
|
|
10
|
+
if (options.eventTypes.length === 0 || options.keys.length === 0) {
|
|
11
|
+
throw new Error("scrubEventHeaderKeys: name at least one event type and one header key");
|
|
12
|
+
}
|
|
13
|
+
return runInSystemRowLevelSecurityScope("event header scrub rewrites event types across all tenants", async () => {
|
|
14
|
+
let removedKeys = 0;
|
|
15
|
+
for (const key of options.keys) {
|
|
16
|
+
removedKeys += await removeEventHeaderKey(db, {
|
|
17
|
+
eventTypes: options.eventTypes,
|
|
18
|
+
key,
|
|
19
|
+
dryRun: options.dryRun === true,
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
const log = createFallbackLogger("event-header-scrub");
|
|
23
|
+
// A real run is an audit trail of a destructive cross-tenant rewrite, so it must survive prod log levels.
|
|
24
|
+
(options.dryRun ? log.debug : log.info)?.("Event header scrub finished", {
|
|
25
|
+
dryRun: options.dryRun === true,
|
|
26
|
+
eventTypes: options.eventTypes,
|
|
27
|
+
keys: options.keys,
|
|
28
|
+
reason: options.reason,
|
|
29
|
+
removedKeys,
|
|
30
|
+
});
|
|
31
|
+
return { removedKeys };
|
|
32
|
+
});
|
|
33
|
+
}
|
|
@@ -18,7 +18,7 @@ export type ScrubEventPayloadOptions = {
|
|
|
18
18
|
readonly shouldScrub?: (value: unknown, candidate: EventScrubCandidate) => boolean | Promise<boolean>;
|
|
19
19
|
readonly dryRun?: boolean;
|
|
20
20
|
readonly batchSize?: number;
|
|
21
|
-
readonly logger?: Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "debug">>;
|
|
21
|
+
readonly logger?: Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "info" | "debug">>;
|
|
22
22
|
};
|
|
23
23
|
export type ScrubEventPayloadResult = {
|
|
24
24
|
readonly scannedEvents: number;
|
|
@@ -16,10 +16,35 @@ const DEFAULT_BATCH_SIZE = 500;
|
|
|
16
16
|
function hasValue(value) {
|
|
17
17
|
return value !== undefined && value !== null && value !== PII_ERASED_SENTINEL;
|
|
18
18
|
}
|
|
19
|
+
const TEXT_FIELD_TYPES = new Set(["text", "longText"]);
|
|
20
|
+
// The sentinel is text; written into a typed projection column (integer, date, uuid, jsonb) it
|
|
21
|
+
// would fail every later rebuild of the table, and the event can no longer be repaired. So only
|
|
22
|
+
// text fields take it, other declared fields go null, and a required non-text field has no
|
|
23
|
+
// valid replacement. Fields the registry does not declare fall back to the value's own type.
|
|
24
|
+
function replacementFor(registry, aggregateType, field, value) {
|
|
25
|
+
const def = registry.getEntity(aggregateType)?.fields[field];
|
|
26
|
+
if (def === undefined)
|
|
27
|
+
return typeof value === "string" ? PII_ERASED_SENTINEL : null;
|
|
28
|
+
if (TEXT_FIELD_TYPES.has(def.type))
|
|
29
|
+
return typeof value === "string" ? PII_ERASED_SENTINEL : null;
|
|
30
|
+
if ("required" in def && def.required === true) {
|
|
31
|
+
throw new Error(`scrubEventPayloadFields: "${aggregateType}.${field}" is a required ${def.type} field; ` +
|
|
32
|
+
"it can neither hold the erase placeholder nor null, so its events cannot be scrubbed.");
|
|
33
|
+
}
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
function assertLifecycleFieldsScrubbable(registry, eventType, fields) {
|
|
37
|
+
for (const [entityName] of registry.getAllEntities()) {
|
|
38
|
+
if (!isLifecycleEventOf(eventType, entityName))
|
|
39
|
+
continue;
|
|
40
|
+
for (const field of fields)
|
|
41
|
+
replacementFor(registry, entityName, field, "");
|
|
42
|
+
}
|
|
43
|
+
}
|
|
19
44
|
function payloadSections(row, payload) {
|
|
20
45
|
return isLifecycleEventOf(row.type, row.aggregate_type) ? lifecycleSections(payload) : [payload];
|
|
21
46
|
}
|
|
22
|
-
async function scrubPayload(row, options) {
|
|
47
|
+
async function scrubPayload(registry, row, options) {
|
|
23
48
|
const payload = structuredClone(row.payload);
|
|
24
49
|
let scrubbed = 0;
|
|
25
50
|
for (const section of payloadSections(row, payload)) {
|
|
@@ -38,7 +63,7 @@ async function scrubPayload(row, options) {
|
|
|
38
63
|
};
|
|
39
64
|
if (options.shouldScrub && !(await options.shouldScrub(value, candidate)))
|
|
40
65
|
continue;
|
|
41
|
-
section[field] =
|
|
66
|
+
section[field] = replacementFor(registry, row.aggregate_type, field, value);
|
|
42
67
|
scrubbed++;
|
|
43
68
|
}
|
|
44
69
|
}
|
|
@@ -57,7 +82,7 @@ async function scrubOneBatch(db, registry, args) {
|
|
|
57
82
|
const touchedAggregateTypes = new Set();
|
|
58
83
|
for (const row of rows) {
|
|
59
84
|
totals.scannedEvents++;
|
|
60
|
-
const { payload, scrubbed } = await scrubPayload(row, options);
|
|
85
|
+
const { payload, scrubbed } = await scrubPayload(registry, row, options);
|
|
61
86
|
if (scrubbed === 0)
|
|
62
87
|
continue;
|
|
63
88
|
totals.scrubbedEvents++;
|
|
@@ -115,7 +140,10 @@ async function scrubAllBatches(db, registry, options) {
|
|
|
115
140
|
log,
|
|
116
141
|
failureMessage: "Projection rebuild after an event payload scrub failed; the tables stay queued in kumiko_pending_rebuilds",
|
|
117
142
|
});
|
|
118
|
-
|
|
143
|
+
// A real run is an audit trail of a destructive cross-tenant rewrite, so it must survive prod log levels.
|
|
144
|
+
(options.dryRun ? log.debug : log.info)?.("Event payload scrub finished", {
|
|
145
|
+
migrationId,
|
|
146
|
+
dryRun: options.dryRun === true,
|
|
119
147
|
eventType: options.eventType,
|
|
120
148
|
fields: options.fields,
|
|
121
149
|
reason: options.reason,
|
|
@@ -123,10 +151,14 @@ async function scrubAllBatches(db, registry, options) {
|
|
|
123
151
|
});
|
|
124
152
|
return { ...totals, queuedTables: [...queuedTables], rebuild };
|
|
125
153
|
}
|
|
154
|
+
// Run it in a maintenance window without running writes on the affected event types: batches
|
|
155
|
+
// page by id (`"id" > afterEventId`) over committed rows, and bigserial ids are handed out before
|
|
156
|
+
// commit, so a parallel append whose id is below the cursor commits unseen and is skipped.
|
|
126
157
|
export function scrubEventPayloadFields(db, registry, options) {
|
|
127
158
|
assertReason("scrubEventPayloadFields", options.reason);
|
|
128
159
|
if (options.fields.length === 0) {
|
|
129
160
|
throw new Error("scrubEventPayloadFields: name at least one payload field to scrub");
|
|
130
161
|
}
|
|
162
|
+
assertLifecycleFieldsScrubbable(registry, options.eventType, options.fields);
|
|
131
163
|
return runInSystemRowLevelSecurityScope("event payload scrub rewrites one event type across all tenants", () => withoutStatementTimeout(db, (unboundedDb) => scrubAllBatches(unboundedDb, registry, options)));
|
|
132
164
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { type ScrubEventHeaderOptions, type ScrubEventHeaderResult, scrubEventHeaderKeys, } from "./event-header-scrub.ts";
|
|
1
2
|
export { type EventScrubCandidate, type ScrubEventPayloadOptions, type ScrubEventPayloadResult, scrubEventPayloadFields, } from "./event-payload-scrub.ts";
|
|
2
3
|
export { assertKumikoSchemaCurrent, SchemaDriftError, } from "./kumiko-drift.ts";
|
|
3
4
|
export { enqueueProjectionRebuild, listPendingRebuilds, PROJECTION_REBUILD_JOB, queueRebuildsFromMarkers, rebuildProjectionOrMultiStream, reportPendingRebuildRun, runPendingRebuilds, } from "./pending-rebuilds.ts";
|
package/dist/migrations/index.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
// Drizzle-free gate (kumiko/migrations system) — the canonical boot-gate.
|
|
2
|
+
// Removes chosen keys from the headers of stored events.
|
|
3
|
+
export { scrubEventHeaderKeys, } from "./event-header-scrub.js";
|
|
2
4
|
// Erases chosen payload fields of stored events and rebuilds the projections fed by them.
|
|
3
5
|
export { scrubEventPayloadFields, } from "./event-payload-scrub.js";
|
|
4
6
|
export { assertKumikoSchemaCurrent, SchemaDriftError, } from "./kumiko-drift.js";
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DbConnection } from "../db/connection.ts";
|
|
2
|
+
import type { FeatureDefinition } from "../engine/types/index.ts";
|
|
2
3
|
export type ChecksumMismatch = {
|
|
3
4
|
readonly id: string;
|
|
4
5
|
readonly expected: string;
|
|
@@ -14,13 +15,15 @@ export type KumikoDriftReport = {
|
|
|
14
15
|
readonly checksumMismatches: readonly ChecksumMismatch[];
|
|
15
16
|
readonly missingTables: readonly string[];
|
|
16
17
|
readonly missingColumns: readonly MissingColumns[];
|
|
18
|
+
/** Tables the booted features declare that the snapshot lacks. */
|
|
19
|
+
readonly tablesMissingFromSnapshot: readonly string[];
|
|
17
20
|
};
|
|
18
21
|
export declare class SchemaDriftError extends Error {
|
|
19
22
|
readonly report: KumikoDriftReport;
|
|
20
23
|
constructor(message: string, report: KumikoDriftReport);
|
|
21
24
|
}
|
|
22
|
-
export declare function detectKumikoDrift(db: DbConnection, migrationsDir: string): Promise<KumikoDriftReport>;
|
|
25
|
+
export declare function detectKumikoDrift(db: DbConnection, migrationsDir: string, bootedFeatures?: readonly FeatureDefinition[]): Promise<KumikoDriftReport>;
|
|
23
26
|
export declare function formatKumikoDriftReport(report: KumikoDriftReport): string;
|
|
24
27
|
/** Throws SchemaDriftError with a human-readable message when the DB is not
|
|
25
28
|
* current with the checked-in kumiko/migrations. */
|
|
26
|
-
export declare function assertKumikoSchemaCurrent(db: DbConnection, migrationsDir: string): Promise<void>;
|
|
29
|
+
export declare function assertKumikoSchemaCurrent(db: DbConnection, migrationsDir: string, bootedFeatures?: readonly FeatureDefinition[]): Promise<void>;
|
|
@@ -10,13 +10,19 @@
|
|
|
10
10
|
// is present in the live schema. A migrated-but-incomplete table (snapshot
|
|
11
11
|
// richer than the DB — e.g. a ride-along column the generator once missed)
|
|
12
12
|
// would otherwise surface only as a runtime-500 on the first write.
|
|
13
|
+
// 4. Booted tables covered: with a snapshot, every table the booted features declare
|
|
14
|
+
// (collectTableMetas, the generator's source) is in it. A tooling feature list that
|
|
15
|
+
// differs from the boot (includeBundled) would otherwise leave tables out of the
|
|
16
|
+
// generated schema unnoticed.
|
|
13
17
|
//
|
|
14
18
|
// Contract (unchanged from the legacy gate): boot VALIDATES only, never
|
|
15
|
-
// applies. Apply is the deploy-step `kumiko schema apply` (runMigrationsFromDir).
|
|
19
|
+
// applies. Apply is the deploy-step `bun /app/kumiko.js schema apply` (runMigrationsFromDir).
|
|
16
20
|
import { existsSync } from "node:fs";
|
|
17
21
|
import { join } from "node:path";
|
|
22
|
+
import { collectTableMetas } from "../db/collect-table-metas.js";
|
|
18
23
|
import { loadSnapshotJson } from "../db/migrate-generator.js";
|
|
19
24
|
import { fetchAppliedMigrations, loadMigrationsFromDir } from "../db/migrate-runner.js";
|
|
25
|
+
import { PRODUCTION_SCHEMA_COMMAND_HINT } from "../db/production-schema-command-hint.js";
|
|
20
26
|
import { columnNamesOf, tableExists } from "../db/schema-inspection.js";
|
|
21
27
|
const SNAPSHOT_FILENAME = ".snapshot.json";
|
|
22
28
|
export class SchemaDriftError extends Error {
|
|
@@ -27,7 +33,7 @@ export class SchemaDriftError extends Error {
|
|
|
27
33
|
this.report = report;
|
|
28
34
|
}
|
|
29
35
|
}
|
|
30
|
-
export async function detectKumikoDrift(db, migrationsDir) {
|
|
36
|
+
export async function detectKumikoDrift(db, migrationsDir, bootedFeatures) {
|
|
31
37
|
// A newer framework version may not have generated `./kumiko/migrations` yet
|
|
32
38
|
// (for example, a first upgrade or custom schema setup without `kumiko schema
|
|
33
39
|
// generate`). Treat this as no local migrations and therefore no drift, matching
|
|
@@ -35,10 +41,17 @@ export async function detectKumikoDrift(db, migrationsDir) {
|
|
|
35
41
|
// calls `readdirSync`, which throws synchronous ENOENT as a plain Error instead
|
|
36
42
|
// of SchemaDriftError and causes an unhandled boot failure.
|
|
37
43
|
if (!existsSync(migrationsDir)) {
|
|
38
|
-
return {
|
|
44
|
+
return {
|
|
45
|
+
ok: true,
|
|
46
|
+
pending: [],
|
|
47
|
+
checksumMismatches: [],
|
|
48
|
+
missingTables: [],
|
|
49
|
+
missingColumns: [],
|
|
50
|
+
tablesMissingFromSnapshot: [],
|
|
51
|
+
};
|
|
39
52
|
}
|
|
40
53
|
const local = loadMigrationsFromDir(migrationsDir);
|
|
41
|
-
// A fresh database may not have the tracking table because `kumiko schema apply`
|
|
54
|
+
// A fresh database may not have the tracking table because `bun /app/kumiko.js schema apply`
|
|
42
55
|
// has never run. Treat that as no applied migrations, so all local migrations are
|
|
43
56
|
// pending rather than reporting an error.
|
|
44
57
|
const trackingExists = await tableExists(db, "_kumiko_migrations");
|
|
@@ -62,7 +75,14 @@ export async function detectKumikoDrift(db, migrationsDir) {
|
|
|
62
75
|
const snapshot = loadSnapshotJson(join(migrationsDir, SNAPSHOT_FILENAME));
|
|
63
76
|
const missingTables = [];
|
|
64
77
|
const missingColumns = [];
|
|
78
|
+
let tablesMissingFromSnapshot = [];
|
|
65
79
|
if (snapshot) {
|
|
80
|
+
if (bootedFeatures !== undefined) {
|
|
81
|
+
const inSnapshot = new Set(snapshot.tables.map((t) => t.tableName));
|
|
82
|
+
tablesMissingFromSnapshot = collectTableMetas(bootedFeatures)
|
|
83
|
+
.map((meta) => meta.tableName)
|
|
84
|
+
.filter((name) => !inSnapshot.has(name));
|
|
85
|
+
}
|
|
66
86
|
const existence = await Promise.all(snapshot.tables.map((t) => tableExists(db, t.tableName).then((exists) => ({ t, exists }))));
|
|
67
87
|
const present = [];
|
|
68
88
|
for (const { t, exists } of existence) {
|
|
@@ -84,11 +104,13 @@ export async function detectKumikoDrift(db, migrationsDir) {
|
|
|
84
104
|
ok: pending.length === 0 &&
|
|
85
105
|
checksumMismatches.length === 0 &&
|
|
86
106
|
missingTables.length === 0 &&
|
|
87
|
-
missingColumns.length === 0
|
|
107
|
+
missingColumns.length === 0 &&
|
|
108
|
+
tablesMissingFromSnapshot.length === 0,
|
|
88
109
|
pending,
|
|
89
110
|
checksumMismatches,
|
|
90
111
|
missingTables,
|
|
91
112
|
missingColumns,
|
|
113
|
+
tablesMissingFromSnapshot,
|
|
92
114
|
};
|
|
93
115
|
}
|
|
94
116
|
export function formatKumikoDriftReport(report) {
|
|
@@ -118,12 +140,20 @@ export function formatKumikoDriftReport(report) {
|
|
|
118
140
|
lines.push(` - ${m.table}: ${m.columns.join(", ")}`);
|
|
119
141
|
}
|
|
120
142
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
lines.
|
|
143
|
+
if (report.tablesMissingFromSnapshot.length > 0) {
|
|
144
|
+
lines.push(` ${report.tablesMissingFromSnapshot.length} booted table(s) missing from the snapshot:`);
|
|
145
|
+
for (const t of report.tablesMissingFromSnapshot)
|
|
146
|
+
lines.push(` - ${t}`);
|
|
147
|
+
}
|
|
148
|
+
return [...lines, "", ...driftRemediationLines(report)].join("\n");
|
|
149
|
+
}
|
|
150
|
+
// Remediation depends on the cause: `bun kumiko-schema apply` handles only pending
|
|
151
|
+
// migrations. A checksum mismatch causes MigrationChecksumMismatchError during
|
|
152
|
+
// apply, and baseline's ON CONFLICT DO NOTHING leaves it in alreadyTracked.
|
|
153
|
+
// A missing table with no pending migration was deleted after apply and cannot be
|
|
154
|
+
// restored by apply.
|
|
155
|
+
function driftRemediationLines(report) {
|
|
156
|
+
const lines = [];
|
|
127
157
|
if (report.pending.length > 0) {
|
|
128
158
|
lines.push("Run 'bun kumiko-schema apply' to apply the pending migration(s).");
|
|
129
159
|
}
|
|
@@ -141,12 +171,19 @@ export function formatKumikoDriftReport(report) {
|
|
|
141
171
|
lines.push("ride-along column the generator now emits). Run 'bun kumiko-schema generate' to");
|
|
142
172
|
lines.push("produce an ALTER-TABLE migration for the new column(s), then 'bun kumiko-schema apply'.");
|
|
143
173
|
}
|
|
144
|
-
|
|
174
|
+
if (report.tablesMissingFromSnapshot.length > 0) {
|
|
175
|
+
lines.push("Booted features declare tables the generated schema lacks — kumiko.config.ts");
|
|
176
|
+
lines.push("includeBundled/feature list differs from the boot; run 'kumiko schema generate'.");
|
|
177
|
+
}
|
|
178
|
+
if (report.pending.length > 0 || report.missingColumns.length > 0) {
|
|
179
|
+
lines.push(PRODUCTION_SCHEMA_COMMAND_HINT);
|
|
180
|
+
}
|
|
181
|
+
return lines;
|
|
145
182
|
}
|
|
146
183
|
/** Throws SchemaDriftError with a human-readable message when the DB is not
|
|
147
184
|
* current with the checked-in kumiko/migrations. */
|
|
148
|
-
export async function assertKumikoSchemaCurrent(db, migrationsDir) {
|
|
149
|
-
const report = await detectKumikoDrift(db, migrationsDir);
|
|
185
|
+
export async function assertKumikoSchemaCurrent(db, migrationsDir, bootedFeatures) {
|
|
186
|
+
const report = await detectKumikoDrift(db, migrationsDir, bootedFeatures);
|
|
150
187
|
if (!report.ok)
|
|
151
188
|
throw new SchemaDriftError(formatKumikoDriftReport(report), report);
|
|
152
189
|
}
|
|
@@ -14,7 +14,7 @@ import { instant, table as pgTable, sql, text } from "../db/dialect.js";
|
|
|
14
14
|
import { deleteMany, selectMany, upsertOnConflict } from "../db/query.js";
|
|
15
15
|
import { readRebuildMarker } from "../db/rebuild-marker.js";
|
|
16
16
|
import { tableExists } from "../db/schema-inspection.js";
|
|
17
|
-
import {
|
|
17
|
+
import { pushTableIfMissing } from "../db/table-push.js";
|
|
18
18
|
import { createFallbackLogger } from "../logging/utils.js";
|
|
19
19
|
import { rebuildMultiStreamProjection, rebuildProjection } from "../pipeline/index.js";
|
|
20
20
|
import { buildProjectionTableIndex } from "./projection-table-index.js";
|
|
@@ -24,10 +24,9 @@ export const pendingRebuildsTable = pgTable("kumiko_pending_rebuilds", {
|
|
|
24
24
|
queuedAt: instant("queued_at", { precision: 3 }).notNull().default(sql `now()`),
|
|
25
25
|
});
|
|
26
26
|
export async function createPendingRebuildsTable(db) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
await unsafePushTables(db, { kumikoPendingRebuilds: pendingRebuildsTable });
|
|
27
|
+
await pushTableIfMissing(db, "public.kumiko_pending_rebuilds", {
|
|
28
|
+
kumikoPendingRebuilds: pendingRebuildsTable,
|
|
29
|
+
});
|
|
31
30
|
}
|
|
32
31
|
/** Reads rebuild markers from freshly applied migrations and queues the
|
|
33
32
|
* affected tables. Upsert: an already-pending table keeps its queue slot
|