@cosmicdrift/kumiko-framework 0.413.0 → 0.414.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -2
- package/agents/api/framework-db.md +14 -3
- package/agents/api/framework-engine.md +64 -13
- package/agents/api/framework-event-store.md +14 -0
- package/agents/api/framework-ui-types.md +149 -21
- package/agents/api/renderer-web.md +93 -9
- package/agents/api/testing-helpers.md +3 -2
- package/agents/api/testing-stack.md +3 -0
- package/agents/api/types.md +205 -14
- package/agents/api.md +7 -7
- package/agents/boot-rules.md +63 -9
- package/agents/examples/api-docs.md +2 -6
- package/agents/examples/auth-mfa.md +57 -10
- package/agents/examples/chat-channels.md +2 -0
- package/agents/examples/currencies-global.md +1 -0
- package/agents/examples/env-composition.md +7 -11
- package/agents/examples/helpdesk.md +15 -8
- package/agents/examples/managed-pages.md +90 -1
- package/agents/examples/pwa-shell.md +210 -0
- package/agents/examples/rate-limiting.md +3 -3
- package/agents/examples/record-detail-layout.md +1537 -67
- package/agents/examples/workflow-engine.md +16 -12
- package/agents/examples.md +1 -0
- package/agents/features-index.md +4 -2
- package/agents/features.md +22 -7
- package/agents/widgets.md +19 -1
- package/dist/api/api-constants.d.ts +1 -0
- package/dist/api/api-constants.js +4 -0
- package/dist/api/auth-middleware.js +4 -3
- package/dist/api/auth-routes-shared.js +2 -1
- package/dist/api/entity-read-access.d.ts +1 -0
- package/dist/api/entity-read-access.js +18 -0
- package/dist/api/export/export-route.d.ts +6 -1
- package/dist/api/export/export-route.js +27 -1
- package/dist/api/export/export-rows.js +19 -3
- package/dist/api/routes.d.ts +2 -0
- package/dist/api/routes.js +9 -1
- package/dist/api/server-extra-routes.js +2 -1
- package/dist/api/server-feature-http-routes.d.ts +9 -0
- package/dist/api/server-feature-http-routes.js +109 -0
- package/dist/api/server-middleware.js +2 -19
- package/dist/api/server-pat-rate-limit-guard.d.ts +3 -0
- package/dist/api/server-pat-rate-limit-guard.js +19 -0
- package/dist/api/server-rate-limit-wiring.d.ts +10 -0
- package/dist/api/server-rate-limit-wiring.js +37 -0
- package/dist/api/server-routes.d.ts +4 -12
- package/dist/api/server-routes.js +12 -171
- package/dist/api/server-schema-route.d.ts +3 -0
- package/dist/api/server-schema-route.js +58 -0
- package/dist/api/server-wiring.js +2 -37
- package/dist/api/server.js +3 -1
- package/dist/compliance/override-schema.js +5 -5
- package/dist/compliance/sub-processors.js +10 -1
- package/dist/crypto/index.d.ts +1 -1
- package/dist/crypto/index.js +1 -1
- package/dist/crypto/pii-field-encryption.d.ts +2 -0
- package/dist/crypto/pii-field-encryption.js +16 -1
- package/dist/db/apply-entity-event.js +4 -0
- package/dist/db/bun-sql-connection-codes.d.ts +1 -0
- package/dist/db/bun-sql-connection-codes.js +7 -0
- package/dist/db/collect-table-metas.js +4 -1
- package/dist/db/config-seed.js +1 -1
- package/dist/db/derived-field-sql.js +15 -13
- package/dist/db/derived-related-sql.d.ts +2 -2
- package/dist/db/derived-related-sql.js +4 -2
- package/dist/db/dialect.d.ts +3 -0
- package/dist/db/dialect.js +1 -3
- package/dist/db/driver-row-coercion.js +2 -2
- package/dist/db/driver-value-preparation.js +1 -1
- package/dist/db/eagerload.js +15 -12
- package/dist/db/entity-table-meta.d.ts +2 -0
- package/dist/db/entity-table-meta.js +7 -0
- package/dist/db/event-store-executor-context.js +13 -8
- package/dist/db/event-store-executor-read-aggregate-currencies.js +2 -2
- package/dist/db/event-store-executor-read-aggregate.js +2 -2
- package/dist/db/event-store-executor-read-filter.d.ts +4 -0
- package/dist/db/event-store-executor-read-filter.js +28 -11
- package/dist/db/event-store-executor-read-search.d.ts +1 -0
- package/dist/db/event-store-executor-read-search.js +6 -6
- package/dist/db/event-store-executor-read-sort.js +3 -3
- package/dist/db/event-store-executor-read-where.js +14 -5
- package/dist/db/event-store-executor-read.js +7 -7
- package/dist/db/event-store-executor-write.js +50 -95
- package/dist/db/executor-managed-tables.js +4 -1
- package/dist/db/index.d.ts +4 -2
- package/dist/db/index.js +4 -2
- package/dist/db/known-table-names.d.ts +1 -0
- package/dist/db/known-table-names.js +7 -5
- package/dist/db/list-aggregate-labels.d.ts +1 -0
- package/dist/db/list-aggregate-labels.js +9 -1
- package/dist/db/migrate-generator.js +23 -4
- package/dist/db/migrate-rls-diff.d.ts +1 -0
- package/dist/db/migrate-rls-diff.js +15 -1
- package/dist/db/migrate-runner.js +2 -2
- package/dist/db/money.d.ts +1 -1
- package/dist/db/money.js +1 -4
- package/dist/db/parent-ref-clause.d.ts +6 -2
- package/dist/db/parent-ref-clause.js +76 -23
- package/dist/db/parent-ref-global-hosts.d.ts +20 -0
- package/dist/db/parent-ref-global-hosts.js +47 -0
- package/dist/db/production-schema-command-hint.d.ts +1 -0
- package/dist/db/production-schema-command-hint.js +3 -0
- package/dist/db/queries/backfill-pii.js +55 -29
- package/dist/db/queries/ddl.js +13 -2
- package/dist/db/queries/event-store-open-streams.d.ts +15 -0
- package/dist/db/queries/event-store-open-streams.js +33 -0
- package/dist/db/queries/event-store-payload-scrub.d.ts +5 -0
- package/dist/db/queries/event-store-payload-scrub.js +11 -0
- package/dist/db/queries/event-store.js +0 -1
- package/dist/db/queries/raw-sql.js +0 -1
- package/dist/db/queries/rebuild-rls-guard.d.ts +2 -1
- package/dist/db/queries/rebuild-rls-guard.js +24 -11
- package/dist/db/queries/rls-boot-catalog.d.ts +1 -0
- package/dist/db/queries/rls-boot-catalog.js +13 -8
- package/dist/db/queries/rls-shape.d.ts +2 -1
- package/dist/db/queries/rls-shape.js +10 -0
- package/dist/db/queries/session-advisory-lock.js +2 -1
- package/dist/db/queries/test-stack.js +7 -7
- package/dist/db/query.js +16 -11
- package/dist/db/rebuild-marker.js +11 -2
- package/dist/db/render-rls-ddl.d.ts +2 -0
- package/dist/db/render-rls-ddl.js +16 -11
- package/dist/db/replay-migration-sql.js +1 -1
- package/dist/db/row-level-security-boot-check.js +18 -5
- package/dist/db/row-level-security-connection.js +25 -12
- package/dist/db/row-level-security-scope.d.ts +2 -1
- package/dist/db/row-level-security-scope.js +9 -5
- package/dist/db/sql-identifier.d.ts +1 -0
- package/dist/db/sql-identifier.js +3 -0
- package/dist/db/system-tenant-read-db.d.ts +5 -0
- package/dist/db/system-tenant-read-db.js +28 -0
- package/dist/db/table-builder.d.ts +3 -1
- package/dist/db/table-builder.js +18 -8
- package/dist/db/table-info.d.ts +1 -2
- package/dist/db/table-introspection.d.ts +3 -0
- package/dist/db/table-introspection.js +6 -0
- package/dist/db/table-push.d.ts +1 -0
- package/dist/db/table-push.js +7 -1
- package/dist/db/tenant-db-runner.d.ts +3 -0
- package/dist/db/tenant-db-runner.js +19 -1
- package/dist/db/transaction-serialization.d.ts +1 -0
- package/dist/db/transaction-serialization.js +20 -3
- package/dist/db/where-clause.js +11 -2
- package/dist/db/write-ownership-failures.d.ts +25 -0
- package/dist/db/write-ownership-failures.js +82 -0
- package/dist/engine/boot-validator/boot-check.js +7 -1
- package/dist/engine/boot-validator/boot-phases-app.js +5 -5
- package/dist/engine/boot-validator/boot-phases.js +3 -2
- package/dist/engine/boot-validator/boot-rules.d.ts +293 -26
- package/dist/engine/boot-validator/boot-rules.js +4 -0
- package/dist/engine/boot-validator/bundled-lists-table.js +2 -2
- package/dist/engine/boot-validator/card-meta-overflow.js +1 -0
- package/dist/engine/boot-validator/config-deps.d.ts +0 -1
- package/dist/engine/boot-validator/config-deps.js +0 -28
- package/dist/engine/boot-validator/config-key-rules.d.ts +0 -15
- package/dist/engine/boot-validator/config-key-rules.js +0 -15
- package/dist/engine/boot-validator/dashboard-screen-target.d.ts +22 -0
- package/dist/engine/boot-validator/dashboard-screen-target.js +20 -0
- package/dist/engine/boot-validator/dashboard-stat-navigate.d.ts +1 -0
- package/dist/engine/boot-validator/dashboard-stat-navigate.js +25 -17
- package/dist/engine/boot-validator/definition-webhook-events-rules.d.ts +5 -0
- package/dist/engine/boot-validator/definition-webhook-events-rules.js +5 -0
- package/dist/engine/boot-validator/derived-sql-fields.js +1 -2
- package/dist/engine/boot-validator/entity-declaration-rules.d.ts +5 -0
- package/dist/engine/boot-validator/entity-declaration-rules.js +5 -0
- package/dist/engine/boot-validator/entity-list-screens.d.ts +1 -0
- package/dist/engine/boot-validator/entity-list-screens.js +33 -6
- package/dist/engine/boot-validator/entity-structure-rules.d.ts +5 -0
- package/dist/engine/boot-validator/entity-structure-rules.js +5 -0
- package/dist/engine/boot-validator/list-column-layout.js +10 -6
- package/dist/engine/boot-validator/list-row-guidance-rules.d.ts +67 -0
- package/dist/engine/boot-validator/list-row-guidance-rules.js +67 -0
- package/dist/engine/boot-validator/list-row-guidance.d.ts +20 -0
- package/dist/engine/boot-validator/list-row-guidance.js +129 -0
- package/dist/engine/boot-validator/list-sort-summary-rules.d.ts +42 -0
- package/dist/engine/boot-validator/list-sort-summary-rules.js +42 -0
- package/dist/engine/boot-validator/list-sort-summary.d.ts +5 -0
- package/dist/engine/boot-validator/list-sort-summary.js +149 -0
- package/dist/engine/boot-validator/list-table-options.js +15 -1
- package/dist/engine/boot-validator/list-table-rules.d.ts +10 -5
- package/dist/engine/boot-validator/list-table-rules.js +10 -5
- package/dist/engine/boot-validator/list-toolbar-options.d.ts +10 -0
- package/dist/engine/boot-validator/list-toolbar-options.js +31 -0
- package/dist/engine/boot-validator/named-filter-rules.d.ts +32 -2
- package/dist/engine/boot-validator/named-filter-rules.js +32 -2
- package/dist/engine/boot-validator/ownership.js +10 -1
- package/dist/engine/boot-validator/parent-ref.js +87 -50
- package/dist/engine/boot-validator/projection-list-screens.d.ts +1 -0
- package/dist/engine/boot-validator/projection-list-screens.js +31 -1
- package/dist/engine/boot-validator/query-output-columns.js +62 -9
- package/dist/engine/boot-validator/query-output-guidance.d.ts +7 -0
- package/dist/engine/boot-validator/query-output-guidance.js +39 -0
- package/dist/engine/boot-validator/query-refs.js +24 -1
- package/dist/engine/boot-validator/row-decoration.d.ts +11 -0
- package/dist/engine/boot-validator/row-decoration.js +16 -0
- package/dist/engine/boot-validator/screens-dashboard-links.d.ts +10 -0
- package/dist/engine/boot-validator/screens-dashboard-links.js +36 -0
- package/dist/engine/boot-validator/screens-dashboard-narrow.d.ts +4 -0
- package/dist/engine/boot-validator/screens-dashboard-narrow.js +211 -0
- package/dist/engine/boot-validator/screens-dashboard-rules.d.ts +84 -4
- package/dist/engine/boot-validator/screens-dashboard-rules.js +84 -4
- package/dist/engine/boot-validator/screens-dashboard.js +29 -7
- package/dist/engine/boot-validator/screens-kind-entity-list.js +32 -27
- package/dist/engine/boot-validator/screens-kind-projection-detail-guidance.d.ts +3 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail-guidance.js +46 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.d.ts +20 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail-rules.js +20 -0
- package/dist/engine/boot-validator/screens-kind-projection-detail.js +2 -0
- package/dist/engine/boot-validator/screens-kind-projection-list.js +4 -0
- package/dist/engine/boot-validator/screens-kind-rules.d.ts +25 -0
- package/dist/engine/boot-validator/screens-kind-rules.js +25 -0
- package/dist/engine/boot-validator/screens-lists.d.ts +8 -2
- package/dist/engine/boot-validator/screens-lists.js +19 -3
- package/dist/engine/boot-validator/screens.js +1 -1
- package/dist/engine/boot-validator/sort-nulls.d.ts +2 -0
- package/dist/engine/boot-validator/sort-nulls.js +15 -0
- package/dist/engine/boot-validator/zod-shape.d.ts +2 -1
- package/dist/engine/boot-validator/zod-shape.js +19 -1
- package/dist/engine/build-app-schema.js +4 -0
- package/dist/engine/config-helpers.d.ts +1 -2
- package/dist/engine/config-helpers.js +6 -7
- package/dist/engine/define-feature.js +1 -0
- package/dist/engine/define-handler.d.ts +1 -0
- package/dist/engine/define-handler.js +4 -2
- package/dist/engine/definition-violations.js +2 -2
- package/dist/engine/entity-handlers.js +3 -8
- package/dist/engine/entity-list-runtime-options.js +9 -8
- package/dist/engine/factories.d.ts +2 -0
- package/dist/engine/feature-config-events-jobs.js +1 -0
- package/dist/engine/feature-entity-handlers.js +5 -2
- package/dist/engine/field-access.d.ts +1 -1
- package/dist/engine/field-access.js +7 -10
- package/dist/engine/find-entity.js +3 -3
- package/dist/engine/form-derivations.js +3 -3
- package/dist/engine/index.d.ts +3 -3
- package/dist/engine/index.js +2 -2
- package/dist/engine/named-filter-roles.d.ts +3 -0
- package/dist/engine/named-filter-roles.js +23 -0
- package/dist/engine/pipeline.js +0 -1
- package/dist/engine/project-app-schema-for-roles.js +17 -2
- package/dist/engine/qualified-name.js +0 -1
- package/dist/engine/registry-facade.js +1 -2
- package/dist/engine/registry-state.d.ts +1 -1
- package/dist/engine/registry-state.js +5 -2
- package/dist/engine/registry-validate.d.ts +1 -0
- package/dist/engine/registry-validate.js +9 -1
- package/dist/engine/registry.js +2 -1
- package/dist/engine/resolve-named-list-filters.js +5 -2
- package/dist/engine/schema-builder.js +6 -10
- package/dist/engine/screen-helpers.d.ts +12 -0
- package/dist/engine/screen-helpers.js +17 -1
- package/dist/engine/soft-delete-cleanup.js +5 -8
- package/dist/engine/state-machine.js +0 -1
- package/dist/engine/steps/_database-clock.d.ts +2 -0
- package/dist/engine/steps/_database-clock.js +6 -0
- package/dist/engine/steps/compute.js +1 -1
- package/dist/engine/steps/retry.js +5 -1
- package/dist/engine/steps/return.js +0 -1
- package/dist/engine/steps/wait-for-event.js +2 -1
- package/dist/engine/steps/wait.js +2 -1
- package/dist/engine/system-user.d.ts +2 -2
- package/dist/engine/system-user.js +2 -2
- package/dist/engine/types/handlers.d.ts +1 -1
- package/dist/engine/types/handlers.js +1 -1
- package/dist/engine/types/screen.d.ts +1 -1
- package/dist/engine/types/screen.js +1 -1
- package/dist/engine/webhook-contracts.d.ts +2 -3
- package/dist/engine/webhook-contracts.js +2 -25
- package/dist/engine/webhook-events.js +48 -29
- package/dist/engine/webhook-payload-strictness.d.ts +7 -0
- package/dist/engine/webhook-payload-strictness.js +53 -0
- package/dist/env/_zod-introspect.d.ts +1 -0
- package/dist/env/dry-run.d.ts +1 -1
- package/dist/env/dry-run.js +1 -1
- package/dist/env/field-meta.d.ts +4 -0
- package/dist/env/field-meta.js +67 -0
- package/dist/env/field-rules.d.ts +9 -0
- package/dist/env/field-rules.js +46 -0
- package/dist/env/index.d.ts +3 -3
- package/dist/env/index.js +16 -71
- package/dist/env/internals.d.ts +3 -0
- package/dist/env/internals.js +1 -2
- package/dist/env/required-env.d.ts +3 -1
- package/dist/env/required-env.js +5 -2
- package/dist/errors/serialize.js +2 -2
- package/dist/errors/write-error-info.js +0 -2
- package/dist/es-ops/operations-schema.js +18 -7
- package/dist/event-store/admin-api.js +3 -3
- package/dist/event-store/event-store.js +0 -3
- package/dist/event-store/index.d.ts +1 -0
- package/dist/event-store/index.js +1 -0
- package/dist/event-store/transfer.js +1 -1
- package/dist/files/assert-file-refs-belong-to-tenant.d.ts +2 -1
- package/dist/files/assert-file-refs-belong-to-tenant.js +15 -13
- package/dist/files/bind-pending-file-refs.d.ts +3 -1
- package/dist/files/bind-pending-file-refs.js +54 -27
- package/dist/files/copy-pending-file-refs-to-tenant.js +7 -3
- package/dist/files/create-file-ref.d.ts +4 -1
- package/dist/files/create-file-ref.js +14 -3
- package/dist/files/default-file-guard.js +4 -1
- package/dist/files/feature.js +1 -0
- package/dist/files/file-claim.d.ts +6 -0
- package/dist/files/file-claim.js +17 -0
- package/dist/files/file-field-slots.d.ts +2 -0
- package/dist/files/file-field-slots.js +28 -0
- package/dist/files/file-ref-entity.d.ts +3 -0
- package/dist/files/file-ref-entity.js +2 -0
- package/dist/files/file-ref-table.d.ts +3 -0
- package/dist/files/file-routes.d.ts +4 -1
- package/dist/files/file-routes.js +80 -16
- package/dist/files/in-memory-provider.d.ts +1 -1
- package/dist/files/in-memory-provider.js +2 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +1 -1
- package/dist/files/invalid-file-ref-error.d.ts +2 -0
- package/dist/files/invalid-file-ref-error.js +15 -0
- package/dist/files/local-provider.js +9 -2
- package/dist/files/provider-resolver.js +1 -1
- package/dist/files/rebind-file-ref.d.ts +3 -2
- package/dist/files/rebind-file-ref.js +24 -18
- package/dist/files/resolve-file-handles.d.ts +13 -0
- package/dist/files/resolve-file-handles.js +57 -0
- package/dist/files/storage-tracking.d.ts +1 -1
- package/dist/files/storage-tracking.js +5 -2
- package/dist/files/sweep-stale-pending-file-refs.js +10 -4
- package/dist/files/types.d.ts +4 -0
- package/dist/files/types.js +10 -0
- package/dist/files/upload-field-resolution.d.ts +1 -0
- package/dist/files/upload-field-resolution.js +16 -2
- package/dist/files/upload-ledger.js +1 -1
- package/dist/i18n/required-surface-guidance-keys.d.ts +3 -0
- package/dist/i18n/required-surface-guidance-keys.js +9 -0
- package/dist/i18n/required-surface-keys.js +19 -4
- package/dist/internal/boot-rule-table-expectations.d.ts +2 -0
- package/dist/internal/boot-rule-table-expectations.js +29 -0
- package/dist/internal/row-level-security.d.ts +1 -0
- package/dist/internal/row-level-security.js +1 -0
- package/dist/internal/testing.d.ts +1 -0
- package/dist/internal/testing.js +1 -0
- package/dist/jobs/index.d.ts +1 -0
- package/dist/jobs/index.js +1 -0
- package/dist/jobs/job-load-events.d.ts +10 -0
- package/dist/jobs/job-load-events.js +12 -0
- package/dist/jobs/job-runner-context.js +6 -158
- package/dist/jobs/job-runner-dispatch.d.ts +7 -0
- package/dist/jobs/job-runner-dispatch.js +57 -0
- package/dist/jobs/job-runner-event-triggers.js +18 -9
- package/dist/jobs/job-runner-execution.js +2 -2
- package/dist/jobs/job-runner-facets.d.ts +48 -0
- package/dist/jobs/job-runner-facets.js +156 -0
- package/dist/jobs/job-runner-gates.js +24 -12
- package/dist/jobs/job-runner-helpers.d.ts +5 -1
- package/dist/jobs/job-runner-helpers.js +18 -3
- package/dist/jobs/job-runner-scope.d.ts +1 -0
- package/dist/jobs/job-runner-scope.js +2 -0
- package/dist/jobs/job-runner-tenant-cap-lock.d.ts +2 -0
- package/dist/jobs/job-runner-tenant-cap-lock.js +30 -0
- package/dist/jobs/job-runner.js +31 -70
- package/dist/jobs/obliterate-job-queues.d.ts +6 -0
- package/dist/jobs/obliterate-job-queues.js +23 -0
- package/dist/lifecycle/lifecycle.js +1 -1
- package/dist/migrations/event-header-scrub.d.ts +11 -0
- package/dist/migrations/event-header-scrub.js +33 -0
- package/dist/migrations/event-payload-scrub.d.ts +1 -1
- package/dist/migrations/event-payload-scrub.js +36 -4
- package/dist/migrations/index.d.ts +1 -0
- package/dist/migrations/index.js +2 -0
- package/dist/migrations/kumiko-drift.d.ts +5 -2
- package/dist/migrations/kumiko-drift.js +51 -14
- package/dist/migrations/pending-rebuilds.js +4 -5
- package/dist/observability/sensitive-filter.js +0 -12
- package/dist/pipeline/aggregate-type-removal.js +17 -4
- package/dist/pipeline/context-services.d.ts +2 -0
- package/dist/pipeline/context-services.js +5 -2
- package/dist/pipeline/dispatch-batch.js +4 -3
- package/dist/pipeline/dispatch-context-helpers.d.ts +11 -0
- package/dist/pipeline/dispatch-context-helpers.js +58 -0
- package/dist/pipeline/dispatch-nested-write.d.ts +7 -0
- package/dist/pipeline/dispatch-nested-write.js +119 -0
- package/dist/pipeline/dispatch-query.js +10 -3
- package/dist/pipeline/dispatch-scope.js +4 -3
- package/dist/pipeline/dispatch-shared.d.ts +2 -6
- package/dist/pipeline/dispatch-shared.js +6 -38
- package/dist/pipeline/dispatch-write.d.ts +0 -1
- package/dist/pipeline/dispatch-write.js +3 -302
- package/dist/pipeline/dispatcher-utils.js +6 -0
- package/dist/pipeline/effective-system-scope.d.ts +3 -0
- package/dist/pipeline/effective-system-scope.js +22 -0
- package/dist/pipeline/event-dispatcher-delivery.d.ts +1 -0
- package/dist/pipeline/event-dispatcher-delivery.js +17 -9
- package/dist/pipeline/event-dispatcher.d.ts +4 -1
- package/dist/pipeline/event-dispatcher.js +6 -7
- package/dist/pipeline/handler-context-auth.d.ts +3 -2
- package/dist/pipeline/handler-context-auth.js +3 -4
- package/dist/pipeline/handler-context-db.js +11 -4
- package/dist/pipeline/handler-context-dispatch.js +2 -2
- package/dist/pipeline/handler-context-events.d.ts +8 -3
- package/dist/pipeline/handler-context-events.js +20 -65
- package/dist/pipeline/handler-context-member-resolution.js +16 -8
- package/dist/pipeline/handler-context-notify.d.ts +1 -0
- package/dist/pipeline/handler-context-notify.js +2 -0
- package/dist/pipeline/handler-context-phase-denials.js +3 -2
- package/dist/pipeline/handler-context-projection.d.ts +3 -0
- package/dist/pipeline/handler-context-projection.js +39 -0
- package/dist/pipeline/handler-context-scope.d.ts +1 -1
- package/dist/pipeline/handler-context-scope.js +14 -2
- package/dist/pipeline/index.d.ts +1 -1
- package/dist/pipeline/index.js +1 -1
- package/dist/pipeline/lifecycle-pipeline.d.ts +1 -0
- package/dist/pipeline/lifecycle-pipeline.js +7 -4
- package/dist/pipeline/member-reader.js +4 -2
- package/dist/pipeline/msp-rebuild.js +1 -1
- package/dist/pipeline/nested-write-parent-guard.d.ts +5 -0
- package/dist/pipeline/nested-write-parent-guard.js +81 -0
- package/dist/pipeline/projection-rebuild.d.ts +2 -0
- package/dist/pipeline/projection-rebuild.js +16 -7
- package/dist/pipeline/projection-state.js +4 -6
- package/dist/pipeline/system-hooks.js +36 -6
- package/dist/pipeline/write-result-masking.js +31 -3
- package/dist/pipeline/write-transition-guard.d.ts +7 -0
- package/dist/pipeline/write-transition-guard.js +95 -0
- package/dist/schema-cli/shared.d.ts +2 -0
- package/dist/schema-cli/shared.js +4 -2
- package/dist/schema-cli/validate-command.js +43 -28
- package/dist/search/meilisearch-adapter.js +14 -9
- package/dist/search/reindex-entity.d.ts +2 -0
- package/dist/search/reindex-entity.js +32 -4
- package/dist/search/reindex-tenants-search.js +20 -4
- package/dist/time/tz-context.js +1 -1
- package/dist/ui-types/card-column-roles.d.ts +4 -0
- package/dist/ui-types/card-column-roles.js +6 -3
- package/dist/ui-types/index.d.ts +2 -2
- package/dist/ui-types/index.js +1 -1
- package/dist/utils/ids.js +0 -1
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.js +1 -0
- package/dist/utils/return-path-limit.d.ts +1 -0
- package/dist/utils/return-path-limit.js +1 -0
- package/package.json +5 -3
- package/presets/biome-app.json +39 -21
- package/presets/biome-plugins/admin-api.grit +10 -0
- package/presets/biome-plugins/error-reasons.grit +53 -0
- package/presets/biome-plugins/es-literal-writes-tests.grit +18 -0
- package/presets/biome-plugins/es-literal-writes.grit +22 -0
- package/presets/biome-plugins/i18n-locale-terminology-de.grit +15 -0
- package/presets/biome-plugins/i18n-locale-terminology-romance.grit +10 -0
- package/presets/biome-plugins/no-create-test-dispatcher.grit +5 -0
- package/presets/biome-plugins/no-css-properties-type.grit +7 -0
- package/presets/biome-plugins/no-date-api.grit +9 -0
- package/presets/biome-plugins/no-direct-fetch.grit +16 -0
- package/presets/biome-plugins/no-direct-fs.grit +16 -0
- package/presets/biome-plugins/no-external-font-cdn-css.grit +9 -0
- package/presets/biome-plugins/no-external-font-cdn.grit +12 -0
- package/presets/biome-plugins/no-raw-hooks.grit +10 -0
- package/presets/biome-plugins/no-same-folder-barrel-import.grit +11 -0
- package/presets/biome-plugins/no-screen-container-classes.grit +7 -0
- package/presets/biome-plugins/no-skipped-or-retried-tests.grit +34 -0
- package/presets/biome-plugins/no-temporal-polyfill-require.grit +10 -0
- package/presets/biome-plugins/raw-html-in-web.grit +40 -0
- package/presets/biome-plugins/raw-ui-in-app.grit +142 -0
- package/presets/biome-plugins/restricted-event-store-symbols.grit +18 -0
- package/presets/biome-plugins/test-title.grit +20 -0
- package/presets/biome-plugins/unsafe-json-parse.grit +6 -0
- package/presets/biome-rules.json +465 -0
- package/src/changes.json +309 -0
- package/dist/engine/minor-units.d.ts +0 -2
- package/dist/engine/minor-units.js +0 -8
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/// <reference types="temporal-polyfill/global" preserve="true" />
|
|
2
|
+
import { crossTenantReadsGrantFor, escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
|
|
3
|
+
import { createCrossTenantReads } from "../db/cross-tenant-reads.js";
|
|
4
|
+
import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
|
|
5
|
+
import { createDerivativesContext } from "../derivatives/derivatives-context.js";
|
|
6
|
+
import { InternalError } from "../errors/index.js";
|
|
7
|
+
import { createTenantFileContext } from "../files/file-handle.js";
|
|
8
|
+
import { createEscapeHatchReporter } from "../observability/escape-hatch-report.js";
|
|
9
|
+
import { configAccessorFactoryOf } from "../pipeline/config-accessor-factory.js";
|
|
10
|
+
import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js";
|
|
11
|
+
import { buildPersonalDataGate } from "../pipeline/write-origin.js";
|
|
12
|
+
import { isScheduledTrigger } from "./cron-repeat-options.js";
|
|
13
|
+
import { isQueuedRun } from "./job-runner-helpers.js";
|
|
14
|
+
export function createJobEscapeHatchReporter(scope, input, systemUser) {
|
|
15
|
+
const { jobName, jobDef, tenantId, triggeredById } = input;
|
|
16
|
+
// One reporter for ctx.systemDb and ctx.db.unsafeRaw() so both dedupe in the same window.
|
|
17
|
+
const isSystemCronRun = triggeredById === null &&
|
|
18
|
+
jobDef.escapeHatch !== undefined &&
|
|
19
|
+
isScheduledTrigger(jobDef.trigger);
|
|
20
|
+
return createEscapeHatchReporter({
|
|
21
|
+
handler: jobName,
|
|
22
|
+
tenantId,
|
|
23
|
+
actor: systemUser.id,
|
|
24
|
+
sink: scope.context._escapeHatchAuditSink,
|
|
25
|
+
log: scope.errorLogger,
|
|
26
|
+
window: scope.escapeHatchReportWindow,
|
|
27
|
+
...(triggeredById !== null && { caller: triggeredById }),
|
|
28
|
+
...(isSystemCronRun && { processDedup: scope.systemCronEscapeHatchDedup }),
|
|
29
|
+
...(scope.context.meter && { meter: scope.context.meter }),
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
export function buildJobDbFacets(scope, facetInput) {
|
|
33
|
+
const { context, registry } = scope;
|
|
34
|
+
const { input, isSystemJob, reportEscapeHatch } = facetInput;
|
|
35
|
+
const { jobDef, tenantId, writeOrigin } = input;
|
|
36
|
+
const personalDataGate = writeOrigin ? buildPersonalDataGate(registry, writeOrigin) : undefined;
|
|
37
|
+
const configDb = context.db; // @cast-boundary db-operator
|
|
38
|
+
// Shared by the config accessor and ctx.services.derivatives below — both need the
|
|
39
|
+
// same tenant-scoped db, and building it twice would let the two calls
|
|
40
|
+
// drift apart. Always "system" mode regardless of the job's own
|
|
41
|
+
// systemScope() status (pre-existing, not something this change alters)
|
|
42
|
+
// — isSystemJob below is what actually keeps ctx.systemDb off a
|
|
43
|
+
// non-system job; it is the only thing standing between this db and an
|
|
44
|
+
// unchecked cross-tenant escape hatch for such a job. Gated like jobDb so
|
|
45
|
+
// ctx.systemDb, built from it, is gated too.
|
|
46
|
+
const tenantScopedDb = configDb
|
|
47
|
+
? createTenantDb({
|
|
48
|
+
db: configDb,
|
|
49
|
+
tenantId,
|
|
50
|
+
mode: "system",
|
|
51
|
+
reason: "job context db for cross-tenant jobs, ctx.systemDb exposure is gated by isSystemJob",
|
|
52
|
+
grants: {
|
|
53
|
+
projectionRegistry: registry,
|
|
54
|
+
...(personalDataGate && { personalDataGate }),
|
|
55
|
+
},
|
|
56
|
+
})
|
|
57
|
+
: undefined;
|
|
58
|
+
const systemDb = isSystemJob && tenantScopedDb
|
|
59
|
+
? createUncheckedSystemDb(tenantScopedDb, undefined, reportEscapeHatch)
|
|
60
|
+
: undefined;
|
|
61
|
+
const jobDb = configDb
|
|
62
|
+
? createTenantDb({
|
|
63
|
+
db: configDb,
|
|
64
|
+
tenantId,
|
|
65
|
+
mode: "tenant",
|
|
66
|
+
tracer: context.tracer,
|
|
67
|
+
meter: context.meter,
|
|
68
|
+
grants: {
|
|
69
|
+
unsafeRaw: escapeHatchFor(jobDef.escapeHatch, "unsafeRaw"),
|
|
70
|
+
globalWrites: escapeHatchFor(jobDef.escapeHatch, "globalWrites"),
|
|
71
|
+
report: reportEscapeHatch,
|
|
72
|
+
projectionRegistry: registry,
|
|
73
|
+
...(personalDataGate && { personalDataGate }),
|
|
74
|
+
},
|
|
75
|
+
})
|
|
76
|
+
: undefined;
|
|
77
|
+
const crossTenantReadsReason = crossTenantReadsGrantFor(jobDef.escapeHatch)?.reason;
|
|
78
|
+
const crossTenantReads = tenantScopedDb && crossTenantReadsReason !== undefined
|
|
79
|
+
? createCrossTenantReads(tenantScopedDb, crossTenantReadsReason, reportEscapeHatch)
|
|
80
|
+
: undefined;
|
|
81
|
+
return { tenantScopedDb, jobDb, systemDb, crossTenantReads };
|
|
82
|
+
}
|
|
83
|
+
export function buildJobServiceFacets(scope, facetInput, tenantScopedDb) {
|
|
84
|
+
const { context, registry } = scope;
|
|
85
|
+
const { input, systemUser } = facetInput;
|
|
86
|
+
const { tenantId } = input;
|
|
87
|
+
// Mirror dispatch-shared.ts buildHandlerContext: ctx.services.files must resolve
|
|
88
|
+
// through the same _fileProviderResolver for jobs as for write-handlers,
|
|
89
|
+
// otherwise event-triggered jobs silently get an unresolved ctx.services.files.
|
|
90
|
+
const fileResolver = context._fileProviderResolver;
|
|
91
|
+
const files = fileResolver ? createTenantFileContext(fileResolver, tenantId) : context.files;
|
|
92
|
+
// Same buildHandlerContext parity as ctx.services.files above: ctx.notify/ctx.config
|
|
93
|
+
// must resolve for jobs the same way they do for write-handlers, or an
|
|
94
|
+
// app-author job calling ctx.notify(...)/ctx.config(...) hits a TypeError
|
|
95
|
+
// at runtime the write-handler path never would.
|
|
96
|
+
const notify = context._notifyFactory?.(systemUser, tenantId, scope.getSelfRunner(), undefined, context._fileProviderResolver);
|
|
97
|
+
const configFactory = configAccessorFactoryOf(context);
|
|
98
|
+
const config = configFactory && tenantScopedDb
|
|
99
|
+
? configFactory({
|
|
100
|
+
user: { id: systemUser.id, tenantId },
|
|
101
|
+
db: tenantScopedDb,
|
|
102
|
+
secrets: context.secrets,
|
|
103
|
+
})
|
|
104
|
+
: undefined;
|
|
105
|
+
// Mirror dispatch-shared.ts: ctx.services.derivatives needs files+db, same
|
|
106
|
+
// tenant-scoped db the config accessor above uses.
|
|
107
|
+
const derivatives = files && tenantScopedDb
|
|
108
|
+
? createDerivativesContext({ files, registry, db: tenantScopedDb, tenantId })
|
|
109
|
+
: context.derivatives;
|
|
110
|
+
return { files, notify, config, derivatives };
|
|
111
|
+
}
|
|
112
|
+
function requireDispatchRef(scope, method) {
|
|
113
|
+
const ref = scope.getDispatchWriteRef();
|
|
114
|
+
if (!ref) {
|
|
115
|
+
throw new Error(`JobContext.${method} called before dispatcher attached — call attachDispatcher() first`);
|
|
116
|
+
}
|
|
117
|
+
return ref;
|
|
118
|
+
}
|
|
119
|
+
export function buildJobDispatchFacets(scope, facetInput) {
|
|
120
|
+
const { input, systemUser, isSystemJob, reportEscapeHatch } = facetInput;
|
|
121
|
+
const { jobName, jobDef, tenantId, writeOrigin } = input;
|
|
122
|
+
// Per-job-run reader, created lazily on queryAsMember's first call and
|
|
123
|
+
// cached for the rest of this run.
|
|
124
|
+
let memberReader;
|
|
125
|
+
const identitySwitchHatch = escapeHatchFor(jobDef.escapeHatch, "systemIdentity");
|
|
126
|
+
const identitySwitchGrantReason = identitySwitchHatch?.reason ??
|
|
127
|
+
(isSystemJob ? `r.systemScope() feature of job "${jobName}"` : undefined);
|
|
128
|
+
const gatedIdentitySwitch = createGatedIdentitySwitch(`job "${jobName}"`, systemUser, isSystemJob || identitySwitchHatch !== undefined, {
|
|
129
|
+
queryAs: (user, qn, payload) => requireDispatchRef(scope, "queryAs").queryAs(user, qn, payload, writeOrigin),
|
|
130
|
+
writeAs: async (user, qn, payload) => requireDispatchRef(scope, "writeAs").write(user, qn, payload, writeOrigin),
|
|
131
|
+
}, { reason: identitySwitchGrantReason, report: reportEscapeHatch });
|
|
132
|
+
return {
|
|
133
|
+
write: (qn, payload) => requireDispatchRef(scope, "write").write(systemUser, qn, payload, writeOrigin),
|
|
134
|
+
writeAs: gatedIdentitySwitch.writeAs,
|
|
135
|
+
queryAs: gatedIdentitySwitch.queryAs,
|
|
136
|
+
queryAsMember: (userId, qn, payload) => {
|
|
137
|
+
const ref = requireDispatchRef(scope, "queryAsMember");
|
|
138
|
+
// Lazy per-run reader, mirroring the handler-context reader.
|
|
139
|
+
memberReader ??= ref.createMemberReader(tenantId);
|
|
140
|
+
return memberReader(userId, qn, payload);
|
|
141
|
+
},
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
export function createJobDefer(jobName, bullJob, token) {
|
|
145
|
+
return (delayMs) => {
|
|
146
|
+
if (!Number.isInteger(delayMs) || delayMs <= 0) {
|
|
147
|
+
throw new RangeError(`Job "${jobName}": ctx.defer needs a positive integer delay in ms (got ${delayMs})`);
|
|
148
|
+
}
|
|
149
|
+
if (!isQueuedRun(bullJob, token)) {
|
|
150
|
+
throw new InternalError({
|
|
151
|
+
message: `Job "${jobName}": ctx.defer needs a queued run; it is not available inline (boot gate).`,
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
return { kind: "job-deferral", delayMs };
|
|
155
|
+
};
|
|
156
|
+
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
|
|
2
|
+
import { scrubLogText } from "../logging/index.js";
|
|
1
3
|
import { prepareJobRun } from "./job-runner-context.js";
|
|
2
|
-
import { attemptNumberOf, buildRetryBullOpts, DEFER_COUNT_KEY, deferDelayMs, isFinalAttempt, priorAttemptsOf, priorDeferCountOf, } from "./job-runner-helpers.js";
|
|
4
|
+
import { attemptNumberOf, buildRetryBullOpts, DEFER_COUNT_KEY, deferDelayMs, isFinalAttempt, isQueuedRun, priorAttemptsOf, priorDeferCountOf, } from "./job-runner-helpers.js";
|
|
3
5
|
import { ACTIVE_TENANT_IDS_QUERY_NAME, perTenantChildJobId } from "./job-runner-schedulers.js";
|
|
4
6
|
import { announceDropped, announceQueued, isOverPerTenantLimit, laneForJob, logFailedAttempt, } from "./job-runner-scope.js";
|
|
5
7
|
import { resolveMaxPerTenant } from "./max-per-tenant.js";
|
|
@@ -35,7 +37,10 @@ async function enqueuePerTenantChildren(scope, bullJob, actualName) {
|
|
|
35
37
|
if (!actualDef) {
|
|
36
38
|
throw new Error(`Unknown job: ${actualName}`);
|
|
37
39
|
}
|
|
38
|
-
const
|
|
40
|
+
const activeTenantIds = await getActiveTenantIds();
|
|
41
|
+
const tenantIds = actualDef.includeSystemTenant === true && !activeTenantIds.includes(SYSTEM_TENANT_ID)
|
|
42
|
+
? [...activeTenantIds, SYSTEM_TENANT_ID]
|
|
43
|
+
: activeTenantIds;
|
|
39
44
|
const targetQueue = scope.queues[laneForJob(actualDef)];
|
|
40
45
|
// wrapperJobId is only absent for a Job BullMQ somehow never assigned
|
|
41
46
|
// an id to (not observed in practice) — falling back to a constant
|
|
@@ -61,7 +66,7 @@ async function enqueuePerTenantChildren(scope, bullJob, actualName) {
|
|
|
61
66
|
// the _perTenant wrapper itself always lives in the consumer-lane
|
|
62
67
|
// (it's picked up by this runner's own worker).
|
|
63
68
|
export async function fanOutPerTenantJob(scope, bullJob, actualName) {
|
|
64
|
-
// These throws happen before
|
|
69
|
+
// These throws happen before executeJobInSpan, so recordJobFailure never sees them
|
|
65
70
|
// and a drained test stack would stay green over a fan-out that never ran.
|
|
66
71
|
try {
|
|
67
72
|
await enqueuePerTenantChildren(scope, bullJob, actualName);
|
|
@@ -69,12 +74,11 @@ export async function fanOutPerTenantJob(scope, bullJob, actualName) {
|
|
|
69
74
|
catch (err) {
|
|
70
75
|
// A retry may still succeed, and a succeeding wrapper reports no completion
|
|
71
76
|
// that could clear an earlier failure, so only the final attempt is reported.
|
|
72
|
-
const
|
|
73
|
-
const finalAttempt = bullJob.attemptsMade + 1 >= retries + 1;
|
|
77
|
+
const finalAttempt = isFinalAttempt(bullJob, scope.allJobs.get(actualName) ?? { retries: 0 });
|
|
74
78
|
logFailedAttempt(scope, {
|
|
75
79
|
job: bullJob.name,
|
|
76
80
|
jobId: bullJob.id ?? "unknown",
|
|
77
|
-
attempt: bullJob
|
|
81
|
+
attempt: attemptNumberOf(bullJob),
|
|
78
82
|
finalAttempt,
|
|
79
83
|
err,
|
|
80
84
|
});
|
|
@@ -86,9 +90,18 @@ export async function fanOutPerTenantJob(scope, bullJob, actualName) {
|
|
|
86
90
|
}
|
|
87
91
|
async function reportPreRunFailure(scope, jobName, bullJob, err) {
|
|
88
92
|
const message = err instanceof Error ? err.message : String(err);
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
93
|
+
try {
|
|
94
|
+
await scope.options.onJobFailed?.(jobName, bullJob.id ?? "unknown", message, [
|
|
95
|
+
{ level: "error", message, timestamp: Temporal.Now.instant() },
|
|
96
|
+
]);
|
|
97
|
+
}
|
|
98
|
+
catch (hookErr) {
|
|
99
|
+
// The caller rethrows the original error; a failing hook (often the same unreachable store) must not replace it as BullMQ's failedReason.
|
|
100
|
+
scope.errorLogger.warn("onJobFailed hook threw while reporting a pre-run failure", {
|
|
101
|
+
job: jobName,
|
|
102
|
+
error: scrubLogText(hookErr instanceof Error ? hookErr.message : String(hookErr)),
|
|
103
|
+
});
|
|
104
|
+
}
|
|
92
105
|
}
|
|
93
106
|
// prepareJobRun and onJobStart throw before the handler span exists, so the
|
|
94
107
|
// span's own failure recording never sees them.
|
|
@@ -123,7 +136,7 @@ export async function resolveJobDefinitionOrFail(scope, bullJob) {
|
|
|
123
136
|
logFailedAttempt(scope, {
|
|
124
137
|
job: jobName,
|
|
125
138
|
jobId: bullJob.id ?? "unknown",
|
|
126
|
-
attempt: bullJob
|
|
139
|
+
attempt: attemptNumberOf(bullJob),
|
|
127
140
|
finalAttempt: true,
|
|
128
141
|
err,
|
|
129
142
|
});
|
|
@@ -138,8 +151,7 @@ export async function deferOverTenantLimit(scope, jobDef, bullJob, token) {
|
|
|
138
151
|
const deferTenantId = bullJob.data["_tenantId"];
|
|
139
152
|
if (maxPerTenant?.onLimit === "defer" &&
|
|
140
153
|
typeof deferTenantId === "string" &&
|
|
141
|
-
token
|
|
142
|
-
bullJob.moveToDelayed &&
|
|
154
|
+
isQueuedRun(bullJob, token) &&
|
|
143
155
|
(await isOverPerTenantLimit(scope, bullJob.name, deferTenantId, maxPerTenant.limit, {
|
|
144
156
|
activeOnly: true,
|
|
145
157
|
self: { id: bullJob.id, processedOn: bullJob.processedOn },
|
|
@@ -16,8 +16,12 @@ export declare function jobSubjectKey(jobName: string, payload: Record<string, u
|
|
|
16
16
|
export declare function resolveJobTz(config: ConfigAccessor | undefined, context: Pick<AppContext, "geoTzProvider" | "clock">, logger: Pick<Logger, "warn">): Promise<TzContext>;
|
|
17
17
|
export declare function missingDbProxy(jobName: string): TenantDb;
|
|
18
18
|
export declare function priorAttemptsOf(data: Record<string, unknown>): number;
|
|
19
|
+
type DeferrableInvocation = JobInvocation & {
|
|
20
|
+
readonly moveToDelayed: NonNullable<JobInvocation["moveToDelayed"]>;
|
|
21
|
+
};
|
|
22
|
+
export declare function isQueuedRun(bullJob: JobInvocation, token: string | undefined): bullJob is DeferrableInvocation;
|
|
19
23
|
export declare function attemptNumberOf(bullJob: JobInvocation): number;
|
|
20
|
-
export declare function isFinalAttempt(bullJob: JobInvocation, jobDef: JobDefinition): boolean;
|
|
24
|
+
export declare function isFinalAttempt(bullJob: JobInvocation, jobDef: Pick<JobDefinition, "retries">): boolean;
|
|
21
25
|
export declare function tenantFailureMessageKey(err: unknown, declaration: JobDefinition["tenantVisibleFailure"]): string | null;
|
|
22
26
|
export declare const TRACE_CONTEXT_KEY = "_traceContext";
|
|
23
27
|
export declare const DEFER_COUNT_KEY = "_deferCount";
|
|
@@ -129,6 +129,10 @@ export function priorAttemptsOf(data) {
|
|
|
129
129
|
const value = data["_priorAttempts"];
|
|
130
130
|
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
|
|
131
131
|
}
|
|
132
|
+
// Only a run picked up from the queue can be moved to delayed; an inline boot-gate run has neither token nor moveToDelayed.
|
|
133
|
+
export function isQueuedRun(bullJob, token) {
|
|
134
|
+
return token !== undefined && bullJob.moveToDelayed !== undefined;
|
|
135
|
+
}
|
|
132
136
|
export function attemptNumberOf(bullJob) {
|
|
133
137
|
return bullJob.attemptsMade + 1 + priorAttemptsOf(bullJob.data);
|
|
134
138
|
}
|
|
@@ -202,19 +206,30 @@ export async function startQueueDepthPolling(queue, lane, meter) {
|
|
|
202
206
|
await pollQueueDepth();
|
|
203
207
|
return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
|
|
204
208
|
}
|
|
209
|
+
// A credential with a stray "%" is accepted by URL but not decodable; ioredis then uses it as written.
|
|
210
|
+
function decodeCredential(raw) {
|
|
211
|
+
try {
|
|
212
|
+
return decodeURIComponent(raw);
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
// skip: not percent-encoded, the raw value is the credential
|
|
216
|
+
return raw;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
205
219
|
export function parseRedisOpts(url) {
|
|
206
220
|
const parsed = new URL(url);
|
|
207
221
|
const result = {
|
|
208
|
-
|
|
222
|
+
// URL keeps the brackets of an IPv6 literal; ioredis and BullMQ need the bare address.
|
|
223
|
+
host: parsed.hostname.replace(/^\[(.*)\]$/, "$1"),
|
|
209
224
|
port: Number(parsed.port) || 6379,
|
|
210
225
|
};
|
|
211
226
|
if (parsed.pathname.length > 1) {
|
|
212
227
|
result.db = Number(parsed.pathname.slice(1));
|
|
213
228
|
}
|
|
214
229
|
if (parsed.username)
|
|
215
|
-
result.username =
|
|
230
|
+
result.username = decodeCredential(parsed.username);
|
|
216
231
|
if (parsed.password)
|
|
217
|
-
result.password =
|
|
232
|
+
result.password = decodeCredential(parsed.password);
|
|
218
233
|
if (parsed.protocol === "rediss:")
|
|
219
234
|
result.tls = {};
|
|
220
235
|
return result;
|
|
@@ -17,6 +17,7 @@ export type JobRunnerScope = {
|
|
|
17
17
|
readonly tracer: Tracer;
|
|
18
18
|
readonly errorLogger: ReturnType<typeof createFallbackLogger>;
|
|
19
19
|
readonly sequentialLock: DistributedLock | null;
|
|
20
|
+
readonly tenantCapLock: DistributedLock | null;
|
|
20
21
|
readonly getActiveTenantIds: (() => Promise<TenantId[]>) | undefined;
|
|
21
22
|
readonly escapeHatchReportWindow: EscapeHatchReportWindow;
|
|
22
23
|
readonly systemCronEscapeHatchDedup: EscapeHatchProcessDedup;
|
|
@@ -16,6 +16,8 @@ export function laneForJob(def) {
|
|
|
16
16
|
// One getJobs per queue reads the requested lists in a single Lua script, so a job
|
|
17
17
|
// moving from waiting to active between two separate reads can no longer vanish
|
|
18
18
|
// from the count; that gap let a third dispatch pass the cap in CI.
|
|
19
|
+
// Only this snapshot is atomic: dispatch and handleEvent hold the tenant-cap lock across the
|
|
20
|
+
// check and the following queue.add, otherwise concurrent dispatches could pass together.
|
|
19
21
|
// `activeOnly` + `self` serve the defer check, which runs inside the job being
|
|
20
22
|
// counted: it must not see waiting jobs or itself, and only counts jobs that
|
|
21
23
|
// became active before it. Without that order, workers that fetch a burst of
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
const CAP_LOCK_TTL_SEC = 10;
|
|
2
|
+
const CAP_LOCK_POLL_MS = 15;
|
|
3
|
+
const CAP_LOCK_MAX_WAIT_MS = 3_000;
|
|
4
|
+
const releaseNothing = async () => { };
|
|
5
|
+
// Count-then-add is two steps; holding this lock across both makes the "skip" cap exact for
|
|
6
|
+
// every dispatcher sharing the Redis. The lock only guards the short check+add window, so a
|
|
7
|
+
// holder that crashes frees it via the TTL. A lock that stays contended past the wait budget
|
|
8
|
+
// degrades to the soft cap instead of failing the dispatch.
|
|
9
|
+
export async function acquireTenantCapLock(scope, jobName, tenantId) {
|
|
10
|
+
const { tenantCapLock } = scope;
|
|
11
|
+
if (!tenantCapLock)
|
|
12
|
+
return releaseNothing;
|
|
13
|
+
const key = `${jobName}:${tenantId}`;
|
|
14
|
+
const deadline = Date.now() + CAP_LOCK_MAX_WAIT_MS;
|
|
15
|
+
for (;;) {
|
|
16
|
+
const token = await tenantCapLock.acquire(key, { ttlSeconds: CAP_LOCK_TTL_SEC });
|
|
17
|
+
if (token) {
|
|
18
|
+
return async () => {
|
|
19
|
+
await tenantCapLock.release(key, token);
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
if (Date.now() >= deadline) {
|
|
23
|
+
scope.errorLogger.warn("maxPerTenant lock stayed contended, enforcing the cap softly", {
|
|
24
|
+
job: jobName,
|
|
25
|
+
});
|
|
26
|
+
return releaseNothing;
|
|
27
|
+
}
|
|
28
|
+
await new Promise((resolve) => setTimeout(resolve, CAP_LOCK_POLL_MS));
|
|
29
|
+
}
|
|
30
|
+
}
|
package/dist/jobs/job-runner.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { DelayedError, Queue, Worker } from "bullmq";
|
|
2
2
|
import { Redis } from "ioredis";
|
|
3
|
-
import { requestContext } from "../api/request-context.js";
|
|
4
3
|
import { assertNoActiveRowLevelSecurityScope, runInSystemRowLevelSecurityScope, } from "../db/row-level-security-scope.js";
|
|
5
4
|
import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
|
|
6
5
|
import { bridgeStub } from "../engine/bridge-stub.js";
|
|
@@ -15,12 +14,14 @@ import { attachOutsideTransaction } from "../pipeline/attach-outside-transaction
|
|
|
15
14
|
import { createDistributedLock } from "../pipeline/distributed-lock.js";
|
|
16
15
|
import { RedisKeys } from "../pipeline/redis-keys.js";
|
|
17
16
|
import { isScheduledTrigger, scheduleRepeatOptions } from "./cron-repeat-options.js";
|
|
17
|
+
import { applyConcurrencyMode, buildEnqueueInputs } from "./job-runner-dispatch.js";
|
|
18
18
|
import { createJobEventTriggers } from "./job-runner-event-triggers.js";
|
|
19
19
|
import { applyHandlerDeferral, executeJobInSpan } from "./job-runner-execution.js";
|
|
20
20
|
import { claimSequentialSlot, deferOverTenantLimit, fanOutPerTenantJob, PER_TENANT_PREFIX, perTenantTargetOf, releaseSequentialSlot, resolveJobDefinitionOrFail, startJobRun, } from "./job-runner-gates.js";
|
|
21
|
-
import { BOOT_GATE_SLOW_WARN_MS, BOOT_REDIS_TIMEOUT_MS, buildRetryBullOpts, COMPLETED_JOB_RETENTION_AGE_SEC,
|
|
21
|
+
import { BOOT_GATE_SLOW_WARN_MS, BOOT_REDIS_TIMEOUT_MS, buildRetryBullOpts, COMPLETED_JOB_RETENTION_AGE_SEC, FAILED_JOB_RETENTION_AGE_SEC, maxRetryWindowMs, parseRedisOpts, STOP_QUEUE_READY_TIMEOUT_MS, stampGatedWriteOrigin, startQueueDepthPolling, timeoutReject, } from "./job-runner-helpers.js";
|
|
22
22
|
import { ACTIVE_TENANT_IDS_QUERY_NAME, bootJobIdForJobName, DEFAULT_QUEUE_NAME_PREFIX, markSchedulersSeen, ORPHAN_SCHEDULER_GRACE_MS, queueNameFor, removeOrphanJobSchedulers, schedulerIdForJobName, } from "./job-runner-schedulers.js";
|
|
23
|
-
import {
|
|
23
|
+
import { announceQueued, isOverPerTenantLimit, laneForJob, } from "./job-runner-scope.js";
|
|
24
|
+
import { acquireTenantCapLock } from "./job-runner-tenant-cap-lock.js";
|
|
24
25
|
import { resolveMaxPerTenant } from "./max-per-tenant.js";
|
|
25
26
|
export { serializeJobSubject } from "./job-runner-helpers.js";
|
|
26
27
|
export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.js";
|
|
@@ -133,13 +134,15 @@ export function createJobRunner(options) {
|
|
|
133
134
|
// Sequential coordination: BullMQ OSS has no `group`, so we serialise
|
|
134
135
|
// same-name jobs ourselves with a per-name Redis lock. Only built when at
|
|
135
136
|
// least one job actually requested it — keeps the no-sequential boot path
|
|
136
|
-
// free of the extra Redis client. Scoped under the consumer lane so two
|
|
137
|
+
// free of the extra Redis client (also built for maxPerTenant "skip" caps). Scoped under the consumer lane so two
|
|
137
138
|
// runners on different lanes cannot collide on the same lock-key for
|
|
138
139
|
// unrelated jobs.
|
|
139
140
|
const hasSequential = [...allJobs.values()].some((def) => def.concurrency === "sequential");
|
|
141
|
+
const hasSkipCap = [...allJobs.values()].some((def) => resolveMaxPerTenant(def)?.onLimit === "skip");
|
|
140
142
|
let lockRedis = null;
|
|
141
143
|
let sequentialLock = null;
|
|
142
|
-
|
|
144
|
+
let tenantCapLock = null;
|
|
145
|
+
if (hasSequential || hasSkipCap) {
|
|
143
146
|
lockRedis = new Redis({ ...redisOpts, keyPrefix });
|
|
144
147
|
// Without a listener, a post-close 'error' (e.g. a teardown-race
|
|
145
148
|
// "Connection is closed") is unhandled and crashes the process — in
|
|
@@ -149,6 +152,8 @@ export function createJobRunner(options) {
|
|
|
149
152
|
// Scoped by queueNamePrefix like the queues themselves, so runners
|
|
150
153
|
// isolated by prefix on a shared Redis don't contend for the same lock.
|
|
151
154
|
sequentialLock = createDistributedLock(lockRedis, `${RedisKeys.lock}seq:${queueNamePrefix}:${lockScope}:`);
|
|
155
|
+
// No lane in the prefix: api and worker runners dispatch the same job and must share the lock.
|
|
156
|
+
tenantCapLock = createDistributedLock(lockRedis, `${RedisKeys.lock}cap:${queueNamePrefix}:`);
|
|
152
157
|
}
|
|
153
158
|
// Two queue-clients — one per lane. Every runner holds both, regardless of
|
|
154
159
|
// its own consumerLane, so dispatch()/handleEvent() always write to the
|
|
@@ -204,6 +209,7 @@ export function createJobRunner(options) {
|
|
|
204
209
|
tracer,
|
|
205
210
|
errorLogger,
|
|
206
211
|
sequentialLock,
|
|
212
|
+
tenantCapLock,
|
|
207
213
|
getActiveTenantIds,
|
|
208
214
|
escapeHatchReportWindow,
|
|
209
215
|
systemCronEscapeHatchDedup,
|
|
@@ -536,75 +542,30 @@ export function createJobRunner(options) {
|
|
|
536
542
|
// dispatch). Fan-out children of perTenant jobs land here on their
|
|
537
543
|
// recursive queue.add and DO carry _tenantId.
|
|
538
544
|
const maxPerTenant = resolveMaxPerTenant(jobDef);
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
545
|
+
const capTenantId = maxPerTenant?.onLimit === "skip"
|
|
546
|
+
? payload?._tenantId // @cast-boundary dynamic-key
|
|
547
|
+
: undefined;
|
|
548
|
+
const releaseCapLock = capTenantId === undefined
|
|
549
|
+
? undefined
|
|
550
|
+
: await acquireTenantCapLock(scope, jobName, capTenantId);
|
|
551
|
+
try {
|
|
552
|
+
if (maxPerTenant !== undefined &&
|
|
553
|
+
capTenantId !== undefined &&
|
|
554
|
+
(await isOverPerTenantLimit(scope, jobName, capTenantId, maxPerTenant.limit))) {
|
|
543
555
|
return "skipped:max-per-tenant";
|
|
544
556
|
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
switch (concurrency) {
|
|
549
|
-
case "skip": {
|
|
550
|
-
const active = await targetQueue.getActive();
|
|
551
|
-
const waiting = await targetQueue.getWaiting();
|
|
552
|
-
const isRunning = [...active, ...waiting].some((j) => j.name === jobName);
|
|
553
|
-
if (isRunning) {
|
|
554
|
-
return "skipped";
|
|
555
|
-
}
|
|
556
|
-
break;
|
|
557
|
-
}
|
|
558
|
-
case "replace": {
|
|
559
|
-
const waiting = await targetQueue.getWaiting();
|
|
560
|
-
for (const j of waiting) {
|
|
561
|
-
if (j.name === jobName && j.id) {
|
|
562
|
-
await j.remove();
|
|
563
|
-
await announceDropped(scope, jobName, jobDef, j.id, j.data);
|
|
564
|
-
}
|
|
565
|
-
}
|
|
566
|
-
break;
|
|
557
|
+
const bullOpts = {};
|
|
558
|
+
if ((await applyConcurrencyMode(scope, targetQueue, jobName, jobDef, bullOpts)) === "skipped") {
|
|
559
|
+
return "skipped";
|
|
567
560
|
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
}
|
|
576
|
-
default:
|
|
577
|
-
break;
|
|
561
|
+
const data = buildEnqueueInputs(tracer, jobDef, payload, meta, bullOpts);
|
|
562
|
+
const job = await targetQueue.add(jobName, data, bullOpts);
|
|
563
|
+
await announceQueued(scope, jobName, jobDef, job, data);
|
|
564
|
+
return job.id ?? "unknown";
|
|
565
|
+
}
|
|
566
|
+
finally {
|
|
567
|
+
await releaseCapLock?.();
|
|
578
568
|
}
|
|
579
|
-
Object.assign(bullOpts, buildRetryBullOpts(jobDef));
|
|
580
|
-
if (jobDef.timeout)
|
|
581
|
-
bullOpts["timeout"] = jobDef.timeout;
|
|
582
|
-
if (meta?.priority !== undefined)
|
|
583
|
-
bullOpts["priority"] = meta.priority;
|
|
584
|
-
if (meta?.jobId !== undefined)
|
|
585
|
-
bullOpts["jobId"] = meta.jobId;
|
|
586
|
-
// Pack meta into job data with _ prefix
|
|
587
|
-
const data = { ...payload };
|
|
588
|
-
if (meta?.triggeredById !== undefined)
|
|
589
|
-
data["_triggeredById"] = meta.triggeredById;
|
|
590
|
-
if (meta?.payload !== undefined)
|
|
591
|
-
data["_payload"] = meta.payload;
|
|
592
|
-
// Carry the enqueuing span context into the worker so job.execute shows
|
|
593
|
-
// as a child of the caller.
|
|
594
|
-
const traceCtx = captureTraceContext(tracer);
|
|
595
|
-
if (traceCtx)
|
|
596
|
-
data[TRACE_CONTEXT_KEY] = traceCtx;
|
|
597
|
-
// Propagate correlation from the scheduling request into the job
|
|
598
|
-
// execution context. The worker re-enters requestContext.run with
|
|
599
|
-
// this value so ctx.events.append / executor writes during the job
|
|
600
|
-
// stamp the same correlation as the HTTP request that scheduled it.
|
|
601
|
-
const reqCtx = requestContext.get();
|
|
602
|
-
if (reqCtx?.correlationId)
|
|
603
|
-
data["_correlationId"] = reqCtx.correlationId;
|
|
604
|
-
stampGatedWriteOrigin(data);
|
|
605
|
-
const job = await targetQueue.add(jobName, data, bullOpts);
|
|
606
|
-
await announceQueued(scope, jobName, jobDef, job, data);
|
|
607
|
-
return job.id ?? "unknown";
|
|
608
569
|
},
|
|
609
570
|
...createJobEventTriggers({ scope, queues, tracer, allJobs }),
|
|
610
571
|
attachDispatcher(ref) {
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { Queue } from "bullmq";
|
|
2
|
+
import { parseRedisOpts } from "./job-runner-helpers.js";
|
|
3
|
+
import { queueNameFor } from "./job-runner-schedulers.js";
|
|
4
|
+
const JOB_RUNNER_LANES = ["api", "worker"];
|
|
5
|
+
// Deletes the lane queues of a disposable runner by name, so the cost is the queues' own size
|
|
6
|
+
// instead of a SCAN over the whole Redis. The runner must be stopped first.
|
|
7
|
+
export async function obliterateJobQueues(options) {
|
|
8
|
+
const connection = parseRedisOpts(options.redisUrl);
|
|
9
|
+
await Promise.all(JOB_RUNNER_LANES.map(async (lane) => {
|
|
10
|
+
// skipMetasUpdate: a fresh Queue otherwise writes its meta key again after obliterate removed it.
|
|
11
|
+
const queue = new Queue(queueNameFor(options.queueNamePrefix, lane), {
|
|
12
|
+
connection,
|
|
13
|
+
prefix: options.bullmqPrefix ?? "bull",
|
|
14
|
+
skipMetasUpdate: true,
|
|
15
|
+
});
|
|
16
|
+
try {
|
|
17
|
+
await queue.obliterate({ force: true });
|
|
18
|
+
}
|
|
19
|
+
finally {
|
|
20
|
+
await queue.close();
|
|
21
|
+
}
|
|
22
|
+
}));
|
|
23
|
+
}
|
|
@@ -92,7 +92,7 @@ export function createLifecycle(opts = {}) {
|
|
|
92
92
|
const timeoutMs = drainOpts.timeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
|
|
93
93
|
// Re-entrant drain: concurrent calls share the in-flight promise.
|
|
94
94
|
// Post-stop calls are a no-op so double-SIGTERM doesn't double-run hooks.
|
|
95
|
-
if (drainPromise)
|
|
95
|
+
if (drainPromise !== null)
|
|
96
96
|
return drainPromise;
|
|
97
97
|
// skip: already fully stopped (e.g. SIGTERM arrives after drain finished).
|
|
98
98
|
// Returning without action keeps the post-drain state pristine.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { DbRunner } from "../db/connection.ts";
|
|
2
|
+
export type ScrubEventHeaderOptions = {
|
|
3
|
+
readonly eventTypes: readonly string[];
|
|
4
|
+
readonly keys: readonly string[];
|
|
5
|
+
readonly reason: string;
|
|
6
|
+
readonly dryRun?: boolean;
|
|
7
|
+
};
|
|
8
|
+
export type ScrubEventHeaderResult = {
|
|
9
|
+
readonly removedKeys: number;
|
|
10
|
+
};
|
|
11
|
+
export declare function scrubEventHeaderKeys(db: DbRunner, options: ScrubEventHeaderOptions): Promise<ScrubEventHeaderResult>;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Removes chosen keys from the headers of stored events. For personal data that was written into
|
|
2
|
+
// event headers, which crypto-shredding cannot reach (only payload fields are encrypted).
|
|
3
|
+
import { removeEventHeaderKey } from "../db/queries/event-store-payload-scrub.js";
|
|
4
|
+
import { runInSystemRowLevelSecurityScope } from "../db/row-level-security-scope.js";
|
|
5
|
+
import { assertReason } from "../engine/escape-hatch-reason.js";
|
|
6
|
+
import { createFallbackLogger } from "../logging/utils.js";
|
|
7
|
+
// Headers are not part of any projection or snapshot, so no rebuild follows.
|
|
8
|
+
export function scrubEventHeaderKeys(db, options) {
|
|
9
|
+
assertReason("scrubEventHeaderKeys", options.reason);
|
|
10
|
+
if (options.eventTypes.length === 0 || options.keys.length === 0) {
|
|
11
|
+
throw new Error("scrubEventHeaderKeys: name at least one event type and one header key");
|
|
12
|
+
}
|
|
13
|
+
return runInSystemRowLevelSecurityScope("event header scrub rewrites event types across all tenants", async () => {
|
|
14
|
+
let removedKeys = 0;
|
|
15
|
+
for (const key of options.keys) {
|
|
16
|
+
removedKeys += await removeEventHeaderKey(db, {
|
|
17
|
+
eventTypes: options.eventTypes,
|
|
18
|
+
key,
|
|
19
|
+
dryRun: options.dryRun === true,
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
const log = createFallbackLogger("event-header-scrub");
|
|
23
|
+
// A real run is an audit trail of a destructive cross-tenant rewrite, so it must survive prod log levels.
|
|
24
|
+
(options.dryRun ? log.debug : log.info)?.("Event header scrub finished", {
|
|
25
|
+
dryRun: options.dryRun === true,
|
|
26
|
+
eventTypes: options.eventTypes,
|
|
27
|
+
keys: options.keys,
|
|
28
|
+
reason: options.reason,
|
|
29
|
+
removedKeys,
|
|
30
|
+
});
|
|
31
|
+
return { removedKeys };
|
|
32
|
+
});
|
|
33
|
+
}
|
|
@@ -18,7 +18,7 @@ export type ScrubEventPayloadOptions = {
|
|
|
18
18
|
readonly shouldScrub?: (value: unknown, candidate: EventScrubCandidate) => boolean | Promise<boolean>;
|
|
19
19
|
readonly dryRun?: boolean;
|
|
20
20
|
readonly batchSize?: number;
|
|
21
|
-
readonly logger?: Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "debug">>;
|
|
21
|
+
readonly logger?: Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "info" | "debug">>;
|
|
22
22
|
};
|
|
23
23
|
export type ScrubEventPayloadResult = {
|
|
24
24
|
readonly scannedEvents: number;
|