@cosmicdrift/kumiko-framework 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -38
- package/package.json +21 -2
- package/src/__tests__/anonymous-access.integration.test.ts +185 -0
- package/src/__tests__/consumer-cli.integration.test.ts +172 -0
- package/src/__tests__/entity-permalink-open.integration.test.ts +94 -0
- package/src/__tests__/full-stack.integration.test.ts +1 -1
- package/src/__tests__/schema-cli-temporal-polyfill.test.ts +48 -0
- package/src/__tests__/schema-cli.integration.test.ts +50 -0
- package/src/__tests__/store-table.integration.test.ts +93 -0
- package/src/api/__tests__/api.test.ts +373 -0
- package/src/api/__tests__/auth-middleware-anonymous-access-boot.test.ts +40 -0
- package/src/api/__tests__/auth-routes-cookie.test.ts +17 -1
- package/src/api/__tests__/auth-routes-invalid-body-invite.test.ts +253 -0
- package/src/api/__tests__/auth-routes-mfa-preauth-confirm.test.ts +222 -0
- package/src/api/__tests__/auth-routes-mfa-preauth-enable-start.test.ts +249 -0
- package/src/api/__tests__/auth-routes-mfa-verify.test.ts +202 -0
- package/src/api/__tests__/auth-routes-trusted-proxy.test.ts +135 -0
- package/src/api/__tests__/batch.integration.test.ts +101 -11
- package/src/api/__tests__/csrf-constants-sync.test.ts +20 -0
- package/src/api/__tests__/dispatcher-live.integration.test.ts +74 -0
- package/src/api/__tests__/jwt.test.ts +200 -1
- package/src/api/__tests__/login-rate-limiter-sweep.test.ts +50 -0
- package/src/api/__tests__/pat-scope.test.ts +36 -0
- package/src/api/__tests__/pii-leak-guard.integration.test.ts +103 -0
- package/src/api/__tests__/redis-login-rate-limiter.integration.test.ts +138 -0
- package/src/api/__tests__/request-id-middleware.test.ts +51 -0
- package/src/api/__tests__/server-boot-guards.test.ts +71 -0
- package/src/api/__tests__/server-jwt-ttl.test.ts +58 -0
- package/src/api/__tests__/sse-broker.test.ts +57 -0
- package/src/api/__tests__/sse-route.test.ts +4 -0
- package/src/api/api-constants.ts +11 -0
- package/src/api/auth-middleware.ts +248 -40
- package/src/api/auth-routes.ts +576 -95
- package/src/api/index.ts +13 -4
- package/src/api/jwt.ts +170 -11
- package/src/api/pat-scope.ts +14 -0
- package/src/api/pii-leak-guard.ts +47 -0
- package/src/api/request-context.ts +3 -0
- package/src/api/request-id-middleware.ts +2 -0
- package/src/api/routes.ts +169 -2
- package/src/api/server.ts +112 -16
- package/src/api/sse-broker.ts +39 -0
- package/src/bun-db/__tests__/PATTERN.md +0 -1
- package/src/bun-db/__tests__/coerce-row-temporal.test.ts +41 -0
- package/src/bun-db/__tests__/select-many-retry.test.ts +79 -0
- package/src/bun-db/__tests__/write-brand.test.ts +21 -0
- package/src/bun-db/connection.ts +3 -3
- package/src/bun-db/index.ts +2 -0
- package/src/bun-db/query.ts +105 -32
- package/src/consumer-cli.ts +134 -0
- package/src/crypto/__tests__/blind-index.test.ts +130 -0
- package/src/crypto/__tests__/event-pii.test.ts +161 -0
- package/src/crypto/__tests__/kek-rotation.integration.test.ts +180 -0
- package/src/crypto/__tests__/kms-adapter-contract.ts +134 -0
- package/src/crypto/__tests__/kms-adapter.contract.test.ts +4 -0
- package/src/crypto/__tests__/pg-kms-adapter.integration.test.ts +117 -0
- package/src/crypto/__tests__/pii-field-encryption.test.ts +376 -0
- package/src/crypto/__tests__/request-kms-cache.test.ts +74 -0
- package/src/crypto/__tests__/subject-resolver.test.ts +108 -0
- package/src/crypto/blind-index.ts +118 -0
- package/src/crypto/event-pii.ts +70 -0
- package/src/crypto/in-memory-kms-adapter.ts +50 -0
- package/src/crypto/index.ts +67 -0
- package/src/crypto/kms-adapter.ts +2 -0
- package/src/crypto/pg-kms-adapter.ts +295 -0
- package/src/crypto/pii-field-encryption.ts +248 -0
- package/src/crypto/request-kms-cache.ts +38 -0
- package/src/crypto/subject-resolver.ts +91 -0
- package/src/db/__tests__/assert-no-unreachable-live-rows.integration.test.ts +191 -0
- package/src/db/__tests__/blind-index.integration.test.ts +248 -0
- package/src/db/__tests__/build-filter-where.test.ts +34 -0
- package/src/db/__tests__/collect-table-metas.test.ts +10 -10
- package/src/db/__tests__/config-seed.integration.test.ts +13 -5
- package/src/db/__tests__/dialect-instant.test.ts +1 -4
- package/src/db/__tests__/entity-field-encryption.test.ts +7 -7
- package/src/db/__tests__/entity-table-meta-source.test.ts +50 -0
- package/src/db/__tests__/event-store-executor-context.pii-roundtrip.test.ts +67 -0
- package/src/db/__tests__/event-store-executor-write-verbs.integration.test.ts +402 -0
- package/src/db/__tests__/event-store-executor.integration.test.ts +299 -29
- package/src/db/__tests__/feature-table-sources.test.ts +34 -0
- package/src/db/__tests__/implicit-projection-equivalence.integration.test.ts +118 -39
- package/src/db/__tests__/instant-to-driver-temporal.test.ts +21 -0
- package/src/db/__tests__/located-timestamp.test.ts +19 -0
- package/src/db/__tests__/migrate-generator.test.ts +21 -0
- package/src/db/__tests__/migrate-runner.test.ts +61 -0
- package/src/db/__tests__/number-field-fractional.integration.test.ts +58 -0
- package/src/db/__tests__/rebuild-marker.test.ts +13 -3
- package/src/db/__tests__/replay-migration-sql.test.ts +384 -0
- package/src/db/__tests__/schema-inspection.test.ts +7 -0
- package/src/db/__tests__/schema-migration.integration.test.ts +1 -1
- package/src/db/__tests__/table-builder-meta-lockstep.test.ts +39 -0
- package/src/db/__tests__/tenant-db-where-merge.test.ts +49 -3
- package/src/db/__tests__/tenant-db.integration.test.ts +6 -2
- package/src/db/api.ts +2 -2
- package/src/db/apply-entity-event.ts +18 -14
- package/src/db/blind-index-cleanup.ts +55 -0
- package/src/db/bun-provider.ts +2 -2
- package/src/db/collect-table-metas.ts +6 -7
- package/src/db/config-seed.ts +9 -9
- package/src/db/connection.ts +7 -12
- package/src/db/cursor.ts +1 -18
- package/src/db/dialect.ts +20 -26
- package/src/db/entity-field-encryption.ts +81 -24
- package/src/db/entity-table-meta-types.ts +2 -0
- package/src/db/entity-table-meta.ts +63 -90
- package/src/db/event-store-executor-context.ts +404 -0
- package/src/db/event-store-executor-read.ts +275 -0
- package/src/db/event-store-executor-write.ts +644 -0
- package/src/db/event-store-executor.ts +28 -1155
- package/src/db/feature-table-sources.ts +6 -5
- package/src/db/index.ts +20 -3
- package/src/db/located-timestamp.ts +4 -0
- package/src/db/migrate-generator.ts +39 -6
- package/src/db/migrate-runner.ts +107 -11
- package/src/db/pg-error.ts +9 -1
- package/src/db/postgres-provider.ts +2 -2
- package/src/db/queries/__tests__/event-store-idempotency-index.integration.test.ts +80 -0
- package/src/db/queries/backfill-pii.ts +277 -0
- package/src/db/queries/ddl.ts +45 -0
- package/src/db/queries/event-consumer.ts +35 -2
- package/src/db/queries/event-store.ts +126 -19
- package/src/db/queries/projection-rebuild.ts +22 -8
- package/src/db/queries/shadow-swap.ts +183 -0
- package/src/db/queries/test-stack.ts +4 -30
- package/src/db/query-api.ts +1 -0
- package/src/db/query.ts +1 -0
- package/src/db/rebuild-marker.ts +18 -2
- package/src/db/reference-data.ts +2 -3
- package/src/db/replay-migration-sql.ts +257 -0
- package/src/db/schema-inspection.ts +1 -1
- package/src/db/table-builder.ts +102 -71
- package/src/db/tenant-db.ts +15 -53
- package/src/engine/__tests__/boot-validator-action-wiring.test.ts +242 -0
- package/src/engine/__tests__/boot-validator-boot-check.test.ts +99 -0
- package/src/engine/__tests__/boot-validator-dashboard.test.ts +237 -0
- package/src/engine/__tests__/boot-validator-entity-list.test.ts +104 -0
- package/src/engine/__tests__/boot-validator-gdpr-storage.test.ts +7 -69
- package/src/engine/__tests__/boot-validator-i18n-keys.test.ts +26 -5
- package/src/engine/__tests__/boot-validator-located-timestamps.test.ts +3 -19
- package/src/engine/__tests__/boot-validator-pii-retention.test.ts +246 -4
- package/src/engine/__tests__/boot-validator.test.ts +219 -15
- package/src/engine/__tests__/build-app-schema.test.ts +82 -0
- package/src/engine/__tests__/build-config-feature-schema.test.ts +125 -0
- package/src/engine/__tests__/build-target.test.ts +3 -11
- package/src/engine/__tests__/codemod-pipeline.test.ts +152 -21
- package/src/engine/__tests__/config-helpers.test.ts +16 -0
- package/src/engine/__tests__/define-feature-entity-mapping.test.ts +6 -0
- package/src/engine/__tests__/define-roles.test.ts +21 -0
- package/src/engine/__tests__/engine.test.ts +163 -1
- package/src/engine/__tests__/entity-handlers.test.ts +80 -0
- package/src/engine/__tests__/event-migration-declarative.test.ts +65 -0
- package/src/engine/__tests__/event-type-map-augmentation.test.ts +24 -0
- package/src/engine/__tests__/extend-entity-projection.test.ts +123 -0
- package/src/engine/__tests__/factories-time.test.ts +2 -66
- package/src/engine/__tests__/feature-crud-shorthand.test.ts +28 -0
- package/src/engine/__tests__/feature-manifest.test.ts +31 -1
- package/src/engine/__tests__/field-access.test.ts +23 -1
- package/src/engine/__tests__/hook-phases.test.ts +5 -5
- package/src/engine/__tests__/membership-roles.test.ts +4 -10
- package/src/engine/__tests__/nav.test.ts +86 -1
- package/src/engine/__tests__/pipeline-engine.test.ts +9 -9
- package/src/engine/__tests__/pipeline-handler.integration.test.ts +18 -18
- package/src/engine/__tests__/pipeline-observability.integration.test.ts +2 -2
- package/src/engine/__tests__/pipeline-performance.integration.test.ts +3 -3
- package/src/engine/__tests__/pipeline-sub-pipelines.test.ts +9 -9
- package/src/engine/__tests__/post-query-hook.test.ts +7 -7
- package/src/engine/__tests__/registrar-object-form.test.ts +141 -0
- package/src/engine/__tests__/registry.test.ts +144 -0
- package/src/engine/__tests__/schema-builder.test.ts +18 -0
- package/src/engine/__tests__/screen.test.ts +147 -1
- package/src/engine/__tests__/soft-delete-cleanup.test.ts +3 -0
- package/src/engine/__tests__/store-table.test.ts +229 -0
- package/src/engine/__tests__/tier-resolver-extension.test.ts +19 -1
- package/src/engine/__tests__/{visual-tree-patterns.test.ts → tree-actions-patterns.test.ts} +7 -95
- package/src/engine/__tests__/validate-projection-allowlist.test.ts +15 -15
- package/src/engine/boot-validator/__tests__/config-deps.test.ts +92 -0
- package/src/engine/boot-validator/action-wiring.ts +149 -0
- package/src/engine/boot-validator/boot-check.ts +21 -0
- package/src/engine/boot-validator/config-deps.ts +40 -4
- package/src/engine/boot-validator/entity-handler.ts +20 -21
- package/src/engine/boot-validator/entity-list-screens.ts +88 -0
- package/src/engine/boot-validator/gdpr-storage.ts +0 -20
- package/src/engine/boot-validator/i18n-keys.ts +58 -4
- package/src/engine/boot-validator/index.ts +33 -12
- package/src/engine/boot-validator/nav.ts +125 -0
- package/src/engine/boot-validator/pii-retention.ts +95 -10
- package/src/engine/boot-validator/{screens-nav.ts → screens.ts} +214 -191
- package/src/engine/boot-validator/workspaces.ts +68 -0
- package/src/engine/build-app-schema.ts +16 -0
- package/src/engine/build-config-feature-schema.ts +44 -7
- package/src/engine/codemod/pipeline-codemod.ts +5 -5
- package/src/engine/config-helpers.ts +15 -0
- package/src/engine/constants.ts +32 -6
- package/src/engine/create-app.ts +11 -0
- package/src/engine/define-feature.ts +95 -947
- package/src/engine/define-handler.ts +28 -94
- package/src/engine/define-workflow.ts +1 -1
- package/src/engine/effective-features.ts +12 -2
- package/src/engine/entity-handlers.ts +76 -11
- package/src/engine/extensions/tenant-data.ts +19 -0
- package/src/engine/extensions/user-data.ts +29 -2
- package/src/engine/factories.ts +8 -49
- package/src/engine/feature-ast/__tests__/canonical-form.test.ts +26 -29
- package/src/engine/feature-ast/__tests__/fixtures/cross-file-registrar/feature.ts +9 -0
- package/src/engine/feature-ast/__tests__/fixtures/cross-file-registrar/screens.ts +4 -0
- package/src/engine/feature-ast/__tests__/parse-happy-path.test.ts +1 -2
- package/src/engine/feature-ast/__tests__/parse-real-features.test.ts +18 -8
- package/src/engine/feature-ast/__tests__/parse.test.ts +1640 -160
- package/src/engine/feature-ast/__tests__/patch.test.ts +206 -12
- package/src/engine/feature-ast/__tests__/patcher.test.ts +20 -23
- package/src/engine/feature-ast/__tests__/render-roundtrip.test.ts +380 -5
- package/src/engine/feature-ast/extractors/events.ts +322 -0
- package/src/engine/feature-ast/extractors/handlers.ts +234 -0
- package/src/engine/feature-ast/extractors/hooks.ts +243 -0
- package/src/engine/feature-ast/extractors/index.ts +34 -31
- package/src/engine/feature-ast/extractors/jobs-routes.ts +221 -0
- package/src/engine/feature-ast/extractors/projections-screens.ts +269 -0
- package/src/engine/feature-ast/extractors/round1.ts +3 -3
- package/src/engine/feature-ast/extractors/round5.ts +3 -3
- package/src/engine/feature-ast/extractors/round6.ts +2 -36
- package/src/engine/feature-ast/extractors/shared.ts +64 -6
- package/src/engine/feature-ast/index.ts +2 -4
- package/src/engine/feature-ast/parse.ts +130 -23
- package/src/engine/feature-ast/patch.ts +39 -50
- package/src/engine/feature-ast/patcher.ts +37 -38
- package/src/engine/feature-ast/patterns.ts +50 -62
- package/src/engine/feature-ast/render.ts +67 -44
- package/src/engine/feature-builder-state.ts +168 -0
- package/src/engine/feature-config-events-jobs.ts +403 -0
- package/src/engine/feature-entity-handlers.ts +208 -0
- package/src/engine/feature-manifest.ts +2 -1
- package/src/engine/feature-ui-extensions.ts +500 -0
- package/src/engine/field-access.ts +13 -2
- package/src/engine/field-helpers.ts +31 -0
- package/src/engine/handler-helpers.ts +26 -0
- package/src/engine/hook-helpers.ts +16 -0
- package/src/engine/index.ts +23 -7
- package/src/engine/membership-roles.ts +13 -0
- package/src/engine/object-form.ts +27 -0
- package/src/engine/ownership.ts +26 -79
- package/src/engine/pattern-library/__tests__/library.test.ts +7 -20
- package/src/engine/pattern-library/library.ts +44 -1152
- package/src/engine/pattern-library/mixed-schemas.ts +450 -0
- package/src/engine/pattern-library/opaque-schemas.ts +124 -0
- package/src/engine/pattern-library/shared-fields.ts +75 -0
- package/src/engine/pattern-library/static-schemas.ts +456 -0
- package/src/engine/pipeline.ts +6 -11
- package/src/engine/registry-facade.ts +362 -0
- package/src/engine/registry-ingest.ts +478 -0
- package/src/engine/registry-state.ts +388 -0
- package/src/engine/registry-validate.ts +642 -0
- package/src/engine/registry.ts +78 -1658
- package/src/engine/run-pipeline.ts +1 -1
- package/src/engine/schema-builder.ts +1 -0
- package/src/engine/screen-helpers.ts +54 -0
- package/src/engine/soft-delete-cleanup.ts +6 -2
- package/src/engine/steps/__tests__/duration-utils.test.ts +20 -0
- package/src/engine/steps/_duration-utils.ts +2 -0
- package/src/engine/steps/unsafe-projection-upsert.ts +1 -4
- package/src/engine/tier-resolver-extension.ts +3 -2
- package/src/engine/types/config.ts +2 -482
- package/src/engine/types/define-handler.ts +2 -0
- package/src/engine/types/entity-handlers.ts +2 -0
- package/src/engine/types/event-type-map.ts +1 -37
- package/src/engine/types/feature.ts +2 -972
- package/src/engine/types/fields.ts +2 -675
- package/src/engine/types/handlers.ts +2 -774
- package/src/engine/types/hooks.ts +2 -184
- package/src/engine/types/http-route.ts +1 -54
- package/src/engine/types/identifiers.ts +1 -47
- package/src/engine/types/index.ts +86 -40
- package/src/engine/types/nav.ts +2 -63
- package/src/engine/types/ownership.ts +2 -0
- package/src/engine/types/projection.ts +2 -138
- package/src/engine/types/relations.ts +1 -51
- package/src/engine/types/screen.ts +2 -574
- package/src/engine/types/step.ts +2 -334
- package/src/engine/types/target-ref.ts +1 -21
- package/src/engine/types/tree-node.ts +1 -132
- package/src/engine/types/workspace.ts +2 -49
- package/src/engine/validate-projection-allowlist.ts +6 -6
- package/src/entrypoint/__tests__/entrypoint-job-wiring.integration.test.ts +120 -1
- package/src/entrypoint/index.ts +49 -6
- package/src/errors/classes.ts +22 -1
- package/src/errors/field-issue.ts +0 -3
- package/src/errors/index.ts +1 -1
- package/src/errors/write-error-info.ts +10 -22
- package/src/es-ops/README.md +1 -1
- package/src/es-ops/__tests__/runner.integration.test.ts +74 -0
- package/src/es-ops/context.ts +4 -2
- package/src/es-ops/types.ts +8 -1
- package/src/event-store/__tests__/admin-api.integration.test.ts +27 -1
- package/src/event-store/__tests__/backfill-pii.integration.test.ts +279 -0
- package/src/event-store/__tests__/event-store.integration.test.ts +201 -0
- package/src/event-store/__tests__/row-to-stored-event.test.ts +2 -2
- package/src/event-store/__tests__/snapshot.integration.test.ts +86 -0
- package/src/event-store/__tests__/unscoped-stream-primitives.guard.test.ts +58 -0
- package/src/event-store/__tests__/upcaster.integration.test.ts +77 -45
- package/src/event-store/admin-api.ts +11 -4
- package/src/event-store/errors.ts +2 -35
- package/src/event-store/event-store.ts +64 -78
- package/src/event-store/events-schema.ts +11 -13
- package/src/event-store/index.ts +18 -3
- package/src/event-store/rebuild-dead-letter.ts +111 -0
- package/src/event-store/snapshot.ts +63 -40
- package/src/event-store/types.ts +2 -0
- package/src/files/__tests__/build-storage-key.test.ts +28 -0
- package/src/files/__tests__/file-ref-entity.test.ts +8 -0
- package/src/files/__tests__/files.integration.test.ts +24 -0
- package/src/files/__tests__/in-memory-provider.contract.test.ts +4 -0
- package/src/files/__tests__/local-provider.test.ts +31 -0
- package/src/files/__tests__/provider-resolver.test.ts +70 -0
- package/src/files/__tests__/write-stream.test.ts +13 -0
- package/src/files/file-handle.ts +2 -19
- package/src/files/file-ref-entity.ts +2 -2
- package/src/files/file-routes.ts +13 -0
- package/src/files/index.ts +1 -1
- package/src/files/local-provider.ts +27 -8
- package/src/files/provider-resolver.ts +31 -20
- package/src/files/types.ts +13 -55
- package/src/i18n/__tests__/required-surface-keys.test.ts +17 -0
- package/src/i18n/required-surface-keys.ts +120 -1
- package/src/jobs/__tests__/job-queue-depth.integration.test.ts +82 -0
- package/src/jobs/__tests__/jobs.integration.test.ts +329 -3
- package/src/jobs/job-runner.ts +82 -12
- package/src/logging/types.ts +1 -7
- package/src/migrations/pending-rebuilds.ts +1 -1
- package/src/observability/__tests__/observability.integration.test.ts +4 -1
- package/src/observability/__tests__/recording-tracer.test.ts +6 -4
- package/src/observability/index.ts +2 -0
- package/src/observability/noop-provider.ts +0 -9
- package/src/observability/recording-tracer.ts +0 -12
- package/src/observability/standard-metrics.ts +64 -2
- package/src/observability/types/index.ts +1 -29
- package/src/observability/types/metric.ts +1 -56
- package/src/observability/types/provider.ts +1 -32
- package/src/observability/types/span.ts +1 -64
- package/src/pipeline/__tests__/ctx-bridge.integration.test.ts +2 -2
- package/src/pipeline/__tests__/dispatcher.test.ts +366 -1
- package/src/pipeline/__tests__/event-consumer-state.integration.test.ts +31 -0
- package/src/pipeline/__tests__/event-dispatcher-delivery-max-attempts.test.ts +126 -0
- package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts +263 -0
- package/src/pipeline/__tests__/event-dispatcher.integration.test.ts +10 -0
- package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts +106 -0
- package/src/pipeline/__tests__/lifecycle-pipeline.test.ts +308 -77
- package/src/pipeline/__tests__/load-aggregate-query.integration.test.ts +16 -8
- package/src/pipeline/__tests__/post-query-hook.integration.test.ts +3 -3
- package/src/pipeline/__tests__/projection-rebuild.integration.test.ts +80 -2
- package/src/pipeline/__tests__/rebuild-poison-quarantine.integration.test.ts +274 -0
- package/src/pipeline/__tests__/try-append-event.integration.test.ts +166 -0
- package/src/pipeline/dispatch-batch.ts +187 -0
- package/src/pipeline/dispatch-query.ts +165 -0
- package/src/pipeline/dispatch-shared.ts +826 -0
- package/src/pipeline/dispatch-stream.ts +90 -0
- package/src/pipeline/dispatch-write.ts +451 -0
- package/src/pipeline/dispatcher-utils.ts +1 -1
- package/src/pipeline/dispatcher.ts +44 -1372
- package/src/pipeline/entity-cache.ts +2 -33
- package/src/pipeline/event-consumer-state.ts +30 -2
- package/src/pipeline/event-dispatcher-admin.ts +293 -0
- package/src/pipeline/event-dispatcher-delivery.ts +305 -0
- package/src/pipeline/event-dispatcher.ts +85 -542
- package/src/pipeline/index.ts +2 -0
- package/src/pipeline/msp-rebuild.ts +42 -3
- package/src/pipeline/multi-stream-apply-context.ts +4 -42
- package/src/pipeline/projection-rebuild.ts +105 -3
- package/src/pipeline/system-hooks.ts +144 -1
- package/src/rate-limit/__tests__/resolver.integration.test.ts +18 -0
- package/src/rate-limit/resolver.ts +16 -32
- package/src/schema-cli.ts +76 -16
- package/src/search/__tests__/meilisearch-adapter.integration.test.ts +207 -185
- package/src/search/__tests__/meilisearch-ids.test.ts +30 -0
- package/src/search/__tests__/reindex-entity.integration.test.ts +144 -0
- package/src/search/index.ts +6 -0
- package/src/search/meilisearch-adapter.ts +13 -12
- package/src/search/reindex-entity.ts +177 -0
- package/src/search/types.ts +1 -39
- package/src/secrets/__tests__/contains-secret.test.ts +34 -0
- package/src/secrets/__tests__/envelope-cipher.test.ts +59 -0
- package/src/secrets/__tests__/envelope.test.ts +1 -1
- package/src/secrets/dek-cache.ts +26 -5
- package/src/secrets/envelope-cipher.ts +53 -0
- package/src/secrets/envelope.ts +5 -3
- package/src/secrets/index.ts +13 -1
- package/src/secrets/stored-envelope.ts +46 -0
- package/src/secrets/types.ts +2 -162
- package/src/stack/__tests__/event-collector.test.ts +42 -0
- package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts +125 -0
- package/src/stack/db.ts +2 -1
- package/src/stack/push-entity-projection-tables.ts +4 -3
- package/src/stack/redis.ts +8 -0
- package/src/stack/request-helper.ts +38 -2
- package/src/stack/table-helpers.ts +6 -4
- package/src/stack/test-stack.ts +249 -131
- package/src/testing/__tests__/late-bound.test.ts +32 -0
- package/src/testing/__tests__/wait-for.test.ts +59 -0
- package/src/testing/boot-validator-fixture.ts +121 -0
- package/src/testing/e2e-generator.ts +8 -0
- package/src/testing/file-provider-contract.ts +104 -0
- package/src/testing/handler-context.ts +3 -1
- package/src/testing/index.ts +5 -0
- package/src/testing/late-bound.ts +5 -3
- package/src/testing/mutable-master-key-provider.ts +29 -3
- package/src/testing/wait-for.ts +3 -0
- package/src/testing/without-ambient-temporal.ts +14 -0
- package/src/time/__tests__/tz-dateline.test.ts +8 -10
- package/src/time/geo-tz.ts +1 -32
- package/src/time/index.ts +1 -0
- package/src/time/legacy-date.ts +18 -0
- package/src/time/polyfill.ts +21 -38
- package/src/time/tz-context.ts +44 -85
- package/src/ui-types/app-schema.ts +12 -0
- package/src/ui-types/index.ts +20 -7
- package/src/utils/__tests__/safe-json-temporal.test.ts +18 -0
- package/src/utils/safe-json.ts +13 -1
- package/src/__tests__/raw-table.integration.test.ts +0 -116
- package/src/bun-db/__tests__/bun-test-stack.ts +0 -6
- package/src/db/__tests__/encryption.test.ts +0 -39
- package/src/db/encryption.ts +0 -39
- package/src/db/row-helpers.ts +0 -4
- package/src/engine/__tests__/raw-table.test.ts +0 -150
- package/src/engine/__tests__/unmanaged-table.test.ts +0 -127
- package/src/engine/feature-ast/__tests__/visual-tree-parse.test.ts +0 -184
- package/src/engine/feature-ast/extractors/round4.ts +0 -1366
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
// reindexEntity — Integration Test
|
|
2
|
+
// Proves: rows written before a search consumer ever indexed them (the
|
|
3
|
+
// #1206 scenario — searchable:true added retroactively) become findable
|
|
4
|
+
// after a backfill run, soft-deleted rows stay excluded, and dryRun writes
|
|
5
|
+
// nothing.
|
|
6
|
+
|
|
7
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
8
|
+
import {
|
|
9
|
+
asRawClient,
|
|
10
|
+
buildEntityTable,
|
|
11
|
+
createEventStoreExecutor,
|
|
12
|
+
createTenantDb,
|
|
13
|
+
} from "@cosmicdrift/kumiko-framework/db";
|
|
14
|
+
import { createEntity, createTextField, defineFeature } from "@cosmicdrift/kumiko-framework/engine";
|
|
15
|
+
import { createEventsTable } from "@cosmicdrift/kumiko-framework/event-store";
|
|
16
|
+
import { reindexEntity } from "@cosmicdrift/kumiko-framework/search";
|
|
17
|
+
import {
|
|
18
|
+
setupTestStack,
|
|
19
|
+
type TestStack,
|
|
20
|
+
TestUsers,
|
|
21
|
+
unsafeCreateEntityTable,
|
|
22
|
+
} from "@cosmicdrift/kumiko-framework/stack";
|
|
23
|
+
|
|
24
|
+
const widgetEntity = createEntity({
|
|
25
|
+
table: "read_reindex_widgets",
|
|
26
|
+
softDelete: true,
|
|
27
|
+
fields: {
|
|
28
|
+
name: createTextField({ required: true, searchable: true }),
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const widgetTable = buildEntityTable("widget", widgetEntity);
|
|
33
|
+
|
|
34
|
+
const widgetFeature = defineFeature("reindex-test", (r) => {
|
|
35
|
+
r.entity("widget", widgetEntity);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
let stack: TestStack;
|
|
39
|
+
const admin = TestUsers.admin;
|
|
40
|
+
|
|
41
|
+
beforeAll(async () => {
|
|
42
|
+
stack = await setupTestStack({ features: [widgetFeature] });
|
|
43
|
+
await unsafeCreateEntityTable(stack.db, widgetEntity);
|
|
44
|
+
await createEventsTable(stack.db);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
afterAll(async () => {
|
|
48
|
+
await stack.cleanup();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
function seedExecutor() {
|
|
52
|
+
return createEventStoreExecutor(widgetTable, widgetEntity, { entityName: "widget" });
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function tenantDb() {
|
|
56
|
+
return createTenantDb(stack.db, admin.tenantId, "system");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
describe("reindexEntity", () => {
|
|
60
|
+
test("indexes rows that were never drained through the search consumer", async () => {
|
|
61
|
+
const executor = seedExecutor();
|
|
62
|
+
const created = await executor.create({ name: "Backfillable Widget" }, admin, tenantDb());
|
|
63
|
+
if (!created.isSuccess) throw new Error("seed failed");
|
|
64
|
+
|
|
65
|
+
// No stack.eventDispatcher.runOnce() call — simulates rows that existed
|
|
66
|
+
// before search indexing ever ran for this entity.
|
|
67
|
+
const preResults = await stack.search.search(admin.tenantId, "backfillable", {
|
|
68
|
+
filterType: "widget",
|
|
69
|
+
});
|
|
70
|
+
expect(preResults).toHaveLength(0);
|
|
71
|
+
|
|
72
|
+
const result = await reindexEntity(
|
|
73
|
+
stack.db,
|
|
74
|
+
stack.registry,
|
|
75
|
+
stack.search,
|
|
76
|
+
"widget",
|
|
77
|
+
admin.tenantId,
|
|
78
|
+
);
|
|
79
|
+
expect(result.indexedRows).toBe(1);
|
|
80
|
+
expect(result.failures).toHaveLength(0);
|
|
81
|
+
|
|
82
|
+
const postResults = await stack.search.search(admin.tenantId, "backfillable", {
|
|
83
|
+
filterType: "widget",
|
|
84
|
+
});
|
|
85
|
+
expect(postResults.some((r) => r.entityId === created.data.id)).toBe(true);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test("skips soft-deleted rows", async () => {
|
|
89
|
+
const executor = seedExecutor();
|
|
90
|
+
const created = await executor.create({ name: "Erased Widget" }, admin, tenantDb());
|
|
91
|
+
if (!created.isSuccess) throw new Error("seed failed");
|
|
92
|
+
const deleted = await executor.delete({ id: created.data.id }, admin, tenantDb());
|
|
93
|
+
if (!deleted.isSuccess) throw new Error("delete failed");
|
|
94
|
+
|
|
95
|
+
await reindexEntity(stack.db, stack.registry, stack.search, "widget", admin.tenantId);
|
|
96
|
+
|
|
97
|
+
const postResults = await stack.search.search(admin.tenantId, "erased", {
|
|
98
|
+
filterType: "widget",
|
|
99
|
+
});
|
|
100
|
+
expect(postResults.some((r) => r.entityId === created.data.id)).toBe(false);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("dryRun scans without writing to the index", async () => {
|
|
104
|
+
const executor = seedExecutor();
|
|
105
|
+
await executor.create({ name: "DryRun Widget" }, admin, tenantDb());
|
|
106
|
+
|
|
107
|
+
const result = await reindexEntity(
|
|
108
|
+
stack.db,
|
|
109
|
+
stack.registry,
|
|
110
|
+
stack.search,
|
|
111
|
+
"widget",
|
|
112
|
+
admin.tenantId,
|
|
113
|
+
{ dryRun: true },
|
|
114
|
+
);
|
|
115
|
+
expect(result.indexedRows).toBe(0);
|
|
116
|
+
expect(result.wouldIndexRows).toBeGreaterThan(0);
|
|
117
|
+
|
|
118
|
+
const postResults = await stack.search.search(admin.tenantId, "dryrun", {
|
|
119
|
+
filterType: "widget",
|
|
120
|
+
});
|
|
121
|
+
expect(postResults).toHaveLength(0);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
// kumiko-framework#1549: a searchable field with no matching read-table
|
|
125
|
+
// column (dropped/never-migrated) used to push one identical failures[]
|
|
126
|
+
// entry per scanned row instead of failing fast — fails loud on the first
|
|
127
|
+
// row's column set instead, which is invariant across the whole scan.
|
|
128
|
+
test("throws immediately when a searchable field has no matching column, without scanning every row", async () => {
|
|
129
|
+
const executor = seedExecutor();
|
|
130
|
+
await executor.create({ name: "Row A" }, admin, tenantDb());
|
|
131
|
+
await executor.create({ name: "Row B" }, admin, tenantDb());
|
|
132
|
+
|
|
133
|
+
await asRawClient(stack.db).unsafe(`ALTER TABLE "read_reindex_widgets" DROP COLUMN "name"`);
|
|
134
|
+
try {
|
|
135
|
+
await expect(
|
|
136
|
+
reindexEntity(stack.db, stack.registry, stack.search, "widget", admin.tenantId),
|
|
137
|
+
).rejects.toThrow(/searchable field "name" is not mappable/);
|
|
138
|
+
} finally {
|
|
139
|
+
await asRawClient(stack.db).unsafe(
|
|
140
|
+
`ALTER TABLE "read_reindex_widgets" ADD COLUMN "name" text`,
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
});
|
package/src/search/index.ts
CHANGED
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
// von SearchAdapter-Types. Apps die Meilisearch nicht nutzen, ziehen den
|
|
4
4
|
// Client-Code nicht mit rein.
|
|
5
5
|
export { createInMemorySearchAdapter } from "./in-memory-adapter";
|
|
6
|
+
export type {
|
|
7
|
+
ReindexEntityFailure,
|
|
8
|
+
ReindexEntityOptions,
|
|
9
|
+
ReindexEntityResult,
|
|
10
|
+
} from "./reindex-entity";
|
|
11
|
+
export { reindexEntity } from "./reindex-entity";
|
|
6
12
|
export type {
|
|
7
13
|
SearchAdapter,
|
|
8
14
|
SearchAdapterConfig,
|
|
@@ -8,13 +8,14 @@ export type MeilisearchAdapterOptions = {
|
|
|
8
8
|
indexPrefix?: string;
|
|
9
9
|
};
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
// Exported for unit tests (index-name / primary-key shape) without a live Meili.
|
|
12
|
+
export function meilisearchTenantIndex(prefix: string, tenantId: TenantId): string {
|
|
12
13
|
return `${prefix}t${tenantId}`;
|
|
13
14
|
}
|
|
14
15
|
|
|
15
16
|
// Meilisearch primary-key-ids: alphanumerics, `-`, `_`. UUIDs contain `-` —
|
|
16
17
|
// legal. Replace anything else just in case callers pass unexpected shapes.
|
|
17
|
-
function
|
|
18
|
+
export function meilisearchDocId(entityType: string, entityId: EntityId): string {
|
|
18
19
|
return `${entityType}_${String(entityId).replace(/[^0-9A-Za-z_-]/g, "_")}`;
|
|
19
20
|
}
|
|
20
21
|
|
|
@@ -24,7 +25,7 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
|
|
|
24
25
|
|
|
25
26
|
return {
|
|
26
27
|
async configure(tenantId, config) {
|
|
27
|
-
const index = client.index(
|
|
28
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
28
29
|
const fields = config.rankingFields ?? config.searchableFields;
|
|
29
30
|
await index.updateSearchableAttributes([...fields]).waitTask();
|
|
30
31
|
await index.updateFilterableAttributes(["_type", "_weight"]).waitTask();
|
|
@@ -32,12 +33,12 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
|
|
|
32
33
|
},
|
|
33
34
|
|
|
34
35
|
async index(tenantId, doc) {
|
|
35
|
-
const index = client.index(
|
|
36
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
36
37
|
await index
|
|
37
38
|
.addDocuments(
|
|
38
39
|
[
|
|
39
40
|
{
|
|
40
|
-
_id:
|
|
41
|
+
_id: meilisearchDocId(doc.entityType, doc.entityId),
|
|
41
42
|
_type: doc.entityType,
|
|
42
43
|
_weight: doc.weight,
|
|
43
44
|
_entityId: doc.entityId,
|
|
@@ -52,9 +53,9 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
|
|
|
52
53
|
async indexBatch(tenantId, docs) {
|
|
53
54
|
// skip: empty batch — avoid an unnecessary Meilisearch round-trip
|
|
54
55
|
if (docs.length === 0) return;
|
|
55
|
-
const index = client.index(
|
|
56
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
56
57
|
const payload = docs.map((doc) => ({
|
|
57
|
-
_id:
|
|
58
|
+
_id: meilisearchDocId(doc.entityType, doc.entityId),
|
|
58
59
|
_type: doc.entityType,
|
|
59
60
|
_weight: doc.weight,
|
|
60
61
|
_entityId: doc.entityId,
|
|
@@ -69,13 +70,13 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
|
|
|
69
70
|
async removeBatch(tenantId, items) {
|
|
70
71
|
// skip: empty batch — avoid an unnecessary Meilisearch round-trip
|
|
71
72
|
if (items.length === 0) return;
|
|
72
|
-
const index = client.index(
|
|
73
|
-
const ids = items.map((i) =>
|
|
73
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
74
|
+
const ids = items.map((i) => meilisearchDocId(i.entityType, i.entityId));
|
|
74
75
|
await index.deleteDocuments(ids).waitTask();
|
|
75
76
|
},
|
|
76
77
|
|
|
77
78
|
async search(tenantId, query, options) {
|
|
78
|
-
const index = client.index(
|
|
79
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
79
80
|
|
|
80
81
|
const filter: string[] = [];
|
|
81
82
|
if (options?.filterType) {
|
|
@@ -99,8 +100,8 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
|
|
|
99
100
|
},
|
|
100
101
|
|
|
101
102
|
async remove(tenantId, entityType, entityId) {
|
|
102
|
-
const index = client.index(
|
|
103
|
-
await index.deleteDocument(
|
|
103
|
+
const index = client.index(meilisearchTenantIndex(prefix, tenantId));
|
|
104
|
+
await index.deleteDocument(meilisearchDocId(entityType, entityId)).waitTask();
|
|
104
105
|
},
|
|
105
106
|
};
|
|
106
107
|
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// One-time backfill for entities that only got `searchable: true` after rows
|
|
2
|
+
// already existed (#1206). buildSearchDocument() only runs on the write path
|
|
3
|
+
// (createSearchEventConsumer, system-hooks.ts) — pre-existing rows never
|
|
4
|
+
// pass through it, so they stay unfindable until their next write. This
|
|
5
|
+
// re-derives a SearchDocument straight from the read-table row for every
|
|
6
|
+
// existing row and indexes it, same as a live write would.
|
|
7
|
+
|
|
8
|
+
import type { DbRunner } from "../db/connection";
|
|
9
|
+
import { resolveTableName } from "../db/entity-table-meta";
|
|
10
|
+
import { executeRawQuery } from "../db/queries/raw-sql";
|
|
11
|
+
import type { Registry, TenantId } from "../engine/types";
|
|
12
|
+
import { buildSearchDocument } from "../pipeline/system-hooks";
|
|
13
|
+
import { toSnakeCase } from "../utils/case";
|
|
14
|
+
import type { SearchAdapter, SearchDocument } from "./types";
|
|
15
|
+
|
|
16
|
+
export type ReindexEntityFailure = {
|
|
17
|
+
readonly entityId: string;
|
|
18
|
+
readonly reason: string;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export type ReindexEntityResult = {
|
|
22
|
+
readonly scannedRows: number;
|
|
23
|
+
readonly indexedRows: number;
|
|
24
|
+
// Only nonzero in a dry run — docs that would have been indexed. In a
|
|
25
|
+
// real run indexedRows already counts these; wouldIndexRows stays 0.
|
|
26
|
+
readonly wouldIndexRows: number;
|
|
27
|
+
readonly failures: readonly ReindexEntityFailure[];
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export type ReindexEntityOptions = {
|
|
31
|
+
readonly batchSize?: number;
|
|
32
|
+
// Scan + build docs, write nothing.
|
|
33
|
+
readonly dryRun?: boolean;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
function quoteIdent(name: string): string {
|
|
37
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Read-table state, forward-mapped from entity.fields (fieldName →
|
|
41
|
+
// toSnakeCase(fieldName)). SELECT * + forward-map instead of building an
|
|
42
|
+
// explicit column list: some field types (files/images) emit no column at
|
|
43
|
+
// all, others (locatedTimestamp, money) emit companion columns under a
|
|
44
|
+
// different name — an explicit alias list breaks on those. Missing columns
|
|
45
|
+
// are just skipped, matching what a `.created` event payload would carry.
|
|
46
|
+
function rowToState(
|
|
47
|
+
row: Record<string, unknown>,
|
|
48
|
+
fieldNames: readonly string[],
|
|
49
|
+
): Record<string, unknown> {
|
|
50
|
+
const state: Record<string, unknown> = {};
|
|
51
|
+
for (const fieldName of fieldNames) {
|
|
52
|
+
const column = toSnakeCase(fieldName);
|
|
53
|
+
if (Object.hasOwn(row, column)) state[fieldName] = row[column];
|
|
54
|
+
}
|
|
55
|
+
return state;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function reindexEntity(
|
|
59
|
+
db: DbRunner,
|
|
60
|
+
registry: Registry,
|
|
61
|
+
searchAdapter: SearchAdapter,
|
|
62
|
+
entityName: string,
|
|
63
|
+
tenantId: TenantId,
|
|
64
|
+
options: ReindexEntityOptions = {},
|
|
65
|
+
): Promise<ReindexEntityResult> {
|
|
66
|
+
const entity = registry.getEntity(entityName);
|
|
67
|
+
if (!entity) {
|
|
68
|
+
throw new Error(`reindexEntity: unknown entity "${entityName}"`);
|
|
69
|
+
}
|
|
70
|
+
const searchableFields = registry.getSearchableFields(entityName);
|
|
71
|
+
const extensions = registry.getSearchPayloadExtensions(entityName);
|
|
72
|
+
if (searchableFields.length === 0 && extensions.length === 0) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`reindexEntity: entity "${entityName}" has no searchable fields and no search-payload ` +
|
|
75
|
+
`extensions — nothing to index.`,
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const batchSize = options.batchSize ?? 500;
|
|
80
|
+
const tableName = resolveTableName(entityName, entity, undefined);
|
|
81
|
+
const fieldNames = Object.keys(entity.fields);
|
|
82
|
+
// Soft-deleted rows already left the live index (the .deleted consumer
|
|
83
|
+
// calls searchAdapter.remove()) — resurrecting them here would make
|
|
84
|
+
// erased entities findable again.
|
|
85
|
+
const deletedFilter =
|
|
86
|
+
entity.softDelete === true ? `AND ${quoteIdent("is_deleted")} IS NOT TRUE` : "";
|
|
87
|
+
|
|
88
|
+
const result = {
|
|
89
|
+
scannedRows: 0,
|
|
90
|
+
indexedRows: 0,
|
|
91
|
+
wouldIndexRows: 0,
|
|
92
|
+
failures: [] as ReindexEntityFailure[],
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
let offset = 0;
|
|
96
|
+
// Whether a searchable field lacks a matching read-table column depends
|
|
97
|
+
// only on the table's column set (rowToState/Object.hasOwn), not on any
|
|
98
|
+
// particular row's data — every row from the same SELECT * has the same
|
|
99
|
+
// columns. Check once against the first row instead of every row, so a
|
|
100
|
+
// dropped/never-migrated column fails fast with one clear error instead
|
|
101
|
+
// of one identical failures[] entry per scanned row (1M rows → 1M
|
|
102
|
+
// entries, and the real schema problem is buried in "x failed").
|
|
103
|
+
let uncheckedSchema = true;
|
|
104
|
+
|
|
105
|
+
for (;;) {
|
|
106
|
+
// ponytail: LIMIT/OFFSET, not a keyset cursor — id can be uuid or
|
|
107
|
+
// serial depending on the entity, and a uniform cursor comparison
|
|
108
|
+
// across both types needs a text cast that breaks integer ordering.
|
|
109
|
+
// This is a one-time backfill over existing rows, not a live hot path;
|
|
110
|
+
// switch to keyset if it ever needs to run against a churning table.
|
|
111
|
+
const rows = await executeRawQuery<Record<string, unknown>>(
|
|
112
|
+
db,
|
|
113
|
+
`SELECT * FROM ${quoteIdent(tableName)}
|
|
114
|
+
WHERE ${quoteIdent("tenant_id")} = $1 ${deletedFilter}
|
|
115
|
+
ORDER BY ${quoteIdent("id")} ASC
|
|
116
|
+
LIMIT $2 OFFSET $3`,
|
|
117
|
+
[tenantId, batchSize, offset],
|
|
118
|
+
);
|
|
119
|
+
if (rows.length === 0) break;
|
|
120
|
+
|
|
121
|
+
if (uncheckedSchema) {
|
|
122
|
+
uncheckedSchema = false;
|
|
123
|
+
const sampleState = rowToState(rows[0] as Record<string, unknown>, fieldNames);
|
|
124
|
+
const unmappedField = searchableFields.find((fieldName) => !(fieldName in sampleState));
|
|
125
|
+
if (unmappedField !== undefined) {
|
|
126
|
+
throw new Error(
|
|
127
|
+
`reindexEntity: searchable field "${unmappedField}" is not mappable from read-table ` +
|
|
128
|
+
`column ${quoteIdent(tableName)} — likely a dropped/never-migrated column.`,
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const docs: Array<{ entityId: string; doc: SearchDocument }> = [];
|
|
134
|
+
for (const row of rows) {
|
|
135
|
+
result.scannedRows++;
|
|
136
|
+
const entityId = String(row["id"]);
|
|
137
|
+
try {
|
|
138
|
+
const state = rowToState(row, fieldNames);
|
|
139
|
+
const doc = await buildSearchDocument(entityName, entityId, state, registry);
|
|
140
|
+
if (doc) docs.push({ entityId, doc });
|
|
141
|
+
} catch (e) {
|
|
142
|
+
result.failures.push({ entityId, reason: e instanceof Error ? e.message : String(e) });
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (!options.dryRun && docs.length > 0) {
|
|
147
|
+
if (searchAdapter.indexBatch) {
|
|
148
|
+
try {
|
|
149
|
+
await searchAdapter.indexBatch(
|
|
150
|
+
tenantId,
|
|
151
|
+
docs.map((d) => d.doc),
|
|
152
|
+
);
|
|
153
|
+
result.indexedRows += docs.length;
|
|
154
|
+
} catch (e) {
|
|
155
|
+
const reason = e instanceof Error ? e.message : String(e);
|
|
156
|
+
for (const { entityId } of docs) result.failures.push({ entityId, reason });
|
|
157
|
+
}
|
|
158
|
+
} else {
|
|
159
|
+
for (const { entityId, doc } of docs) {
|
|
160
|
+
try {
|
|
161
|
+
await searchAdapter.index(tenantId, doc);
|
|
162
|
+
result.indexedRows++;
|
|
163
|
+
} catch (e) {
|
|
164
|
+
result.failures.push({ entityId, reason: e instanceof Error ? e.message : String(e) });
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
} else if (options.dryRun) {
|
|
169
|
+
result.wouldIndexRows += docs.length;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
offset += rows.length;
|
|
173
|
+
if (rows.length < batchSize) break;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
return result;
|
|
177
|
+
}
|
package/src/search/types.ts
CHANGED
|
@@ -1,39 +1 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
export type SearchAdapterConfig = {
|
|
4
|
-
searchableFields: readonly string[];
|
|
5
|
-
rankingFields?: readonly string[];
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
export type SearchDocument = {
|
|
9
|
-
entityType: string;
|
|
10
|
-
entityId: EntityId;
|
|
11
|
-
weight: number;
|
|
12
|
-
fields: Record<string, unknown>;
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
export type SearchResult = {
|
|
16
|
-
entityType: string;
|
|
17
|
-
entityId: EntityId;
|
|
18
|
-
};
|
|
19
|
-
|
|
20
|
-
export type SearchOptions = {
|
|
21
|
-
limit?: number;
|
|
22
|
-
filterType?: string;
|
|
23
|
-
};
|
|
24
|
-
|
|
25
|
-
export type SearchAdapter = {
|
|
26
|
-
configure(tenantId: TenantId, config: SearchAdapterConfig): Promise<void>;
|
|
27
|
-
index(tenantId: TenantId, doc: SearchDocument): Promise<void>;
|
|
28
|
-
search(tenantId: TenantId, query: string, options?: SearchOptions): Promise<SearchResult[]>;
|
|
29
|
-
remove(tenantId: TenantId, entityType: string, entityId: EntityId): Promise<void>;
|
|
30
|
-
// Bulk variants. Default implementations loop over the single-doc methods —
|
|
31
|
-
// adapters should override when the backend supports a real batch call
|
|
32
|
-
// (Meilisearch, Elasticsearch, Typesense all do). Cuts a batch-write from
|
|
33
|
-
// N sequential HTTP + waitTask round-trips to one.
|
|
34
|
-
indexBatch?(tenantId: TenantId, docs: readonly SearchDocument[]): Promise<void>;
|
|
35
|
-
removeBatch?(
|
|
36
|
-
tenantId: TenantId,
|
|
37
|
-
items: readonly { entityType: string; entityId: EntityId }[],
|
|
38
|
-
): Promise<void>;
|
|
39
|
-
};
|
|
1
|
+
export type * from "@cosmicdrift/kumiko-types/search-adapter";
|
|
@@ -32,6 +32,18 @@ export type _R6TypeAssertions = [
|
|
|
32
32
|
// the backstop. This is what keeps generic-over-response handlers compiling.
|
|
33
33
|
Expect<Equal<ContainsSecret<unknown>, false>>,
|
|
34
34
|
Expect<Equal<ContainsSecret<never>, false>>,
|
|
35
|
+
// 556/3: ContainsSecret<A | B> distributes over the union (naked type
|
|
36
|
+
// param) to `ContainsSecret<A> | ContainsSecret<B>` = `boolean`, not the
|
|
37
|
+
// literal `true` — pinned here so a future change to this predicate can't
|
|
38
|
+
// silently regress it. The actual leak-guard in define-handler.ts compares
|
|
39
|
+
// via `true extends ContainsSecret<TData>` (556/1) instead of `ContainsSecret<
|
|
40
|
+
// TData> extends true`, which is fail-closed for `boolean` either way.
|
|
41
|
+
Expect<Equal<ContainsSecret<{ ok: true } | { s: Secret<string> }>, boolean>>,
|
|
42
|
+
// 556/2: Map/Set are explicit SafeLeaf members now — pins that the runtime
|
|
43
|
+
// guard's "walk Map/Set entries separately" intent is mirrored, not an
|
|
44
|
+
// accidental compile-time blind spot.
|
|
45
|
+
Expect<Equal<ContainsSecret<Map<string, Secret<string>>>, false>>,
|
|
46
|
+
Expect<Equal<ContainsSecret<Set<Secret<string>>>, false>>,
|
|
35
47
|
];
|
|
36
48
|
|
|
37
49
|
const schema = z.object({ q: z.string() });
|
|
@@ -57,6 +69,28 @@ function genericResponseHandler<K extends string>(kind: K) {
|
|
|
57
69
|
}
|
|
58
70
|
void genericResponseHandler;
|
|
59
71
|
|
|
72
|
+
// 556/1: ContainsSecret<A | B> distributes over the union (naked type param
|
|
73
|
+
// in the condition) to `ContainsSecret<A> | ContainsSecret<B>` = `boolean`,
|
|
74
|
+
// not the literal `true` — a check written as `ContainsSecret<TData> extends
|
|
75
|
+
// true` silently passes a union with a leak hidden in ONE branch. Both
|
|
76
|
+
// assertions below must hold: the guard rejects the leaking union AND still
|
|
77
|
+
// accepts the clean one, proving the fix (membership form on the phantom
|
|
78
|
+
// param) is fail-closed for unions, not just for a bare Secret<>.
|
|
79
|
+
declare const leakingUnion: { ok: true } | { apiKey: Secret<string> };
|
|
80
|
+
// @ts-expect-error — R6: a Secret<> hidden in one union branch must still leak-guard
|
|
81
|
+
defineQueryHandler({
|
|
82
|
+
name: "t:query:union-leak",
|
|
83
|
+
schema,
|
|
84
|
+
handler: async () => leakingUnion,
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
declare const cleanUnion: { ok: true } | { id: string };
|
|
88
|
+
defineQueryHandler({
|
|
89
|
+
name: "t:query:union-clean",
|
|
90
|
+
schema,
|
|
91
|
+
handler: async () => cleanUnion,
|
|
92
|
+
});
|
|
93
|
+
|
|
60
94
|
describe("R6 ContainsSecret", () => {
|
|
61
95
|
test("a clean query handler still builds — the guard param is invisible", () => {
|
|
62
96
|
const def = defineQueryHandler({
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { createEnvMasterKeyProvider } from "../env-master-key-provider";
|
|
4
|
+
import { createEnvelopeCipher } from "../envelope-cipher";
|
|
5
|
+
import { isStoredEnvelope } from "../stored-envelope";
|
|
6
|
+
|
|
7
|
+
function makeProvider(currentVersion = 1) {
|
|
8
|
+
return createEnvMasterKeyProvider({
|
|
9
|
+
env: {
|
|
10
|
+
KUMIKO_SECRETS_MASTER_KEY_CURRENT_VERSION: String(currentVersion),
|
|
11
|
+
[`KUMIKO_SECRETS_MASTER_KEY_V${currentVersion}`]: randomBytes(32).toString("base64"),
|
|
12
|
+
},
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
describe("envelope-cipher — encrypt/decrypt", () => {
|
|
17
|
+
test("round-trips plaintext through the JSON envelope format", async () => {
|
|
18
|
+
const cipher = createEnvelopeCipher(makeProvider());
|
|
19
|
+
const stored = await cipher.encrypt("s3cret-smtp-pass");
|
|
20
|
+
|
|
21
|
+
expect(stored.startsWith("{")).toBe(true);
|
|
22
|
+
const parsed: unknown = JSON.parse(stored);
|
|
23
|
+
expect(isStoredEnvelope(parsed)).toBe(true);
|
|
24
|
+
|
|
25
|
+
expect(await cipher.decrypt(stored)).toBe("s3cret-smtp-pass");
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("stored value carries the kekVersion for later rotation", async () => {
|
|
29
|
+
const cipher = createEnvelopeCipher(makeProvider(7));
|
|
30
|
+
const stored = await cipher.encrypt("x");
|
|
31
|
+
expect((JSON.parse(stored) as { kekVersion: number }).kekVersion).toBe(7);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("tampered ciphertext fails GCM authentication", async () => {
|
|
35
|
+
const cipher = createEnvelopeCipher(makeProvider());
|
|
36
|
+
const stored = JSON.parse(await cipher.encrypt("payload")) as Record<string, unknown>;
|
|
37
|
+
const cipherBytes = Buffer.from(stored["ciphertext"] as string, "base64");
|
|
38
|
+
cipherBytes[0] = (cipherBytes[0] ?? 0) ^ 0xff;
|
|
39
|
+
const tampered = JSON.stringify({ ...stored, ciphertext: cipherBytes.toString("base64") });
|
|
40
|
+
await expect(cipher.decrypt(tampered)).rejects.toThrow();
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
describe("envelope-cipher — malformed input", () => {
|
|
45
|
+
test("rejects invalid JSON that looks like an envelope", async () => {
|
|
46
|
+
const cipher = createEnvelopeCipher(makeProvider());
|
|
47
|
+
await expect(cipher.decrypt("{garbage")).rejects.toThrow(/not valid JSON/);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("rejects JSON that is not a StoredEnvelope", async () => {
|
|
51
|
+
const cipher = createEnvelopeCipher(makeProvider());
|
|
52
|
+
await expect(cipher.decrypt('{"foo":"bar"}')).rejects.toThrow(/not a StoredEnvelope/);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("empty string is not valid JSON", async () => {
|
|
56
|
+
const cipher = createEnvelopeCipher(makeProvider());
|
|
57
|
+
await expect(cipher.decrypt("")).rejects.toThrow(/not valid JSON/);
|
|
58
|
+
});
|
|
59
|
+
});
|
|
@@ -53,7 +53,7 @@ describe("envelope — encryptValue/decryptValue", () => {
|
|
|
53
53
|
...envelope,
|
|
54
54
|
ciphertext: Buffer.concat([
|
|
55
55
|
envelope.ciphertext.subarray(0, 1),
|
|
56
|
-
Buffer.from([envelope.ciphertext[
|
|
56
|
+
Buffer.from([envelope.ciphertext[1]! ^ 0xff]),
|
|
57
57
|
envelope.ciphertext.subarray(2),
|
|
58
58
|
]),
|
|
59
59
|
};
|
package/src/secrets/dek-cache.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// - maxEntries (default 1000): LRU eviction kicks in on insert when full.
|
|
10
10
|
|
|
11
11
|
import { createHash } from "node:crypto";
|
|
12
|
-
import type { MasterKeyProvider } from "./types";
|
|
12
|
+
import type { KeyScope, MasterKeyProvider } from "./types";
|
|
13
13
|
|
|
14
14
|
const DEFAULT_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
|
15
15
|
const DEFAULT_MAX_ENTRIES = 1000;
|
|
@@ -25,8 +25,16 @@ export type DekCacheOptions = {
|
|
|
25
25
|
|
|
26
26
|
export type DekCache = {
|
|
27
27
|
// Unwrap via the provider, caching the result. Second call within TTL
|
|
28
|
-
// returns the cached DEK without hitting the provider.
|
|
29
|
-
|
|
28
|
+
// returns the cached DEK without hitting the provider. The cache key is
|
|
29
|
+
// (encryptedDek, kekVersion) — scope is only forwarded to the provider;
|
|
30
|
+
// encryptedDek bytes are unique per value, so scoped providers can't
|
|
31
|
+
// collide on the key either.
|
|
32
|
+
unwrapDek(
|
|
33
|
+
encryptedDek: Buffer,
|
|
34
|
+
kekVersion: number,
|
|
35
|
+
provider: MasterKeyProvider,
|
|
36
|
+
scope?: KeyScope,
|
|
37
|
+
): Promise<Buffer>;
|
|
30
38
|
|
|
31
39
|
// Drop every entry. Call after KEK rotation so old cached DEKs (still
|
|
32
40
|
// valid, but referencing the old kekVersion) don't serve reads that
|
|
@@ -37,6 +45,19 @@ export type DekCache = {
|
|
|
37
45
|
size(): number;
|
|
38
46
|
};
|
|
39
47
|
|
|
48
|
+
// Wrap a provider so its unwrapDek goes through the cache. Callers keep the
|
|
49
|
+
// full MasterKeyProvider contract without knowing about caching —
|
|
50
|
+
// decryptValue handles crypto, the cache handles cost.
|
|
51
|
+
export function withDekCache(provider: MasterKeyProvider, cache: DekCache): MasterKeyProvider {
|
|
52
|
+
return {
|
|
53
|
+
wrapDek: (dek, scope) => provider.wrapDek(dek, scope),
|
|
54
|
+
unwrapDek: (encryptedDek, version, scope) =>
|
|
55
|
+
cache.unwrapDek(encryptedDek, version, provider, scope),
|
|
56
|
+
currentVersion: () => provider.currentVersion(),
|
|
57
|
+
isAvailable: () => provider.isAvailable(),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
40
61
|
export function createDekCache(opts: DekCacheOptions = {}): DekCache {
|
|
41
62
|
const ttlMs = opts.ttlMs ?? DEFAULT_TTL_MS;
|
|
42
63
|
const maxEntries = opts.maxEntries ?? DEFAULT_MAX_ENTRIES;
|
|
@@ -74,7 +95,7 @@ export function createDekCache(opts: DekCacheOptions = {}): DekCache {
|
|
|
74
95
|
}
|
|
75
96
|
|
|
76
97
|
return {
|
|
77
|
-
async unwrapDek(encryptedDek, kekVersion, provider) {
|
|
98
|
+
async unwrapDek(encryptedDek, kekVersion, provider, scope) {
|
|
78
99
|
const key = cacheKey(encryptedDek, kekVersion);
|
|
79
100
|
const hit = entries.get(key);
|
|
80
101
|
if (hit && hit.expiresAt > now()) {
|
|
@@ -92,7 +113,7 @@ export function createDekCache(opts: DekCacheOptions = {}): DekCache {
|
|
|
92
113
|
entries.delete(key);
|
|
93
114
|
}
|
|
94
115
|
|
|
95
|
-
const dek = await provider.unwrapDek(encryptedDek, kekVersion);
|
|
116
|
+
const dek = await provider.unwrapDek(encryptedDek, kekVersion, scope);
|
|
96
117
|
evictOldestIfFull();
|
|
97
118
|
// Store a copy — caller can zero its own buffer after use.
|
|
98
119
|
entries.set(key, { dek: Buffer.from(dek), expiresAt: now() + ttlMs });
|