@open-mercato/core 0.5.1-develop.2691.d8a0934b37 → 0.5.1-develop.2699.f8b50c8046
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/api_keys/data/entities.js +1 -1
- package/dist/modules/api_keys/data/entities.js.map +1 -1
- package/dist/modules/api_keys/services/apiKeyService.js +5 -5
- package/dist/modules/api_keys/services/apiKeyService.js.map +2 -2
- package/dist/modules/attachments/api/library/[id]/route.js +1 -1
- package/dist/modules/attachments/api/library/[id]/route.js.map +2 -2
- package/dist/modules/attachments/api/library/route.js +7 -9
- package/dist/modules/attachments/api/library/route.js.map +2 -2
- package/dist/modules/attachments/api/partitions/route.js +3 -3
- package/dist/modules/attachments/api/partitions/route.js.map +2 -2
- package/dist/modules/attachments/api/route.js +6 -5
- package/dist/modules/attachments/api/route.js.map +2 -2
- package/dist/modules/attachments/api/transfer/route.js +1 -1
- package/dist/modules/attachments/api/transfer/route.js.map +2 -2
- package/dist/modules/attachments/data/entities.js +2 -1
- package/dist/modules/attachments/data/entities.js.map +2 -2
- package/dist/modules/attachments/lib/ocrQueue.js +1 -1
- package/dist/modules/attachments/lib/ocrQueue.js.map +2 -2
- package/dist/modules/audit_logs/api/audit-logs/actions/export/route.js.map +2 -2
- package/dist/modules/audit_logs/api/audit-logs/actions/route.js.map +2 -2
- package/dist/modules/audit_logs/data/entities.js +1 -1
- package/dist/modules/audit_logs/data/entities.js.map +1 -1
- package/dist/modules/audit_logs/services/actionLogService.js +77 -70
- package/dist/modules/audit_logs/services/actionLogService.js.map +2 -2
- package/dist/modules/auth/api/roles/acl/route.js +1 -1
- package/dist/modules/auth/api/roles/acl/route.js.map +2 -2
- package/dist/modules/auth/api/users/acl/route.js +2 -2
- package/dist/modules/auth/api/users/acl/route.js.map +2 -2
- package/dist/modules/auth/api/users/resend-invite/route.js +1 -1
- package/dist/modules/auth/api/users/resend-invite/route.js.map +2 -2
- package/dist/modules/auth/cli.js +12 -6
- package/dist/modules/auth/cli.js.map +2 -2
- package/dist/modules/auth/commands/users.js +1 -1
- package/dist/modules/auth/commands/users.js.map +2 -2
- package/dist/modules/auth/data/entities.js +1 -1
- package/dist/modules/auth/data/entities.js.map +2 -2
- package/dist/modules/auth/lib/setup-app.js +3 -3
- package/dist/modules/auth/lib/setup-app.js.map +2 -2
- package/dist/modules/auth/services/authService.js +2 -2
- package/dist/modules/auth/services/authService.js.map +2 -2
- package/dist/modules/business_rules/api/rules/route.js +3 -3
- package/dist/modules/business_rules/api/rules/route.js.map +2 -2
- package/dist/modules/business_rules/api/sets/[id]/members/route.js +7 -4
- package/dist/modules/business_rules/api/sets/[id]/members/route.js.map +2 -2
- package/dist/modules/business_rules/api/sets/route.js +3 -3
- package/dist/modules/business_rules/api/sets/route.js.map +2 -2
- package/dist/modules/business_rules/cli.js +1 -1
- package/dist/modules/business_rules/cli.js.map +2 -2
- package/dist/modules/business_rules/data/entities.js +2 -9
- package/dist/modules/business_rules/data/entities.js.map +2 -2
- package/dist/modules/business_rules/lib/rule-engine.js +1 -1
- package/dist/modules/business_rules/lib/rule-engine.js.map +2 -2
- package/dist/modules/catalog/api/option-schemas/route.js +0 -1
- package/dist/modules/catalog/api/option-schemas/route.js.map +2 -2
- package/dist/modules/catalog/data/entities.js +2 -11
- package/dist/modules/catalog/data/entities.js.map +2 -2
- package/dist/modules/configs/data/entities.js +2 -1
- package/dist/modules/configs/data/entities.js.map +2 -2
- package/dist/modules/currencies/commands/fetch-configs.js +3 -3
- package/dist/modules/currencies/commands/fetch-configs.js.map +2 -2
- package/dist/modules/currencies/data/entities.js +1 -1
- package/dist/modules/currencies/data/entities.js.map +2 -2
- package/dist/modules/customer_accounts/api/signup.js +1 -1
- package/dist/modules/customer_accounts/api/signup.js.map +2 -2
- package/dist/modules/customer_accounts/data/entities.js +1 -1
- package/dist/modules/customer_accounts/data/entities.js.map +2 -2
- package/dist/modules/customer_accounts/services/customerInvitationService.js +1 -1
- package/dist/modules/customer_accounts/services/customerInvitationService.js.map +2 -2
- package/dist/modules/customer_accounts/services/customerSessionService.js +1 -1
- package/dist/modules/customer_accounts/services/customerSessionService.js.map +2 -2
- package/dist/modules/customer_accounts/services/customerTokenService.js +12 -7
- package/dist/modules/customer_accounts/services/customerTokenService.js.map +2 -2
- package/dist/modules/customers/api/interactions/conflicts/route.js +19 -17
- package/dist/modules/customers/api/interactions/conflicts/route.js.map +2 -2
- package/dist/modules/customers/api/interactions/counts/route.js +7 -6
- package/dist/modules/customers/api/interactions/counts/route.js.map +2 -2
- package/dist/modules/customers/api/interactions/route.js +28 -42
- package/dist/modules/customers/api/interactions/route.js.map +2 -2
- package/dist/modules/customers/api/utils.js +29 -24
- package/dist/modules/customers/api/utils.js.map +2 -2
- package/dist/modules/customers/cli.js +45 -40
- package/dist/modules/customers/cli.js.map +2 -2
- package/dist/modules/customers/commands/dictionaries.js +1 -1
- package/dist/modules/customers/commands/dictionaries.js.map +2 -2
- package/dist/modules/customers/commands/tags.js +1 -1
- package/dist/modules/customers/commands/tags.js.map +2 -2
- package/dist/modules/customers/data/entities.js +2 -12
- package/dist/modules/customers/data/entities.js.map +2 -2
- package/dist/modules/customers/lib/interactionProjection.js +18 -15
- package/dist/modules/customers/lib/interactionProjection.js.map +2 -2
- package/dist/modules/customers/lib/personCompanyLinkTable.js +6 -8
- package/dist/modules/customers/lib/personCompanyLinkTable.js.map +2 -2
- package/dist/modules/dashboards/api/roles/widgets/route.js +1 -1
- package/dist/modules/dashboards/api/roles/widgets/route.js.map +2 -2
- package/dist/modules/dashboards/api/users/widgets/route.js +1 -1
- package/dist/modules/dashboards/api/users/widgets/route.js.map +2 -2
- package/dist/modules/dashboards/data/entities.js +1 -1
- package/dist/modules/dashboards/data/entities.js.map +1 -1
- package/dist/modules/data_sync/api/mappings/route.js +1 -1
- package/dist/modules/data_sync/api/mappings/route.js.map +2 -2
- package/dist/modules/data_sync/data/entities.js +2 -1
- package/dist/modules/data_sync/data/entities.js.map +2 -2
- package/dist/modules/data_sync/lib/id-mapping.js +1 -1
- package/dist/modules/data_sync/lib/id-mapping.js.map +2 -2
- package/dist/modules/data_sync/lib/sync-run-service.js +1 -1
- package/dist/modules/data_sync/lib/sync-run-service.js.map +2 -2
- package/dist/modules/dictionaries/commands/factory.js +1 -1
- package/dist/modules/dictionaries/commands/factory.js.map +2 -2
- package/dist/modules/dictionaries/data/entities.js +2 -9
- package/dist/modules/dictionaries/data/entities.js.map +2 -2
- package/dist/modules/directory/commands/organizations.js +4 -4
- package/dist/modules/directory/commands/organizations.js.map +2 -2
- package/dist/modules/directory/data/entities.js +2 -1
- package/dist/modules/directory/data/entities.js.map +2 -2
- package/dist/modules/entities/api/definitions.js +2 -2
- package/dist/modules/entities/api/definitions.js.map +2 -2
- package/dist/modules/entities/api/encryption.js +2 -2
- package/dist/modules/entities/api/encryption.js.map +2 -2
- package/dist/modules/entities/api/relations/options.js +2 -2
- package/dist/modules/entities/api/relations/options.js.map +2 -2
- package/dist/modules/entities/cli.js +4 -4
- package/dist/modules/entities/cli.js.map +2 -2
- package/dist/modules/entities/data/entities.js +1 -1
- package/dist/modules/entities/data/entities.js.map +2 -2
- package/dist/modules/entities/lib/field-definitions.js +2 -2
- package/dist/modules/entities/lib/field-definitions.js.map +2 -2
- package/dist/modules/entities/lib/register.js +1 -1
- package/dist/modules/entities/lib/register.js.map +2 -2
- package/dist/modules/feature_toggles/data/entities.js +2 -9
- package/dist/modules/feature_toggles/data/entities.js.map +2 -2
- package/dist/modules/inbox_ops/api/proposals/counts/route.js +3 -6
- package/dist/modules/inbox_ops/api/proposals/counts/route.js.map +2 -2
- package/dist/modules/inbox_ops/data/entities.js +2 -8
- package/dist/modules/inbox_ops/data/entities.js.map +2 -2
- package/dist/modules/inbox_ops/lib/messagesIntegration.js +6 -6
- package/dist/modules/inbox_ops/lib/messagesIntegration.js.map +2 -2
- package/dist/modules/integrations/data/entities.js +2 -1
- package/dist/modules/integrations/data/entities.js.map +2 -2
- package/dist/modules/integrations/lib/credentials-service.js +1 -1
- package/dist/modules/integrations/lib/credentials-service.js.map +2 -2
- package/dist/modules/integrations/lib/log-service.js +1 -1
- package/dist/modules/integrations/lib/log-service.js.map +2 -2
- package/dist/modules/integrations/lib/state-service.js +1 -1
- package/dist/modules/integrations/lib/state-service.js.map +2 -2
- package/dist/modules/messages/api/route.js +90 -93
- package/dist/modules/messages/api/route.js.map +2 -2
- package/dist/modules/messages/api/unread-count/route.js +8 -7
- package/dist/modules/messages/api/unread-count/route.js.map +2 -2
- package/dist/modules/messages/commands/confirmations.js +1 -1
- package/dist/modules/messages/commands/confirmations.js.map +2 -2
- package/dist/modules/messages/commands/messages.js +3 -3
- package/dist/modules/messages/commands/messages.js.map +2 -2
- package/dist/modules/messages/data/entities.js +2 -1
- package/dist/modules/messages/data/entities.js.map +2 -2
- package/dist/modules/messages/lib/email-sender.js +1 -1
- package/dist/modules/messages/lib/email-sender.js.map +2 -2
- package/dist/modules/messages/lib/searchLookup.js +8 -8
- package/dist/modules/messages/lib/searchLookup.js.map +2 -2
- package/dist/modules/messages/lib/tokenConsumption.js +9 -4
- package/dist/modules/messages/lib/tokenConsumption.js.map +2 -2
- package/dist/modules/notifications/data/entities.js +2 -1
- package/dist/modules/notifications/data/entities.js.map +2 -2
- package/dist/modules/notifications/lib/notificationRecipients.js +15 -5
- package/dist/modules/notifications/lib/notificationRecipients.js.map +2 -2
- package/dist/modules/notifications/lib/notificationService.js +39 -34
- package/dist/modules/notifications/lib/notificationService.js.map +2 -2
- package/dist/modules/notifications/workers/create-notification.worker.js +14 -13
- package/dist/modules/notifications/workers/create-notification.worker.js.map +2 -2
- package/dist/modules/payment_gateways/api/transactions/route.js +2 -2
- package/dist/modules/payment_gateways/api/transactions/route.js.map +2 -2
- package/dist/modules/payment_gateways/data/entities.js +2 -1
- package/dist/modules/payment_gateways/data/entities.js.map +2 -2
- package/dist/modules/payment_gateways/lib/gateway-service.js +1 -1
- package/dist/modules/payment_gateways/lib/gateway-service.js.map +2 -2
- package/dist/modules/payment_gateways/lib/webhook-utils.js +2 -2
- package/dist/modules/payment_gateways/lib/webhook-utils.js.map +2 -2
- package/dist/modules/perspectives/data/entities.js +1 -1
- package/dist/modules/perspectives/data/entities.js.map +2 -2
- package/dist/modules/planner/data/entities.js +1 -1
- package/dist/modules/planner/data/entities.js.map +2 -2
- package/dist/modules/progress/data/entities.js +2 -1
- package/dist/modules/progress/data/entities.js.map +2 -2
- package/dist/modules/progress/lib/progressServiceImpl.js +1 -1
- package/dist/modules/progress/lib/progressServiceImpl.js.map +2 -2
- package/dist/modules/query_index/api/status.js +66 -57
- package/dist/modules/query_index/api/status.js.map +2 -2
- package/dist/modules/query_index/cli.js +39 -24
- package/dist/modules/query_index/cli.js.map +2 -2
- package/dist/modules/query_index/data/entities.js +1 -1
- package/dist/modules/query_index/data/entities.js.map +2 -2
- package/dist/modules/query_index/di.js +25 -13
- package/dist/modules/query_index/di.js.map +2 -2
- package/dist/modules/query_index/lib/batch.js +31 -33
- package/dist/modules/query_index/lib/batch.js.map +2 -2
- package/dist/modules/query_index/lib/coverage.js +63 -50
- package/dist/modules/query_index/lib/coverage.js.map +2 -2
- package/dist/modules/query_index/lib/engine.js +592 -588
- package/dist/modules/query_index/lib/engine.js.map +2 -2
- package/dist/modules/query_index/lib/indexer.js +74 -47
- package/dist/modules/query_index/lib/indexer.js.map +2 -2
- package/dist/modules/query_index/lib/jobs.js +37 -24
- package/dist/modules/query_index/lib/jobs.js.map +2 -2
- package/dist/modules/query_index/lib/purge.js +19 -11
- package/dist/modules/query_index/lib/purge.js.map +2 -2
- package/dist/modules/query_index/lib/reindexer.js +47 -44
- package/dist/modules/query_index/lib/reindexer.js.map +2 -2
- package/dist/modules/query_index/lib/search-tokens.js +47 -25
- package/dist/modules/query_index/lib/search-tokens.js.map +2 -2
- package/dist/modules/query_index/lib/stale.js +14 -12
- package/dist/modules/query_index/lib/stale.js.map +2 -2
- package/dist/modules/query_index/lib/subscriber-scope.js +2 -2
- package/dist/modules/query_index/lib/subscriber-scope.js.map +2 -2
- package/dist/modules/query_index/subscribers/delete_one.js +3 -2
- package/dist/modules/query_index/subscribers/delete_one.js.map +2 -2
- package/dist/modules/resources/commands/tag-assignments.js +1 -1
- package/dist/modules/resources/commands/tag-assignments.js.map +2 -2
- package/dist/modules/resources/commands/tags.js +1 -1
- package/dist/modules/resources/commands/tags.js.map +2 -2
- package/dist/modules/resources/data/entities.js +2 -1
- package/dist/modules/resources/data/entities.js.map +2 -2
- package/dist/modules/sales/commands/documentAddresses.js +2 -2
- package/dist/modules/sales/commands/documentAddresses.js.map +2 -2
- package/dist/modules/sales/commands/notes.js.map +2 -2
- package/dist/modules/sales/commands/tags.js +1 -1
- package/dist/modules/sales/commands/tags.js.map +2 -2
- package/dist/modules/sales/data/enrichers.js +9 -8
- package/dist/modules/sales/data/enrichers.js.map +2 -2
- package/dist/modules/sales/data/entities.js +2 -11
- package/dist/modules/sales/data/entities.js.map +2 -2
- package/dist/modules/shipping_carriers/data/entities.js +2 -1
- package/dist/modules/shipping_carriers/data/entities.js.map +2 -2
- package/dist/modules/shipping_carriers/lib/shipping-service.js +1 -1
- package/dist/modules/shipping_carriers/lib/shipping-service.js.map +2 -2
- package/dist/modules/shipping_carriers/lib/webhook-utils.js +2 -2
- package/dist/modules/shipping_carriers/lib/webhook-utils.js.map +2 -2
- package/dist/modules/staff/data/entities.js +1 -1
- package/dist/modules/staff/data/entities.js.map +2 -2
- package/dist/modules/translations/api/[entityType]/[entityId]/route.js +3 -5
- package/dist/modules/translations/api/[entityType]/[entityId]/route.js.map +2 -2
- package/dist/modules/translations/api/context.js +2 -2
- package/dist/modules/translations/api/context.js.map +2 -2
- package/dist/modules/translations/commands/translations.js +46 -39
- package/dist/modules/translations/commands/translations.js.map +2 -2
- package/dist/modules/translations/components/TranslationManager.js +19 -10
- package/dist/modules/translations/components/TranslationManager.js.map +2 -2
- package/dist/modules/translations/data/entities.js +1 -1
- package/dist/modules/translations/data/entities.js.map +2 -2
- package/dist/modules/translations/lib/apply.js +4 -4
- package/dist/modules/translations/lib/apply.js.map +2 -2
- package/dist/modules/translations/lib/batch.js +3 -2
- package/dist/modules/translations/lib/batch.js.map +2 -2
- package/dist/modules/translations/subscribers/cleanup.js +3 -5
- package/dist/modules/translations/subscribers/cleanup.js.map +2 -2
- package/dist/modules/workflows/api/definitions/route.js +1 -1
- package/dist/modules/workflows/api/definitions/route.js.map +2 -2
- package/dist/modules/workflows/cli.js +5 -5
- package/dist/modules/workflows/cli.js.map +2 -2
- package/dist/modules/workflows/data/entities.js +2 -1
- package/dist/modules/workflows/data/entities.js.map +2 -2
- package/dist/modules/workflows/lib/event-logger.js +2 -2
- package/dist/modules/workflows/lib/event-logger.js.map +2 -2
- package/dist/modules/workflows/lib/seeds.js +16 -1
- package/dist/modules/workflows/lib/seeds.js.map +2 -2
- package/dist/modules/workflows/lib/step-handler.js +3 -3
- package/dist/modules/workflows/lib/step-handler.js.map +2 -2
- package/dist/modules/workflows/lib/task-handler.js +1 -1
- package/dist/modules/workflows/lib/task-handler.js.map +2 -2
- package/dist/modules/workflows/lib/transition-handler.js +1 -1
- package/dist/modules/workflows/lib/transition-handler.js.map +2 -2
- package/dist/modules/workflows/lib/workflow-executor.js +2 -2
- package/dist/modules/workflows/lib/workflow-executor.js.map +2 -2
- package/jest.config.cjs +4 -2
- package/package.json +3 -3
- package/src/modules/api_keys/data/entities.ts +1 -1
- package/src/modules/api_keys/services/apiKeyService.ts +5 -5
- package/src/modules/attachments/api/library/[id]/route.ts +1 -1
- package/src/modules/attachments/api/library/route.ts +10 -12
- package/src/modules/attachments/api/partitions/route.ts +3 -3
- package/src/modules/attachments/api/route.ts +10 -8
- package/src/modules/attachments/api/transfer/route.ts +1 -1
- package/src/modules/attachments/data/entities.ts +2 -1
- package/src/modules/attachments/lib/ocrQueue.ts +1 -1
- package/src/modules/audit_logs/api/audit-logs/actions/export/route.ts +4 -4
- package/src/modules/audit_logs/api/audit-logs/actions/route.ts +4 -4
- package/src/modules/audit_logs/data/entities.ts +1 -1
- package/src/modules/audit_logs/services/actionLogService.ts +96 -87
- package/src/modules/auth/api/roles/acl/route.ts +1 -1
- package/src/modules/auth/api/users/acl/route.ts +2 -2
- package/src/modules/auth/api/users/resend-invite/route.ts +1 -1
- package/src/modules/auth/cli.ts +46 -40
- package/src/modules/auth/commands/users.ts +1 -1
- package/src/modules/auth/data/entities.ts +1 -1
- package/src/modules/auth/lib/setup-app.ts +3 -3
- package/src/modules/auth/services/authService.ts +2 -2
- package/src/modules/business_rules/api/rules/route.ts +3 -3
- package/src/modules/business_rules/api/sets/[id]/members/route.ts +7 -4
- package/src/modules/business_rules/api/sets/route.ts +3 -3
- package/src/modules/business_rules/cli.ts +1 -1
- package/src/modules/business_rules/data/entities.ts +2 -9
- package/src/modules/business_rules/lib/rule-engine.ts +1 -1
- package/src/modules/catalog/api/option-schemas/route.ts +0 -1
- package/src/modules/catalog/data/entities.ts +2 -11
- package/src/modules/configs/data/entities.ts +2 -1
- package/src/modules/currencies/commands/fetch-configs.ts +3 -3
- package/src/modules/currencies/data/entities.ts +1 -1
- package/src/modules/customer_accounts/api/signup.ts +1 -1
- package/src/modules/customer_accounts/data/entities.ts +1 -1
- package/src/modules/customer_accounts/services/customerInvitationService.ts +1 -1
- package/src/modules/customer_accounts/services/customerSessionService.ts +1 -1
- package/src/modules/customer_accounts/services/customerTokenService.ts +26 -15
- package/src/modules/customers/api/interactions/conflicts/route.ts +26 -23
- package/src/modules/customers/api/interactions/counts/route.ts +13 -11
- package/src/modules/customers/api/interactions/route.ts +32 -44
- package/src/modules/customers/api/utils.ts +45 -37
- package/src/modules/customers/cli.ts +88 -67
- package/src/modules/customers/commands/dictionaries.ts +1 -1
- package/src/modules/customers/commands/tags.ts +1 -1
- package/src/modules/customers/data/entities.ts +2 -12
- package/src/modules/customers/lib/interactionProjection.ts +36 -25
- package/src/modules/customers/lib/personCompanyLinkTable.ts +13 -18
- package/src/modules/dashboards/api/roles/widgets/route.ts +1 -1
- package/src/modules/dashboards/api/users/widgets/route.ts +1 -1
- package/src/modules/dashboards/data/entities.ts +1 -1
- package/src/modules/data_sync/api/mappings/route.ts +1 -1
- package/src/modules/data_sync/data/entities.ts +2 -1
- package/src/modules/data_sync/lib/id-mapping.ts +1 -1
- package/src/modules/data_sync/lib/sync-run-service.ts +1 -1
- package/src/modules/dictionaries/commands/factory.ts +1 -1
- package/src/modules/dictionaries/data/entities.ts +2 -9
- package/src/modules/directory/commands/organizations.ts +4 -4
- package/src/modules/directory/data/entities.ts +2 -1
- package/src/modules/entities/api/definitions.ts +2 -2
- package/src/modules/entities/api/encryption.ts +2 -2
- package/src/modules/entities/api/relations/options.ts +8 -3
- package/src/modules/entities/cli.ts +4 -4
- package/src/modules/entities/data/entities.ts +1 -1
- package/src/modules/entities/lib/field-definitions.ts +2 -2
- package/src/modules/entities/lib/register.ts +1 -1
- package/src/modules/feature_toggles/data/entities.ts +2 -9
- package/src/modules/inbox_ops/api/proposals/counts/route.ts +10 -10
- package/src/modules/inbox_ops/data/entities.ts +2 -8
- package/src/modules/inbox_ops/lib/messagesIntegration.ts +12 -11
- package/src/modules/integrations/data/entities.ts +2 -1
- package/src/modules/integrations/lib/credentials-service.ts +1 -1
- package/src/modules/integrations/lib/log-service.ts +1 -1
- package/src/modules/integrations/lib/state-service.ts +1 -1
- package/src/modules/messages/api/route.ts +134 -123
- package/src/modules/messages/api/unread-count/route.ts +19 -16
- package/src/modules/messages/commands/confirmations.ts +1 -1
- package/src/modules/messages/commands/messages.ts +3 -3
- package/src/modules/messages/data/entities.ts +2 -1
- package/src/modules/messages/lib/email-sender.ts +1 -1
- package/src/modules/messages/lib/searchLookup.ts +16 -13
- package/src/modules/messages/lib/tokenConsumption.ts +16 -8
- package/src/modules/notifications/data/entities.ts +2 -1
- package/src/modules/notifications/lib/notificationRecipients.ts +42 -26
- package/src/modules/notifications/lib/notificationService.ts +53 -42
- package/src/modules/notifications/workers/create-notification.worker.ts +20 -17
- package/src/modules/payment_gateways/api/transactions/route.ts +2 -2
- package/src/modules/payment_gateways/data/entities.ts +2 -1
- package/src/modules/payment_gateways/lib/gateway-service.ts +1 -1
- package/src/modules/payment_gateways/lib/webhook-utils.ts +2 -2
- package/src/modules/perspectives/data/entities.ts +1 -1
- package/src/modules/planner/data/entities.ts +1 -1
- package/src/modules/progress/data/entities.ts +2 -1
- package/src/modules/progress/lib/progressServiceImpl.ts +1 -1
- package/src/modules/query_index/api/status.ts +85 -71
- package/src/modules/query_index/cli.ts +51 -31
- package/src/modules/query_index/data/entities.ts +1 -1
- package/src/modules/query_index/di.ts +41 -16
- package/src/modules/query_index/lib/batch.ts +68 -55
- package/src/modules/query_index/lib/coverage.ts +115 -88
- package/src/modules/query_index/lib/engine.ts +1036 -1096
- package/src/modules/query_index/lib/indexer.ts +115 -79
- package/src/modules/query_index/lib/jobs.ts +51 -31
- package/src/modules/query_index/lib/purge.ts +25 -19
- package/src/modules/query_index/lib/reindexer.ts +97 -84
- package/src/modules/query_index/lib/search-tokens.ts +67 -36
- package/src/modules/query_index/lib/stale.ts +14 -17
- package/src/modules/query_index/lib/subscriber-scope.ts +6 -5
- package/src/modules/query_index/subscribers/delete_one.ts +9 -6
- package/src/modules/resources/commands/tag-assignments.ts +1 -1
- package/src/modules/resources/commands/tags.ts +1 -1
- package/src/modules/resources/data/entities.ts +2 -1
- package/src/modules/sales/commands/documentAddresses.ts +2 -2
- package/src/modules/sales/commands/notes.ts +1 -1
- package/src/modules/sales/commands/tags.ts +1 -1
- package/src/modules/sales/data/enrichers.ts +17 -13
- package/src/modules/sales/data/entities.ts +2 -11
- package/src/modules/shipping_carriers/data/entities.ts +2 -1
- package/src/modules/shipping_carriers/lib/shipping-service.ts +1 -1
- package/src/modules/shipping_carriers/lib/webhook-utils.ts +2 -2
- package/src/modules/staff/data/entities.ts +1 -1
- package/src/modules/translations/api/[entityType]/[entityId]/route.ts +14 -11
- package/src/modules/translations/api/context.ts +4 -4
- package/src/modules/translations/commands/translations.ts +116 -81
- package/src/modules/translations/components/TranslationManager.tsx +23 -14
- package/src/modules/translations/data/entities.ts +1 -1
- package/src/modules/translations/i18n/de.json +1 -0
- package/src/modules/translations/i18n/en.json +1 -0
- package/src/modules/translations/i18n/es.json +1 -0
- package/src/modules/translations/i18n/pl.json +1 -0
- package/src/modules/translations/lib/apply.ts +6 -6
- package/src/modules/translations/lib/batch.ts +9 -7
- package/src/modules/translations/subscribers/cleanup.ts +10 -11
- package/src/modules/workflows/api/definitions/route.ts +1 -1
- package/src/modules/workflows/cli.ts +5 -5
- package/src/modules/workflows/data/entities.ts +2 -1
- package/src/modules/workflows/lib/event-logger.ts +2 -2
- package/src/modules/workflows/lib/seeds.ts +16 -1
- package/src/modules/workflows/lib/step-handler.ts +3 -3
- package/src/modules/workflows/lib/task-handler.ts +1 -1
- package/src/modules/workflows/lib/transition-handler.ts +1 -1
- package/src/modules/workflows/lib/workflow-executor.ts +2 -2
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { sql } from "kysely";
|
|
1
2
|
import { resolveEntityTableName } from "@open-mercato/shared/lib/query/engine";
|
|
2
3
|
import { resolveTenantEncryptionService } from "@open-mercato/shared/lib/encryption/customFieldValues";
|
|
3
4
|
import { decryptIndexDocForSearch, encryptIndexDocForStorage } from "@open-mercato/shared/lib/encryption/indexDoc";
|
|
@@ -11,8 +12,8 @@ const DEFAULT_BATCH_SIZE = 500;
|
|
|
11
12
|
const deriveOrgFromId = /* @__PURE__ */ new Set(["directory:organization"]);
|
|
12
13
|
const COVERAGE_REFRESH_THROTTLE_MS = 5 * 60 * 1e3;
|
|
13
14
|
const lastCoverageReset = /* @__PURE__ */ new Map();
|
|
14
|
-
async function cleanupLegacyJobScopes(
|
|
15
|
-
await
|
|
15
|
+
async function cleanupLegacyJobScopes(db, options) {
|
|
16
|
+
await db.deleteFrom("entity_index_jobs").where("entity_type", "=", options.entityType).where(sql`organization_id is not distinct from ${options.organizationId}`).where(sql`tenant_id is not distinct from ${options.tenantId}`).where(sql`partition_count is distinct from ${options.activePartitionCount}`).execute();
|
|
16
17
|
}
|
|
17
18
|
function toNumber(value) {
|
|
18
19
|
if (typeof value === "number") return Number.isFinite(value) ? value : 0;
|
|
@@ -22,10 +23,10 @@ function toNumber(value) {
|
|
|
22
23
|
}
|
|
23
24
|
return 0;
|
|
24
25
|
}
|
|
25
|
-
async function getColumnSet(
|
|
26
|
+
async function getColumnSet(db, tableName) {
|
|
26
27
|
try {
|
|
27
|
-
const
|
|
28
|
-
return new Set(
|
|
28
|
+
const rows = await db.selectFrom("information_schema.columns").select(["column_name"]).where(sql`table_schema = current_schema()`).where("table_name", "=", tableName).execute();
|
|
29
|
+
return new Set(rows.map((row) => String(row.column_name).toLowerCase()));
|
|
29
30
|
} catch {
|
|
30
31
|
return /* @__PURE__ */ new Set();
|
|
31
32
|
}
|
|
@@ -54,7 +55,7 @@ async function reindexEntity(em, options) {
|
|
|
54
55
|
const partitionIndexRaw = Number.isFinite(options?.partitionIndex) ? Math.max(0, Math.trunc(options.partitionIndex)) : 0;
|
|
55
56
|
const partitionIndex = usingPartitions ? Math.min(partitionIndexRaw, partitionCountRaw - 1) : null;
|
|
56
57
|
const resetCoverage = options?.resetCoverage ?? (!usingPartitions || partitionIndex === 0);
|
|
57
|
-
const
|
|
58
|
+
const db = em.getKysely();
|
|
58
59
|
const table = resolveEntityTableName(em, entityType);
|
|
59
60
|
if (entityType === "query_index:search_token" || table === "search_tokens") {
|
|
60
61
|
return {
|
|
@@ -64,7 +65,7 @@ async function reindexEntity(em, options) {
|
|
|
64
65
|
scopes: []
|
|
65
66
|
};
|
|
66
67
|
}
|
|
67
|
-
const columns = await getColumnSet(
|
|
68
|
+
const columns = await getColumnSet(db, table);
|
|
68
69
|
const hasOrgCol = columns.has("organization_id");
|
|
69
70
|
const hasTenantCol = columns.has("tenant_id");
|
|
70
71
|
const hasDeletedCol = columns.has("deleted_at");
|
|
@@ -76,14 +77,7 @@ async function reindexEntity(em, options) {
|
|
|
76
77
|
partitionCount: usingPartitions ? partitionCountRaw : null
|
|
77
78
|
};
|
|
78
79
|
if (!force) {
|
|
79
|
-
const activeJob = await (
|
|
80
|
-
let query = knex("entity_index_jobs").where("entity_type", entityType).whereNull("finished_at");
|
|
81
|
-
query = query.whereRaw("organization_id is not distinct from ?", [null]);
|
|
82
|
-
query = query.whereRaw("tenant_id is not distinct from ?", [tenantId ?? null]);
|
|
83
|
-
query = query.whereRaw("partition_index is not distinct from ?", [partitionIndex]);
|
|
84
|
-
query = query.whereRaw("partition_count is not distinct from ?", [usingPartitions ? partitionCountRaw : null]);
|
|
85
|
-
return query.first();
|
|
86
|
-
})();
|
|
80
|
+
const activeJob = await db.selectFrom("entity_index_jobs").select(["id"]).where("entity_type", "=", entityType).where("finished_at", "is", null).where(sql`organization_id is not distinct from ${null}`).where(sql`tenant_id is not distinct from ${tenantId ?? null}`).where(sql`partition_index is not distinct from ${partitionIndex}`).where(sql`partition_count is not distinct from ${usingPartitions ? partitionCountRaw : null}`).executeTakeFirst();
|
|
87
81
|
if (activeJob) {
|
|
88
82
|
return {
|
|
89
83
|
processed: 0,
|
|
@@ -94,7 +88,7 @@ async function reindexEntity(em, options) {
|
|
|
94
88
|
}
|
|
95
89
|
}
|
|
96
90
|
if (resetCoverage) {
|
|
97
|
-
await cleanupLegacyJobScopes(
|
|
91
|
+
await cleanupLegacyJobScopes(db, {
|
|
98
92
|
entityType,
|
|
99
93
|
organizationId: jobScope.organizationId ?? null,
|
|
100
94
|
tenantId: jobScope.tenantId ?? null,
|
|
@@ -102,19 +96,19 @@ async function reindexEntity(em, options) {
|
|
|
102
96
|
});
|
|
103
97
|
}
|
|
104
98
|
const scopeKey = (tenantValue, orgValue) => `${tenantValue ?? "__null__"}|${orgValue ?? "__null__"}`;
|
|
105
|
-
const
|
|
106
|
-
|
|
99
|
+
const applyBaseWhere = (q) => {
|
|
100
|
+
let chain = q;
|
|
101
|
+
if (hasDeletedCol) chain = chain.where("b.deleted_at", "is", null);
|
|
107
102
|
if (tenantId !== void 0 && hasTenantCol) {
|
|
108
|
-
|
|
109
|
-
else builder.where("b.tenant_id", tenantId);
|
|
103
|
+
chain = tenantId === null ? chain.where("b.tenant_id", "is", null) : chain.where("b.tenant_id", "=", tenantId);
|
|
110
104
|
}
|
|
111
105
|
if (organizationId !== void 0 && hasOrgCol) {
|
|
112
|
-
|
|
113
|
-
else builder.where("b.organization_id", organizationId);
|
|
106
|
+
chain = organizationId === null ? chain.where("b.organization_id", "is", null) : chain.where("b.organization_id", "=", organizationId);
|
|
114
107
|
}
|
|
115
108
|
if (usingPartitions && partitionIndex !== null) {
|
|
116
|
-
|
|
109
|
+
chain = chain.where(sql`mod(abs(hashtext(b.id::text)), ${partitionCountRaw}) = ${partitionIndex}`);
|
|
117
110
|
}
|
|
111
|
+
return chain;
|
|
118
112
|
};
|
|
119
113
|
const baseCounts = /* @__PURE__ */ new Map();
|
|
120
114
|
const registerBaseCount = (tenantValue, orgValue, count) => {
|
|
@@ -124,27 +118,32 @@ async function reindexEntity(em, options) {
|
|
|
124
118
|
const groupByTenant = hasTenantCol && tenantId === void 0;
|
|
125
119
|
const groupByOrg = hasOrgCol && organizationId === void 0;
|
|
126
120
|
if (groupByTenant || groupByOrg) {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
121
|
+
let groupQuery = applyBaseWhere(
|
|
122
|
+
db.selectFrom(`${table} as b`).select(sql`count(*)`.as("count"))
|
|
123
|
+
);
|
|
124
|
+
if (groupByTenant) {
|
|
125
|
+
groupQuery = groupQuery.select("b.tenant_id as tenant_id").groupBy("b.tenant_id");
|
|
126
|
+
}
|
|
127
|
+
if (groupByOrg) {
|
|
128
|
+
groupQuery = groupQuery.select("b.organization_id as organization_id").groupBy("b.organization_id");
|
|
129
|
+
}
|
|
130
|
+
const rows = await groupQuery.execute();
|
|
134
131
|
for (const row of rows) {
|
|
135
132
|
const bucketTenant = groupByTenant ? row?.tenant_id ?? null : tenantId === void 0 ? null : tenantId ?? null;
|
|
136
133
|
const bucketOrg = groupByOrg ? row?.organization_id ?? null : organizationId === void 0 ? null : organizationId ?? null;
|
|
137
134
|
registerBaseCount(bucketTenant, bucketOrg, toNumber(row?.count));
|
|
138
135
|
}
|
|
139
136
|
} else {
|
|
140
|
-
const row = await
|
|
137
|
+
const row = await applyBaseWhere(
|
|
138
|
+
db.selectFrom(`${table} as b`).select(sql`count(*)`.as("count"))
|
|
139
|
+
).executeTakeFirst();
|
|
141
140
|
const bucketTenant = tenantId === void 0 ? null : tenantId ?? null;
|
|
142
141
|
const bucketOrg = organizationId === void 0 ? null : organizationId ?? null;
|
|
143
142
|
registerBaseCount(bucketTenant, bucketOrg, toNumber(row?.count));
|
|
144
143
|
}
|
|
145
144
|
const total = Array.from(baseCounts.values()).reduce((acc, value) => acc + (Number.isFinite(value.count) ? value.count : 0), 0);
|
|
146
|
-
await prepareJob(
|
|
147
|
-
const jobRow = await
|
|
145
|
+
await prepareJob(db, jobScope, "reindexing", { totalCount: total });
|
|
146
|
+
const jobRow = await db.selectFrom("entity_index_jobs").select(["started_at"]).where("entity_type", "=", entityType).where("organization_id", "is", null).where(sql`tenant_id is not distinct from ${tenantId ?? null}`).where(sql`partition_index is not distinct from ${partitionIndex}`).where(sql`partition_count is not distinct from ${usingPartitions ? partitionCountRaw : null}`).orderBy("started_at", "desc").executeTakeFirst();
|
|
148
147
|
const jobStartedAt = jobRow?.started_at ? new Date(jobRow.started_at) : /* @__PURE__ */ new Date();
|
|
149
148
|
const deriveOrg = deriveOrgFromId.has(entityType) ? (row) => String(row.id) : void 0;
|
|
150
149
|
const scopeOverrides = {};
|
|
@@ -166,21 +165,23 @@ async function reindexEntity(em, options) {
|
|
|
166
165
|
options?.onProgress?.({ processed, total, chunkSize: 0 });
|
|
167
166
|
if (resetCoverage) {
|
|
168
167
|
if (force) {
|
|
169
|
-
|
|
168
|
+
try {
|
|
169
|
+
let purgeQuery = db.deleteFrom("entity_indexes").where("entity_type", "=", entityType);
|
|
170
170
|
if (tenantId !== void 0) {
|
|
171
|
-
|
|
171
|
+
purgeQuery = purgeQuery.where(sql`tenant_id is not distinct from ${tenantId ?? null}`);
|
|
172
172
|
}
|
|
173
173
|
if (organizationId !== void 0) {
|
|
174
|
-
|
|
174
|
+
purgeQuery = purgeQuery.where(sql`organization_id is not distinct from ${organizationId ?? null}`);
|
|
175
175
|
}
|
|
176
|
-
|
|
176
|
+
await purgeQuery.execute();
|
|
177
|
+
} catch (error) {
|
|
177
178
|
console.warn("[HybridQueryEngine] Failed to purge index rows before force reindex", {
|
|
178
179
|
entityType,
|
|
179
180
|
tenantId: tenantId ?? null,
|
|
180
181
|
organizationId: organizationId ?? null,
|
|
181
182
|
error: error instanceof Error ? error.message : error
|
|
182
183
|
});
|
|
183
|
-
}
|
|
184
|
+
}
|
|
184
185
|
if (emitVectorize && eventBus) {
|
|
185
186
|
if (tenantId !== void 0) {
|
|
186
187
|
const payload = {
|
|
@@ -226,11 +227,13 @@ async function reindexEntity(em, options) {
|
|
|
226
227
|
}
|
|
227
228
|
try {
|
|
228
229
|
while (true) {
|
|
229
|
-
let query =
|
|
230
|
+
let query = applyBaseWhere(
|
|
231
|
+
db.selectFrom(`${table} as b`).selectAll("b").orderBy("b.id", "asc").limit(batchSize)
|
|
232
|
+
);
|
|
230
233
|
if (lastId !== null) {
|
|
231
234
|
query = query.where("b.id", ">", lastId);
|
|
232
235
|
}
|
|
233
|
-
const rows = await query;
|
|
236
|
+
const rows = await query.execute();
|
|
234
237
|
if (!rows.length) break;
|
|
235
238
|
const encryption = resolveTenantEncryptionService(em);
|
|
236
239
|
const dekKeyCache = /* @__PURE__ */ new Map();
|
|
@@ -266,7 +269,7 @@ async function reindexEntity(em, options) {
|
|
|
266
269
|
}
|
|
267
270
|
return result;
|
|
268
271
|
};
|
|
269
|
-
await upsertIndexBatch(
|
|
272
|
+
await upsertIndexBatch(db, entityType, rows, scopeOverrides, { deriveOrganizationId: deriveOrg, encryptDoc, decryptDoc });
|
|
270
273
|
const coverageDeltas = /* @__PURE__ */ new Map();
|
|
271
274
|
for (const row of rows) {
|
|
272
275
|
const scopeTenant = tenantId !== void 0 ? tenantId ?? null : hasTenantCol ? row.tenant_id ?? null : null;
|
|
@@ -310,9 +313,9 @@ async function reindexEntity(em, options) {
|
|
|
310
313
|
processed += rows.length;
|
|
311
314
|
lastId = String(rows[rows.length - 1].id);
|
|
312
315
|
options?.onProgress?.({ processed, total, chunkSize: rows.length });
|
|
313
|
-
await updateJobProgress(
|
|
316
|
+
await updateJobProgress(db, jobScope, rows.length);
|
|
314
317
|
}
|
|
315
|
-
await purgeOrphans(
|
|
318
|
+
await purgeOrphans(db, {
|
|
316
319
|
entityType,
|
|
317
320
|
tenantId,
|
|
318
321
|
organizationId,
|
|
@@ -349,7 +352,7 @@ async function reindexEntity(em, options) {
|
|
|
349
352
|
);
|
|
350
353
|
}
|
|
351
354
|
} finally {
|
|
352
|
-
await finalizeJob(
|
|
355
|
+
await finalizeJob(db, jobScope);
|
|
353
356
|
}
|
|
354
357
|
return {
|
|
355
358
|
processed,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/query_index/lib/reindexer.ts"],
|
|
4
|
-
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport type { Knex } from 'knex'\nimport { resolveEntityTableName } from '@open-mercato/shared/lib/query/engine'\nimport { resolveTenantEncryptionService } from '@open-mercato/shared/lib/encryption/customFieldValues'\nimport { decryptIndexDocForSearch, encryptIndexDocForStorage } from '@open-mercato/shared/lib/encryption/indexDoc'\nimport { upsertIndexBatch, type AnyRow } from './batch'\nimport { refreshCoverageSnapshot, writeCoverageCounts, applyCoverageAdjustments } from './coverage'\nimport { prepareJob, updateJobProgress, finalizeJob, type JobScope } from './jobs'\nimport { purgeOrphans } from './stale'\nimport type { VectorIndexService } from '@open-mercato/search/vector'\nimport { isSearchDebugEnabled } from './search-tokens'\n\nexport type ReindexJobOptions = {\n entityType: string\n tenantId?: string | null\n organizationId?: string | null\n force?: boolean\n batchSize?: number\n emitVectorizeEvents?: boolean\n eventBus?: {\n emitEvent(event: string, payload: any, options?: any): Promise<void>\n }\n partitionCount?: number\n partitionIndex?: number\n resetCoverage?: boolean\n onProgress?: (info: { processed: number; total: number; chunkSize: number }) => void\n vectorService?: VectorIndexService | null\n}\n\nexport type ReindexJobResult = {\n processed: number\n total: number\n tenantScopes: Array<string | null>\n scopes: Array<{ tenantId: string | null; organizationId: string | null }>\n}\n\nexport const DEFAULT_REINDEX_PARTITIONS = 5\nconst DEFAULT_BATCH_SIZE = 500\nconst deriveOrgFromId = new Set<string>(['directory:organization'])\nconst COVERAGE_REFRESH_THROTTLE_MS = 5 * 60 * 1000\nconst lastCoverageReset = new Map<string, number>()\n\nasync function cleanupLegacyJobScopes(\n knex: Knex,\n options: {\n entityType: string\n organizationId: string | null\n tenantId: string | null\n activePartitionCount: number | null\n },\n): Promise<void> {\n await knex('entity_index_jobs')\n .where('entity_type', options.entityType)\n .andWhereRaw('organization_id is not distinct from ?', [options.organizationId])\n .andWhereRaw('tenant_id is not distinct from ?', [options.tenantId])\n .andWhereRaw('partition_count is distinct from ?', [options.activePartitionCount])\n .del()\n}\n\nfunction toNumber(value: unknown): number {\n if (typeof value === 'number') return Number.isFinite(value) ? value : 0\n if (typeof value === 'string') {\n const parsed = Number(value)\n return Number.isFinite(parsed) ? parsed : 0\n }\n return 0\n}\n\nasync function getColumnSet(knex: Knex, tableName: string): Promise<Set<string>> {\n try {\n const info = await knex(tableName).columnInfo()\n return new Set(Object.keys(info).map((key) => key.toLowerCase()))\n } catch {\n return new Set<string>()\n }\n}\n\nexport async function reindexEntity(\n em: EntityManager,\n options: ReindexJobOptions,\n): Promise<ReindexJobResult> {\n const entityType = String(options?.entityType || '')\n if (!entityType) {\n return {\n processed: 0,\n total: 0,\n tenantScopes: [],\n scopes: [],\n }\n }\n const tenantIdInput = options?.tenantId\n const tenantId = tenantIdInput === 'undefined' ? undefined : tenantIdInput\n const organizationIdInput = options?.organizationId\n const organizationId = organizationIdInput === 'undefined' ? undefined : organizationIdInput\n const force = options?.force === true\n const batchSize = Number.isFinite(options?.batchSize) && options!.batchSize! > 0\n ? Math.max(1, Math.trunc(options!.batchSize!))\n : DEFAULT_BATCH_SIZE\n const emitVectorize = options?.emitVectorizeEvents === true\n const eventBus = options?.eventBus\n const vectorService = options?.vectorService ?? null\n const partitionCountRaw = Number.isFinite(options?.partitionCount)\n ? Math.max(1, Math.trunc(options!.partitionCount!))\n : 1\n const usingPartitions = partitionCountRaw > 1\n const partitionIndexRaw = Number.isFinite(options?.partitionIndex)\n ? Math.max(0, Math.trunc(options!.partitionIndex!))\n : 0\n const partitionIndex = usingPartitions\n ? Math.min(partitionIndexRaw, partitionCountRaw - 1)\n : null\n const resetCoverage = options?.resetCoverage ?? (!usingPartitions || partitionIndex === 0)\n\n const knex = (em as any).getConnection().getKnex() as Knex\n const table = resolveEntityTableName(em, entityType)\n if (entityType === 'query_index:search_token' || table === 'search_tokens') {\n return {\n processed: 0,\n total: 0,\n tenantScopes: [],\n scopes: [],\n }\n }\n const columns = await getColumnSet(knex, table)\n const hasOrgCol = columns.has('organization_id')\n const hasTenantCol = columns.has('tenant_id')\n const hasDeletedCol = columns.has('deleted_at')\n\n const jobScope: JobScope = {\n entityType,\n organizationId: organizationId ?? null,\n tenantId: tenantId ?? null,\n partitionIndex,\n partitionCount: usingPartitions ? partitionCountRaw : null,\n }\n\n if (!force) {\n const activeJob = await (async () => {\n let query = knex('entity_index_jobs')\n .where('entity_type', entityType)\n .whereNull('finished_at')\n query = query.whereRaw('organization_id is not distinct from ?', [null])\n query = query.whereRaw('tenant_id is not distinct from ?', [tenantId ?? null])\n query = query.whereRaw('partition_index is not distinct from ?', [partitionIndex])\n query = query.whereRaw('partition_count is not distinct from ?', [usingPartitions ? partitionCountRaw : null])\n return query.first()\n })()\n if (activeJob) {\n return {\n processed: 0,\n total: 0,\n tenantScopes: [],\n scopes: [],\n }\n }\n }\n\n if (resetCoverage) {\n await cleanupLegacyJobScopes(knex, {\n entityType,\n organizationId: jobScope.organizationId ?? null,\n tenantId: jobScope.tenantId ?? null,\n activePartitionCount: jobScope.partitionCount ?? null,\n })\n }\n\n const scopeKey = (tenantValue: string | null, orgValue: string | null) => `${tenantValue ?? '__null__'}|${orgValue ?? '__null__'}`\n const baseWhere = (builder: Knex.QueryBuilder<any, any>) => {\n if (hasDeletedCol) builder.whereNull('b.deleted_at')\n if (tenantId !== undefined && hasTenantCol) {\n if (tenantId === null) builder.whereNull('b.tenant_id')\n else builder.where('b.tenant_id', tenantId)\n }\n if (organizationId !== undefined && hasOrgCol) {\n if (organizationId === null) builder.whereNull('b.organization_id')\n else builder.where('b.organization_id', organizationId)\n }\n if (usingPartitions && partitionIndex !== null) {\n builder.whereRaw('mod(abs(hashtext(b.id::text)), ?) = ?', [partitionCountRaw, partitionIndex])\n }\n }\n\n type ScopeStats = { tenantId: string | null; organizationId: string | null; count: number }\n const baseCounts = new Map<string, ScopeStats>()\n const registerBaseCount = (tenantValue: string | null, orgValue: string | null, count: number) => {\n const key = scopeKey(tenantValue, orgValue)\n baseCounts.set(key, { tenantId: tenantValue, organizationId: orgValue, count })\n }\n\n const groupByTenant = hasTenantCol && tenantId === undefined\n const groupByOrg = hasOrgCol && organizationId === undefined\n\n if (groupByTenant || groupByOrg) {\n const rows = await knex({ b: table })\n .modify(baseWhere)\n .modify((qb) => {\n if (groupByTenant) qb.select(knex.raw('b.tenant_id as tenant_id'))\n if (groupByOrg) qb.select(knex.raw('b.organization_id as organization_id'))\n })\n .count<{ count: unknown }[]>({ count: '*' })\n .modify((qb) => {\n if (groupByTenant) qb.groupBy('b.tenant_id')\n if (groupByOrg) qb.groupBy('b.organization_id')\n })\n for (const row of rows) {\n const bucketTenant = groupByTenant\n ? ((row as any)?.tenant_id ?? null)\n : (tenantId === undefined ? null : tenantId ?? null)\n const bucketOrg = groupByOrg\n ? ((row as any)?.organization_id ?? null)\n : (organizationId === undefined ? null : organizationId ?? null)\n registerBaseCount(bucketTenant, bucketOrg, toNumber((row as any)?.count))\n }\n } else {\n const row = await knex({ b: table })\n .modify(baseWhere)\n .count({ count: '*' })\n .first()\n const bucketTenant = tenantId === undefined ? null : tenantId ?? null\n const bucketOrg = organizationId === undefined ? null : organizationId ?? null\n registerBaseCount(bucketTenant, bucketOrg, toNumber(row?.count))\n }\n\n const total = Array.from(baseCounts.values()).reduce((acc, value) => acc + (Number.isFinite(value.count) ? value.count : 0), 0)\n await prepareJob(knex, jobScope, 'reindexing', { totalCount: total })\n const jobRow = await knex('entity_index_jobs')\n .where({ entity_type: entityType })\n .whereNull('organization_id')\n .andWhereRaw('tenant_id is not distinct from ?', [tenantId ?? null])\n .andWhereRaw('partition_index is not distinct from ?', [partitionIndex])\n .andWhereRaw('partition_count is not distinct from ?', [usingPartitions ? partitionCountRaw : null])\n .orderBy('started_at', 'desc')\n .first<{ started_at: Date }>()\n const jobStartedAt = jobRow?.started_at ? new Date(jobRow.started_at) : new Date()\n const deriveOrg = deriveOrgFromId.has(entityType)\n ? (row: AnyRow) => String(row.id)\n : undefined\n\n const scopeOverrides: { tenantId?: string; orgId?: string } = {}\n if (tenantId !== undefined && tenantId !== null) {\n scopeOverrides.tenantId = String(tenantId)\n }\n if (organizationId !== undefined && organizationId !== null) {\n scopeOverrides.orgId = String(organizationId)\n }\n\n const scopeEntries = Array.from(baseCounts.values()).map((entry) => ({\n tenantId: entry.tenantId,\n organizationId: entry.organizationId,\n }))\n const tenantScopes = Array.from(\n new Set(scopeEntries.map((entry) => entry.tenantId ?? null)),\n )\n\n let processed = 0\n let lastId: string | null = null\n\n options?.onProgress?.({ processed, total, chunkSize: 0 })\n\n if (resetCoverage) {\n if (force) {\n await knex('entity_indexes')\n .where('entity_type', entityType)\n .modify((qb) => {\n if (tenantId !== undefined) {\n qb.andWhereRaw('tenant_id is not distinct from ?', [tenantId ?? null])\n }\n if (organizationId !== undefined) {\n qb.andWhereRaw('organization_id is not distinct from ?', [organizationId ?? null])\n }\n })\n .del()\n .catch((error) => {\n console.warn('[HybridQueryEngine] Failed to purge index rows before force reindex', {\n entityType,\n tenantId: tenantId ?? null,\n organizationId: organizationId ?? null,\n error: error instanceof Error ? error.message : error,\n })\n })\n\n if (emitVectorize && eventBus) {\n if (tenantId !== undefined) {\n const payload: Record<string, unknown> = {\n entityType,\n tenantId: tenantId ?? null,\n }\n if (organizationId !== undefined) payload.organizationId = organizationId ?? null\n try {\n await eventBus.emitEvent('query_index.vectorize_purge', payload)\n } catch (err) {\n console.warn('[HybridQueryEngine] Failed to queue vector purge before force reindex', {\n entityType,\n tenantId: tenantId ?? null,\n organizationId: organizationId ?? null,\n error: err instanceof Error ? err.message : err,\n })\n }\n } else {\n console.warn('[HybridQueryEngine] Skipping vector purge for force reindex without tenant scope', {\n entityType,\n })\n }\n }\n }\n\n const nowTs = Date.now()\n for (const scope of baseCounts.values()) {\n const key = `${entityType}|${scopeKey(scope.tenantId, scope.organizationId)}`\n const last = lastCoverageReset.get(key) ?? 0\n if (force || nowTs - last >= COVERAGE_REFRESH_THROTTLE_MS) {\n await writeCoverageCounts(em, {\n entityType,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n withDeleted: false,\n }, {\n baseCount: scope.count,\n indexedCount: 0,\n vectorCount: emitVectorize ? 0 : undefined,\n })\n lastCoverageReset.set(key, nowTs)\n }\n }\n }\n\n try {\n while (true) {\n let query = knex({ b: table })\n .modify(baseWhere)\n .select('b.*')\n .orderBy('b.id', 'asc')\n .limit(batchSize)\n if (lastId !== null) {\n query = query.where('b.id', '>', lastId)\n }\n const rows = await query as AnyRow[]\n if (!rows.length) break\n\n const encryption = resolveTenantEncryptionService(em as any)\n const dekKeyCache = new Map<string | null, string | null>()\n const encryptDoc = async (\n targetEntity: string,\n doc: Record<string, unknown>,\n scope: { organizationId: string | null; tenantId: string | null },\n ) => {\n return await encryptIndexDocForStorage(\n targetEntity,\n doc,\n { tenantId: scope.tenantId ?? null, organizationId: scope.organizationId ?? null },\n encryption,\n )\n }\n const decryptDoc = async (\n targetEntity: string,\n doc: Record<string, unknown>,\n scope: { organizationId: string | null; tenantId: string | null },\n ) => {\n const result = await decryptIndexDocForSearch(\n targetEntity,\n doc,\n { tenantId: scope.tenantId ?? null, organizationId: scope.organizationId ?? null },\n encryption,\n dekKeyCache,\n )\n if (isSearchDebugEnabled()) {\n const keysOfInterest = ['display_name', 'first_name', 'last_name', 'brand_name', 'legal_name', 'primary_email', 'primary_phone']\n const snapshot: Record<string, unknown> = {}\n for (const key of keysOfInterest) {\n if (key in result) snapshot[key] = (result as Record<string, unknown>)[key]\n }\n console.info('[reindex:decrypt]', {\n entityType: targetEntity,\n tenantId: scope.tenantId ?? null,\n organizationId: scope.organizationId ?? null,\n keys: Object.keys(snapshot),\n sample: snapshot,\n })\n }\n return result\n }\n\n await upsertIndexBatch(knex, entityType, rows, scopeOverrides, { deriveOrganizationId: deriveOrg, encryptDoc, decryptDoc })\n\n const coverageDeltas = new Map<string, { tenantId: string | null; organizationId: string | null; delta: number }>()\n for (const row of rows) {\n const scopeTenant = tenantId !== undefined\n ? tenantId ?? null\n : (hasTenantCol ? ((row as AnyRow).tenant_id ?? null) : null)\n const scopeOrg = organizationId !== undefined\n ? organizationId ?? null\n : (hasOrgCol ? ((row as AnyRow).organization_id ?? null) : (deriveOrg ? deriveOrg(row) ?? null : null))\n const key = scopeKey(scopeTenant ?? null, scopeOrg ?? null)\n const existingDelta = coverageDeltas.get(key)\n if (existingDelta) existingDelta.delta += 1\n else coverageDeltas.set(key, {\n tenantId: scopeTenant ?? null,\n organizationId: scopeOrg ?? null,\n delta: 1,\n })\n }\n if (coverageDeltas.size > 0) {\n await applyCoverageAdjustments(\n em,\n Array.from(coverageDeltas.values()).map((entry) => ({\n entityType,\n tenantId: entry.tenantId,\n organizationId: entry.organizationId,\n withDeleted: false,\n deltaBase: 0,\n deltaIndex: entry.delta,\n })),\n )\n }\n\n if (emitVectorize && eventBus) {\n await Promise.all(\n rows.map((row) => {\n const scopeOrg = organizationId !== undefined\n ? organizationId ?? null\n : hasOrgCol\n ? ((row as AnyRow).organization_id ?? null)\n : (deriveOrg ? deriveOrg(row) ?? null : null)\n const scopeTenant = tenantId !== undefined\n ? tenantId ?? null\n : (hasTenantCol ? ((row as AnyRow).tenant_id ?? null) : null)\n return eventBus\n .emitEvent('query_index.vectorize_one', {\n entityType,\n recordId: String(row.id),\n organizationId: scopeOrg,\n tenantId: scopeTenant,\n })\n .catch(() => undefined)\n }),\n )\n }\n\n processed += rows.length\n lastId = String(rows[rows.length - 1]!.id)\n options?.onProgress?.({ processed, total, chunkSize: rows.length })\n await updateJobProgress(knex, jobScope, rows.length)\n }\n\n await purgeOrphans(knex, {\n entityType,\n tenantId,\n organizationId,\n partitionIndex: usingPartitions ? partitionIndex : null,\n partitionCount: usingPartitions ? partitionCountRaw : null,\n startedAt: jobStartedAt,\n })\n\n if (force && vectorService && (!usingPartitions || partitionIndex === null)) {\n try {\n await vectorService.removeOrphans({\n entityId: entityType,\n tenantId,\n organizationId,\n olderThan: jobStartedAt,\n })\n } catch (error) {\n console.warn('[HybridQueryEngine] Failed to prune vector orphans after reindex', {\n entityType,\n tenantId: tenantId ?? null,\n organizationId: organizationId ?? null,\n error: error instanceof Error ? error.message : error,\n })\n }\n }\n\n for (const scope of scopeEntries) {\n await refreshCoverageSnapshot(\n em,\n {\n entityType,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n withDeleted: false,\n },\n )\n }\n } finally {\n await finalizeJob(knex, jobScope)\n }\n\n return {\n processed,\n total,\n scopes: scopeEntries,\n tenantScopes,\n }\n}\n"],
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { type Kysely, sql } from 'kysely'\nimport { resolveEntityTableName } from '@open-mercato/shared/lib/query/engine'\nimport { resolveTenantEncryptionService } from '@open-mercato/shared/lib/encryption/customFieldValues'\nimport { decryptIndexDocForSearch, encryptIndexDocForStorage } from '@open-mercato/shared/lib/encryption/indexDoc'\nimport { upsertIndexBatch, type AnyRow } from './batch'\nimport { refreshCoverageSnapshot, writeCoverageCounts, applyCoverageAdjustments } from './coverage'\nimport { prepareJob, updateJobProgress, finalizeJob, type JobScope } from './jobs'\nimport { purgeOrphans } from './stale'\nimport type { VectorIndexService } from '@open-mercato/search/vector'\nimport { isSearchDebugEnabled } from './search-tokens'\n\nexport type ReindexJobOptions = {\n entityType: string\n tenantId?: string | null\n organizationId?: string | null\n force?: boolean\n batchSize?: number\n emitVectorizeEvents?: boolean\n eventBus?: {\n emitEvent(event: string, payload: any, options?: any): Promise<void>\n }\n partitionCount?: number\n partitionIndex?: number\n resetCoverage?: boolean\n onProgress?: (info: { processed: number; total: number; chunkSize: number }) => void\n vectorService?: VectorIndexService | null\n}\n\nexport type ReindexJobResult = {\n processed: number\n total: number\n tenantScopes: Array<string | null>\n scopes: Array<{ tenantId: string | null; organizationId: string | null }>\n}\n\nexport const DEFAULT_REINDEX_PARTITIONS = 5\nconst DEFAULT_BATCH_SIZE = 500\nconst deriveOrgFromId = new Set<string>(['directory:organization'])\nconst COVERAGE_REFRESH_THROTTLE_MS = 5 * 60 * 1000\nconst lastCoverageReset = new Map<string, number>()\n\nasync function cleanupLegacyJobScopes(\n db: Kysely<any>,\n options: {\n entityType: string\n organizationId: string | null\n tenantId: string | null\n activePartitionCount: number | null\n },\n): Promise<void> {\n await db\n .deleteFrom('entity_index_jobs' as any)\n .where('entity_type' as any, '=', options.entityType)\n .where(sql<boolean>`organization_id is not distinct from ${options.organizationId}`)\n .where(sql<boolean>`tenant_id is not distinct from ${options.tenantId}`)\n .where(sql<boolean>`partition_count is distinct from ${options.activePartitionCount}`)\n .execute()\n}\n\nfunction toNumber(value: unknown): number {\n if (typeof value === 'number') return Number.isFinite(value) ? value : 0\n if (typeof value === 'string') {\n const parsed = Number(value)\n return Number.isFinite(parsed) ? parsed : 0\n }\n return 0\n}\n\nasync function getColumnSet(db: Kysely<any>, tableName: string): Promise<Set<string>> {\n try {\n const rows = await db\n .selectFrom('information_schema.columns' as any)\n .select(['column_name' as any])\n .where(sql<boolean>`table_schema = current_schema()`)\n .where('table_name' as any, '=', tableName)\n .execute() as Array<{ column_name: string }>\n return new Set(rows.map((row) => String(row.column_name).toLowerCase()))\n } catch {\n return new Set<string>()\n }\n}\n\nexport async function reindexEntity(\n em: EntityManager,\n options: ReindexJobOptions,\n): Promise<ReindexJobResult> {\n const entityType = String(options?.entityType || '')\n if (!entityType) {\n return {\n processed: 0,\n total: 0,\n tenantScopes: [],\n scopes: [],\n }\n }\n const tenantIdInput = options?.tenantId\n const tenantId = tenantIdInput === 'undefined' ? undefined : tenantIdInput\n const organizationIdInput = options?.organizationId\n const organizationId = organizationIdInput === 'undefined' ? undefined : organizationIdInput\n const force = options?.force === true\n const batchSize = Number.isFinite(options?.batchSize) && options!.batchSize! > 0\n ? Math.max(1, Math.trunc(options!.batchSize!))\n : DEFAULT_BATCH_SIZE\n const emitVectorize = options?.emitVectorizeEvents === true\n const eventBus = options?.eventBus\n const vectorService = options?.vectorService ?? null\n const partitionCountRaw = Number.isFinite(options?.partitionCount)\n ? Math.max(1, Math.trunc(options!.partitionCount!))\n : 1\n const usingPartitions = partitionCountRaw > 1\n const partitionIndexRaw = Number.isFinite(options?.partitionIndex)\n ? Math.max(0, Math.trunc(options!.partitionIndex!))\n : 0\n const partitionIndex = usingPartitions\n ? Math.min(partitionIndexRaw, partitionCountRaw - 1)\n : null\n const resetCoverage = options?.resetCoverage ?? (!usingPartitions || partitionIndex === 0)\n\n const db = (em as any).getKysely() as Kysely<any>\n const table = resolveEntityTableName(em, entityType)\n if (entityType === 'query_index:search_token' || table === 'search_tokens') {\n return {\n processed: 0,\n total: 0,\n tenantScopes: [],\n scopes: [],\n }\n }\n const columns = await getColumnSet(db, table)\n const hasOrgCol = columns.has('organization_id')\n const hasTenantCol = columns.has('tenant_id')\n const hasDeletedCol = columns.has('deleted_at')\n\n const jobScope: JobScope = {\n entityType,\n organizationId: organizationId ?? null,\n tenantId: tenantId ?? null,\n partitionIndex,\n partitionCount: usingPartitions ? partitionCountRaw : null,\n }\n\n if (!force) {\n const activeJob = await db\n .selectFrom('entity_index_jobs' as any)\n .select(['id' as any])\n .where('entity_type' as any, '=', entityType)\n .where('finished_at' as any, 'is', null as any)\n .where(sql<boolean>`organization_id is not distinct from ${null}`)\n .where(sql<boolean>`tenant_id is not distinct from ${tenantId ?? null}`)\n .where(sql<boolean>`partition_index is not distinct from ${partitionIndex}`)\n .where(sql<boolean>`partition_count is not distinct from ${usingPartitions ? partitionCountRaw : null}`)\n .executeTakeFirst()\n if (activeJob) {\n return {\n processed: 0,\n total: 0,\n tenantScopes: [],\n scopes: [],\n }\n }\n }\n\n if (resetCoverage) {\n await cleanupLegacyJobScopes(db, {\n entityType,\n organizationId: jobScope.organizationId ?? null,\n tenantId: jobScope.tenantId ?? null,\n activePartitionCount: jobScope.partitionCount ?? null,\n })\n }\n\n const scopeKey = (tenantValue: string | null, orgValue: string | null) => `${tenantValue ?? '__null__'}|${orgValue ?? '__null__'}`\n\n const applyBaseWhere = <QB extends { where: (...args: any[]) => QB }>(q: QB): QB => {\n let chain = q\n if (hasDeletedCol) chain = chain.where('b.deleted_at' as any, 'is', null as any)\n if (tenantId !== undefined && hasTenantCol) {\n chain = tenantId === null\n ? chain.where('b.tenant_id' as any, 'is', null as any)\n : chain.where('b.tenant_id' as any, '=', tenantId)\n }\n if (organizationId !== undefined && hasOrgCol) {\n chain = organizationId === null\n ? chain.where('b.organization_id' as any, 'is', null as any)\n : chain.where('b.organization_id' as any, '=', organizationId)\n }\n if (usingPartitions && partitionIndex !== null) {\n chain = chain.where(sql<boolean>`mod(abs(hashtext(b.id::text)), ${partitionCountRaw}) = ${partitionIndex}`)\n }\n return chain\n }\n\n type ScopeStats = { tenantId: string | null; organizationId: string | null; count: number }\n const baseCounts = new Map<string, ScopeStats>()\n const registerBaseCount = (tenantValue: string | null, orgValue: string | null, count: number) => {\n const key = scopeKey(tenantValue, orgValue)\n baseCounts.set(key, { tenantId: tenantValue, organizationId: orgValue, count })\n }\n\n const groupByTenant = hasTenantCol && tenantId === undefined\n const groupByOrg = hasOrgCol && organizationId === undefined\n\n if (groupByTenant || groupByOrg) {\n let groupQuery = applyBaseWhere(\n db.selectFrom(`${table} as b` as any).select(sql<number>`count(*)`.as('count')),\n )\n if (groupByTenant) {\n groupQuery = groupQuery.select('b.tenant_id as tenant_id' as any).groupBy('b.tenant_id' as any)\n }\n if (groupByOrg) {\n groupQuery = groupQuery.select('b.organization_id as organization_id' as any).groupBy('b.organization_id' as any)\n }\n const rows = await groupQuery.execute() as Array<Record<string, unknown>>\n for (const row of rows) {\n const bucketTenant = groupByTenant\n ? ((row as any)?.tenant_id ?? null)\n : (tenantId === undefined ? null : tenantId ?? null)\n const bucketOrg = groupByOrg\n ? ((row as any)?.organization_id ?? null)\n : (organizationId === undefined ? null : organizationId ?? null)\n registerBaseCount(bucketTenant, bucketOrg, toNumber((row as any)?.count))\n }\n } else {\n const row = await applyBaseWhere(\n db.selectFrom(`${table} as b` as any).select(sql<number>`count(*)`.as('count')),\n ).executeTakeFirst() as { count: unknown } | undefined\n const bucketTenant = tenantId === undefined ? null : tenantId ?? null\n const bucketOrg = organizationId === undefined ? null : organizationId ?? null\n registerBaseCount(bucketTenant, bucketOrg, toNumber(row?.count))\n }\n\n const total = Array.from(baseCounts.values()).reduce((acc, value) => acc + (Number.isFinite(value.count) ? value.count : 0), 0)\n await prepareJob(db, jobScope, 'reindexing', { totalCount: total })\n const jobRow = await db\n .selectFrom('entity_index_jobs' as any)\n .select(['started_at' as any])\n .where('entity_type' as any, '=', entityType)\n .where('organization_id' as any, 'is', null as any)\n .where(sql<boolean>`tenant_id is not distinct from ${tenantId ?? null}`)\n .where(sql<boolean>`partition_index is not distinct from ${partitionIndex}`)\n .where(sql<boolean>`partition_count is not distinct from ${usingPartitions ? partitionCountRaw : null}`)\n .orderBy('started_at' as any, 'desc')\n .executeTakeFirst() as { started_at: Date | string } | undefined\n const jobStartedAt = jobRow?.started_at ? new Date(jobRow.started_at) : new Date()\n const deriveOrg = deriveOrgFromId.has(entityType)\n ? (row: AnyRow) => String(row.id)\n : undefined\n\n const scopeOverrides: { tenantId?: string; orgId?: string } = {}\n if (tenantId !== undefined && tenantId !== null) {\n scopeOverrides.tenantId = String(tenantId)\n }\n if (organizationId !== undefined && organizationId !== null) {\n scopeOverrides.orgId = String(organizationId)\n }\n\n const scopeEntries = Array.from(baseCounts.values()).map((entry) => ({\n tenantId: entry.tenantId,\n organizationId: entry.organizationId,\n }))\n const tenantScopes = Array.from(\n new Set(scopeEntries.map((entry) => entry.tenantId ?? null)),\n )\n\n let processed = 0\n let lastId: string | null = null\n\n options?.onProgress?.({ processed, total, chunkSize: 0 })\n\n if (resetCoverage) {\n if (force) {\n try {\n let purgeQuery = db\n .deleteFrom('entity_indexes' as any)\n .where('entity_type' as any, '=', entityType)\n if (tenantId !== undefined) {\n purgeQuery = purgeQuery.where(sql<boolean>`tenant_id is not distinct from ${tenantId ?? null}`)\n }\n if (organizationId !== undefined) {\n purgeQuery = purgeQuery.where(sql<boolean>`organization_id is not distinct from ${organizationId ?? null}`)\n }\n await purgeQuery.execute()\n } catch (error) {\n console.warn('[HybridQueryEngine] Failed to purge index rows before force reindex', {\n entityType,\n tenantId: tenantId ?? null,\n organizationId: organizationId ?? null,\n error: error instanceof Error ? error.message : error,\n })\n }\n\n if (emitVectorize && eventBus) {\n if (tenantId !== undefined) {\n const payload: Record<string, unknown> = {\n entityType,\n tenantId: tenantId ?? null,\n }\n if (organizationId !== undefined) payload.organizationId = organizationId ?? null\n try {\n await eventBus.emitEvent('query_index.vectorize_purge', payload)\n } catch (err) {\n console.warn('[HybridQueryEngine] Failed to queue vector purge before force reindex', {\n entityType,\n tenantId: tenantId ?? null,\n organizationId: organizationId ?? null,\n error: err instanceof Error ? err.message : err,\n })\n }\n } else {\n console.warn('[HybridQueryEngine] Skipping vector purge for force reindex without tenant scope', {\n entityType,\n })\n }\n }\n }\n\n const nowTs = Date.now()\n for (const scope of baseCounts.values()) {\n const key = `${entityType}|${scopeKey(scope.tenantId, scope.organizationId)}`\n const last = lastCoverageReset.get(key) ?? 0\n if (force || nowTs - last >= COVERAGE_REFRESH_THROTTLE_MS) {\n await writeCoverageCounts(em, {\n entityType,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n withDeleted: false,\n }, {\n baseCount: scope.count,\n indexedCount: 0,\n vectorCount: emitVectorize ? 0 : undefined,\n })\n lastCoverageReset.set(key, nowTs)\n }\n }\n }\n\n try {\n while (true) {\n let query = applyBaseWhere(\n db\n .selectFrom(`${table} as b` as any)\n .selectAll('b' as any)\n .orderBy('b.id' as any, 'asc')\n .limit(batchSize),\n )\n if (lastId !== null) {\n query = query.where('b.id' as any, '>', lastId)\n }\n const rows = await query.execute() as AnyRow[]\n if (!rows.length) break\n\n const encryption = resolveTenantEncryptionService(em as any)\n const dekKeyCache = new Map<string | null, string | null>()\n const encryptDoc = async (\n targetEntity: string,\n doc: Record<string, unknown>,\n scope: { organizationId: string | null; tenantId: string | null },\n ) => {\n return await encryptIndexDocForStorage(\n targetEntity,\n doc,\n { tenantId: scope.tenantId ?? null, organizationId: scope.organizationId ?? null },\n encryption,\n )\n }\n const decryptDoc = async (\n targetEntity: string,\n doc: Record<string, unknown>,\n scope: { organizationId: string | null; tenantId: string | null },\n ) => {\n const result = await decryptIndexDocForSearch(\n targetEntity,\n doc,\n { tenantId: scope.tenantId ?? null, organizationId: scope.organizationId ?? null },\n encryption,\n dekKeyCache,\n )\n if (isSearchDebugEnabled()) {\n const keysOfInterest = ['display_name', 'first_name', 'last_name', 'brand_name', 'legal_name', 'primary_email', 'primary_phone']\n const snapshot: Record<string, unknown> = {}\n for (const key of keysOfInterest) {\n if (key in result) snapshot[key] = (result as Record<string, unknown>)[key]\n }\n console.info('[reindex:decrypt]', {\n entityType: targetEntity,\n tenantId: scope.tenantId ?? null,\n organizationId: scope.organizationId ?? null,\n keys: Object.keys(snapshot),\n sample: snapshot,\n })\n }\n return result\n }\n\n await upsertIndexBatch(db, entityType, rows, scopeOverrides, { deriveOrganizationId: deriveOrg, encryptDoc, decryptDoc })\n\n const coverageDeltas = new Map<string, { tenantId: string | null; organizationId: string | null; delta: number }>()\n for (const row of rows) {\n const scopeTenant = tenantId !== undefined\n ? tenantId ?? null\n : (hasTenantCol ? ((row as AnyRow).tenant_id ?? null) : null)\n const scopeOrg = organizationId !== undefined\n ? organizationId ?? null\n : (hasOrgCol ? ((row as AnyRow).organization_id ?? null) : (deriveOrg ? deriveOrg(row) ?? null : null))\n const key = scopeKey(scopeTenant ?? null, scopeOrg ?? null)\n const existingDelta = coverageDeltas.get(key)\n if (existingDelta) existingDelta.delta += 1\n else coverageDeltas.set(key, {\n tenantId: scopeTenant ?? null,\n organizationId: scopeOrg ?? null,\n delta: 1,\n })\n }\n if (coverageDeltas.size > 0) {\n await applyCoverageAdjustments(\n em,\n Array.from(coverageDeltas.values()).map((entry) => ({\n entityType,\n tenantId: entry.tenantId,\n organizationId: entry.organizationId,\n withDeleted: false,\n deltaBase: 0,\n deltaIndex: entry.delta,\n })),\n )\n }\n\n if (emitVectorize && eventBus) {\n await Promise.all(\n rows.map((row) => {\n const scopeOrg = organizationId !== undefined\n ? organizationId ?? null\n : hasOrgCol\n ? ((row as AnyRow).organization_id ?? null)\n : (deriveOrg ? deriveOrg(row) ?? null : null)\n const scopeTenant = tenantId !== undefined\n ? tenantId ?? null\n : (hasTenantCol ? ((row as AnyRow).tenant_id ?? null) : null)\n return eventBus\n .emitEvent('query_index.vectorize_one', {\n entityType,\n recordId: String(row.id),\n organizationId: scopeOrg,\n tenantId: scopeTenant,\n })\n .catch(() => undefined)\n }),\n )\n }\n\n processed += rows.length\n lastId = String(rows[rows.length - 1]!.id)\n options?.onProgress?.({ processed, total, chunkSize: rows.length })\n await updateJobProgress(db, jobScope, rows.length)\n }\n\n await purgeOrphans(db, {\n entityType,\n tenantId,\n organizationId,\n partitionIndex: usingPartitions ? partitionIndex : null,\n partitionCount: usingPartitions ? partitionCountRaw : null,\n startedAt: jobStartedAt,\n })\n\n if (force && vectorService && (!usingPartitions || partitionIndex === null)) {\n try {\n await vectorService.removeOrphans({\n entityId: entityType,\n tenantId,\n organizationId,\n olderThan: jobStartedAt,\n })\n } catch (error) {\n console.warn('[HybridQueryEngine] Failed to prune vector orphans after reindex', {\n entityType,\n tenantId: tenantId ?? null,\n organizationId: organizationId ?? null,\n error: error instanceof Error ? error.message : error,\n })\n }\n }\n\n for (const scope of scopeEntries) {\n await refreshCoverageSnapshot(\n em,\n {\n entityType,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n withDeleted: false,\n },\n )\n }\n } finally {\n await finalizeJob(db, jobScope)\n }\n\n return {\n processed,\n total,\n scopes: scopeEntries,\n tenantScopes,\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAsB,WAAW;AACjC,SAAS,8BAA8B;AACvC,SAAS,sCAAsC;AAC/C,SAAS,0BAA0B,iCAAiC;AACpE,SAAS,wBAAqC;AAC9C,SAAS,yBAAyB,qBAAqB,gCAAgC;AACvF,SAAS,YAAY,mBAAmB,mBAAkC;AAC1E,SAAS,oBAAoB;AAE7B,SAAS,4BAA4B;AA0B9B,MAAM,6BAA6B;AAC1C,MAAM,qBAAqB;AAC3B,MAAM,kBAAkB,oBAAI,IAAY,CAAC,wBAAwB,CAAC;AAClE,MAAM,+BAA+B,IAAI,KAAK;AAC9C,MAAM,oBAAoB,oBAAI,IAAoB;AAElD,eAAe,uBACb,IACA,SAMe;AACf,QAAM,GACH,WAAW,mBAA0B,EACrC,MAAM,eAAsB,KAAK,QAAQ,UAAU,EACnD,MAAM,2CAAoD,QAAQ,cAAc,EAAE,EAClF,MAAM,qCAA8C,QAAQ,QAAQ,EAAE,EACtE,MAAM,uCAAgD,QAAQ,oBAAoB,EAAE,EACpF,QAAQ;AACb;AAEA,SAAS,SAAS,OAAwB;AACxC,MAAI,OAAO,UAAU,SAAU,QAAO,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,SAAS,OAAO,KAAK;AAC3B,WAAO,OAAO,SAAS,MAAM,IAAI,SAAS;AAAA,EAC5C;AACA,SAAO;AACT;AAEA,eAAe,aAAa,IAAiB,WAAyC;AACpF,MAAI;AACF,UAAM,OAAO,MAAM,GAChB,WAAW,4BAAmC,EAC9C,OAAO,CAAC,aAAoB,CAAC,EAC7B,MAAM,oCAA6C,EACnD,MAAM,cAAqB,KAAK,SAAS,EACzC,QAAQ;AACX,WAAO,IAAI,IAAI,KAAK,IAAI,CAAC,QAAQ,OAAO,IAAI,WAAW,EAAE,YAAY,CAAC,CAAC;AAAA,EACzE,QAAQ;AACN,WAAO,oBAAI,IAAY;AAAA,EACzB;AACF;AAEA,eAAsB,cACpB,IACA,SAC2B;AAC3B,QAAM,aAAa,OAAO,SAAS,cAAc,EAAE;AACnD,MAAI,CAAC,YAAY;AACf,WAAO;AAAA,MACL,WAAW;AAAA,MACX,OAAO;AAAA,MACP,cAAc,CAAC;AAAA,MACf,QAAQ,CAAC;AAAA,IACX;AAAA,EACF;AACA,QAAM,gBAAgB,SAAS;AAC/B,QAAM,WAAW,kBAAkB,cAAc,SAAY;AAC7D,QAAM,sBAAsB,SAAS;AACrC,QAAM,iBAAiB,wBAAwB,cAAc,SAAY;AACzE,QAAM,QAAQ,SAAS,UAAU;AACjC,QAAM,YAAY,OAAO,SAAS,SAAS,SAAS,KAAK,QAAS,YAAa,IAC3E,KAAK,IAAI,GAAG,KAAK,MAAM,QAAS,SAAU,CAAC,IAC3C;AACJ,QAAM,gBAAgB,SAAS,wBAAwB;AACvD,QAAM,WAAW,SAAS;AAC1B,QAAM,gBAAgB,SAAS,iBAAiB;AAChD,QAAM,oBAAoB,OAAO,SAAS,SAAS,cAAc,IAC7D,KAAK,IAAI,GAAG,KAAK,MAAM,QAAS,cAAe,CAAC,IAChD;AACJ,QAAM,kBAAkB,oBAAoB;AAC5C,QAAM,oBAAoB,OAAO,SAAS,SAAS,cAAc,IAC7D,KAAK,IAAI,GAAG,KAAK,MAAM,QAAS,cAAe,CAAC,IAChD;AACJ,QAAM,iBAAiB,kBACnB,KAAK,IAAI,mBAAmB,oBAAoB,CAAC,IACjD;AACJ,QAAM,gBAAgB,SAAS,kBAAkB,CAAC,mBAAmB,mBAAmB;AAExF,QAAM,KAAM,GAAW,UAAU;AACjC,QAAM,QAAQ,uBAAuB,IAAI,UAAU;AACnD,MAAI,eAAe,8BAA8B,UAAU,iBAAiB;AAC1E,WAAO;AAAA,MACL,WAAW;AAAA,MACX,OAAO;AAAA,MACP,cAAc,CAAC;AAAA,MACf,QAAQ,CAAC;AAAA,IACX;AAAA,EACF;AACA,QAAM,UAAU,MAAM,aAAa,IAAI,KAAK;AAC5C,QAAM,YAAY,QAAQ,IAAI,iBAAiB;AAC/C,QAAM,eAAe,QAAQ,IAAI,WAAW;AAC5C,QAAM,gBAAgB,QAAQ,IAAI,YAAY;AAE9C,QAAM,WAAqB;AAAA,IACzB;AAAA,IACA,gBAAgB,kBAAkB;AAAA,IAClC,UAAU,YAAY;AAAA,IACtB;AAAA,IACA,gBAAgB,kBAAkB,oBAAoB;AAAA,EACxD;AAEA,MAAI,CAAC,OAAO;AACV,UAAM,YAAY,MAAM,GACrB,WAAW,mBAA0B,EACrC,OAAO,CAAC,IAAW,CAAC,EACpB,MAAM,eAAsB,KAAK,UAAU,EAC3C,MAAM,eAAsB,MAAM,IAAW,EAC7C,MAAM,2CAAoD,IAAI,EAAE,EAChE,MAAM,qCAA8C,YAAY,IAAI,EAAE,EACtE,MAAM,2CAAoD,cAAc,EAAE,EAC1E,MAAM,2CAAoD,kBAAkB,oBAAoB,IAAI,EAAE,EACtG,iBAAiB;AACpB,QAAI,WAAW;AACb,aAAO;AAAA,QACL,WAAW;AAAA,QACX,OAAO;AAAA,QACP,cAAc,CAAC;AAAA,QACf,QAAQ,CAAC;AAAA,MACX;AAAA,IACF;AAAA,EACF;AAEA,MAAI,eAAe;AACjB,UAAM,uBAAuB,IAAI;AAAA,MAC/B;AAAA,MACA,gBAAgB,SAAS,kBAAkB;AAAA,MAC3C,UAAU,SAAS,YAAY;AAAA,MAC/B,sBAAsB,SAAS,kBAAkB;AAAA,IACnD,CAAC;AAAA,EACH;AAEA,QAAM,WAAW,CAAC,aAA4B,aAA4B,GAAG,eAAe,UAAU,IAAI,YAAY,UAAU;AAEhI,QAAM,iBAAiB,CAA+C,MAAc;AAClF,QAAI,QAAQ;AACZ,QAAI,cAAe,SAAQ,MAAM,MAAM,gBAAuB,MAAM,IAAW;AAC/E,QAAI,aAAa,UAAa,cAAc;AAC1C,cAAQ,aAAa,OACjB,MAAM,MAAM,eAAsB,MAAM,IAAW,IACnD,MAAM,MAAM,eAAsB,KAAK,QAAQ;AAAA,IACrD;AACA,QAAI,mBAAmB,UAAa,WAAW;AAC7C,cAAQ,mBAAmB,OACvB,MAAM,MAAM,qBAA4B,MAAM,IAAW,IACzD,MAAM,MAAM,qBAA4B,KAAK,cAAc;AAAA,IACjE;AACA,QAAI,mBAAmB,mBAAmB,MAAM;AAC9C,cAAQ,MAAM,MAAM,qCAA8C,iBAAiB,OAAO,cAAc,EAAE;AAAA,IAC5G;AACA,WAAO;AAAA,EACT;AAGA,QAAM,aAAa,oBAAI,IAAwB;AAC/C,QAAM,oBAAoB,CAAC,aAA4B,UAAyB,UAAkB;AAChG,UAAM,MAAM,SAAS,aAAa,QAAQ;AAC1C,eAAW,IAAI,KAAK,EAAE,UAAU,aAAa,gBAAgB,UAAU,MAAM,CAAC;AAAA,EAChF;AAEA,QAAM,gBAAgB,gBAAgB,aAAa;AACnD,QAAM,aAAa,aAAa,mBAAmB;AAEnD,MAAI,iBAAiB,YAAY;AAC/B,QAAI,aAAa;AAAA,MACf,GAAG,WAAW,GAAG,KAAK,OAAc,EAAE,OAAO,cAAsB,GAAG,OAAO,CAAC;AAAA,IAChF;AACA,QAAI,eAAe;AACjB,mBAAa,WAAW,OAAO,0BAAiC,EAAE,QAAQ,aAAoB;AAAA,IAChG;AACA,QAAI,YAAY;AACd,mBAAa,WAAW,OAAO,sCAA6C,EAAE,QAAQ,mBAA0B;AAAA,IAClH;AACA,UAAM,OAAO,MAAM,WAAW,QAAQ;AACtC,eAAW,OAAO,MAAM;AACtB,YAAM,eAAe,gBACf,KAAa,aAAa,OAC3B,aAAa,SAAY,OAAO,YAAY;AACjD,YAAM,YAAY,aACZ,KAAa,mBAAmB,OACjC,mBAAmB,SAAY,OAAO,kBAAkB;AAC7D,wBAAkB,cAAc,WAAW,SAAU,KAAa,KAAK,CAAC;AAAA,IAC1E;AAAA,EACF,OAAO;AACL,UAAM,MAAM,MAAM;AAAA,MAChB,GAAG,WAAW,GAAG,KAAK,OAAc,EAAE,OAAO,cAAsB,GAAG,OAAO,CAAC;AAAA,IAChF,EAAE,iBAAiB;AACnB,UAAM,eAAe,aAAa,SAAY,OAAO,YAAY;AACjE,UAAM,YAAY,mBAAmB,SAAY,OAAO,kBAAkB;AAC1E,sBAAkB,cAAc,WAAW,SAAS,KAAK,KAAK,CAAC;AAAA,EACjE;AAEA,QAAM,QAAQ,MAAM,KAAK,WAAW,OAAO,CAAC,EAAE,OAAO,CAAC,KAAK,UAAU,OAAO,OAAO,SAAS,MAAM,KAAK,IAAI,MAAM,QAAQ,IAAI,CAAC;AAC9H,QAAM,WAAW,IAAI,UAAU,cAAc,EAAE,YAAY,MAAM,CAAC;AAClE,QAAM,SAAS,MAAM,GAClB,WAAW,mBAA0B,EACrC,OAAO,CAAC,YAAmB,CAAC,EAC5B,MAAM,eAAsB,KAAK,UAAU,EAC3C,MAAM,mBAA0B,MAAM,IAAW,EACjD,MAAM,qCAA8C,YAAY,IAAI,EAAE,EACtE,MAAM,2CAAoD,cAAc,EAAE,EAC1E,MAAM,2CAAoD,kBAAkB,oBAAoB,IAAI,EAAE,EACtG,QAAQ,cAAqB,MAAM,EACnC,iBAAiB;AACpB,QAAM,eAAe,QAAQ,aAAa,IAAI,KAAK,OAAO,UAAU,IAAI,oBAAI,KAAK;AACjF,QAAM,YAAY,gBAAgB,IAAI,UAAU,IAC5C,CAAC,QAAgB,OAAO,IAAI,EAAE,IAC9B;AAEJ,QAAM,iBAAwD,CAAC;AAC/D,MAAI,aAAa,UAAa,aAAa,MAAM;AAC/C,mBAAe,WAAW,OAAO,QAAQ;AAAA,EAC3C;AACA,MAAI,mBAAmB,UAAa,mBAAmB,MAAM;AAC3D,mBAAe,QAAQ,OAAO,cAAc;AAAA,EAC9C;AAEA,QAAM,eAAe,MAAM,KAAK,WAAW,OAAO,CAAC,EAAE,IAAI,CAAC,WAAW;AAAA,IACnE,UAAU,MAAM;AAAA,IAChB,gBAAgB,MAAM;AAAA,EACxB,EAAE;AACF,QAAM,eAAe,MAAM;AAAA,IACzB,IAAI,IAAI,aAAa,IAAI,CAAC,UAAU,MAAM,YAAY,IAAI,CAAC;AAAA,EAC7D;AAEA,MAAI,YAAY;AAChB,MAAI,SAAwB;AAE5B,WAAS,aAAa,EAAE,WAAW,OAAO,WAAW,EAAE,CAAC;AAExD,MAAI,eAAe;AACjB,QAAI,OAAO;AACT,UAAI;AACF,YAAI,aAAa,GACd,WAAW,gBAAuB,EAClC,MAAM,eAAsB,KAAK,UAAU;AAC9C,YAAI,aAAa,QAAW;AAC1B,uBAAa,WAAW,MAAM,qCAA8C,YAAY,IAAI,EAAE;AAAA,QAChG;AACA,YAAI,mBAAmB,QAAW;AAChC,uBAAa,WAAW,MAAM,2CAAoD,kBAAkB,IAAI,EAAE;AAAA,QAC5G;AACA,cAAM,WAAW,QAAQ;AAAA,MAC3B,SAAS,OAAO;AACd,gBAAQ,KAAK,uEAAuE;AAAA,UAClF;AAAA,UACA,UAAU,YAAY;AAAA,UACtB,gBAAgB,kBAAkB;AAAA,UAClC,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,QAClD,CAAC;AAAA,MACH;AAEA,UAAI,iBAAiB,UAAU;AAC7B,YAAI,aAAa,QAAW;AAC1B,gBAAM,UAAmC;AAAA,YACvC;AAAA,YACA,UAAU,YAAY;AAAA,UACxB;AACA,cAAI,mBAAmB,OAAW,SAAQ,iBAAiB,kBAAkB;AAC7E,cAAI;AACF,kBAAM,SAAS,UAAU,+BAA+B,OAAO;AAAA,UACjE,SAAS,KAAK;AACZ,oBAAQ,KAAK,yEAAyE;AAAA,cACpF;AAAA,cACA,UAAU,YAAY;AAAA,cACtB,gBAAgB,kBAAkB;AAAA,cAClC,OAAO,eAAe,QAAQ,IAAI,UAAU;AAAA,YAC9C,CAAC;AAAA,UACH;AAAA,QACF,OAAO;AACL,kBAAQ,KAAK,oFAAoF;AAAA,YAC/F;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,IAAI;AACvB,eAAW,SAAS,WAAW,OAAO,GAAG;AACvC,YAAM,MAAM,GAAG,UAAU,IAAI,SAAS,MAAM,UAAU,MAAM,cAAc,CAAC;AAC3E,YAAM,OAAO,kBAAkB,IAAI,GAAG,KAAK;AAC3C,UAAI,SAAS,QAAQ,QAAQ,8BAA8B;AACzD,cAAM,oBAAoB,IAAI;AAAA,UAC5B;AAAA,UACA,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,aAAa;AAAA,QACf,GAAG;AAAA,UACD,WAAW,MAAM;AAAA,UACjB,cAAc;AAAA,UACd,aAAa,gBAAgB,IAAI;AAAA,QACnC,CAAC;AACD,0BAAkB,IAAI,KAAK,KAAK;AAAA,MAClC;AAAA,IACF;AAAA,EACF;AAEA,MAAI;AACF,WAAO,MAAM;AACX,UAAI,QAAQ;AAAA,QACV,GACG,WAAW,GAAG,KAAK,OAAc,EACjC,UAAU,GAAU,EACpB,QAAQ,QAAe,KAAK,EAC5B,MAAM,SAAS;AAAA,MACpB;AACA,UAAI,WAAW,MAAM;AACnB,gBAAQ,MAAM,MAAM,QAAe,KAAK,MAAM;AAAA,MAChD;AACA,YAAM,OAAO,MAAM,MAAM,QAAQ;AACjC,UAAI,CAAC,KAAK,OAAQ;AAElB,YAAM,aAAa,+BAA+B,EAAS;AAC3D,YAAM,cAAc,oBAAI,IAAkC;AAC1D,YAAM,aAAa,OACjB,cACA,KACA,UACG;AACH,eAAO,MAAM;AAAA,UACX;AAAA,UACA;AAAA,UACA,EAAE,UAAU,MAAM,YAAY,MAAM,gBAAgB,MAAM,kBAAkB,KAAK;AAAA,UACjF;AAAA,QACF;AAAA,MACF;AACA,YAAM,aAAa,OACjB,cACA,KACA,UACG;AACH,cAAM,SAAS,MAAM;AAAA,UACnB;AAAA,UACA;AAAA,UACA,EAAE,UAAU,MAAM,YAAY,MAAM,gBAAgB,MAAM,kBAAkB,KAAK;AAAA,UACjF;AAAA,UACA;AAAA,QACF;AACA,YAAI,qBAAqB,GAAG;AAC1B,gBAAM,iBAAiB,CAAC,gBAAgB,cAAc,aAAa,cAAc,cAAc,iBAAiB,eAAe;AAC/H,gBAAM,WAAoC,CAAC;AAC3C,qBAAW,OAAO,gBAAgB;AAChC,gBAAI,OAAO,OAAQ,UAAS,GAAG,IAAK,OAAmC,GAAG;AAAA,UAC5E;AACA,kBAAQ,KAAK,qBAAqB;AAAA,YAChC,YAAY;AAAA,YACZ,UAAU,MAAM,YAAY;AAAA,YAC5B,gBAAgB,MAAM,kBAAkB;AAAA,YACxC,MAAM,OAAO,KAAK,QAAQ;AAAA,YAC1B,QAAQ;AAAA,UACV,CAAC;AAAA,QACH;AACA,eAAO;AAAA,MACT;AAEA,YAAM,iBAAiB,IAAI,YAAY,MAAM,gBAAgB,EAAE,sBAAsB,WAAW,YAAY,WAAW,CAAC;AAExH,YAAM,iBAAiB,oBAAI,IAAuF;AAClH,iBAAW,OAAO,MAAM;AACtB,cAAM,cAAc,aAAa,SAC7B,YAAY,OACX,eAAiB,IAAe,aAAa,OAAQ;AAC1D,cAAM,WAAW,mBAAmB,SAChC,kBAAkB,OACjB,YAAc,IAAe,mBAAmB,OAAS,YAAY,UAAU,GAAG,KAAK,OAAO;AACnG,cAAM,MAAM,SAAS,eAAe,MAAM,YAAY,IAAI;AAC1D,cAAM,gBAAgB,eAAe,IAAI,GAAG;AAC5C,YAAI,cAAe,eAAc,SAAS;AAAA,YACrC,gBAAe,IAAI,KAAK;AAAA,UAC3B,UAAU,eAAe;AAAA,UACzB,gBAAgB,YAAY;AAAA,UAC5B,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AACA,UAAI,eAAe,OAAO,GAAG;AAC3B,cAAM;AAAA,UACJ;AAAA,UACA,MAAM,KAAK,eAAe,OAAO,CAAC,EAAE,IAAI,CAAC,WAAW;AAAA,YAClD;AAAA,YACA,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,aAAa;AAAA,YACb,WAAW;AAAA,YACX,YAAY,MAAM;AAAA,UACpB,EAAE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,iBAAiB,UAAU;AAC7B,cAAM,QAAQ;AAAA,UACZ,KAAK,IAAI,CAAC,QAAQ;AAChB,kBAAM,WAAW,mBAAmB,SAChC,kBAAkB,OAClB,YACI,IAAe,mBAAmB,OACnC,YAAY,UAAU,GAAG,KAAK,OAAO;AAC5C,kBAAM,cAAc,aAAa,SAC7B,YAAY,OACX,eAAiB,IAAe,aAAa,OAAQ;AAC1D,mBAAO,SACJ,UAAU,6BAA6B;AAAA,cACtC;AAAA,cACA,UAAU,OAAO,IAAI,EAAE;AAAA,cACvB,gBAAgB;AAAA,cAChB,UAAU;AAAA,YACZ,CAAC,EACA,MAAM,MAAM,MAAS;AAAA,UAC1B,CAAC;AAAA,QACH;AAAA,MACF;AAEA,mBAAa,KAAK;AAClB,eAAS,OAAO,KAAK,KAAK,SAAS,CAAC,EAAG,EAAE;AACzC,eAAS,aAAa,EAAE,WAAW,OAAO,WAAW,KAAK,OAAO,CAAC;AAClE,YAAM,kBAAkB,IAAI,UAAU,KAAK,MAAM;AAAA,IACnD;AAEA,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,kBAAkB,iBAAiB;AAAA,MACnD,gBAAgB,kBAAkB,oBAAoB;AAAA,MACtD,WAAW;AAAA,IACb,CAAC;AAED,QAAI,SAAS,kBAAkB,CAAC,mBAAmB,mBAAmB,OAAO;AAC3E,UAAI;AACF,cAAM,cAAc,cAAc;AAAA,UAChC,UAAU;AAAA,UACV;AAAA,UACA;AAAA,UACA,WAAW;AAAA,QACb,CAAC;AAAA,MACH,SAAS,OAAO;AACd,gBAAQ,KAAK,oEAAoE;AAAA,UAC/E;AAAA,UACA,UAAU,YAAY;AAAA,UACtB,gBAAgB,kBAAkB;AAAA,UAClC,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,QAClD,CAAC;AAAA,MACH;AAAA,IACF;AAEA,eAAW,SAAS,cAAc;AAChC,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,UACE;AAAA,UACA,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,aAAa;AAAA,QACf;AAAA,MACF;AAAA,IACF;AAAA,EACF,UAAE;AACA,UAAM,YAAY,IAAI,QAAQ;AAAA,EAChC;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1,6 +1,14 @@
|
|
|
1
|
+
import { sql } from "kysely";
|
|
1
2
|
import { resolveSearchConfig } from "@open-mercato/shared/lib/search/config";
|
|
2
3
|
import { tokenizeText } from "@open-mercato/shared/lib/search/tokenize";
|
|
3
4
|
import { parseBooleanToken } from "@open-mercato/shared/lib/boolean";
|
|
5
|
+
const INSERT_BATCH_SIZE = 500;
|
|
6
|
+
function chunk(items, size) {
|
|
7
|
+
if (size <= 0) return [items];
|
|
8
|
+
const out = [];
|
|
9
|
+
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
|
|
10
|
+
return out;
|
|
11
|
+
}
|
|
4
12
|
const DEFAULT_SCOPE = { organizationId: null, tenantId: null };
|
|
5
13
|
const isSearchDebugEnabled = () => {
|
|
6
14
|
return parseBooleanToken(process.env.OM_SEARCH_DEBUG ?? "") === true;
|
|
@@ -96,32 +104,39 @@ function buildFieldPairs(recordId, doc) {
|
|
|
96
104
|
}
|
|
97
105
|
return pairs;
|
|
98
106
|
}
|
|
99
|
-
async function replaceSearchTokensForRecord(
|
|
107
|
+
async function replaceSearchTokensForRecord(db, params) {
|
|
100
108
|
const rows = buildSearchTokenRows(params);
|
|
101
109
|
const config = params.config ?? resolveSearchConfig();
|
|
102
110
|
if (!config.enabled) return;
|
|
103
111
|
const organizationId = params.organizationId ?? null;
|
|
104
112
|
const tenantId = params.tenantId ?? null;
|
|
105
113
|
const fieldPairs = buildFieldPairs(String(params.recordId), params.doc);
|
|
106
|
-
await
|
|
107
|
-
|
|
108
|
-
if (fieldPairs.length)
|
|
109
|
-
|
|
110
|
-
|
|
114
|
+
await db.transaction().execute(async (trx) => {
|
|
115
|
+
let deleteQuery = trx.deleteFrom("search_tokens").where("entity_type", "=", params.entityType).where(sql`organization_id is not distinct from ${organizationId}`).where(sql`tenant_id is not distinct from ${tenantId}`);
|
|
116
|
+
if (fieldPairs.length) {
|
|
117
|
+
deleteQuery = deleteQuery.where((eb) => eb.or(
|
|
118
|
+
fieldPairs.map(([rid, field]) => eb.and([
|
|
119
|
+
eb("entity_id", "=", rid),
|
|
120
|
+
eb("field", "=", field)
|
|
121
|
+
]))
|
|
122
|
+
));
|
|
123
|
+
} else {
|
|
124
|
+
deleteQuery = deleteQuery.where("entity_id", "=", String(params.recordId));
|
|
125
|
+
}
|
|
126
|
+
await deleteQuery.execute();
|
|
111
127
|
if (!rows.length) return;
|
|
112
|
-
const payloads = rows.map((row) => ({
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
}
|
|
116
|
-
await trx.batchInsert("search_tokens", payloads, 500);
|
|
128
|
+
const payloads = rows.map((row) => ({ ...row, created_at: sql`now()` }));
|
|
129
|
+
for (const batch of chunk(payloads, INSERT_BATCH_SIZE)) {
|
|
130
|
+
await trx.insertInto("search_tokens").values(batch).execute();
|
|
131
|
+
}
|
|
117
132
|
});
|
|
118
133
|
}
|
|
119
|
-
async function deleteSearchTokensForRecord(
|
|
134
|
+
async function deleteSearchTokensForRecord(db, params) {
|
|
120
135
|
const organizationId = params.organizationId ?? null;
|
|
121
136
|
const tenantId = params.tenantId ?? null;
|
|
122
|
-
await
|
|
137
|
+
await db.deleteFrom("search_tokens").where("entity_type", "=", params.entityType).where("entity_id", "=", String(params.recordId)).where(sql`organization_id is not distinct from ${organizationId}`).where(sql`tenant_id is not distinct from ${tenantId}`).execute();
|
|
123
138
|
}
|
|
124
|
-
async function replaceSearchTokensForBatch(
|
|
139
|
+
async function replaceSearchTokensForBatch(db, payloads) {
|
|
125
140
|
if (!payloads.length) return;
|
|
126
141
|
const config = resolveSearchConfig();
|
|
127
142
|
if (!config.enabled) return;
|
|
@@ -130,7 +145,7 @@ async function replaceSearchTokensForBatch(knex, payloads) {
|
|
|
130
145
|
const entityType = payloads[0]?.entityType;
|
|
131
146
|
if (!entityType) return;
|
|
132
147
|
const ids = payloads.map((p) => String(p.recordId));
|
|
133
|
-
await
|
|
148
|
+
await db.deleteFrom("search_tokens").where("entity_type", "=", entityType).where("entity_id", "in", ids).execute();
|
|
134
149
|
return;
|
|
135
150
|
}
|
|
136
151
|
const scopeKey = (org, tenant) => `${org ?? "__null__"}|${tenant ?? "__null__"}`;
|
|
@@ -161,19 +176,26 @@ async function replaceSearchTokensForBatch(knex, payloads) {
|
|
|
161
176
|
fieldPairsByScope.set(key, pairs);
|
|
162
177
|
seenPairsByScope.set(key, seen);
|
|
163
178
|
}
|
|
164
|
-
await
|
|
179
|
+
await db.transaction().execute(async (trx) => {
|
|
165
180
|
for (const [key, bucket] of scopeBuckets.entries()) {
|
|
166
181
|
const pairs = fieldPairsByScope.get(key) ?? [];
|
|
167
|
-
|
|
168
|
-
if (pairs.length)
|
|
169
|
-
|
|
170
|
-
|
|
182
|
+
let deleteQuery = trx.deleteFrom("search_tokens").where("entity_type", "=", payloads[0].entityType).where(sql`organization_id is not distinct from ${bucket.organizationId}`).where(sql`tenant_id is not distinct from ${bucket.tenantId}`);
|
|
183
|
+
if (pairs.length) {
|
|
184
|
+
deleteQuery = deleteQuery.where((eb) => eb.or(
|
|
185
|
+
pairs.map(([rid, field]) => eb.and([
|
|
186
|
+
eb("entity_id", "=", rid),
|
|
187
|
+
eb("field", "=", field)
|
|
188
|
+
]))
|
|
189
|
+
));
|
|
190
|
+
} else {
|
|
191
|
+
deleteQuery = deleteQuery.where("entity_id", "in", Array.from(bucket.ids));
|
|
192
|
+
}
|
|
193
|
+
await deleteQuery.execute();
|
|
194
|
+
}
|
|
195
|
+
const payloadWithTimestamps = rows.map((row) => ({ ...row, created_at: sql`now()` }));
|
|
196
|
+
for (const batch of chunk(payloadWithTimestamps, INSERT_BATCH_SIZE)) {
|
|
197
|
+
await trx.insertInto("search_tokens").values(batch).execute();
|
|
171
198
|
}
|
|
172
|
-
const payloadWithTimestamps = rows.map((row) => ({
|
|
173
|
-
...row,
|
|
174
|
-
created_at: trx.fn.now()
|
|
175
|
-
}));
|
|
176
|
-
await trx.batchInsert("search_tokens", payloadWithTimestamps, 500);
|
|
177
199
|
});
|
|
178
200
|
}
|
|
179
201
|
export {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/query_index/lib/search-tokens.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Knex } from 'knex'\nimport { resolveSearchConfig, type SearchConfig } from '@open-mercato/shared/lib/search/config'\nimport { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'\nimport { parseBooleanToken } from '@open-mercato/shared/lib/boolean'\n\nexport type SearchTokenRow = {\n entity_type: string\n entity_id: string\n organization_id: string | null\n tenant_id: string | null\n field: string\n token_hash: string\n token?: string | null\n}\n\ntype BuildTokenOptions = {\n entityType: string\n recordId: string\n organizationId?: string | null\n tenantId?: string | null\n doc?: Record<string, unknown> | null\n config?: SearchConfig\n}\n\nconst DEFAULT_SCOPE = { organizationId: null, tenantId: null }\ntype EntityFieldPair = [string, string]\n\nexport const isSearchDebugEnabled = (): boolean => {\n return parseBooleanToken(process.env.OM_SEARCH_DEBUG ?? '') === true\n}\n\nconst debug = (event: string, payload: Record<string, unknown>) => {\n if (!isSearchDebugEnabled()) return\n try {\n // eslint-disable-next-line no-console\n console.debug(`[search-tokens] ${event}`, payload)\n } catch {\n // ignore\n }\n}\n\nfunction collectTextValues(value: unknown): string[] {\n if (typeof value === 'string') return [value]\n if (Array.isArray(value)) {\n const out: string[] = []\n for (const entry of value) {\n if (typeof entry === 'string') out.push(entry)\n }\n return out\n }\n return []\n}\n\nfunction shouldIndexField(field: string, value: unknown, config: SearchConfig): boolean {\n if (typeof value !== 'string' && !Array.isArray(value)) return false\n const lower = field.toLowerCase()\n if (lower === 'id' || lower.endsWith('_id') || lower.endsWith('.id')) return false\n if (lower.endsWith('_at')) return false\n if (['created_at', 'updated_at', 'deleted_at', 'tenant_id', 'organization_id'].includes(lower)) return false\n if (config.blocklistedFields.some((blocked) => lower.includes(blocked))) return false\n const values = collectTextValues(value)\n if (!values.length) return false\n return values.some((text) => tokenizeText(text, config).tokens.length > 0)\n}\n\nexport function buildSearchTokenRows(params: BuildTokenOptions): SearchTokenRow[] {\n const config = params.config ?? resolveSearchConfig()\n if (!config.enabled) return []\n if (!params.doc) return []\n const tokens: SearchTokenRow[] = []\n const capturePairs = isSearchDebugEnabled() && params.entityType === 'customers:customer_deal'\n const debugPairs: Array<{ field: string; token: string; hash: string }> = []\n const scope = {\n organizationId: params.organizationId ?? DEFAULT_SCOPE.organizationId,\n tenantId: params.tenantId ?? DEFAULT_SCOPE.tenantId,\n }\n\n for (const [field, rawValue] of Object.entries(params.doc)) {\n if (!shouldIndexField(field, rawValue, config)) continue\n const values = collectTextValues(rawValue)\n const seen = new Set<string>()\n for (const text of values) {\n const { tokens: textTokens, hashes } = tokenizeText(text, config)\n for (let i = 0; i < textTokens.length; i += 1) {\n const token = textTokens[i]\n const hash = hashes[i]\n const dedupeKey = `${field}|${hash}`\n if (seen.has(dedupeKey)) continue\n seen.add(dedupeKey)\n debug('token.generated', { entityType: params.entityType, recordId: params.recordId, field, token, hash })\n tokens.push({\n entity_type: params.entityType,\n entity_id: String(params.recordId),\n organization_id: scope.organizationId,\n tenant_id: scope.tenantId,\n field,\n token_hash: hash,\n token: config.storeRawTokens ? token : null,\n })\n if (capturePairs) {\n debugPairs.push({ field, token, hash })\n }\n }\n }\n }\n if (capturePairs) {\n debug('deal.tokens', {\n entityType: params.entityType,\n recordId: params.recordId,\n title: params.doc?.title ?? null,\n tokens: debugPairs,\n })\n }\n debug('doc.completed', { entityType: params.entityType, recordId: params.recordId, tokenCount: tokens.length })\n\n return tokens\n}\n\nfunction buildFieldPairs(recordId: string, doc?: Record<string, unknown> | null): EntityFieldPair[] {\n if (!doc) return []\n const pairs: EntityFieldPair[] = []\n const dedupe = new Set<string>()\n for (const field of Object.keys(doc)) {\n const key = `${recordId}|${field}`\n if (dedupe.has(key)) continue\n dedupe.add(key)\n pairs.push([recordId, field])\n }\n return pairs\n}\n\nexport async function replaceSearchTokensForRecord(\n knex: Knex,\n params: BuildTokenOptions\n): Promise<void> {\n const rows = buildSearchTokenRows(params)\n const config = params.config ?? resolveSearchConfig()\n if (!config.enabled) return\n const organizationId = params.organizationId ?? null\n const tenantId = params.tenantId ?? null\n const fieldPairs = buildFieldPairs(String(params.recordId), params.doc)\n\n await knex.transaction(async (trx) => {\n const deleteQuery = trx('search_tokens')\n .where({ entity_type: params.entityType })\n .andWhereRaw('organization_id is not distinct from ?', [organizationId])\n .andWhereRaw('tenant_id is not distinct from ?', [tenantId])\n if (fieldPairs.length) deleteQuery.whereIn(['entity_id', 'field'], fieldPairs)\n else deleteQuery.where('entity_id', String(params.recordId))\n await deleteQuery.del()\n if (!rows.length) return\n const payloads = rows.map((row) => ({\n ...row,\n created_at: trx.fn.now(),\n }))\n await trx.batchInsert('search_tokens', payloads, 500)\n })\n}\n\nexport async function deleteSearchTokensForRecord(\n knex: Knex,\n params: { entityType: string; recordId: string; organizationId?: string | null; tenantId?: string | null }\n): Promise<void> {\n const organizationId = params.organizationId ?? null\n const tenantId = params.tenantId ?? null\n await knex('search_tokens')\n .where({ entity_type: params.entityType, entity_id: String(params.recordId) })\n .andWhereRaw('organization_id is not distinct from ?', [organizationId])\n .andWhereRaw('tenant_id is not distinct from ?', [tenantId])\n .del()\n}\n\nexport async function replaceSearchTokensForBatch(\n knex: Knex,\n payloads: Array<BuildTokenOptions & { doc: Record<string, unknown> }>\n): Promise<void> {\n if (!payloads.length) return\n const config = resolveSearchConfig()\n if (!config.enabled) return\n\n const rows = payloads.flatMap((payload) => buildSearchTokenRows({ ...payload, config }))\n if (!rows.length) {\n const entityType = payloads[0]?.entityType\n if (!entityType) return\n const ids = payloads.map((p) => String(p.recordId))\n await knex('search_tokens').where({ entity_type: entityType }).whereIn('entity_id', ids).del()\n return\n }\n\n const scopeKey = (org: string | null, tenant: string | null) => `${org ?? '__null__'}|${tenant ?? '__null__'}`\n const scopeBuckets = new Map<string, { organizationId: string | null; tenantId: string | null; ids: Set<string> }>()\n const fieldPairsByScope = new Map<string, EntityFieldPair[]>()\n const seenPairsByScope = new Map<string, Set<string>>()\n\n for (const payload of payloads) {\n const org = payload.organizationId ?? null\n const tenant = payload.tenantId ?? null\n const key = scopeKey(org, tenant)\n const bucket = scopeBuckets.get(key) ?? { organizationId: org, tenantId: tenant, ids: new Set<string>() }\n bucket.ids.add(String(payload.recordId))\n scopeBuckets.set(key, bucket)\n }\n\n for (const payload of payloads) {\n const org = payload.organizationId ?? null\n const tenant = payload.tenantId ?? null\n const key = scopeKey(org, tenant)\n const pairs = fieldPairsByScope.get(key) ?? []\n const seen = seenPairsByScope.get(key) ?? new Set<string>()\n const fieldPairs = buildFieldPairs(String(payload.recordId), payload.doc)\n for (const pair of fieldPairs) {\n const dedupeKey = `${pair[0]}|${pair[1]}`\n if (seen.has(dedupeKey)) continue\n seen.add(dedupeKey)\n pairs.push(pair)\n }\n fieldPairsByScope.set(key, pairs)\n seenPairsByScope.set(key, seen)\n }\n\n await knex.transaction(async (trx) => {\n for (const [key, bucket] of scopeBuckets.entries()) {\n const pairs = fieldPairsByScope.get(key) ?? []\n const deleteQuery = trx('search_tokens')\n .where({ entity_type: payloads[0].entityType })\n .andWhereRaw('organization_id is not distinct from ?', [bucket.organizationId])\n .andWhereRaw('tenant_id is not distinct from ?', [bucket.tenantId])\n if (pairs.length) deleteQuery.whereIn(['entity_id', 'field'], pairs)\n else deleteQuery.whereIn('entity_id', Array.from(bucket.ids))\n await deleteQuery.del()\n }\n const payloadWithTimestamps = rows.map((row) => ({\n ...row,\n created_at: trx.fn.now(),\n }))\n await trx.batchInsert('search_tokens', payloadWithTimestamps, 500)\n })\n}\n"],
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["import { type Kysely, sql } from 'kysely'\nimport { resolveSearchConfig, type SearchConfig } from '@open-mercato/shared/lib/search/config'\nimport { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'\nimport { parseBooleanToken } from '@open-mercato/shared/lib/boolean'\n\nconst INSERT_BATCH_SIZE = 500\n\nfunction chunk<T>(items: T[], size: number): T[][] {\n if (size <= 0) return [items]\n const out: T[][] = []\n for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size))\n return out\n}\n\nexport type SearchTokenRow = {\n entity_type: string\n entity_id: string\n organization_id: string | null\n tenant_id: string | null\n field: string\n token_hash: string\n token?: string | null\n}\n\ntype BuildTokenOptions = {\n entityType: string\n recordId: string\n organizationId?: string | null\n tenantId?: string | null\n doc?: Record<string, unknown> | null\n config?: SearchConfig\n}\n\nconst DEFAULT_SCOPE = { organizationId: null, tenantId: null }\ntype EntityFieldPair = [string, string]\n\nexport const isSearchDebugEnabled = (): boolean => {\n return parseBooleanToken(process.env.OM_SEARCH_DEBUG ?? '') === true\n}\n\nconst debug = (event: string, payload: Record<string, unknown>) => {\n if (!isSearchDebugEnabled()) return\n try {\n // eslint-disable-next-line no-console\n console.debug(`[search-tokens] ${event}`, payload)\n } catch {\n // ignore\n }\n}\n\nfunction collectTextValues(value: unknown): string[] {\n if (typeof value === 'string') return [value]\n if (Array.isArray(value)) {\n const out: string[] = []\n for (const entry of value) {\n if (typeof entry === 'string') out.push(entry)\n }\n return out\n }\n return []\n}\n\nfunction shouldIndexField(field: string, value: unknown, config: SearchConfig): boolean {\n if (typeof value !== 'string' && !Array.isArray(value)) return false\n const lower = field.toLowerCase()\n if (lower === 'id' || lower.endsWith('_id') || lower.endsWith('.id')) return false\n if (lower.endsWith('_at')) return false\n if (['created_at', 'updated_at', 'deleted_at', 'tenant_id', 'organization_id'].includes(lower)) return false\n if (config.blocklistedFields.some((blocked) => lower.includes(blocked))) return false\n const values = collectTextValues(value)\n if (!values.length) return false\n return values.some((text) => tokenizeText(text, config).tokens.length > 0)\n}\n\nexport function buildSearchTokenRows(params: BuildTokenOptions): SearchTokenRow[] {\n const config = params.config ?? resolveSearchConfig()\n if (!config.enabled) return []\n if (!params.doc) return []\n const tokens: SearchTokenRow[] = []\n const capturePairs = isSearchDebugEnabled() && params.entityType === 'customers:customer_deal'\n const debugPairs: Array<{ field: string; token: string; hash: string }> = []\n const scope = {\n organizationId: params.organizationId ?? DEFAULT_SCOPE.organizationId,\n tenantId: params.tenantId ?? DEFAULT_SCOPE.tenantId,\n }\n\n for (const [field, rawValue] of Object.entries(params.doc)) {\n if (!shouldIndexField(field, rawValue, config)) continue\n const values = collectTextValues(rawValue)\n const seen = new Set<string>()\n for (const text of values) {\n const { tokens: textTokens, hashes } = tokenizeText(text, config)\n for (let i = 0; i < textTokens.length; i += 1) {\n const token = textTokens[i]\n const hash = hashes[i]\n const dedupeKey = `${field}|${hash}`\n if (seen.has(dedupeKey)) continue\n seen.add(dedupeKey)\n debug('token.generated', { entityType: params.entityType, recordId: params.recordId, field, token, hash })\n tokens.push({\n entity_type: params.entityType,\n entity_id: String(params.recordId),\n organization_id: scope.organizationId,\n tenant_id: scope.tenantId,\n field,\n token_hash: hash,\n token: config.storeRawTokens ? token : null,\n })\n if (capturePairs) {\n debugPairs.push({ field, token, hash })\n }\n }\n }\n }\n if (capturePairs) {\n debug('deal.tokens', {\n entityType: params.entityType,\n recordId: params.recordId,\n title: params.doc?.title ?? null,\n tokens: debugPairs,\n })\n }\n debug('doc.completed', { entityType: params.entityType, recordId: params.recordId, tokenCount: tokens.length })\n\n return tokens\n}\n\nfunction buildFieldPairs(recordId: string, doc?: Record<string, unknown> | null): EntityFieldPair[] {\n if (!doc) return []\n const pairs: EntityFieldPair[] = []\n const dedupe = new Set<string>()\n for (const field of Object.keys(doc)) {\n const key = `${recordId}|${field}`\n if (dedupe.has(key)) continue\n dedupe.add(key)\n pairs.push([recordId, field])\n }\n return pairs\n}\n\nexport async function replaceSearchTokensForRecord(\n db: Kysely<any>,\n params: BuildTokenOptions\n): Promise<void> {\n const rows = buildSearchTokenRows(params)\n const config = params.config ?? resolveSearchConfig()\n if (!config.enabled) return\n const organizationId = params.organizationId ?? null\n const tenantId = params.tenantId ?? null\n const fieldPairs = buildFieldPairs(String(params.recordId), params.doc)\n\n await db.transaction().execute(async (trx) => {\n let deleteQuery = trx\n .deleteFrom('search_tokens' as any)\n .where('entity_type' as any, '=', params.entityType)\n .where(sql<boolean>`organization_id is not distinct from ${organizationId}`)\n .where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)\n if (fieldPairs.length) {\n deleteQuery = deleteQuery.where((eb: any) => eb.or(\n fieldPairs.map(([rid, field]) => eb.and([\n eb('entity_id' as any, '=', rid),\n eb('field' as any, '=', field),\n ])),\n ))\n } else {\n deleteQuery = deleteQuery.where('entity_id' as any, '=', String(params.recordId))\n }\n await deleteQuery.execute()\n if (!rows.length) return\n const payloads = rows.map((row) => ({ ...row, created_at: sql`now()` }))\n for (const batch of chunk(payloads, INSERT_BATCH_SIZE)) {\n await trx.insertInto('search_tokens' as any).values(batch as any).execute()\n }\n })\n}\n\nexport async function deleteSearchTokensForRecord(\n db: Kysely<any>,\n params: { entityType: string; recordId: string; organizationId?: string | null; tenantId?: string | null }\n): Promise<void> {\n const organizationId = params.organizationId ?? null\n const tenantId = params.tenantId ?? null\n await db\n .deleteFrom('search_tokens' as any)\n .where('entity_type' as any, '=', params.entityType)\n .where('entity_id' as any, '=', String(params.recordId))\n .where(sql<boolean>`organization_id is not distinct from ${organizationId}`)\n .where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)\n .execute()\n}\n\nexport async function replaceSearchTokensForBatch(\n db: Kysely<any>,\n payloads: Array<BuildTokenOptions & { doc: Record<string, unknown> }>\n): Promise<void> {\n if (!payloads.length) return\n const config = resolveSearchConfig()\n if (!config.enabled) return\n\n const rows = payloads.flatMap((payload) => buildSearchTokenRows({ ...payload, config }))\n if (!rows.length) {\n const entityType = payloads[0]?.entityType\n if (!entityType) return\n const ids = payloads.map((p) => String(p.recordId))\n await db\n .deleteFrom('search_tokens' as any)\n .where('entity_type' as any, '=', entityType)\n .where('entity_id' as any, 'in', ids)\n .execute()\n return\n }\n\n const scopeKey = (org: string | null, tenant: string | null) => `${org ?? '__null__'}|${tenant ?? '__null__'}`\n const scopeBuckets = new Map<string, { organizationId: string | null; tenantId: string | null; ids: Set<string> }>()\n const fieldPairsByScope = new Map<string, EntityFieldPair[]>()\n const seenPairsByScope = new Map<string, Set<string>>()\n\n for (const payload of payloads) {\n const org = payload.organizationId ?? null\n const tenant = payload.tenantId ?? null\n const key = scopeKey(org, tenant)\n const bucket = scopeBuckets.get(key) ?? { organizationId: org, tenantId: tenant, ids: new Set<string>() }\n bucket.ids.add(String(payload.recordId))\n scopeBuckets.set(key, bucket)\n }\n\n for (const payload of payloads) {\n const org = payload.organizationId ?? null\n const tenant = payload.tenantId ?? null\n const key = scopeKey(org, tenant)\n const pairs = fieldPairsByScope.get(key) ?? []\n const seen = seenPairsByScope.get(key) ?? new Set<string>()\n const fieldPairs = buildFieldPairs(String(payload.recordId), payload.doc)\n for (const pair of fieldPairs) {\n const dedupeKey = `${pair[0]}|${pair[1]}`\n if (seen.has(dedupeKey)) continue\n seen.add(dedupeKey)\n pairs.push(pair)\n }\n fieldPairsByScope.set(key, pairs)\n seenPairsByScope.set(key, seen)\n }\n\n await db.transaction().execute(async (trx) => {\n for (const [key, bucket] of scopeBuckets.entries()) {\n const pairs = fieldPairsByScope.get(key) ?? []\n let deleteQuery = trx\n .deleteFrom('search_tokens' as any)\n .where('entity_type' as any, '=', payloads[0].entityType)\n .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)\n .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)\n if (pairs.length) {\n deleteQuery = deleteQuery.where((eb: any) => eb.or(\n pairs.map(([rid, field]) => eb.and([\n eb('entity_id' as any, '=', rid),\n eb('field' as any, '=', field),\n ])),\n ))\n } else {\n deleteQuery = deleteQuery.where('entity_id' as any, 'in', Array.from(bucket.ids))\n }\n await deleteQuery.execute()\n }\n const payloadWithTimestamps = rows.map((row) => ({ ...row, created_at: sql`now()` }))\n for (const batch of chunk(payloadWithTimestamps, INSERT_BATCH_SIZE)) {\n await trx.insertInto('search_tokens' as any).values(batch as any).execute()\n }\n })\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAsB,WAAW;AACjC,SAAS,2BAA8C;AACvD,SAAS,oBAAoB;AAC7B,SAAS,yBAAyB;AAElC,MAAM,oBAAoB;AAE1B,SAAS,MAAS,OAAY,MAAqB;AACjD,MAAI,QAAQ,EAAG,QAAO,CAAC,KAAK;AAC5B,QAAM,MAAa,CAAC;AACpB,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,KAAM,KAAI,KAAK,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC;AAC9E,SAAO;AACT;AAqBA,MAAM,gBAAgB,EAAE,gBAAgB,MAAM,UAAU,KAAK;AAGtD,MAAM,uBAAuB,MAAe;AACjD,SAAO,kBAAkB,QAAQ,IAAI,mBAAmB,EAAE,MAAM;AAClE;AAEA,MAAM,QAAQ,CAAC,OAAe,YAAqC;AACjE,MAAI,CAAC,qBAAqB,EAAG;AAC7B,MAAI;AAEF,YAAQ,MAAM,mBAAmB,KAAK,IAAI,OAAO;AAAA,EACnD,QAAQ;AAAA,EAER;AACF;AAEA,SAAS,kBAAkB,OAA0B;AACnD,MAAI,OAAO,UAAU,SAAU,QAAO,CAAC,KAAK;AAC5C,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,UAAM,MAAgB,CAAC;AACvB,eAAW,SAAS,OAAO;AACzB,UAAI,OAAO,UAAU,SAAU,KAAI,KAAK,KAAK;AAAA,IAC/C;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEA,SAAS,iBAAiB,OAAe,OAAgB,QAA+B;AACtF,MAAI,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO;AAC/D,QAAM,QAAQ,MAAM,YAAY;AAChC,MAAI,UAAU,QAAQ,MAAM,SAAS,KAAK,KAAK,MAAM,SAAS,KAAK,EAAG,QAAO;AAC7E,MAAI,MAAM,SAAS,KAAK,EAAG,QAAO;AAClC,MAAI,CAAC,cAAc,cAAc,cAAc,aAAa,iBAAiB,EAAE,SAAS,KAAK,EAAG,QAAO;AACvG,MAAI,OAAO,kBAAkB,KAAK,CAAC,YAAY,MAAM,SAAS,OAAO,CAAC,EAAG,QAAO;AAChF,QAAM,SAAS,kBAAkB,KAAK;AACtC,MAAI,CAAC,OAAO,OAAQ,QAAO;AAC3B,SAAO,OAAO,KAAK,CAAC,SAAS,aAAa,MAAM,MAAM,EAAE,OAAO,SAAS,CAAC;AAC3E;AAEO,SAAS,qBAAqB,QAA6C;AAChF,QAAM,SAAS,OAAO,UAAU,oBAAoB;AACpD,MAAI,CAAC,OAAO,QAAS,QAAO,CAAC;AAC7B,MAAI,CAAC,OAAO,IAAK,QAAO,CAAC;AACzB,QAAM,SAA2B,CAAC;AAClC,QAAM,eAAe,qBAAqB,KAAK,OAAO,eAAe;AACrE,QAAM,aAAoE,CAAC;AAC3E,QAAM,QAAQ;AAAA,IACZ,gBAAgB,OAAO,kBAAkB,cAAc;AAAA,IACvD,UAAU,OAAO,YAAY,cAAc;AAAA,EAC7C;AAEA,aAAW,CAAC,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,GAAG,GAAG;AAC1D,QAAI,CAAC,iBAAiB,OAAO,UAAU,MAAM,EAAG;AAChD,UAAM,SAAS,kBAAkB,QAAQ;AACzC,UAAM,OAAO,oBAAI,IAAY;AAC7B,eAAW,QAAQ,QAAQ;AACzB,YAAM,EAAE,QAAQ,YAAY,OAAO,IAAI,aAAa,MAAM,MAAM;AAChE,eAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK,GAAG;AAC7C,cAAM,QAAQ,WAAW,CAAC;AAC1B,cAAM,OAAO,OAAO,CAAC;AACrB,cAAM,YAAY,GAAG,KAAK,IAAI,IAAI;AAClC,YAAI,KAAK,IAAI,SAAS,EAAG;AACzB,aAAK,IAAI,SAAS;AAClB,cAAM,mBAAmB,EAAE,YAAY,OAAO,YAAY,UAAU,OAAO,UAAU,OAAO,OAAO,KAAK,CAAC;AACzG,eAAO,KAAK;AAAA,UACV,aAAa,OAAO;AAAA,UACpB,WAAW,OAAO,OAAO,QAAQ;AAAA,UACjC,iBAAiB,MAAM;AAAA,UACvB,WAAW,MAAM;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,OAAO,OAAO,iBAAiB,QAAQ;AAAA,QACzC,CAAC;AACD,YAAI,cAAc;AAChB,qBAAW,KAAK,EAAE,OAAO,OAAO,KAAK,CAAC;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,cAAc;AAChB,UAAM,eAAe;AAAA,MACnB,YAAY,OAAO;AAAA,MACnB,UAAU,OAAO;AAAA,MACjB,OAAO,OAAO,KAAK,SAAS;AAAA,MAC5B,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,QAAM,iBAAiB,EAAE,YAAY,OAAO,YAAY,UAAU,OAAO,UAAU,YAAY,OAAO,OAAO,CAAC;AAE9G,SAAO;AACT;AAEA,SAAS,gBAAgB,UAAkB,KAAyD;AAClG,MAAI,CAAC,IAAK,QAAO,CAAC;AAClB,QAAM,QAA2B,CAAC;AAClC,QAAM,SAAS,oBAAI,IAAY;AAC/B,aAAW,SAAS,OAAO,KAAK,GAAG,GAAG;AACpC,UAAM,MAAM,GAAG,QAAQ,IAAI,KAAK;AAChC,QAAI,OAAO,IAAI,GAAG,EAAG;AACrB,WAAO,IAAI,GAAG;AACd,UAAM,KAAK,CAAC,UAAU,KAAK,CAAC;AAAA,EAC9B;AACA,SAAO;AACT;AAEA,eAAsB,6BACpB,IACA,QACe;AACf,QAAM,OAAO,qBAAqB,MAAM;AACxC,QAAM,SAAS,OAAO,UAAU,oBAAoB;AACpD,MAAI,CAAC,OAAO,QAAS;AACrB,QAAM,iBAAiB,OAAO,kBAAkB;AAChD,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,aAAa,gBAAgB,OAAO,OAAO,QAAQ,GAAG,OAAO,GAAG;AAEtE,QAAM,GAAG,YAAY,EAAE,QAAQ,OAAO,QAAQ;AAC5C,QAAI,cAAc,IACf,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,OAAO,UAAU,EAClD,MAAM,2CAAoD,cAAc,EAAE,EAC1E,MAAM,qCAA8C,QAAQ,EAAE;AACjE,QAAI,WAAW,QAAQ;AACrB,oBAAc,YAAY,MAAM,CAAC,OAAY,GAAG;AAAA,QAC9C,WAAW,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,IAAI;AAAA,UACtC,GAAG,aAAoB,KAAK,GAAG;AAAA,UAC/B,GAAG,SAAgB,KAAK,KAAK;AAAA,QAC/B,CAAC,CAAC;AAAA,MACJ,CAAC;AAAA,IACH,OAAO;AACL,oBAAc,YAAY,MAAM,aAAoB,KAAK,OAAO,OAAO,QAAQ,CAAC;AAAA,IAClF;AACA,UAAM,YAAY,QAAQ;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAClB,UAAM,WAAW,KAAK,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,YAAY,WAAW,EAAE;AACvE,eAAW,SAAS,MAAM,UAAU,iBAAiB,GAAG;AACtD,YAAM,IAAI,WAAW,eAAsB,EAAE,OAAO,KAAY,EAAE,QAAQ;AAAA,IAC5E;AAAA,EACF,CAAC;AACH;AAEA,eAAsB,4BACpB,IACA,QACe;AACf,QAAM,iBAAiB,OAAO,kBAAkB;AAChD,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,GACH,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,OAAO,UAAU,EAClD,MAAM,aAAoB,KAAK,OAAO,OAAO,QAAQ,CAAC,EACtD,MAAM,2CAAoD,cAAc,EAAE,EAC1E,MAAM,qCAA8C,QAAQ,EAAE,EAC9D,QAAQ;AACb;AAEA,eAAsB,4BACpB,IACA,UACe;AACf,MAAI,CAAC,SAAS,OAAQ;AACtB,QAAM,SAAS,oBAAoB;AACnC,MAAI,CAAC,OAAO,QAAS;AAErB,QAAM,OAAO,SAAS,QAAQ,CAAC,YAAY,qBAAqB,EAAE,GAAG,SAAS,OAAO,CAAC,CAAC;AACvF,MAAI,CAAC,KAAK,QAAQ;AAChB,UAAM,aAAa,SAAS,CAAC,GAAG;AAChC,QAAI,CAAC,WAAY;AACjB,UAAM,MAAM,SAAS,IAAI,CAAC,MAAM,OAAO,EAAE,QAAQ,CAAC;AAClD,UAAM,GACH,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,UAAU,EAC3C,MAAM,aAAoB,MAAM,GAAG,EACnC,QAAQ;AACX;AAAA,EACF;AAEA,QAAM,WAAW,CAAC,KAAoB,WAA0B,GAAG,OAAO,UAAU,IAAI,UAAU,UAAU;AAC5G,QAAM,eAAe,oBAAI,IAA0F;AACnH,QAAM,oBAAoB,oBAAI,IAA+B;AAC7D,QAAM,mBAAmB,oBAAI,IAAyB;AAEtD,aAAW,WAAW,UAAU;AAC9B,UAAM,MAAM,QAAQ,kBAAkB;AACtC,UAAM,SAAS,QAAQ,YAAY;AACnC,UAAM,MAAM,SAAS,KAAK,MAAM;AAChC,UAAM,SAAS,aAAa,IAAI,GAAG,KAAK,EAAE,gBAAgB,KAAK,UAAU,QAAQ,KAAK,oBAAI,IAAY,EAAE;AACxG,WAAO,IAAI,IAAI,OAAO,QAAQ,QAAQ,CAAC;AACvC,iBAAa,IAAI,KAAK,MAAM;AAAA,EAC9B;AAEA,aAAW,WAAW,UAAU;AAC9B,UAAM,MAAM,QAAQ,kBAAkB;AACtC,UAAM,SAAS,QAAQ,YAAY;AACnC,UAAM,MAAM,SAAS,KAAK,MAAM;AAChC,UAAM,QAAQ,kBAAkB,IAAI,GAAG,KAAK,CAAC;AAC7C,UAAM,OAAO,iBAAiB,IAAI,GAAG,KAAK,oBAAI,IAAY;AAC1D,UAAM,aAAa,gBAAgB,OAAO,QAAQ,QAAQ,GAAG,QAAQ,GAAG;AACxE,eAAW,QAAQ,YAAY;AAC7B,YAAM,YAAY,GAAG,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC;AACvC,UAAI,KAAK,IAAI,SAAS,EAAG;AACzB,WAAK,IAAI,SAAS;AAClB,YAAM,KAAK,IAAI;AAAA,IACjB;AACA,sBAAkB,IAAI,KAAK,KAAK;AAChC,qBAAiB,IAAI,KAAK,IAAI;AAAA,EAChC;AAEA,QAAM,GAAG,YAAY,EAAE,QAAQ,OAAO,QAAQ;AAC5C,eAAW,CAAC,KAAK,MAAM,KAAK,aAAa,QAAQ,GAAG;AAClD,YAAM,QAAQ,kBAAkB,IAAI,GAAG,KAAK,CAAC;AAC7C,UAAI,cAAc,IACf,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,SAAS,CAAC,EAAE,UAAU,EACvD,MAAM,2CAAoD,OAAO,cAAc,EAAE,EACjF,MAAM,qCAA8C,OAAO,QAAQ,EAAE;AACxE,UAAI,MAAM,QAAQ;AAChB,sBAAc,YAAY,MAAM,CAAC,OAAY,GAAG;AAAA,UAC9C,MAAM,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,IAAI;AAAA,YACjC,GAAG,aAAoB,KAAK,GAAG;AAAA,YAC/B,GAAG,SAAgB,KAAK,KAAK;AAAA,UAC/B,CAAC,CAAC;AAAA,QACJ,CAAC;AAAA,MACH,OAAO;AACL,sBAAc,YAAY,MAAM,aAAoB,MAAM,MAAM,KAAK,OAAO,GAAG,CAAC;AAAA,MAClF;AACA,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,UAAM,wBAAwB,KAAK,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,YAAY,WAAW,EAAE;AACpF,eAAW,SAAS,MAAM,uBAAuB,iBAAiB,GAAG;AACnE,YAAM,IAAI,WAAW,eAAsB,EAAE,OAAO,KAAY,EAAE,QAAQ;AAAA,IAC5E;AAAA,EACF,CAAC;AACH;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1,16 +1,18 @@
|
|
|
1
|
-
|
|
1
|
+
import { sql } from "kysely";
|
|
2
|
+
async function purgeOrphans(db, options) {
|
|
2
3
|
const { entityType, tenantId, partitionIndex, partitionCount, startedAt } = options;
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
4
|
+
let q = db.deleteFrom("entity_indexes").where("entity_type", "=", entityType);
|
|
5
|
+
if (tenantId !== void 0) {
|
|
6
|
+
q = q.where(sql`tenant_id is not distinct from ${tenantId ?? null}`);
|
|
7
|
+
}
|
|
8
|
+
if (options.organizationId !== void 0) {
|
|
9
|
+
q = q.where(sql`organization_id is not distinct from ${options.organizationId ?? null}`);
|
|
10
|
+
}
|
|
11
|
+
if (partitionIndex != null && partitionCount != null) {
|
|
12
|
+
q = q.where(sql`mod(abs(hashtext(entity_id::text)), ${partitionCount}) = ${partitionIndex}`);
|
|
13
|
+
}
|
|
14
|
+
q = q.where("updated_at", "<", startedAt);
|
|
15
|
+
await q.execute();
|
|
14
16
|
}
|
|
15
17
|
export {
|
|
16
18
|
purgeOrphans
|