@acorex-platform/framework-server 0.0.30 → 0.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/background-job/src/index.d.ts +3 -0
- package/background-job/src/index.d.ts.map +1 -1
- package/background-job/src/index.js +3 -0
- package/background-job/src/index.js.map +1 -1
- package/background-job/src/lib/background-job.module.d.ts +3 -0
- package/background-job/src/lib/background-job.module.d.ts.map +1 -1
- package/background-job/src/lib/background-job.module.js +13 -1
- package/background-job/src/lib/background-job.module.js.map +1 -1
- package/background-job/src/lib/background-job.service.d.ts +9 -2
- package/background-job/src/lib/background-job.service.d.ts.map +1 -1
- package/background-job/src/lib/background-job.service.js +42 -2
- package/background-job/src/lib/background-job.service.js.map +1 -1
- package/background-job/src/lib/command-concurrency-lock.registrar.d.ts +14 -0
- package/background-job/src/lib/command-concurrency-lock.registrar.d.ts.map +1 -0
- package/background-job/src/lib/command-concurrency-lock.registrar.js +29 -0
- package/background-job/src/lib/command-concurrency-lock.registrar.js.map +1 -0
- package/background-job/src/lib/execution/background-job-cancellation.service.d.ts +5 -1
- package/background-job/src/lib/execution/background-job-cancellation.service.d.ts.map +1 -1
- package/background-job/src/lib/execution/background-job-cancellation.service.js +11 -2
- package/background-job/src/lib/execution/background-job-cancellation.service.js.map +1 -1
- package/background-job/src/lib/execution/background-job-execution.service.d.ts +15 -2
- package/background-job/src/lib/execution/background-job-execution.service.d.ts.map +1 -1
- package/background-job/src/lib/execution/background-job-execution.service.js +145 -8
- package/background-job/src/lib/execution/background-job-execution.service.js.map +1 -1
- package/background-job/src/lib/notification/background-job-notification.port.d.ts +20 -0
- package/background-job/src/lib/notification/background-job-notification.port.d.ts.map +1 -0
- package/background-job/src/lib/notification/background-job-notification.port.js +31 -0
- package/background-job/src/lib/notification/background-job-notification.port.js.map +1 -0
- package/background-job/src/lib/queue/background-job-recovery.service.d.ts +22 -0
- package/background-job/src/lib/queue/background-job-recovery.service.d.ts.map +1 -0
- package/background-job/src/lib/queue/background-job-recovery.service.js +101 -0
- package/background-job/src/lib/queue/background-job-recovery.service.js.map +1 -0
- package/background-job/src/lib/schedule/background-job-schedule-scanner.service.d.ts.map +1 -1
- package/background-job/src/lib/schedule/background-job-schedule-scanner.service.js +18 -12
- package/background-job/src/lib/schedule/background-job-schedule-scanner.service.js.map +1 -1
- package/background-job/src/lib/store/background-job-attempt-store.service.d.ts +12 -0
- package/background-job/src/lib/store/background-job-attempt-store.service.d.ts.map +1 -1
- package/background-job/src/lib/store/background-job-attempt-store.service.js +30 -0
- package/background-job/src/lib/store/background-job-attempt-store.service.js.map +1 -1
- package/background-job/src/lib/store/background-job-schedule-store.service.d.ts +5 -1
- package/background-job/src/lib/store/background-job-schedule-store.service.d.ts.map +1 -1
- package/background-job/src/lib/store/background-job-schedule-store.service.js +8 -4
- package/background-job/src/lib/store/background-job-schedule-store.service.js.map +1 -1
- package/background-job/src/lib/store/background-job-store.service.d.ts +25 -3
- package/background-job/src/lib/store/background-job-store.service.d.ts.map +1 -1
- package/background-job/src/lib/store/background-job-store.service.js +54 -26
- package/background-job/src/lib/store/background-job-store.service.js.map +1 -1
- package/background-job/src/lib/store/jql-filter.util.d.ts +1 -0
- package/background-job/src/lib/store/jql-filter.util.d.ts.map +1 -1
- package/background-job/src/lib/store/jql-filter.util.js +4 -0
- package/background-job/src/lib/store/jql-filter.util.js.map +1 -1
- package/background-job/src/lib/tray/background-job-tray-realtime.publisher.d.ts +17 -0
- package/background-job/src/lib/tray/background-job-tray-realtime.publisher.d.ts.map +1 -0
- package/background-job/src/lib/tray/background-job-tray-realtime.publisher.js +61 -0
- package/background-job/src/lib/tray/background-job-tray-realtime.publisher.js.map +1 -0
- package/core/src/lib/entity-storage/entity-mutation-events/entity-mutation-event-emitter.service.d.ts +2 -1
- package/core/src/lib/entity-storage/entity-mutation-events/entity-mutation-event-emitter.service.d.ts.map +1 -1
- package/core/src/lib/entity-storage/entity-mutation-events/entity-mutation-event-emitter.service.js +3 -0
- package/core/src/lib/entity-storage/entity-mutation-events/entity-mutation-event-emitter.service.js.map +1 -1
- package/core/src/lib/entity-storage/entity-mutation-events/entity-mutation-event.types.d.ts +12 -0
- package/core/src/lib/entity-storage/entity-mutation-events/entity-mutation-event.types.d.ts.map +1 -1
- package/core/src/lib/entity-storage/entity-mutation-events/entity-mutation-event.types.js +6 -1
- package/core/src/lib/entity-storage/entity-mutation-events/entity-mutation-event.types.js.map +1 -1
- package/core/src/lib/entity-storage/entity-storage-middleware.service.d.ts +3 -2
- package/core/src/lib/entity-storage/entity-storage-middleware.service.d.ts.map +1 -1
- package/core/src/lib/entity-storage/entity-storage-middleware.service.js +6 -2
- package/core/src/lib/entity-storage/entity-storage-middleware.service.js.map +1 -1
- package/core/src/lib/runtime/index.d.ts +3 -0
- package/core/src/lib/runtime/index.d.ts.map +1 -1
- package/core/src/lib/runtime/index.js +3 -0
- package/core/src/lib/runtime/index.js.map +1 -1
- package/core/src/lib/runtime/runtime-authz.guard.d.ts +1 -0
- package/core/src/lib/runtime/runtime-authz.guard.d.ts.map +1 -1
- package/core/src/lib/runtime/runtime-authz.guard.js +10 -0
- package/core/src/lib/runtime/runtime-authz.guard.js.map +1 -1
- package/core/src/lib/runtime/runtime-bulk-command.service.d.ts +4 -1
- package/core/src/lib/runtime/runtime-bulk-command.service.d.ts.map +1 -1
- package/core/src/lib/runtime/runtime-bulk-command.service.js +64 -17
- package/core/src/lib/runtime/runtime-bulk-command.service.js.map +1 -1
- package/core/src/lib/runtime/runtime-bulk-progress.realtime.port.d.ts +9 -0
- package/core/src/lib/runtime/runtime-bulk-progress.realtime.port.d.ts.map +1 -0
- package/core/src/lib/runtime/runtime-bulk-progress.realtime.port.js +13 -0
- package/core/src/lib/runtime/runtime-bulk-progress.realtime.port.js.map +1 -0
- package/core/src/lib/runtime/runtime-bulk-progress.store.d.ts +12 -0
- package/core/src/lib/runtime/runtime-bulk-progress.store.d.ts.map +1 -0
- package/core/src/lib/runtime/runtime-bulk-progress.store.js +61 -0
- package/core/src/lib/runtime/runtime-bulk-progress.store.js.map +1 -0
- package/core/src/lib/runtime/runtime-command.controller.d.ts +5 -1
- package/core/src/lib/runtime/runtime-command.controller.d.ts.map +1 -1
- package/core/src/lib/runtime/runtime-command.controller.js +30 -2
- package/core/src/lib/runtime/runtime-command.controller.js.map +1 -1
- package/core/src/lib/runtime/runtime-dispatcher.service.d.ts +3 -2
- package/core/src/lib/runtime/runtime-dispatcher.service.d.ts.map +1 -1
- package/core/src/lib/runtime/runtime-dispatcher.service.js +23 -2
- package/core/src/lib/runtime/runtime-dispatcher.service.js.map +1 -1
- package/core/src/lib/runtime/runtime-request-session.util.d.ts +6 -0
- package/core/src/lib/runtime/runtime-request-session.util.d.ts.map +1 -0
- package/core/src/lib/runtime/runtime-request-session.util.js +21 -0
- package/core/src/lib/runtime/runtime-request-session.util.js.map +1 -0
- package/core/src/lib/runtime/runtime.module.d.ts.map +1 -1
- package/core/src/lib/runtime/runtime.module.js +3 -0
- package/core/src/lib/runtime/runtime.module.js.map +1 -1
- package/domain/abstract/src/index.d.ts +2 -0
- package/domain/abstract/src/index.d.ts.map +1 -1
- package/domain/abstract/src/index.js +2 -0
- package/domain/abstract/src/index.js.map +1 -1
- package/domain/abstract/src/lib/commands/command-concurrency.types.d.ts +30 -0
- package/domain/abstract/src/lib/commands/command-concurrency.types.d.ts.map +1 -0
- package/domain/abstract/src/lib/commands/command-concurrency.types.js +9 -0
- package/domain/abstract/src/lib/commands/command-concurrency.types.js.map +1 -0
- package/domain/abstract/src/lib/commands/command-concurrency.util.d.ts +11 -0
- package/domain/abstract/src/lib/commands/command-concurrency.util.d.ts.map +1 -0
- package/domain/abstract/src/lib/commands/command-concurrency.util.js +53 -0
- package/domain/abstract/src/lib/commands/command-concurrency.util.js.map +1 -0
- package/domain/abstract/src/lib/commands/server-command-registry.token.d.ts +3 -0
- package/domain/abstract/src/lib/commands/server-command-registry.token.d.ts.map +1 -1
- package/domain/abstract/src/lib/commands/server-command-registry.token.js.map +1 -1
- package/domain/abstract/src/lib/commands/server-handler.decorator.d.ts +1 -0
- package/domain/abstract/src/lib/commands/server-handler.decorator.d.ts.map +1 -1
- package/domain/abstract/src/lib/commands/server-handler.decorator.js +1 -0
- package/domain/abstract/src/lib/commands/server-handler.decorator.js.map +1 -1
- package/domain/abstract/src/lib/commands/server-handler.types.d.ts +7 -0
- package/domain/abstract/src/lib/commands/server-handler.types.d.ts.map +1 -1
- package/domain/abstract/src/lib/commands/server-handler.types.js.map +1 -1
- package/domain/providers/src/lib/command-concurrency.guard.d.ts +11 -0
- package/domain/providers/src/lib/command-concurrency.guard.d.ts.map +1 -0
- package/domain/providers/src/lib/command-concurrency.guard.js +110 -0
- package/domain/providers/src/lib/command-concurrency.guard.js.map +1 -0
- package/domain/providers/src/lib/middlewares/lookup-resolver.middleware.d.ts.map +1 -1
- package/domain/providers/src/lib/middlewares/lookup-resolver.middleware.js +5 -3
- package/domain/providers/src/lib/middlewares/lookup-resolver.middleware.js.map +1 -1
- package/domain/providers/src/lib/server-command.module.d.ts.map +1 -1
- package/domain/providers/src/lib/server-command.module.js +8 -1
- package/domain/providers/src/lib/server-command.module.js.map +1 -1
- package/domain/providers/src/lib/server-command.registry.d.ts +7 -3
- package/domain/providers/src/lib/server-command.registry.d.ts.map +1 -1
- package/domain/providers/src/lib/server-command.registry.js +18 -7
- package/domain/providers/src/lib/server-command.registry.js.map +1 -1
- package/domain/providers/src/lib/utils/entity-query-fields-path-planner.util.d.ts +2 -0
- package/domain/providers/src/lib/utils/entity-query-fields-path-planner.util.d.ts.map +1 -1
- package/domain/providers/src/lib/utils/entity-query-fields-path-planner.util.js +12 -2
- package/domain/providers/src/lib/utils/entity-query-fields-path-planner.util.js.map +1 -1
- package/domain/providers/src/lib/utils/entity-relation-join.planner.d.ts.map +1 -1
- package/domain/providers/src/lib/utils/entity-relation-join.planner.js +1 -1
- package/domain/providers/src/lib/utils/entity-relation-join.planner.js.map +1 -1
- package/jql/elasticsearch/src/lib/compile/filter-compiler.js +14 -0
- package/jql/elasticsearch/src/lib/compile/filter-compiler.js.map +1 -1
- package/jql/elasticsearch/src/lib/jql-elasticsearch.engine.js +3 -0
- package/jql/elasticsearch/src/lib/jql-elasticsearch.engine.js.map +1 -1
- package/jql/schema/src/lib/generated/plugin-contributions.bootstrap.d.ts.map +1 -1
- package/jql/schema/src/lib/generated/plugin-contributions.bootstrap.js +6 -0
- package/jql/schema/src/lib/generated/plugin-contributions.bootstrap.js.map +1 -1
- package/observability/abstract/src/index.d.ts +1 -0
- package/observability/abstract/src/index.d.ts.map +1 -1
- package/observability/abstract/src/index.js +1 -0
- package/observability/abstract/src/index.js.map +1 -1
- package/observability/abstract/src/lib/entity-observability-skip.util.d.ts +11 -0
- package/observability/abstract/src/lib/entity-observability-skip.util.d.ts.map +1 -0
- package/observability/abstract/src/lib/entity-observability-skip.util.js +22 -0
- package/observability/abstract/src/lib/entity-observability-skip.util.js.map +1 -0
- package/observability/abstract/src/lib/observability.config.d.ts +2 -0
- package/observability/abstract/src/lib/observability.config.d.ts.map +1 -1
- package/observability/abstract/src/lib/observability.config.js.map +1 -1
- package/observability/abstract/src/lib/runtime-log-buffer.d.ts.map +1 -1
- package/observability/abstract/src/lib/runtime-log-buffer.js +5 -1
- package/observability/abstract/src/lib/runtime-log-buffer.js.map +1 -1
- package/observability/providers/src/lib/context/correlation-fields.util.d.ts +5 -0
- package/observability/providers/src/lib/context/correlation-fields.util.d.ts.map +1 -1
- package/observability/providers/src/lib/context/correlation-fields.util.js +4 -1
- package/observability/providers/src/lib/context/correlation-fields.util.js.map +1 -1
- package/observability/providers/src/lib/context/resolve-request-observability-entity.util.d.ts +14 -0
- package/observability/providers/src/lib/context/resolve-request-observability-entity.util.d.ts.map +1 -0
- package/observability/providers/src/lib/context/resolve-request-observability-entity.util.js +82 -0
- package/observability/providers/src/lib/context/resolve-request-observability-entity.util.js.map +1 -0
- package/observability/providers/src/lib/execution-trace/execution-trace-recorder.service.d.ts.map +1 -1
- package/observability/providers/src/lib/execution-trace/execution-trace-recorder.service.js +3 -0
- package/observability/providers/src/lib/execution-trace/execution-trace-recorder.service.js.map +1 -1
- package/observability/providers/src/lib/http/observability-context.interceptor.d.ts +3 -1
- package/observability/providers/src/lib/http/observability-context.interceptor.d.ts.map +1 -1
- package/observability/providers/src/lib/http/observability-context.interceptor.js +17 -4
- package/observability/providers/src/lib/http/observability-context.interceptor.js.map +1 -1
- package/observability/providers/src/lib/pino/pino.factory.d.ts.map +1 -1
- package/observability/providers/src/lib/pino/pino.factory.js +9 -1
- package/observability/providers/src/lib/pino/pino.factory.js.map +1 -1
- package/package.json +2 -2
- package/realtime/src/lib/delivery/socket-io-realtime.delivery.d.ts +3 -3
- package/realtime/src/lib/delivery/socket-io-realtime.delivery.d.ts.map +1 -1
- package/realtime/src/lib/delivery/socket-io-realtime.delivery.js +10 -2
- package/realtime/src/lib/delivery/socket-io-realtime.delivery.js.map +1 -1
- package/realtime/src/lib/entity-bridge/realtime-bulk-progress-bridge.listener.d.ts +9 -0
- package/realtime/src/lib/entity-bridge/realtime-bulk-progress-bridge.listener.d.ts.map +1 -0
- package/realtime/src/lib/entity-bridge/realtime-bulk-progress-bridge.listener.js +62 -0
- package/realtime/src/lib/entity-bridge/realtime-bulk-progress-bridge.listener.js.map +1 -0
- package/realtime/src/lib/gateway/realtime.gateway.d.ts +2 -2
- package/realtime/src/lib/gateway/realtime.gateway.d.ts.map +1 -1
- package/realtime/src/lib/gateway/realtime.gateway.js +1 -1
- package/realtime/src/lib/gateway/realtime.gateway.js.map +1 -1
- package/realtime/src/lib/realtime.module.d.ts.map +1 -1
- package/realtime/src/lib/realtime.module.js +8 -0
- package/realtime/src/lib/realtime.module.js.map +1 -1
- package/realtime/src/lib/topics/realtime-topics.d.ts +2 -0
- package/realtime/src/lib/topics/realtime-topics.d.ts.map +1 -1
- package/realtime/src/lib/topics/realtime-topics.js +3 -1
- package/realtime/src/lib/topics/realtime-topics.js.map +1 -1
- package/realtime/src/lib/types/realtime.types.d.ts +26 -0
- package/realtime/src/lib/types/realtime.types.d.ts.map +1 -1
- package/realtime/src/lib/types/realtime.types.js.map +1 -1
- package/search/abstract/src/lib/quick-search-content.util.d.ts.map +1 -1
- package/search/abstract/src/lib/quick-search-content.util.js +14 -1
- package/search/abstract/src/lib/quick-search-content.util.js.map +1 -1
- package/search/providers/src/lib/quick-search-index.service.d.ts +9 -1
- package/search/providers/src/lib/quick-search-index.service.d.ts.map +1 -1
- package/search/providers/src/lib/quick-search-index.service.js +53 -15
- package/search/providers/src/lib/quick-search-index.service.js.map +1 -1
- package/search/providers/src/lib/quick-search-provider.module.d.ts.map +1 -1
- package/search/providers/src/lib/quick-search-provider.module.js +12 -0
- package/search/providers/src/lib/quick-search-provider.module.js.map +1 -1
- package/search/providers/src/lib/quick-search-reindex.service.d.ts +15 -6
- package/search/providers/src/lib/quick-search-reindex.service.d.ts.map +1 -1
- package/search/providers/src/lib/quick-search-reindex.service.js +98 -12
- package/search/providers/src/lib/quick-search-reindex.service.js.map +1 -1
- package/search/providers/src/lib/quick-search-related.util.d.ts +14 -0
- package/search/providers/src/lib/quick-search-related.util.d.ts.map +1 -1
- package/search/providers/src/lib/quick-search-related.util.js +142 -8
- package/search/providers/src/lib/quick-search-related.util.js.map +1 -1
- package/security/src/lib/auth/auth.service.js +1 -1
- package/security/src/lib/auth/auth.service.js.map +1 -1
- package/swagger/src/lib/runtime-openapi.service.d.ts.map +1 -1
- package/swagger/src/lib/runtime-openapi.service.js +43 -0
- package/swagger/src/lib/runtime-openapi.service.js.map +1 -1
- package/tracing/abstract/src/index.d.ts +1 -0
- package/tracing/abstract/src/index.d.ts.map +1 -1
- package/tracing/abstract/src/index.js +1 -0
- package/tracing/abstract/src/index.js.map +1 -1
- package/tracing/abstract/src/lib/entity-log.types.d.ts +14 -0
- package/tracing/abstract/src/lib/entity-log.types.d.ts.map +1 -0
- package/tracing/abstract/src/lib/entity-log.types.js +10 -0
- package/tracing/abstract/src/lib/entity-log.types.js.map +1 -0
- package/tracing/abstract/src/lib/entity-trace.tokens.d.ts +2 -0
- package/tracing/abstract/src/lib/entity-trace.tokens.d.ts.map +1 -1
- package/tracing/abstract/src/lib/entity-trace.tokens.js.map +1 -1
- package/tracing/providers/src/lib/entity-trace-config-resolver.service.d.ts +5 -3
- package/tracing/providers/src/lib/entity-trace-config-resolver.service.d.ts.map +1 -1
- package/tracing/providers/src/lib/entity-trace-config-resolver.service.js +36 -11
- package/tracing/providers/src/lib/entity-trace-config-resolver.service.js.map +1 -1
|
@@ -13,6 +13,7 @@ const background_job_runtime_1 = require("./background-job-runtime");
|
|
|
13
13
|
const background_job_tenant_util_1 = require("./background-job-tenant.util");
|
|
14
14
|
const distributed_lock_port_1 = require("../lock/distributed-lock.port");
|
|
15
15
|
const background_job_observability_publisher_1 = require("../observability/background-job-observability.publisher");
|
|
16
|
+
const background_job_notification_port_1 = require("../notification/background-job-notification.port");
|
|
16
17
|
const background_job_attempt_store_service_1 = require("../store/background-job-attempt-store.service");
|
|
17
18
|
const background_job_schedule_store_service_1 = require("../store/background-job-schedule-store.service");
|
|
18
19
|
const background_job_capabilities_resolver_1 = require("../store/background-job-capabilities.resolver");
|
|
@@ -21,15 +22,19 @@ const background_job_attempt_factory_1 = require("./background-job-attempt.facto
|
|
|
21
22
|
const background_job_cancellation_service_1 = require("./background-job-cancellation.service");
|
|
22
23
|
const background_job_retry_util_1 = require("./background-job-retry.util");
|
|
23
24
|
const background_job_retry_scheduler_service_1 = require("./background-job-retry-scheduler.service");
|
|
25
|
+
const background_job_tray_realtime_publisher_1 = require("../tray/background-job-tray-realtime.publisher");
|
|
24
26
|
/** Default execution-lock TTL when the job declares no timeout. */
|
|
25
27
|
const DEFAULT_RUN_LOCK_TTL_MS = 15 * 60 * 1000;
|
|
26
28
|
const RUN_LOCK_BUFFER_MS = 60 * 1000;
|
|
27
29
|
/** Redelivery delay when the execution lock is held by another run. */
|
|
28
30
|
const LOCK_BUSY_REDELIVERY_DELAY_MS = 30 * 1000;
|
|
31
|
+
/** Heartbeat write interval while an attempt is running. */
|
|
32
|
+
const HEARTBEAT_INTERVAL_MS = 30 * 1000;
|
|
33
|
+
const MIN_LOCK_RENEW_INTERVAL_MS = 5 * 1000;
|
|
29
34
|
//#endregion
|
|
30
35
|
//#region ---- Execution service (spec §7.1 flow) ----
|
|
31
36
|
let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class BackgroundJobExecutionService {
|
|
32
|
-
constructor(jobStore, attemptStore, scheduleStore, capabilitiesResolver, attemptFactory, serverCommands, cancellation, retryScheduler, lock, observability, moduleRef) {
|
|
37
|
+
constructor(jobStore, attemptStore, scheduleStore, capabilitiesResolver, attemptFactory, serverCommands, cancellation, retryScheduler, lock, observability, notifications, trayRealtime, moduleRef) {
|
|
33
38
|
this.jobStore = jobStore;
|
|
34
39
|
this.attemptStore = attemptStore;
|
|
35
40
|
this.scheduleStore = scheduleStore;
|
|
@@ -40,6 +45,8 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
40
45
|
this.retryScheduler = retryScheduler;
|
|
41
46
|
this.lock = lock;
|
|
42
47
|
this.observability = observability;
|
|
48
|
+
this.notifications = notifications;
|
|
49
|
+
this.trayRealtime = trayRealtime;
|
|
43
50
|
this.moduleRef = moduleRef;
|
|
44
51
|
this.logger = new common_1.Logger(BackgroundJobExecutionService_1.name);
|
|
45
52
|
}
|
|
@@ -77,6 +84,7 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
77
84
|
async executeAttemptInContext(input, preloadedJob) {
|
|
78
85
|
let job = preloadedJob;
|
|
79
86
|
let lockHandle = null;
|
|
87
|
+
let lockRenewTimer;
|
|
80
88
|
let attemptStarted = false;
|
|
81
89
|
try {
|
|
82
90
|
job = await this.ensureJobCapabilities(preloadedJob);
|
|
@@ -105,6 +113,16 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
105
113
|
}, LOCK_BUSY_REDELIVERY_DELAY_MS);
|
|
106
114
|
return;
|
|
107
115
|
}
|
|
116
|
+
const renewEveryMs = Math.max(Math.floor(ttl / 3), MIN_LOCK_RENEW_INTERVAL_MS);
|
|
117
|
+
const handle = lockHandle;
|
|
118
|
+
lockRenewTimer = setInterval(() => {
|
|
119
|
+
void this.lock.renew(handle, ttl).then((ok) => {
|
|
120
|
+
if (!ok) {
|
|
121
|
+
this.logger.warn(`Execution lock renew failed for job ${job?.id ?? input.jobId}`);
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
}, renewEveryMs);
|
|
125
|
+
lockRenewTimer.unref?.();
|
|
108
126
|
}
|
|
109
127
|
attemptStarted = true;
|
|
110
128
|
await this.runAttempt(job, input);
|
|
@@ -118,6 +136,9 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
118
136
|
throw error;
|
|
119
137
|
}
|
|
120
138
|
finally {
|
|
139
|
+
if (lockRenewTimer) {
|
|
140
|
+
clearInterval(lockRenewTimer);
|
|
141
|
+
}
|
|
121
142
|
if (lockHandle) {
|
|
122
143
|
await this.lock.release(lockHandle);
|
|
123
144
|
}
|
|
@@ -160,16 +181,65 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
160
181
|
return false;
|
|
161
182
|
}
|
|
162
183
|
}
|
|
184
|
+
/**
|
|
185
|
+
* Marks a stale `running` attempt as `abandoned` and applies the job retry policy.
|
|
186
|
+
* No-op when this process still owns the attempt.
|
|
187
|
+
*/
|
|
188
|
+
async recoverAbandonedAttempt(attempt) {
|
|
189
|
+
if (attempt.status !== 'running') {
|
|
190
|
+
return false;
|
|
191
|
+
}
|
|
192
|
+
if (this.cancellation.isExecutingLocally(attempt.jobId)) {
|
|
193
|
+
return false;
|
|
194
|
+
}
|
|
195
|
+
const job = await this.jobStore.getOneAsEngine(attempt.jobId);
|
|
196
|
+
if (!job) {
|
|
197
|
+
const now = new Date().toISOString();
|
|
198
|
+
await this.attemptStore.patchAsEngine(attempt.id, {
|
|
199
|
+
status: 'abandoned',
|
|
200
|
+
completedAt: now,
|
|
201
|
+
error: {
|
|
202
|
+
code: 'BACKGROUND_JOB_ABANDONED',
|
|
203
|
+
message: 'Attempt abandoned after missed heartbeats (job row missing).',
|
|
204
|
+
retryable: false,
|
|
205
|
+
},
|
|
206
|
+
});
|
|
207
|
+
return true;
|
|
208
|
+
}
|
|
209
|
+
if (this.jobStore.isTerminal(job)) {
|
|
210
|
+
const now = new Date().toISOString();
|
|
211
|
+
await this.attemptStore.patchAsEngine(attempt.id, {
|
|
212
|
+
status: 'abandoned',
|
|
213
|
+
completedAt: now,
|
|
214
|
+
error: {
|
|
215
|
+
code: 'BACKGROUND_JOB_ABANDONED',
|
|
216
|
+
message: 'Attempt abandoned after missed heartbeats (job already terminal).',
|
|
217
|
+
retryable: false,
|
|
218
|
+
},
|
|
219
|
+
});
|
|
220
|
+
return true;
|
|
221
|
+
}
|
|
222
|
+
const startedAt = attempt.startedAt ? new Date(attempt.startedAt) : new Date();
|
|
223
|
+
await this.completeAttemptFailed(job, attempt, startedAt, {
|
|
224
|
+
code: 'BACKGROUND_JOB_ABANDONED',
|
|
225
|
+
message: 'Attempt abandoned after missed heartbeats (worker crash or lock expiry).',
|
|
226
|
+
retryable: true,
|
|
227
|
+
}, false, undefined, 'abandoned');
|
|
228
|
+
return true;
|
|
229
|
+
}
|
|
163
230
|
//#region ---- Attempt lifecycle ----
|
|
164
231
|
async runAttempt(job, input) {
|
|
165
232
|
const attempt = await this.attemptFactory.createAttempt(job, input.attemptTrigger, input.queueJobId);
|
|
166
233
|
const startedAt = new Date();
|
|
234
|
+
const startedAtIso = startedAt.toISOString();
|
|
167
235
|
await this.attemptStore.patchAsEngine(attempt.id, {
|
|
168
236
|
status: 'running',
|
|
169
|
-
startedAt:
|
|
237
|
+
startedAt: startedAtIso,
|
|
238
|
+
heartbeatAt: startedAtIso,
|
|
170
239
|
});
|
|
171
240
|
attempt.status = 'running';
|
|
172
|
-
attempt.startedAt =
|
|
241
|
+
attempt.startedAt = startedAtIso;
|
|
242
|
+
attempt.heartbeatAt = startedAtIso;
|
|
173
243
|
const executionStartedAt = job.firstStartedAt ?? startedAt.toISOString();
|
|
174
244
|
await this.jobStore.patchAsEngine(job.id, {
|
|
175
245
|
status: 'running',
|
|
@@ -179,9 +249,11 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
179
249
|
job.status = 'running';
|
|
180
250
|
job.firstStartedAt = executionStartedAt;
|
|
181
251
|
job.executedAt = job.executedAt ?? executionStartedAt;
|
|
252
|
+
await this.publishTrayRefresh(job, 'updated');
|
|
182
253
|
const controller = new AbortController();
|
|
183
254
|
let abortCause = null;
|
|
184
255
|
let timeoutTimer;
|
|
256
|
+
let heartbeatTimer;
|
|
185
257
|
if (job.executionTimeoutMs && job.executionTimeoutMs > 0) {
|
|
186
258
|
timeoutTimer = setTimeout(() => {
|
|
187
259
|
abortCause = 'timeout';
|
|
@@ -192,6 +264,10 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
192
264
|
abortCause = abortCause ?? 'cancel';
|
|
193
265
|
});
|
|
194
266
|
this.cancellation.registerLocalExecution(job.id, controller);
|
|
267
|
+
heartbeatTimer = setInterval(() => {
|
|
268
|
+
void this.attemptStore.patchAsEngine(attempt.id, { heartbeatAt: new Date().toISOString() });
|
|
269
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
270
|
+
heartbeatTimer.unref?.();
|
|
195
271
|
const runtime = this.createRuntimeContext(job, attempt, controller.signal);
|
|
196
272
|
const commandKey = (0, background_job_1.resolveBackgroundJobCommandKey)(job);
|
|
197
273
|
try {
|
|
@@ -244,6 +320,9 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
244
320
|
await this.completeAttemptFailed(job, attempt, startedAt, normalized.error, false, normalized.stack);
|
|
245
321
|
}
|
|
246
322
|
finally {
|
|
323
|
+
if (heartbeatTimer) {
|
|
324
|
+
clearInterval(heartbeatTimer);
|
|
325
|
+
}
|
|
247
326
|
if (timeoutTimer) {
|
|
248
327
|
clearTimeout(timeoutTimer);
|
|
249
328
|
}
|
|
@@ -272,11 +351,16 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
272
351
|
applicationId: job.applicationId?.trim() || undefined,
|
|
273
352
|
userId: job.userId,
|
|
274
353
|
reportProgress: async (progress, statusText, currentStep) => {
|
|
354
|
+
const nextProgress = Math.max(0, Math.min(100, progress));
|
|
275
355
|
await this.jobStore.patchAsEngine(job.id, {
|
|
276
|
-
progress:
|
|
356
|
+
progress: nextProgress,
|
|
277
357
|
statusText,
|
|
278
358
|
currentStep,
|
|
279
359
|
});
|
|
360
|
+
job.progress = nextProgress;
|
|
361
|
+
job.statusText = statusText;
|
|
362
|
+
job.currentStep = currentStep;
|
|
363
|
+
await this.publishTrayRefresh(job, 'updated');
|
|
280
364
|
},
|
|
281
365
|
isCancellationRequested: () => this.cancellation.isCancellationRequested(job.id),
|
|
282
366
|
heartbeat: async () => {
|
|
@@ -340,9 +424,9 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
340
424
|
await this.publishAttempt(job, attempt);
|
|
341
425
|
await this.publishJob(job);
|
|
342
426
|
}
|
|
343
|
-
async completeAttemptFailed(job, attempt, startedAt, error, timedOut, stack) {
|
|
427
|
+
async completeAttemptFailed(job, attempt, startedAt, error, timedOut, stack, attemptStatusOverride) {
|
|
344
428
|
const { completedAt, durationMs } = finishTimes(startedAt);
|
|
345
|
-
const attemptStatus = timedOut ? 'timed-out' : 'failed';
|
|
429
|
+
const attemptStatus = attemptStatusOverride ?? (timedOut ? 'timed-out' : 'failed');
|
|
346
430
|
await this.attemptStore.patchAsEngine(attempt.id, {
|
|
347
431
|
status: attemptStatus,
|
|
348
432
|
completedAt,
|
|
@@ -366,6 +450,7 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
366
450
|
job.totalDurationMs = totalDurationMs;
|
|
367
451
|
job.result = jobResult;
|
|
368
452
|
await this.publishAttempt(job, attempt);
|
|
453
|
+
await this.publishTrayRefresh(job, 'updated');
|
|
369
454
|
await this.retryScheduler.scheduleExecution({
|
|
370
455
|
jobId: job.id,
|
|
371
456
|
attemptTrigger: 'automatic-retry',
|
|
@@ -434,6 +519,7 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
434
519
|
}
|
|
435
520
|
this.logger.warn(`Schedule ${schedule.id} (${schedule.command.key}) exceeded failure threshold ` +
|
|
436
521
|
`(${consecutiveFailures}/${threshold}) — action: ${action}`);
|
|
522
|
+
await this.notifyScheduleThreshold(schedule, consecutiveFailures, threshold, action);
|
|
437
523
|
}
|
|
438
524
|
await this.scheduleStore.patchAsSystemCommand(schedule.id, patch);
|
|
439
525
|
}
|
|
@@ -458,6 +544,44 @@ let BackgroundJobExecutionService = BackgroundJobExecutionService_1 = class Back
|
|
|
458
544
|
catch (error) {
|
|
459
545
|
this.logger.warn(`Job observability publish failed: ${String(error)}`);
|
|
460
546
|
}
|
|
547
|
+
await this.publishTrayTerminal(job);
|
|
548
|
+
await this.notifyJobTerminal(job);
|
|
549
|
+
}
|
|
550
|
+
async publishTrayTerminal(job) {
|
|
551
|
+
const op = resolveTrayOpFromTerminalStatus(job.status);
|
|
552
|
+
if (!op) {
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
await this.publishTrayRefresh(job, op);
|
|
556
|
+
}
|
|
557
|
+
async publishTrayRefresh(job, op) {
|
|
558
|
+
try {
|
|
559
|
+
await this.trayRealtime.notify(job, op);
|
|
560
|
+
}
|
|
561
|
+
catch (error) {
|
|
562
|
+
this.logger.warn(`Tray realtime publish failed: ${String(error)}`);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
async notifyJobTerminal(job) {
|
|
566
|
+
try {
|
|
567
|
+
await this.notifications.notifyJobTerminal(job);
|
|
568
|
+
}
|
|
569
|
+
catch (error) {
|
|
570
|
+
this.logger.warn(`Job notification failed: ${String(error)}`);
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
async notifyScheduleThreshold(schedule, consecutiveFailures, threshold, action) {
|
|
574
|
+
try {
|
|
575
|
+
await this.notifications.notifyScheduleThreshold({
|
|
576
|
+
schedule,
|
|
577
|
+
consecutiveFailures,
|
|
578
|
+
threshold,
|
|
579
|
+
action,
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
catch (error) {
|
|
583
|
+
this.logger.warn(`Schedule threshold notification failed: ${String(error)}`);
|
|
584
|
+
}
|
|
461
585
|
}
|
|
462
586
|
};
|
|
463
587
|
exports.BackgroundJobExecutionService = BackgroundJobExecutionService;
|
|
@@ -473,10 +597,23 @@ exports.BackgroundJobExecutionService = BackgroundJobExecutionService = Backgrou
|
|
|
473
597
|
background_job_retry_scheduler_service_1.BackgroundJobRetrySchedulerService,
|
|
474
598
|
distributed_lock_port_1.DistributedLockPort,
|
|
475
599
|
background_job_observability_publisher_1.BackgroundJobObservabilityPublisher,
|
|
600
|
+
background_job_notification_port_1.BackgroundJobNotificationPort,
|
|
601
|
+
background_job_tray_realtime_publisher_1.BackgroundJobTrayRealtimePublisher,
|
|
476
602
|
core_1.ModuleRef])
|
|
477
603
|
], BackgroundJobExecutionService);
|
|
478
|
-
|
|
479
|
-
|
|
604
|
+
function resolveTrayOpFromTerminalStatus(status) {
|
|
605
|
+
switch (status) {
|
|
606
|
+
case 'completed':
|
|
607
|
+
return 'completed';
|
|
608
|
+
case 'failed':
|
|
609
|
+
case 'dead-lettered':
|
|
610
|
+
return 'failed';
|
|
611
|
+
case 'cancelled':
|
|
612
|
+
return 'cancelled';
|
|
613
|
+
default:
|
|
614
|
+
return null;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
480
617
|
function finishTimes(startedAt) {
|
|
481
618
|
const finished = new Date();
|
|
482
619
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"background-job-execution.service.js","sourceRoot":"","sources":["../../../../../../../../libs/framework/server/background-job/src/lib/execution/background-job-execution.service.ts"],"names":[],"mappings":";;;;;AAAA,2CAAoD;AACpD,uCAA2D;AAE3D,qFAc0D;AAC1D,kFAA2F;AAC3F,yEAGoD;AACpD,iEAA4F;AAC5F,qEAGkC;AAClC,6EAA4E;AAC5E,yEAAgG;AAChG,oHAA8G;AAC9G,wGAAiG;AACjG,0GAAmG;AACnG,wGAAkG;AAClG,wFAAkF;AAClF,qFAA+E;AAC/E,+FAAyF;AACzF,2EAA4D;AAC5D,qGAA8F;AAe9F,mEAAmE;AACnE,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC/C,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;AACrC,uEAAuE;AACvE,MAAM,6BAA6B,GAAG,EAAE,GAAG,IAAI,CAAC;AAEhD,YAAY;AAEZ,sDAAsD;AAG/C,IAAM,6BAA6B,qCAAnC,MAAM,6BAA6B;IAGxC,YACmB,QAAmC,EACnC,YAA8C,EAC9C,aAAgD,EAChD,oBAAuD,EACvD,cAA2C,EAC3C,cAAqC,EACrC,YAA8C,EAC9C,cAAkD,EAClD,IAAyB,EACzB,aAAkD,EAClD,SAAoB;QAVpB,aAAQ,GAAR,QAAQ,CAA2B;QACnC,iBAAY,GAAZ,YAAY,CAAkC;QAC9C,kBAAa,GAAb,aAAa,CAAmC;QAChD,yBAAoB,GAApB,oBAAoB,CAAmC;QACvD,mBAAc,GAAd,cAAc,CAA6B;QAC3C,mBAAc,GAAd,cAAc,CAAuB;QACrC,iBAAY,GAAZ,YAAY,CAAkC;QAC9C,mBAAc,GAAd,cAAc,CAAoC;QAClD,SAAI,GAAJ,IAAI,CAAqB;QACzB,kBAAa,GAAb,aAAa,CAAqC;QAClD,cAAS,GAAT,SAAS,CAAW;QAbtB,WAAM,GAAG,IAAI,eAAM,CAAC,+BAA6B,CAAC,IAAI,CAAC,CAAC;IActE,CAAC;IAEJ,KAAK,CAAC,cAAc,CAAC,KAA0B;QAC7C,8EAA8E;QAC9E,gDAAgD;QAChD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACnE,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,KAAK,iCAAiC,CAAC,CAAC;YACtE,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAA,yDAA4B,EAAC,UAAU,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,kBAAkB,UAAU,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAC3E,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,eAAe,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC/C,IAAI,eAAe,IAAI,eAAe,KAAK,QAAQ,EAAE,CAAC;YACpD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,4BAA4B,eAAe,aAAa,UAAU,CAAC,EAAE,wBAAwB,QAAQ,GAAG,CACzG,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,gBAAgB,CAAC;QAC7D,MAAM,GAAG,GAAG;YACV,CAAC,8BAAmB,CAAC,EAAE;gBACrB,MAAM;gBACN,QAAQ,EAAE,MAAM;gBAChB,QAAQ;gBACR,aAAa,EAAE,UAAU,CAAC,aAAa;aACR;SACZ,CAAC;QAExB,MAAM,SAAS,GAAG,uBAAgB,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE1D,OAAO,IAAA,yCAAkC,EAAC,GAAG,EAAE,GAAG,EAAE,CAClD,IAAI,CAAC,uBAAuB,CAAC,KAAK,EAAE,UAAU,CAAC,CAChD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAA0B,EAC1B,YAA8B;QAE9B,IAAI,GAAG,GAA4B,YAAY,CAAC;QAChD,IAAI,UAAU,GAAiC,IAAI,CAAC;QACpD,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;YAErD,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,sBAAsB,GAAG,CAAC,MAAM,yBAAyB,CAAC,CAAC;gBAC1F,OAAO;YACT,CAAC;YAED,iEAAiE;YACjE,IAAI,GAAG,CAAC,uBAAuB,EAAE,CAAC;gBAChC,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBAClC,OAAO;YACT,CAAC;YAED,0EAA0E;YAC1E,MAAM,QAAQ,GAAG,IAAA,yDAA4B,EAAC,GAAG,CAAC,CAAC;YACnD,IAAI,GAAG,CAAC,cAAc,IAAI,QAAQ,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,kBAAkB,IAAI,uBAAuB,CAAC,GAAG,kBAAkB,CAAC;gBACrF,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAClC,IAAA,wCAAuB,EAAC,QAAQ,EAAE,GAAG,CAAC,cAAc,CAAC,EACrD,GAAG,CACJ,CAAC;gBACF,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,4BAA4B,GAAG,CAAC,cAAc,wBAAwB,GAAG,CAAC,EAAE,OAAO,6BAA6B,IAAI,CACrH,CAAC;oBACF,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CACzC;wBACE,KAAK,EAAE,GAAG,CAAC,EAAE;wBACb,cAAc,EAAE,KAAK,CAAC,cAAc;wBACpC,QAAQ,EAAE,GAAG,CAAC,QAAQ;wBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;wBAClB,aAAa,EAAE,GAAG,CAAC,aAAa;qBACjC,EACD,6BAA6B,CAC9B,CAAC;oBACF,OAAO;gBACT,CAAC;YACH,CAAC;YAED,cAAc,GAAG,IAAI,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,+EAA+E;YAC/E,mFAAmF;YACnF,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,wBAAwB,CAC5B,KAAa,EACb,KAAc,EACd,GAA6B;QAE7B,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/D,MAAM,QAAQ,GAA0B;YACtC,IAAI,EAAE,iBAAiB;YACvB,OAAO;YACP,SAAS,EAAE,IAAI;SAChB,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAA,gDAA+B,EAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEhE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;YACnE,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;YAC/B,IACE,MAAM;gBACN,MAAM,KAAK,QAAQ;gBACnB,MAAM,KAAK,SAAS;gBACpB,MAAM,KAAK,WAAW;gBACtB,MAAM,KAAK,SAAS,EACpB,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,EAAE;gBACvC,MAAM,EAAE,QAAQ;gBAChB,WAAW;gBACX,MAAM;gBACN,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aAClC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sBAAsB,KAAK,uCAAuC,OAAO,MACvE,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CACtE,EAAE,CACH,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,qCAAqC;IAE7B,KAAK,CAAC,UAAU,CAAC,GAAqB,EAAE,KAA0B;QACxE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,CACrD,GAAG,EACH,KAAK,CAAC,cAAc,EACpB,KAAK,CAAC,UAAU,CACjB,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;YAChD,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;SACnC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;QAC3B,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC,cAAc,IAAI,SAAS,CAAC,WAAW,EAAE,CAAC;QACzE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,SAAS;YACjB,cAAc,EAAE,kBAAkB;YAClC,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,kBAAkB;SACjD,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,GAAG,CAAC,cAAc,GAAG,kBAAkB,CAAC;QACxC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,kBAAkB,CAAC;QAEtD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI,UAAU,GAAsB,IAAI,CAAC;QACzC,IAAI,YAAwC,CAAC;QAE7C,IAAI,GAAG,CAAC,kBAAkB,IAAI,GAAG,CAAC,kBAAkB,GAAG,CAAC,EAAE,CAAC;YACzD,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC7B,UAAU,GAAG,SAAS,CAAC;gBACvB,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,sDAAqC,CAAC,CAAC,CAAC;YACrE,CAAC,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAC7B,CAAC;QAED,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC/C,UAAU,GAAG,UAAU,IAAI,QAAQ,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,sBAAsB,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QAE7D,MAAM,OAAO,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC3E,MAAM,UAAU,GAAG,IAAA,+CAA8B,EAAC,GAAG,CAAC,CAAC;QAEvD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,CAAC,qBAAqB,CAC9B,GAAG,EACH,OAAO,EACP,SAAS,EACT;oBACE,IAAI,EAAE,kCAAkC;oBACxC,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,SAAS,EAAE,KAAK;iBACjB,EACD,KAAK,CACN,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAC5C,6CAAoB,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,CACrC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,IAAA,mDAAkC,EAAC,GAAG,CAAC,CAAC,CACjF,EACD,UAAU,CAAC,MAAM,CAClB,CAAC;YAEF,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;gBAC3B,MAAM,WAAW,GACf,OAAO,aAAa,CAAC,OAAO,EAAE,IAAI,KAAK,QAAQ;oBAC7C,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI;oBAC5B,CAAC,CAAC,YAAY,UAAU,UAAU,CAAC;gBAEvC,MAAM,eAAe,GACnB,UAAU,KAAK,QAAQ;oBACvB,IAAA,mDAAkC,EAAC,WAAW,CAAC;oBAC/C,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC5D,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;oBAC7D,OAAO;gBACT,CAAC;gBAED,MAAM,IAAI,CAAC,qBAAqB,CAC9B,GAAG,EACH,OAAO,EACP,SAAS,EACT;oBACE,IAAI,EAAE,+BAA+B;oBACrC,OAAO,EAAE,WAAW;oBACpB,SAAS,EAAE,KAAK;iBACjB,EACD,KAAK,CACN,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,aAAa,GAAG,IAAA,2DAA0C,EAAC,aAAa,CAAC,CAAC;YAEhF,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;QAC9E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,qBAAqB,CAC9B,GAAG,EACH,OAAO,EACP,SAAS,EACT;oBACE,IAAI,EAAE,sDAAqC;oBAC3C,OAAO,EAAE,2BAA2B,GAAG,CAAC,kBAAkB,IAAI;iBAC/D,EACD,IAAI,CACL,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,eAAe,GACnB,UAAU,KAAK,QAAQ;gBACvB,IAAA,iDAAgC,EAAC,KAAK,CAAC;gBACvC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5D,IAAI,eAAe,EAAE,CAAC;gBACpB,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;gBAC7D,OAAO;YACT,CAAC;YAED,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,IAAI,CAAC,qBAAqB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;QACvG,CAAC;gBAAS,CAAC;YACT,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,CAAC,YAAY,CAAC,CAAC;YAC7B,CAAC;YACD,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,GAAqB;QACvD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;QAEvC,IACE,OAAO,CAAC,+BAA+B,KAAK,YAAY,CAAC,+BAA+B;YACxF,OAAO,CAAC,gBAAgB,KAAK,YAAY,CAAC,gBAAgB,EAC1D,CAAC;YACD,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;YAChC,OAAO,GAAG,CAAC;QACb,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;QAC5D,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;QAChC,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,oBAAoB,CAC1B,GAAqB,EACrB,OAAgC,EAChC,MAAmB;QAEnB,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,MAAM;YACN,OAAO,EAAE,IAAA,mDAAkC,EAAC,GAAG,CAAC;YAChD,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,SAAS;YACrD,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,cAAc,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,EAAE;gBAC1D,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;oBACxC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC9C,UAAU;oBACV,WAAW;iBACZ,CAAC,CAAC;YACL,CAAC;YACD,uBAAuB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC;YAChF,SAAS,EAAE,KAAK,IAAI,EAAE;gBACpB,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YAC/F,CAAC;SACF,CAAC;IACJ,CAAC;IAED,iFAAiF;IACzE,aAAa,CAAI,IAAgB,EAAE,MAAmB;QAC5D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;YACpE,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1D,IAAI;iBACD,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;iBACrB,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC;IAED,YAAY;IAEZ,qCAAqC;IAE7B,KAAK,CAAC,wBAAwB,CACpC,GAAqB,EACrB,OAAgC,EAChC,SAAe,EACf,MAAsC;QAEtC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;QAE3D,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;QACpG,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC;QAC7B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;QAClC,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;QAEhC,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC;QAChE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,WAAW;YACnB,QAAQ,EAAE,GAAG;YACb,MAAM,EAAE,MAAM,EAAE,MAAM;YACtB,eAAe,EAAE,MAAM,EAAE,eAAe;YACxC,WAAW;YACX,eAAe;SAChB,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;QACzB,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC;QAC9B,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,MAAM,EAAE,MAAM,CAAC;QAE5B,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,GAAqB,EACrB,OAAgC,EAChC,SAAe;QAEf,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;QAE3D,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;QACpG,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC;QAC7B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;QAClC,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;QAEhC,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC;QAChE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,WAAW;YACxB,WAAW;YACX,eAAe;SAChB,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;QACzB,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;QAEtC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,qBAAqB,CACjC,GAAqB,EACrB,OAAgC,EAChC,SAAe,EACf,KAA4B,EAC5B,QAAiB,EACjB,KAAc;QAEd,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;QAExD,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;YAChD,MAAM,EAAE,aAAa;YACrB,WAAW;YACX,UAAU;YACV,KAAK;SACN,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,GAAG,aAAa,CAAC;QAC/B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;QAClC,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;QAChC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QAEtB,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC;QAChE,MAAM,QAAQ,GAAG,IAAA,yCAAa,EAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAA,gDAA+B,EAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAEhE,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;gBACxC,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,SAAS;gBACjB,eAAe;aAChB,CAAC,CAAC;YACH,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;YACtB,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;YACtC,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;YAEvB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CACzC;gBACE,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,cAAc,EAAE,iBAAiB;gBACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,aAAa,EAAE,GAAG,CAAC,aAAa;aACjC,EACD,QAAQ,CAAC,OAAO,CACjB,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,QAAQ,CAAC,WAAW;YAC5B,MAAM,EAAE,SAAS;YACjB,WAAW;YACX,eAAe;SAChB,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,WAAW,CAAC;QAClC,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC;QAC9B,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QAEvB,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,GAAqB;QACnD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,GAAG;YAChB,WAAW,EAAE,GAAG;SACjB,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,YAAY;IAEZ,uDAAuD;IAE/C,KAAK,CAAC,oBAAoB,CAAC,GAAqB,EAAE,SAAkB;QAC1E,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACjE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO;YACT,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAErC,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,EAAE,iBAAiB,KAAK,KAAK,CAAC;gBAC1E,MAAM,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,EAAE;oBACzD,aAAa,EAAE,GAAG;oBAClB,mBAAmB,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB;iBACtE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,mBAAmB,GAAG,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACpE,MAAM,KAAK,GAA4B,EAAE,aAAa,EAAE,GAAG,EAAE,mBAAmB,EAAE,CAAC;YAEnF,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,EAAE,2BAA2B,CAAC;YACtE,IAAI,SAAS,IAAI,mBAAmB,IAAI,SAAS,EAAE,CAAC;gBAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,EAAE,mBAAmB,IAAI,YAAY,CAAC;gBAC3E,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBACzB,KAAK,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;oBAC7B,KAAK,CAAC,WAAW,CAAC,GAAG,SAAS,CAAC;gBACjC,CAAC;qBAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;oBAC9B,KAAK,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;oBAC1B,KAAK,CAAC,WAAW,CAAC,GAAG,SAAS,CAAC;gBACjC,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,QAAQ,CAAC,EAAE,KAAK,QAAQ,CAAC,OAAO,CAAC,GAAG,+BAA+B;oBAC7E,IAAI,mBAAmB,IAAI,SAAS,eAAe,MAAM,EAAE,CAC9D,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,4CAA4C,GAAG,CAAC,EAAE,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAChH,CAAC;QACJ,CAAC;IACH,CAAC;IAED,YAAY;IAEZ,iCAAiC;IAEzB,KAAK,CAAC,cAAc,CAC1B,GAAqB,EACrB,OAAgC;QAEhC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,sBAAsB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAChE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,GAAqB;QAC5C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;CAGF,CAAA;AArkBY,sEAA6B;wCAA7B,6BAA6B;IADzC,IAAA,mBAAU,GAAE;6CAKkB,wDAAyB;QACrB,uEAAgC;QAC/B,yEAAiC;QAC1B,wEAAiC;QACvC,4DAA2B;QAC3B,iCAAqB;QACvB,sEAAgC;QAC9B,2EAAkC;QAC5C,2CAAmB;QACV,4EAAmC;QACvC,gBAAS;GAd5B,6BAA6B,CAqkBzC;AAED,YAAY;AAEZ,2BAA2B;AAE3B,SAAS,WAAW,CAAC,SAAe;IAClC,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;IAC5B,OAAO;QACL,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,KAIjB,CAAC;QACF,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACrE,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB;gBACrF,MAAM,EAAE,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBAC3E,SAAS,EAAE,OAAO,SAAS,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;aACtF;YACD,KAAK,EAAE,SAAS,CAAC,KAAK;SACvB,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,KAMjB,CAAC;QACF,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACrE,OAAO,EACL,OAAO,SAAS,CAAC,OAAO,KAAK,QAAQ,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;oBACnE,CAAC,CAAC,SAAS,CAAC,OAAO;oBACnB,CAAC,CAAC,yBAAyB;gBAC/B,MAAM,EAAE,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBAC3E,SAAS,EAAE,OAAO,SAAS,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;aACtF;YACD,KAAK,EAAE,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SACzE,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;AAC/C,CAAC;AAED,YAAY"}
|
|
1
|
+
{"version":3,"file":"background-job-execution.service.js","sourceRoot":"","sources":["../../../../../../../../libs/framework/server/background-job/src/lib/execution/background-job-execution.service.ts"],"names":[],"mappings":";;;;;AAAA,2CAAoD;AACpD,uCAA2D;AAE3D,qFAiB0D;AAC1D,kFAA2F;AAC3F,yEAGoD;AACpD,iEAA4F;AAC5F,qEAGkC;AAClC,6EAA4E;AAC5E,yEAAgG;AAChG,oHAA8G;AAC9G,uGAAiG;AACjG,wGAAiG;AACjG,0GAAmG;AACnG,wGAAkG;AAClG,wFAAkF;AAClF,qFAA+E;AAC/E,+FAAyF;AACzF,2EAA4D;AAC5D,qGAA8F;AAC9F,2GAGwD;AAexD,mEAAmE;AACnE,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC/C,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;AACrC,uEAAuE;AACvE,MAAM,6BAA6B,GAAG,EAAE,GAAG,IAAI,CAAC;AAChD,4DAA4D;AAC5D,MAAM,qBAAqB,GAAG,EAAE,GAAG,IAAI,CAAC;AACxC,MAAM,0BAA0B,GAAG,CAAC,GAAG,IAAI,CAAC;AAE5C,YAAY;AAEZ,sDAAsD;AAG/C,IAAM,6BAA6B,qCAAnC,MAAM,6BAA6B;IAGxC,YACmB,QAAmC,EACnC,YAA8C,EAC9C,aAAgD,EAChD,oBAAuD,EACvD,cAA2C,EAC3C,cAAqC,EACrC,YAA8C,EAC9C,cAAkD,EAClD,IAAyB,EACzB,aAAkD,EAClD,aAA4C,EAC5C,YAAgD,EAChD,SAAoB;QAZpB,aAAQ,GAAR,QAAQ,CAA2B;QACnC,iBAAY,GAAZ,YAAY,CAAkC;QAC9C,kBAAa,GAAb,aAAa,CAAmC;QAChD,yBAAoB,GAApB,oBAAoB,CAAmC;QACvD,mBAAc,GAAd,cAAc,CAA6B;QAC3C,mBAAc,GAAd,cAAc,CAAuB;QACrC,iBAAY,GAAZ,YAAY,CAAkC;QAC9C,mBAAc,GAAd,cAAc,CAAoC;QAClD,SAAI,GAAJ,IAAI,CAAqB;QACzB,kBAAa,GAAb,aAAa,CAAqC;QAClD,kBAAa,GAAb,aAAa,CAA+B;QAC5C,iBAAY,GAAZ,YAAY,CAAoC;QAChD,cAAS,GAAT,SAAS,CAAW;QAftB,WAAM,GAAG,IAAI,eAAM,CAAC,+BAA6B,CAAC,IAAI,CAAC,CAAC;IAgBtE,CAAC;IAEJ,KAAK,CAAC,cAAc,CAAC,KAA0B;QAC7C,8EAA8E;QAC9E,gDAAgD;QAChD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACnE,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,KAAK,iCAAiC,CAAC,CAAC;YACtE,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAA,yDAA4B,EAAC,UAAU,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,kBAAkB,UAAU,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAC3E,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,eAAe,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC/C,IAAI,eAAe,IAAI,eAAe,KAAK,QAAQ,EAAE,CAAC;YACpD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,4BAA4B,eAAe,aAAa,UAAU,CAAC,EAAE,wBAAwB,QAAQ,GAAG,CACzG,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,gBAAgB,CAAC;QAC7D,MAAM,GAAG,GAAG;YACV,CAAC,8BAAmB,CAAC,EAAE;gBACrB,MAAM;gBACN,QAAQ,EAAE,MAAM;gBAChB,QAAQ;gBACR,aAAa,EAAE,UAAU,CAAC,aAAa;aACR;SACZ,CAAC;QAExB,MAAM,SAAS,GAAG,uBAAgB,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE1D,OAAO,IAAA,yCAAkC,EAAC,GAAG,EAAE,GAAG,EAAE,CAClD,IAAI,CAAC,uBAAuB,CAAC,KAAK,EAAE,UAAU,CAAC,CAChD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAA0B,EAC1B,YAA8B;QAE9B,IAAI,GAAG,GAA4B,YAAY,CAAC;QAChD,IAAI,UAAU,GAAiC,IAAI,CAAC;QACpD,IAAI,cAA0C,CAAC;QAC/C,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;YAErD,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,sBAAsB,GAAG,CAAC,MAAM,yBAAyB,CAAC,CAAC;gBAC1F,OAAO;YACT,CAAC;YAED,iEAAiE;YACjE,IAAI,GAAG,CAAC,uBAAuB,EAAE,CAAC;gBAChC,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBAClC,OAAO;YACT,CAAC;YAED,0EAA0E;YAC1E,MAAM,QAAQ,GAAG,IAAA,yDAA4B,EAAC,GAAG,CAAC,CAAC;YACnD,IAAI,GAAG,CAAC,cAAc,IAAI,QAAQ,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,kBAAkB,IAAI,uBAAuB,CAAC,GAAG,kBAAkB,CAAC;gBACrF,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAClC,IAAA,wCAAuB,EAAC,QAAQ,EAAE,GAAG,CAAC,cAAc,CAAC,EACrD,GAAG,CACJ,CAAC;gBACF,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,4BAA4B,GAAG,CAAC,cAAc,wBAAwB,GAAG,CAAC,EAAE,OAAO,6BAA6B,IAAI,CACrH,CAAC;oBACF,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CACzC;wBACE,KAAK,EAAE,GAAG,CAAC,EAAE;wBACb,cAAc,EAAE,KAAK,CAAC,cAAc;wBACpC,QAAQ,EAAE,GAAG,CAAC,QAAQ;wBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;wBAClB,aAAa,EAAE,GAAG,CAAC,aAAa;qBACjC,EACD,6BAA6B,CAC9B,CAAC;oBACF,OAAO;gBACT,CAAC;gBAED,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC;gBAC/E,MAAM,MAAM,GAAG,UAAU,CAAC;gBAC1B,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;oBAChC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;wBAC5C,IAAI,CAAC,EAAE,EAAE,CAAC;4BACR,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,GAAG,EAAE,EAAE,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;wBACpF,CAAC;oBACH,CAAC,CAAC,CAAC;gBACL,CAAC,EAAE,YAAY,CAAC,CAAC;gBACjB,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC;YAC3B,CAAC;YAED,cAAc,GAAG,IAAI,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,+EAA+E;YAC/E,mFAAmF;YACnF,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,IAAI,cAAc,EAAE,CAAC;gBACnB,aAAa,CAAC,cAAc,CAAC,CAAC;YAChC,CAAC;YACD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,wBAAwB,CAC5B,KAAa,EACb,KAAc,EACd,GAA6B;QAE7B,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/D,MAAM,QAAQ,GAA0B;YACtC,IAAI,EAAE,iBAAiB;YACvB,OAAO;YACP,SAAS,EAAE,IAAI;SAChB,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAA,gDAA+B,EAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEhE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;YACnE,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;YAC/B,IACE,MAAM;gBACN,MAAM,KAAK,QAAQ;gBACnB,MAAM,KAAK,SAAS;gBACpB,MAAM,KAAK,WAAW;gBACtB,MAAM,KAAK,SAAS,EACpB,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,EAAE;gBACvC,MAAM,EAAE,QAAQ;gBAChB,WAAW;gBACX,MAAM;gBACN,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aAClC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sBAAsB,KAAK,uCAAuC,OAAO,MACvE,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CACtE,EAAE,CACH,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,uBAAuB,CAAC,OAAgC;QAC5D,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;gBAChD,MAAM,EAAE,WAAW;gBACnB,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE;oBACL,IAAI,EAAE,0BAA0B;oBAChC,OAAO,EAAE,8DAA8D;oBACvE,SAAS,EAAE,KAAK;iBACjB;aACF,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;gBAChD,MAAM,EAAE,WAAW;gBACnB,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE;oBACL,IAAI,EAAE,0BAA0B;oBAChC,OAAO,EAAE,mEAAmE;oBAC5E,SAAS,EAAE,KAAK;iBACjB;aACF,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/E,MAAM,IAAI,CAAC,qBAAqB,CAC9B,GAAG,EACH,OAAO,EACP,SAAS,EACT;YACE,IAAI,EAAE,0BAA0B;YAChC,OAAO,EAAE,0EAA0E;YACnF,SAAS,EAAE,IAAI;SAChB,EACD,KAAK,EACL,SAAS,EACT,WAAW,CACZ,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qCAAqC;IAE7B,KAAK,CAAC,UAAU,CAAC,GAAqB,EAAE,KAA0B;QACxE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,CACrD,GAAG,EACH,KAAK,CAAC,cAAc,EACpB,KAAK,CAAC,UAAU,CACjB,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;YAChD,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,YAAY;YACvB,WAAW,EAAE,YAAY;SAC1B,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;QAC3B,OAAO,CAAC,SAAS,GAAG,YAAY,CAAC;QACjC,OAAO,CAAC,WAAW,GAAG,YAAY,CAAC;QAEnC,MAAM,kBAAkB,GAAG,GAAG,CAAC,cAAc,IAAI,SAAS,CAAC,WAAW,EAAE,CAAC;QACzE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,SAAS;YACjB,cAAc,EAAE,kBAAkB;YAClC,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,kBAAkB;SACjD,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,GAAG,CAAC,cAAc,GAAG,kBAAkB,CAAC;QACxC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,kBAAkB,CAAC;QACtD,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE9C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI,UAAU,GAAsB,IAAI,CAAC;QACzC,IAAI,YAAwC,CAAC;QAC7C,IAAI,cAA0C,CAAC;QAE/C,IAAI,GAAG,CAAC,kBAAkB,IAAI,GAAG,CAAC,kBAAkB,GAAG,CAAC,EAAE,CAAC;YACzD,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC7B,UAAU,GAAG,SAAS,CAAC;gBACvB,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,sDAAqC,CAAC,CAAC,CAAC;YACrE,CAAC,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAC7B,CAAC;QAED,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC/C,UAAU,GAAG,UAAU,IAAI,QAAQ,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,sBAAsB,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QAE7D,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,KAAK,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC9F,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC1B,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC;QAEzB,MAAM,OAAO,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC3E,MAAM,UAAU,GAAG,IAAA,+CAA8B,EAAC,GAAG,CAAC,CAAC;QAEvD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,CAAC,qBAAqB,CAC9B,GAAG,EACH,OAAO,EACP,SAAS,EACT;oBACE,IAAI,EAAE,kCAAkC;oBACxC,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,SAAS,EAAE,KAAK;iBACjB,EACD,KAAK,CACN,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAC5C,6CAAoB,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,CACrC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,IAAA,mDAAkC,EAAC,GAAG,CAAC,CAAC,CACjF,EACD,UAAU,CAAC,MAAM,CAClB,CAAC;YAEF,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;gBAC3B,MAAM,WAAW,GACf,OAAO,aAAa,CAAC,OAAO,EAAE,IAAI,KAAK,QAAQ;oBAC7C,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI;oBAC5B,CAAC,CAAC,YAAY,UAAU,UAAU,CAAC;gBAEvC,MAAM,eAAe,GACnB,UAAU,KAAK,QAAQ;oBACvB,IAAA,mDAAkC,EAAC,WAAW,CAAC;oBAC/C,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC5D,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;oBAC7D,OAAO;gBACT,CAAC;gBAED,MAAM,IAAI,CAAC,qBAAqB,CAC9B,GAAG,EACH,OAAO,EACP,SAAS,EACT;oBACE,IAAI,EAAE,+BAA+B;oBACrC,OAAO,EAAE,WAAW;oBACpB,SAAS,EAAE,KAAK;iBACjB,EACD,KAAK,CACN,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,aAAa,GAAG,IAAA,2DAA0C,EAAC,aAAa,CAAC,CAAC;YAEhF,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;QAC9E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,qBAAqB,CAC9B,GAAG,EACH,OAAO,EACP,SAAS,EACT;oBACE,IAAI,EAAE,sDAAqC;oBAC3C,OAAO,EAAE,2BAA2B,GAAG,CAAC,kBAAkB,IAAI;iBAC/D,EACD,IAAI,CACL,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,eAAe,GACnB,UAAU,KAAK,QAAQ;gBACvB,IAAA,iDAAgC,EAAC,KAAK,CAAC;gBACvC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5D,IAAI,eAAe,EAAE,CAAC;gBACpB,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;gBAC7D,OAAO;YACT,CAAC;YAED,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,IAAI,CAAC,qBAAqB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;QACvG,CAAC;gBAAS,CAAC;YACT,IAAI,cAAc,EAAE,CAAC;gBACnB,aAAa,CAAC,cAAc,CAAC,CAAC;YAChC,CAAC;YACD,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,CAAC,YAAY,CAAC,CAAC;YAC7B,CAAC;YACD,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,GAAqB;QACvD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;QAEvC,IACE,OAAO,CAAC,+BAA+B,KAAK,YAAY,CAAC,+BAA+B;YACxF,OAAO,CAAC,gBAAgB,KAAK,YAAY,CAAC,gBAAgB,EAC1D,CAAC;YACD,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;YAChC,OAAO,GAAG,CAAC;QACb,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;QAC5D,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;QAChC,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,oBAAoB,CAC1B,GAAqB,EACrB,OAAgC,EAChC,MAAmB;QAEnB,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,MAAM;YACN,OAAO,EAAE,IAAA,mDAAkC,EAAC,GAAG,CAAC;YAChD,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,SAAS;YACrD,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,cAAc,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,EAAE;gBAC1D,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAC1D,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;oBACxC,QAAQ,EAAE,YAAY;oBACtB,UAAU;oBACV,WAAW;iBACZ,CAAC,CAAC;gBACH,GAAG,CAAC,QAAQ,GAAG,YAAY,CAAC;gBAC5B,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC;gBAC5B,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC9B,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAChD,CAAC;YACD,uBAAuB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC;YAChF,SAAS,EAAE,KAAK,IAAI,EAAE;gBACpB,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YAC/F,CAAC;SACF,CAAC;IACJ,CAAC;IAED,iFAAiF;IACzE,aAAa,CAAI,IAAgB,EAAE,MAAmB;QAC5D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;YACpE,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1D,IAAI;iBACD,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;iBACrB,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC;IAED,YAAY;IAEZ,qCAAqC;IAE7B,KAAK,CAAC,wBAAwB,CACpC,GAAqB,EACrB,OAAgC,EAChC,SAAe,EACf,MAAsC;QAEtC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;QAE3D,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;QACpG,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC;QAC7B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;QAClC,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;QAEhC,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC;QAChE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,WAAW;YACnB,QAAQ,EAAE,GAAG;YACb,MAAM,EAAE,MAAM,EAAE,MAAM;YACtB,eAAe,EAAE,MAAM,EAAE,eAAe;YACxC,WAAW;YACX,eAAe;SAChB,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;QACzB,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC;QAC9B,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,MAAM,EAAE,MAAM,CAAC;QAE5B,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,GAAqB,EACrB,OAAgC,EAChC,SAAe;QAEf,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;QAE3D,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;QACpG,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC;QAC7B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;QAClC,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;QAEhC,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC;QAChE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,WAAW;YACxB,WAAW;YACX,eAAe;SAChB,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;QACzB,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;QAEtC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,qBAAqB,CACjC,GAAqB,EACrB,OAAgC,EAChC,SAAe,EACf,KAA4B,EAC5B,QAAiB,EACjB,KAAc,EACd,qBAAqD;QAErD,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,qBAAqB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEnF,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;YAChD,MAAM,EAAE,aAAa;YACrB,WAAW;YACX,UAAU;YACV,KAAK;SACN,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,GAAG,aAAa,CAAC;QAC/B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;QAClC,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;QAChC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QAEtB,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC;QAChE,MAAM,QAAQ,GAAG,IAAA,yCAAa,EAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAA,gDAA+B,EAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAEhE,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;gBACxC,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,SAAS;gBACjB,eAAe;aAChB,CAAC,CAAC;YACH,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;YACtB,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;YACtC,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;YAEvB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAC9C,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CACzC;gBACE,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,cAAc,EAAE,iBAAiB;gBACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,aAAa,EAAE,GAAG,CAAC,aAAa;aACjC,EACD,QAAQ,CAAC,OAAO,CACjB,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,QAAQ,CAAC,WAAW;YAC5B,MAAM,EAAE,SAAS;YACjB,WAAW;YACX,eAAe;SAChB,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,WAAW,CAAC;QAClC,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC;QAC9B,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QAEvB,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,GAAqB;QACnD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE;YACxC,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,GAAG;YAChB,WAAW,EAAE,GAAG;SACjB,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,YAAY;IAEZ,uDAAuD;IAE/C,KAAK,CAAC,oBAAoB,CAAC,GAAqB,EAAE,SAAkB;QAC1E,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACjE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO;YACT,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAErC,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,EAAE,iBAAiB,KAAK,KAAK,CAAC;gBAC1E,MAAM,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,EAAE;oBACzD,aAAa,EAAE,GAAG;oBAClB,mBAAmB,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB;iBACtE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,mBAAmB,GAAG,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACpE,MAAM,KAAK,GAA4B,EAAE,aAAa,EAAE,GAAG,EAAE,mBAAmB,EAAE,CAAC;YAEnF,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,EAAE,2BAA2B,CAAC;YACtE,IAAI,SAAS,IAAI,mBAAmB,IAAI,SAAS,EAAE,CAAC;gBAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,EAAE,mBAAmB,IAAI,YAAY,CAAC;gBAC3E,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBACzB,KAAK,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;oBAC7B,KAAK,CAAC,WAAW,CAAC,GAAG,SAAS,CAAC;gBACjC,CAAC;qBAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;oBAC9B,KAAK,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;oBAC1B,KAAK,CAAC,WAAW,CAAC,GAAG,SAAS,CAAC;gBACjC,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,QAAQ,CAAC,EAAE,KAAK,QAAQ,CAAC,OAAO,CAAC,GAAG,+BAA+B;oBAC7E,IAAI,mBAAmB,IAAI,SAAS,eAAe,MAAM,EAAE,CAC9D,CAAC;gBACF,MAAM,IAAI,CAAC,uBAAuB,CAAC,QAAQ,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACvF,CAAC;YAED,MAAM,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,4CAA4C,GAAG,CAAC,EAAE,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAChH,CAAC;QACJ,CAAC;IACH,CAAC;IAED,YAAY;IAEZ,iCAAiC;IAEzB,KAAK,CAAC,cAAc,CAC1B,GAAqB,EACrB,OAAgC;QAEhC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,sBAAsB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAChE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,GAAqB;QAC5C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACpC,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,GAAqB;QACrD,MAAM,EAAE,GAAG,+BAA+B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACvD,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,GAYC,EACD,EAA+B;QAE/B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,GAAqB;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAClD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,QAAkC,EAClC,mBAA2B,EAC3B,SAAiB,EACjB,MAA+C;QAE/C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,uBAAuB,CAAC;gBAC/C,QAAQ;gBACR,mBAAmB;gBACnB,SAAS;gBACT,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;CAGF,CAAA;AAhuBY,sEAA6B;wCAA7B,6BAA6B;IADzC,IAAA,mBAAU,GAAE;6CAKkB,wDAAyB;QACrB,uEAAgC;QAC/B,yEAAiC;QAC1B,wEAAiC;QACvC,4DAA2B;QAC3B,iCAAqB;QACvB,sEAAgC;QAC9B,2EAAkC;QAC5C,2CAAmB;QACV,4EAAmC;QACnC,gEAA6B;QAC9B,2EAAkC;QACrC,gBAAS;GAhB5B,6BAA6B,CAguBzC;AAED,SAAS,+BAA+B,CACtC,MAAkC;IAElC,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,WAAW;YACd,OAAO,WAAW,CAAC;QACrB,KAAK,QAAQ,CAAC;QACd,KAAK,eAAe;YAClB,OAAO,QAAQ,CAAC;QAClB,KAAK,WAAW;YACd,OAAO,WAAW,CAAC;QACrB;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,SAAe;IAClC,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;IAC5B,OAAO;QACL,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,KAIjB,CAAC;QACF,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACrE,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB;gBACrF,MAAM,EAAE,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBAC3E,SAAS,EAAE,OAAO,SAAS,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;aACtF;YACD,KAAK,EAAE,SAAS,CAAC,KAAK;SACvB,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,KAMjB,CAAC;QACF,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACrE,OAAO,EACL,OAAO,SAAS,CAAC,OAAO,KAAK,QAAQ,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;oBACnE,CAAC,CAAC,SAAS,CAAC,OAAO;oBACnB,CAAC,CAAC,yBAAyB;gBAC/B,MAAM,EAAE,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBAC3E,SAAS,EAAE,OAAO,SAAS,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;aACtF;YACD,KAAK,EAAE,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SACzE,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;AAC/C,CAAC;AAED,YAAY"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { AXPBackgroundJob, AXPBackgroundJobSchedule, AXPBackgroundJobScheduleThresholdAction } from '@acorex-platform/framework-shared/background-job';
|
|
2
|
+
export interface BackgroundJobScheduleThresholdAlert {
|
|
3
|
+
schedule: AXPBackgroundJobSchedule;
|
|
4
|
+
consecutiveFailures: number;
|
|
5
|
+
threshold: number;
|
|
6
|
+
action: AXPBackgroundJobScheduleThresholdAction;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Server port for terminal job and schedule-threshold user notifications.
|
|
10
|
+
* Default is no-op; `background-job-management` binds a communication adapter.
|
|
11
|
+
*/
|
|
12
|
+
export declare abstract class BackgroundJobNotificationPort {
|
|
13
|
+
abstract notifyJobTerminal(job: AXPBackgroundJob): Promise<void>;
|
|
14
|
+
abstract notifyScheduleThreshold(alert: BackgroundJobScheduleThresholdAlert): Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
export declare class NoopBackgroundJobNotificationPort extends BackgroundJobNotificationPort {
|
|
17
|
+
notifyJobTerminal(): Promise<void>;
|
|
18
|
+
notifyScheduleThreshold(): Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=background-job-notification.port.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"background-job-notification.port.d.ts","sourceRoot":"","sources":["../../../../../../../../libs/framework/server/background-job/src/lib/notification/background-job-notification.port.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,gBAAgB,EAChB,wBAAwB,EACxB,uCAAuC,EACxC,MAAM,kDAAkD,CAAC;AAI1D,MAAM,WAAW,mCAAmC;IAClD,QAAQ,EAAE,wBAAwB,CAAC;IACnC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,uCAAuC,CAAC;CACjD;AAMD;;;GAGG;AACH,8BACsB,6BAA6B;IACjD,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAChE,QAAQ,CAAC,uBAAuB,CAAC,KAAK,EAAE,mCAAmC,GAAG,OAAO,CAAC,IAAI,CAAC;CAC5F;AAED,qBACa,iCAAkC,SAAQ,6BAA6B;IAC5E,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAIlC,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/C"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NoopBackgroundJobNotificationPort = exports.BackgroundJobNotificationPort = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const common_1 = require("@nestjs/common");
|
|
6
|
+
//#endregion
|
|
7
|
+
//#region ----- Notification port -----
|
|
8
|
+
/**
|
|
9
|
+
* Server port for terminal job and schedule-threshold user notifications.
|
|
10
|
+
* Default is no-op; `background-job-management` binds a communication adapter.
|
|
11
|
+
*/
|
|
12
|
+
let BackgroundJobNotificationPort = class BackgroundJobNotificationPort {
|
|
13
|
+
};
|
|
14
|
+
exports.BackgroundJobNotificationPort = BackgroundJobNotificationPort;
|
|
15
|
+
exports.BackgroundJobNotificationPort = BackgroundJobNotificationPort = tslib_1.__decorate([
|
|
16
|
+
(0, common_1.Injectable)()
|
|
17
|
+
], BackgroundJobNotificationPort);
|
|
18
|
+
let NoopBackgroundJobNotificationPort = class NoopBackgroundJobNotificationPort extends BackgroundJobNotificationPort {
|
|
19
|
+
async notifyJobTerminal() {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
async notifyScheduleThreshold() {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
exports.NoopBackgroundJobNotificationPort = NoopBackgroundJobNotificationPort;
|
|
27
|
+
exports.NoopBackgroundJobNotificationPort = NoopBackgroundJobNotificationPort = tslib_1.__decorate([
|
|
28
|
+
(0, common_1.Injectable)()
|
|
29
|
+
], NoopBackgroundJobNotificationPort);
|
|
30
|
+
//#endregion
|
|
31
|
+
//# sourceMappingURL=background-job-notification.port.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"background-job-notification.port.js","sourceRoot":"","sources":["../../../../../../../../libs/framework/server/background-job/src/lib/notification/background-job-notification.port.ts"],"names":[],"mappings":";;;;AAAA,2CAA4C;AAgB5C,YAAY;AAEZ,uCAAuC;AAEvC;;;GAGG;AAEI,IAAe,6BAA6B,GAA5C,MAAe,6BAA6B;CAGlD,CAAA;AAHqB,sEAA6B;wCAA7B,6BAA6B;IADlD,IAAA,mBAAU,GAAE;GACS,6BAA6B,CAGlD;AAGM,IAAM,iCAAiC,GAAvC,MAAM,iCAAkC,SAAQ,6BAA6B;IAClF,KAAK,CAAC,iBAAiB;QACrB,OAAO;IACT,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,OAAO;IACT,CAAC;CACF,CAAA;AARY,8EAAiC;4CAAjC,iCAAiC;IAD7C,IAAA,mBAAU,GAAE;GACA,iCAAiC,CAQ7C;AAED,YAAY"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { OnApplicationBootstrap, OnModuleDestroy } from '@nestjs/common';
|
|
2
|
+
import { AXPAppStartUpService } from '@acorex-platform/framework-server/core';
|
|
3
|
+
import { BackgroundJobExecutionService } from '../execution/background-job-execution.service';
|
|
4
|
+
import { BackgroundJobAttemptStoreService } from '../store/background-job-attempt-store.service';
|
|
5
|
+
import { BackgroundJobSchemaBootstrap } from '../store/background-job-schema.bootstrap';
|
|
6
|
+
/**
|
|
7
|
+
* Marks `running` attempts with a stale (or missing) heartbeat as `abandoned`,
|
|
8
|
+
* then applies the job retry policy. Skips attempts still executing in this process.
|
|
9
|
+
*/
|
|
10
|
+
export declare class BackgroundJobRecoveryService implements OnApplicationBootstrap, OnModuleDestroy {
|
|
11
|
+
private readonly attempts;
|
|
12
|
+
private readonly execution;
|
|
13
|
+
private readonly schemaBootstrap;
|
|
14
|
+
private readonly logger;
|
|
15
|
+
private timer;
|
|
16
|
+
private recovering;
|
|
17
|
+
constructor(attempts: BackgroundJobAttemptStoreService, execution: BackgroundJobExecutionService, schemaBootstrap: BackgroundJobSchemaBootstrap, appStartUp: AXPAppStartUpService);
|
|
18
|
+
onApplicationBootstrap(): void;
|
|
19
|
+
onModuleDestroy(): void;
|
|
20
|
+
private recoverStaleAttempts;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=background-job-recovery.service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"background-job-recovery.service.d.ts","sourceRoot":"","sources":["../../../../../../../../libs/framework/server/background-job/src/lib/queue/background-job-recovery.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,sBAAsB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC7F,OAAO,EAEL,oBAAoB,EACrB,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAAE,6BAA6B,EAAE,MAAM,+CAA+C,CAAC;AAC9F,OAAO,EAAE,gCAAgC,EAAE,MAAM,+CAA+C,CAAC;AACjG,OAAO,EAAE,4BAA4B,EAAE,MAAM,0CAA0C,CAAC;AAexF;;;GAGG;AACH,qBACa,4BAA6B,YAAW,sBAAsB,EAAE,eAAe;IAMxF,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAPlC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiD;IACxE,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,UAAU,CAAS;gBAGR,QAAQ,EAAE,gCAAgC,EAC1C,SAAS,EAAE,6BAA6B,EACxC,eAAe,EAAE,4BAA4B,EAC9D,UAAU,EAAE,oBAAoB;IAgBlC,sBAAsB,IAAI,IAAI;IAW9B,eAAe,IAAI,IAAI;YAOT,oBAAoB;CAmCnC"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var BackgroundJobRecoveryService_1;
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.BackgroundJobRecoveryService = void 0;
|
|
5
|
+
const tslib_1 = require("tslib");
|
|
6
|
+
const common_1 = require("@nestjs/common");
|
|
7
|
+
const core_1 = require("@acorex-platform/framework-server/core");
|
|
8
|
+
const background_job_execution_service_1 = require("../execution/background-job-execution.service");
|
|
9
|
+
const background_job_attempt_store_service_1 = require("../store/background-job-attempt-store.service");
|
|
10
|
+
const background_job_schema_bootstrap_1 = require("../store/background-job-schema.bootstrap");
|
|
11
|
+
//#region ----- Constants -----
|
|
12
|
+
const RECOVERY_INTERVAL_MS = 60 * 1000;
|
|
13
|
+
/** Running attempts with no heartbeat for this long are treated as abandoned. */
|
|
14
|
+
const STALE_HEARTBEAT_MS = 90 * 1000;
|
|
15
|
+
/** When heartbeat was never written, fall back to startedAt older than the default lock TTL. */
|
|
16
|
+
const NO_HEARTBEAT_FALLBACK_MS = 15 * 60 * 1000;
|
|
17
|
+
const RECOVERY_BATCH_SIZE = 50;
|
|
18
|
+
//#endregion
|
|
19
|
+
//#region ----- Abandoned-attempt recovery -----
|
|
20
|
+
/**
|
|
21
|
+
* Marks `running` attempts with a stale (or missing) heartbeat as `abandoned`,
|
|
22
|
+
* then applies the job retry policy. Skips attempts still executing in this process.
|
|
23
|
+
*/
|
|
24
|
+
let BackgroundJobRecoveryService = BackgroundJobRecoveryService_1 = class BackgroundJobRecoveryService {
|
|
25
|
+
constructor(attempts, execution, schemaBootstrap, appStartUp) {
|
|
26
|
+
this.attempts = attempts;
|
|
27
|
+
this.execution = execution;
|
|
28
|
+
this.schemaBootstrap = schemaBootstrap;
|
|
29
|
+
this.logger = new common_1.Logger(BackgroundJobRecoveryService_1.name);
|
|
30
|
+
this.recovering = false;
|
|
31
|
+
if (process.env['BACKGROUND_JOB_WORKER_ENABLED'] === 'false') {
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
appStartUp.registerTask({
|
|
35
|
+
name: 'background-job-abandoned-recovery',
|
|
36
|
+
priority: core_1.AXP_APP_STARTUP_PRIORITY.postSchemaServices,
|
|
37
|
+
run: async () => {
|
|
38
|
+
await this.schemaBootstrap.registerAllSchemas({ throwOnError: true });
|
|
39
|
+
await this.recoverStaleAttempts();
|
|
40
|
+
},
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
onApplicationBootstrap() {
|
|
44
|
+
if (process.env['BACKGROUND_JOB_WORKER_ENABLED'] === 'false') {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
this.timer = setInterval(() => {
|
|
48
|
+
void this.recoverStaleAttempts();
|
|
49
|
+
}, RECOVERY_INTERVAL_MS);
|
|
50
|
+
this.timer.unref?.();
|
|
51
|
+
}
|
|
52
|
+
onModuleDestroy() {
|
|
53
|
+
if (this.timer) {
|
|
54
|
+
clearInterval(this.timer);
|
|
55
|
+
this.timer = undefined;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
async recoverStaleAttempts() {
|
|
59
|
+
if (this.recovering) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
this.recovering = true;
|
|
63
|
+
try {
|
|
64
|
+
const now = Date.now();
|
|
65
|
+
const attempts = await this.attempts.listStaleRunning({
|
|
66
|
+
heartbeatStaleBeforeIso: new Date(now - STALE_HEARTBEAT_MS).toISOString(),
|
|
67
|
+
startedBeforeIso: new Date(now - NO_HEARTBEAT_FALLBACK_MS).toISOString(),
|
|
68
|
+
take: RECOVERY_BATCH_SIZE,
|
|
69
|
+
});
|
|
70
|
+
if (attempts.length === 0) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
let recovered = 0;
|
|
74
|
+
for (const attempt of attempts) {
|
|
75
|
+
const marked = await this.execution.recoverAbandonedAttempt(attempt);
|
|
76
|
+
if (marked) {
|
|
77
|
+
recovered += 1;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (recovered > 0) {
|
|
81
|
+
this.logger.warn(`Recovered ${recovered} abandoned background job attempt(s)`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
this.logger.warn(`Abandoned-attempt recovery failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
this.recovering = false;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
exports.BackgroundJobRecoveryService = BackgroundJobRecoveryService;
|
|
93
|
+
exports.BackgroundJobRecoveryService = BackgroundJobRecoveryService = BackgroundJobRecoveryService_1 = tslib_1.__decorate([
|
|
94
|
+
(0, common_1.Injectable)(),
|
|
95
|
+
tslib_1.__metadata("design:paramtypes", [background_job_attempt_store_service_1.BackgroundJobAttemptStoreService,
|
|
96
|
+
background_job_execution_service_1.BackgroundJobExecutionService,
|
|
97
|
+
background_job_schema_bootstrap_1.BackgroundJobSchemaBootstrap,
|
|
98
|
+
core_1.AXPAppStartUpService])
|
|
99
|
+
], BackgroundJobRecoveryService);
|
|
100
|
+
//#endregion
|
|
101
|
+
//# sourceMappingURL=background-job-recovery.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"background-job-recovery.service.js","sourceRoot":"","sources":["../../../../../../../../libs/framework/server/background-job/src/lib/queue/background-job-recovery.service.ts"],"names":[],"mappings":";;;;;AAAA,2CAA6F;AAC7F,iEAGgD;AAChD,oGAA8F;AAC9F,wGAAiG;AACjG,8FAAwF;AAExF,+BAA+B;AAE/B,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,CAAC;AACvC,iFAAiF;AACjF,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;AACrC,gGAAgG;AAChG,MAAM,wBAAwB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAChD,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,YAAY;AAEZ,gDAAgD;AAEhD;;;GAGG;AAEI,IAAM,4BAA4B,oCAAlC,MAAM,4BAA4B;IAKvC,YACmB,QAA0C,EAC1C,SAAwC,EACxC,eAA6C,EAC9D,UAAgC;QAHf,aAAQ,GAAR,QAAQ,CAAkC;QAC1C,cAAS,GAAT,SAAS,CAA+B;QACxC,oBAAe,GAAf,eAAe,CAA8B;QAP/C,WAAM,GAAG,IAAI,eAAM,CAAC,8BAA4B,CAAC,IAAI,CAAC,CAAC;QAEhE,eAAU,GAAG,KAAK,CAAC;QAQzB,IAAI,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,KAAK,OAAO,EAAE,CAAC;YAC7D,OAAO;QACT,CAAC;QAED,UAAU,CAAC,YAAY,CAAC;YACtB,IAAI,EAAE,mCAAmC;YACzC,QAAQ,EAAE,+BAAwB,CAAC,kBAAkB;YACrD,GAAG,EAAE,KAAK,IAAI,EAAE;gBACd,MAAM,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;gBACtE,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACpC,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB;QACpB,IAAI,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,KAAK,OAAO,EAAE,CAAC;YAC7D,OAAO;QACT,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,KAAK,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnC,CAAC,EAAE,oBAAoB,CAAC,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACvB,CAAC;IAED,eAAe;QACb,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB;QAChC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBACpD,uBAAuB,EAAE,IAAI,IAAI,CAAC,GAAG,GAAG,kBAAkB,CAAC,CAAC,WAAW,EAAE;gBACzE,gBAAgB,EAAE,IAAI,IAAI,CAAC,GAAG,GAAG,wBAAwB,CAAC,CAAC,WAAW,EAAE;gBACxE,IAAI,EAAE,mBAAmB;aAC1B,CAAC,CAAC;YACH,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,OAAO;YACT,CAAC;YAED,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;gBACrE,IAAI,MAAM,EAAE,CAAC;oBACX,SAAS,IAAI,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;YAED,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,sCAAsC,CAAC,CAAC;YACjF,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC1B,CAAC;IACH,CAAC;CACF,CAAA;AA9EY,oEAA4B;uCAA5B,4BAA4B;IADxC,IAAA,mBAAU,GAAE;6CAOkB,uEAAgC;QAC/B,gEAA6B;QACvB,8DAA4B;QAClD,2BAAoB;GATvB,4BAA4B,CA8ExC;AAED,YAAY"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"background-job-schedule-scanner.service.d.ts","sourceRoot":"","sources":["../../../../../../../../libs/framework/server/background-job/src/lib/schedule/background-job-schedule-scanner.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,sBAAsB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAkB7F,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,iCAAiC,EAAE,MAAM,gDAAgD,CAAC;AACnG,OAAO,EAAE,yBAAyB,EAAE,MAAM,uCAAuC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAcjE;;;;;;;GAOG;AACH,qBACa,mCACX,YAAW,sBAAsB,EAAE,eAAe;IAOhD,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IARvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwD;IAC/E,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,QAAQ,CAAS;gBAGN,aAAa,EAAE,iCAAiC,EAChD,QAAQ,EAAE,yBAAyB,EACnC,UAAU,EAAE,oBAAoB,EAChC,IAAI,EAAE,mBAAmB;IAG5C,sBAAsB,IAAI,IAAI;IAS9B,8EAA8E;YAChE,qBAAqB;IAoDnC,OAAO,CAAC,2BAA2B;IAyBnC,qFAAqF;YACvE,6BAA6B;IAwB3C,eAAe,IAAI,IAAI;IAMjB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"background-job-schedule-scanner.service.d.ts","sourceRoot":"","sources":["../../../../../../../../libs/framework/server/background-job/src/lib/schedule/background-job-schedule-scanner.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,sBAAsB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAkB7F,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,iCAAiC,EAAE,MAAM,gDAAgD,CAAC;AACnG,OAAO,EAAE,yBAAyB,EAAE,MAAM,uCAAuC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAcjE;;;;;;;GAOG;AACH,qBACa,mCACX,YAAW,sBAAsB,EAAE,eAAe;IAOhD,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IARvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwD;IAC/E,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,QAAQ,CAAS;gBAGN,aAAa,EAAE,iCAAiC,EAChD,QAAQ,EAAE,yBAAyB,EACnC,UAAU,EAAE,oBAAoB,EAChC,IAAI,EAAE,mBAAmB;IAG5C,sBAAsB,IAAI,IAAI;IAS9B,8EAA8E;YAChE,qBAAqB;IAoDnC,OAAO,CAAC,2BAA2B;IAyBnC,qFAAqF;YACvE,6BAA6B;IAwB3C,eAAe,IAAI,IAAI;IAMjB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAuCb,YAAY;YAqFZ,eAAe;IAI7B,OAAO,CAAC,oBAAoB;CAc7B"}
|
|
@@ -128,22 +128,28 @@ let BackgroundJobScheduleScannerService = BackgroundJobScheduleScannerService_1
|
|
|
128
128
|
return;
|
|
129
129
|
}
|
|
130
130
|
this.scanning = true;
|
|
131
|
-
const leaderLock = await this.lock.acquire(background_job_1.AXP_BACKGROUND_JOB_SCHEDULER_LEADER_LOCK, LEADER_LOCK_TTL_MS);
|
|
132
|
-
if (!leaderLock) {
|
|
133
|
-
this.scanning = false;
|
|
134
|
-
return;
|
|
135
|
-
}
|
|
136
131
|
try {
|
|
137
|
-
const
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
await this.fireSchedule(schedule, now).catch((error) => {
|
|
141
|
-
this.logger.error(`Failed to fire schedule ${schedule.id} (${schedule.command.key}): ${error instanceof Error ? error.message : String(error)}`);
|
|
142
|
-
});
|
|
132
|
+
const leaderLock = await this.lock.acquire(background_job_1.AXP_BACKGROUND_JOB_SCHEDULER_LEADER_LOCK, LEADER_LOCK_TTL_MS);
|
|
133
|
+
if (!leaderLock) {
|
|
134
|
+
return;
|
|
143
135
|
}
|
|
136
|
+
try {
|
|
137
|
+
const now = new Date();
|
|
138
|
+
const due = await this.scheduleStore.listDue(now);
|
|
139
|
+
for (const schedule of due) {
|
|
140
|
+
await this.fireSchedule(schedule, now).catch((error) => {
|
|
141
|
+
this.logger.error(`Failed to fire schedule ${schedule.id} (${schedule.command.key}): ${error instanceof Error ? error.message : String(error)}`);
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
finally {
|
|
146
|
+
await this.lock.release(leaderLock);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
this.logger.warn(`Schedule scan skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
144
151
|
}
|
|
145
152
|
finally {
|
|
146
|
-
await this.lock.release(leaderLock);
|
|
147
153
|
this.scanning = false;
|
|
148
154
|
}
|
|
149
155
|
}
|