@nest-batch/core 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +368 -0
- package/dist/src/adapters/in-process.adapter.d.ts +140 -0
- package/dist/src/adapters/in-process.adapter.d.ts.map +1 -0
- package/dist/src/adapters/in-process.adapter.js +86 -0
- package/dist/src/adapters/in-process.adapter.js.map +1 -0
- package/dist/src/adapters/index.d.ts +18 -0
- package/dist/src/adapters/index.d.ts.map +1 -0
- package/dist/src/adapters/index.js +35 -0
- package/dist/src/adapters/index.js.map +1 -0
- package/dist/src/builder/batch-builder.d.ts +26 -0
- package/dist/src/builder/batch-builder.d.ts.map +1 -0
- package/dist/src/builder/batch-builder.js +25 -0
- package/dist/src/builder/batch-builder.js.map +1 -0
- package/dist/src/builder/flow-builder.d.ts +59 -0
- package/dist/src/builder/flow-builder.d.ts.map +1 -0
- package/dist/src/builder/flow-builder.js +115 -0
- package/dist/src/builder/flow-builder.js.map +1 -0
- package/dist/src/builder/index.d.ts +5 -0
- package/dist/src/builder/index.d.ts.map +1 -0
- package/dist/src/builder/index.js +23 -0
- package/dist/src/builder/index.js.map +1 -0
- package/dist/src/builder/job-builder.d.ts +76 -0
- package/dist/src/builder/job-builder.d.ts.map +1 -0
- package/dist/src/builder/job-builder.js +166 -0
- package/dist/src/builder/job-builder.js.map +1 -0
- package/dist/src/builder/step-builder.d.ts +74 -0
- package/dist/src/builder/step-builder.d.ts.map +1 -0
- package/dist/src/builder/step-builder.js +144 -0
- package/dist/src/builder/step-builder.js.map +1 -0
- package/dist/src/compiler/builder-types.d.ts +20 -0
- package/dist/src/compiler/builder-types.d.ts.map +1 -0
- package/dist/src/compiler/builder-types.js +6 -0
- package/dist/src/compiler/builder-types.js.map +1 -0
- package/dist/src/compiler/definition-compiler.d.ts +99 -0
- package/dist/src/compiler/definition-compiler.d.ts.map +1 -0
- package/dist/src/compiler/definition-compiler.js +257 -0
- package/dist/src/compiler/definition-compiler.js.map +1 -0
- package/dist/src/compiler/index.d.ts +3 -0
- package/dist/src/compiler/index.d.ts.map +1 -0
- package/dist/src/compiler/index.js +21 -0
- package/dist/src/compiler/index.js.map +1 -0
- package/dist/src/core/errors.d.ts +77 -0
- package/dist/src/core/errors.d.ts.map +1 -0
- package/dist/src/core/errors.js +170 -0
- package/dist/src/core/errors.js.map +1 -0
- package/dist/src/core/execution-context/index.d.ts +4 -0
- package/dist/src/core/execution-context/index.d.ts.map +1 -0
- package/dist/src/core/execution-context/index.js +22 -0
- package/dist/src/core/execution-context/index.js.map +1 -0
- package/dist/src/core/execution-context/json-value.d.ts +5 -0
- package/dist/src/core/execution-context/json-value.d.ts.map +1 -0
- package/dist/src/core/execution-context/json-value.js +6 -0
- package/dist/src/core/execution-context/json-value.js.map +1 -0
- package/dist/src/core/execution-context/serializer.d.ts +4 -0
- package/dist/src/core/execution-context/serializer.d.ts.map +1 -0
- package/dist/src/core/execution-context/serializer.js +34 -0
- package/dist/src/core/execution-context/serializer.js.map +1 -0
- package/dist/src/core/execution-context/validator.d.ts +18 -0
- package/dist/src/core/execution-context/validator.d.ts.map +1 -0
- package/dist/src/core/execution-context/validator.js +90 -0
- package/dist/src/core/execution-context/validator.js.map +1 -0
- package/dist/src/core/index.d.ts +8 -0
- package/dist/src/core/index.d.ts.map +1 -0
- package/dist/src/core/index.js +26 -0
- package/dist/src/core/index.js.map +1 -0
- package/dist/src/core/ir/decider-definition.d.ts +20 -0
- package/dist/src/core/ir/decider-definition.d.ts.map +1 -0
- package/dist/src/core/ir/decider-definition.js +6 -0
- package/dist/src/core/ir/decider-definition.js.map +1 -0
- package/dist/src/core/ir/index.d.ts +8 -0
- package/dist/src/core/ir/index.d.ts.map +1 -0
- package/dist/src/core/ir/index.js +26 -0
- package/dist/src/core/ir/index.js.map +1 -0
- package/dist/src/core/ir/job-definition.d.ts +15 -0
- package/dist/src/core/ir/job-definition.d.ts.map +1 -0
- package/dist/src/core/ir/job-definition.js +6 -0
- package/dist/src/core/ir/job-definition.js.map +1 -0
- package/dist/src/core/ir/listener-definition.d.ts +10 -0
- package/dist/src/core/ir/listener-definition.d.ts.map +1 -0
- package/dist/src/core/ir/listener-definition.js +6 -0
- package/dist/src/core/ir/listener-definition.js.map +1 -0
- package/dist/src/core/ir/policy-config.d.ts +24 -0
- package/dist/src/core/ir/policy-config.d.ts.map +1 -0
- package/dist/src/core/ir/policy-config.js +6 -0
- package/dist/src/core/ir/policy-config.js.map +1 -0
- package/dist/src/core/ir/refs.d.ts +42 -0
- package/dist/src/core/ir/refs.d.ts.map +1 -0
- package/dist/src/core/ir/refs.js +18 -0
- package/dist/src/core/ir/refs.js.map +1 -0
- package/dist/src/core/ir/step-definition.d.ts +59 -0
- package/dist/src/core/ir/step-definition.d.ts.map +1 -0
- package/dist/src/core/ir/step-definition.js +6 -0
- package/dist/src/core/ir/step-definition.js.map +1 -0
- package/dist/src/core/ir/transition-definition.d.ts +8 -0
- package/dist/src/core/ir/transition-definition.d.ts.map +1 -0
- package/dist/src/core/ir/transition-definition.js +6 -0
- package/dist/src/core/ir/transition-definition.js.map +1 -0
- package/dist/src/core/item/index.d.ts +2 -0
- package/dist/src/core/item/index.d.ts.map +1 -0
- package/dist/src/core/item/index.js +20 -0
- package/dist/src/core/item/index.js.map +1 -0
- package/dist/src/core/item/interfaces.d.ts +64 -0
- package/dist/src/core/item/interfaces.d.ts.map +1 -0
- package/dist/src/core/item/interfaces.js +6 -0
- package/dist/src/core/item/interfaces.js.map +1 -0
- package/dist/src/core/repository/index.d.ts +3 -0
- package/dist/src/core/repository/index.d.ts.map +1 -0
- package/dist/src/core/repository/index.js +21 -0
- package/dist/src/core/repository/index.js.map +1 -0
- package/dist/src/core/repository/job-repository.d.ts +60 -0
- package/dist/src/core/repository/job-repository.d.ts.map +1 -0
- package/dist/src/core/repository/job-repository.js +27 -0
- package/dist/src/core/repository/job-repository.js.map +1 -0
- package/dist/src/core/repository/types.d.ts +84 -0
- package/dist/src/core/repository/types.d.ts.map +1 -0
- package/dist/src/core/repository/types.js +6 -0
- package/dist/src/core/repository/types.js.map +1 -0
- package/dist/src/core/status.d.ts +29 -0
- package/dist/src/core/status.d.ts.map +1 -0
- package/dist/src/core/status.js +58 -0
- package/dist/src/core/status.js.map +1 -0
- package/dist/src/core/transaction/index.d.ts +2 -0
- package/dist/src/core/transaction/index.d.ts.map +1 -0
- package/dist/src/core/transaction/index.js +20 -0
- package/dist/src/core/transaction/index.js.map +1 -0
- package/dist/src/core/transaction/transaction-manager.d.ts +8 -0
- package/dist/src/core/transaction/transaction-manager.d.ts.map +1 -0
- package/dist/src/core/transaction/transaction-manager.js +14 -0
- package/dist/src/core/transaction/transaction-manager.js.map +1 -0
- package/dist/src/core/validation/definition-validator.d.ts +46 -0
- package/dist/src/core/validation/definition-validator.d.ts.map +1 -0
- package/dist/src/core/validation/definition-validator.js +177 -0
- package/dist/src/core/validation/definition-validator.js.map +1 -0
- package/dist/src/core/validation/index.d.ts +2 -0
- package/dist/src/core/validation/index.d.ts.map +1 -0
- package/dist/src/core/validation/index.js +20 -0
- package/dist/src/core/validation/index.js.map +1 -0
- package/dist/src/decorators/constants.d.ts +10 -0
- package/dist/src/decorators/constants.d.ts.map +1 -0
- package/dist/src/decorators/constants.js +50 -0
- package/dist/src/decorators/constants.js.map +1 -0
- package/dist/src/decorators/flow.decorator.d.ts +25 -0
- package/dist/src/decorators/flow.decorator.d.ts.map +1 -0
- package/dist/src/decorators/flow.decorator.js +19 -0
- package/dist/src/decorators/flow.decorator.js.map +1 -0
- package/dist/src/decorators/index.d.ts +8 -0
- package/dist/src/decorators/index.d.ts.map +1 -0
- package/dist/src/decorators/index.js +26 -0
- package/dist/src/decorators/index.js.map +1 -0
- package/dist/src/decorators/item.decorators.d.ts +32 -0
- package/dist/src/decorators/item.decorators.d.ts.map +1 -0
- package/dist/src/decorators/item.decorators.js +40 -0
- package/dist/src/decorators/item.decorators.js.map +1 -0
- package/dist/src/decorators/job.decorator.d.ts +11 -0
- package/dist/src/decorators/job.decorator.d.ts.map +1 -0
- package/dist/src/decorators/job.decorator.js +17 -0
- package/dist/src/decorators/job.decorator.js.map +1 -0
- package/dist/src/decorators/listener.decorators.d.ts +56 -0
- package/dist/src/decorators/listener.decorators.d.ts.map +1 -0
- package/dist/src/decorators/listener.decorators.js +157 -0
- package/dist/src/decorators/listener.decorators.js.map +1 -0
- package/dist/src/decorators/step.decorator.d.ts +25 -0
- package/dist/src/decorators/step.decorator.d.ts.map +1 -0
- package/dist/src/decorators/step.decorator.js +21 -0
- package/dist/src/decorators/step.decorator.js.map +1 -0
- package/dist/src/decorators/tasklet.decorator.d.ts +7 -0
- package/dist/src/decorators/tasklet.decorator.d.ts.map +1 -0
- package/dist/src/decorators/tasklet.decorator.js +21 -0
- package/dist/src/decorators/tasklet.decorator.js.map +1 -0
- package/dist/src/execution/batch-worker-runner.d.ts +27 -0
- package/dist/src/execution/batch-worker-runner.d.ts.map +1 -0
- package/dist/src/execution/batch-worker-runner.js +147 -0
- package/dist/src/execution/batch-worker-runner.js.map +1 -0
- package/dist/src/execution/chunk-step-executor.d.ts +86 -0
- package/dist/src/execution/chunk-step-executor.d.ts.map +1 -0
- package/dist/src/execution/chunk-step-executor.js +482 -0
- package/dist/src/execution/chunk-step-executor.js.map +1 -0
- package/dist/src/execution/execution-strategy.d.ts +110 -0
- package/dist/src/execution/execution-strategy.d.ts.map +1 -0
- package/dist/src/execution/execution-strategy.js +13 -0
- package/dist/src/execution/execution-strategy.js.map +1 -0
- package/dist/src/execution/external-task-execution-strategy.d.ts +36 -0
- package/dist/src/execution/external-task-execution-strategy.d.ts.map +1 -0
- package/dist/src/execution/external-task-execution-strategy.js +97 -0
- package/dist/src/execution/external-task-execution-strategy.js.map +1 -0
- package/dist/src/execution/in-process-execution-strategy.d.ts +129 -0
- package/dist/src/execution/in-process-execution-strategy.d.ts.map +1 -0
- package/dist/src/execution/in-process-execution-strategy.js +141 -0
- package/dist/src/execution/in-process-execution-strategy.js.map +1 -0
- package/dist/src/execution/index.d.ts +14 -0
- package/dist/src/execution/index.d.ts.map +1 -0
- package/dist/src/execution/index.js +32 -0
- package/dist/src/execution/index.js.map +1 -0
- package/dist/src/execution/job-executor.d.ts +145 -0
- package/dist/src/execution/job-executor.d.ts.map +1 -0
- package/dist/src/execution/job-executor.js +475 -0
- package/dist/src/execution/job-executor.js.map +1 -0
- package/dist/src/execution/job-explorer.d.ts +15 -0
- package/dist/src/execution/job-explorer.d.ts.map +1 -0
- package/dist/src/execution/job-explorer.js +84 -0
- package/dist/src/execution/job-explorer.js.map +1 -0
- package/dist/src/execution/job-key.d.ts +3 -0
- package/dist/src/execution/job-key.d.ts.map +1 -0
- package/dist/src/execution/job-key.js +43 -0
- package/dist/src/execution/job-key.js.map +1 -0
- package/dist/src/execution/job-launcher.d.ts +75 -0
- package/dist/src/execution/job-launcher.d.ts.map +1 -0
- package/dist/src/execution/job-launcher.js +112 -0
- package/dist/src/execution/job-launcher.js.map +1 -0
- package/dist/src/execution/job-operator.d.ts +22 -0
- package/dist/src/execution/job-operator.d.ts.map +1 -0
- package/dist/src/execution/job-operator.js +125 -0
- package/dist/src/execution/job-operator.js.map +1 -0
- package/dist/src/execution/listener-invoker.d.ts +164 -0
- package/dist/src/execution/listener-invoker.d.ts.map +1 -0
- package/dist/src/execution/listener-invoker.js +246 -0
- package/dist/src/execution/listener-invoker.js.map +1 -0
- package/dist/src/execution/ref-resolver.d.ts +40 -0
- package/dist/src/execution/ref-resolver.d.ts.map +1 -0
- package/dist/src/execution/ref-resolver.js +41 -0
- package/dist/src/execution/ref-resolver.js.map +1 -0
- package/dist/src/execution/tasklet-step-executor.d.ts +79 -0
- package/dist/src/execution/tasklet-step-executor.d.ts.map +1 -0
- package/dist/src/execution/tasklet-step-executor.js +138 -0
- package/dist/src/execution/tasklet-step-executor.js.map +1 -0
- package/dist/src/explorer/batch-explorer.d.ts +138 -0
- package/dist/src/explorer/batch-explorer.d.ts.map +1 -0
- package/dist/src/explorer/batch-explorer.js +167 -0
- package/dist/src/explorer/batch-explorer.js.map +1 -0
- package/dist/src/explorer/index.d.ts +2 -0
- package/dist/src/explorer/index.d.ts.map +1 -0
- package/dist/src/explorer/index.js +20 -0
- package/dist/src/explorer/index.js.map +1 -0
- package/dist/src/flow/flow-evaluator.d.ts +30 -0
- package/dist/src/flow/flow-evaluator.d.ts.map +1 -0
- package/dist/src/flow/flow-evaluator.js +80 -0
- package/dist/src/flow/flow-evaluator.js.map +1 -0
- package/dist/src/flow/index.d.ts +2 -0
- package/dist/src/flow/index.d.ts.map +1 -0
- package/dist/src/flow/index.js +20 -0
- package/dist/src/flow/index.js.map +1 -0
- package/dist/src/index.d.ts +18 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +90 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/io/checkpoint.d.ts +7 -0
- package/dist/src/io/checkpoint.d.ts.map +1 -0
- package/dist/src/io/checkpoint.js +56 -0
- package/dist/src/io/checkpoint.js.map +1 -0
- package/dist/src/io/database.d.ts +50 -0
- package/dist/src/io/database.d.ts.map +1 -0
- package/dist/src/io/database.js +108 -0
- package/dist/src/io/database.js.map +1 -0
- package/dist/src/io/file-readers.d.ts +54 -0
- package/dist/src/io/file-readers.d.ts.map +1 -0
- package/dist/src/io/file-readers.js +167 -0
- package/dist/src/io/file-readers.js.map +1 -0
- package/dist/src/io/file-writers.d.ts +31 -0
- package/dist/src/io/file-writers.d.ts.map +1 -0
- package/dist/src/io/file-writers.js +80 -0
- package/dist/src/io/file-writers.js.map +1 -0
- package/dist/src/io/index.d.ts +6 -0
- package/dist/src/io/index.d.ts.map +1 -0
- package/dist/src/io/index.js +24 -0
- package/dist/src/io/index.js.map +1 -0
- package/dist/src/io/s3.d.ts +50 -0
- package/dist/src/io/s3.d.ts.map +1 -0
- package/dist/src/io/s3.js +96 -0
- package/dist/src/io/s3.js.map +1 -0
- package/dist/src/listeners/builtin-listeners.d.ts +77 -0
- package/dist/src/listeners/builtin-listeners.d.ts.map +1 -0
- package/dist/src/listeners/builtin-listeners.js +108 -0
- package/dist/src/listeners/builtin-listeners.js.map +1 -0
- package/dist/src/listeners/index.d.ts +8 -0
- package/dist/src/listeners/index.d.ts.map +1 -0
- package/dist/src/listeners/index.js +25 -0
- package/dist/src/listeners/index.js.map +1 -0
- package/dist/src/module/adapter-options.d.ts +39 -0
- package/dist/src/module/adapter-options.d.ts.map +1 -0
- package/dist/src/module/adapter-options.js +34 -0
- package/dist/src/module/adapter-options.js.map +1 -0
- package/dist/src/module/adapter.d.ts +157 -0
- package/dist/src/module/adapter.d.ts.map +1 -0
- package/dist/src/module/adapter.js +80 -0
- package/dist/src/module/adapter.js.map +1 -0
- package/dist/src/module/batch-schedule-registry.d.ts +110 -0
- package/dist/src/module/batch-schedule-registry.d.ts.map +1 -0
- package/dist/src/module/batch-schedule-registry.js +0 -0
- package/dist/src/module/batch-schedule-registry.js.map +1 -0
- package/dist/src/module/index.d.ts +14 -0
- package/dist/src/module/index.d.ts.map +1 -0
- package/dist/src/module/index.js +31 -0
- package/dist/src/module/index.js.map +1 -0
- package/dist/src/module/nest-batch.module.d.ts +236 -0
- package/dist/src/module/nest-batch.module.d.ts.map +1 -0
- package/dist/src/module/nest-batch.module.js +475 -0
- package/dist/src/module/nest-batch.module.js.map +1 -0
- package/dist/src/module/tokens.d.ts +83 -0
- package/dist/src/module/tokens.d.ts.map +1 -0
- package/dist/src/module/tokens.js +58 -0
- package/dist/src/module/tokens.js.map +1 -0
- package/dist/src/observability/event-types.d.ts +55 -0
- package/dist/src/observability/event-types.d.ts.map +1 -0
- package/dist/src/observability/event-types.js +36 -0
- package/dist/src/observability/event-types.js.map +1 -0
- package/dist/src/observability/exporters.d.ts +35 -0
- package/dist/src/observability/exporters.d.ts.map +1 -0
- package/dist/src/observability/exporters.js +93 -0
- package/dist/src/observability/exporters.js.map +1 -0
- package/dist/src/observability/index.d.ts +3 -0
- package/dist/src/observability/index.d.ts.map +1 -0
- package/dist/src/observability/index.js +21 -0
- package/dist/src/observability/index.js.map +1 -0
- package/dist/src/partition-helpers.d.ts +127 -0
- package/dist/src/partition-helpers.d.ts.map +1 -0
- package/dist/src/partition-helpers.js +136 -0
- package/dist/src/partition-helpers.js.map +1 -0
- package/dist/src/policies/backoff.d.ts +3 -0
- package/dist/src/policies/backoff.d.ts.map +1 -0
- package/dist/src/policies/backoff.js +34 -0
- package/dist/src/policies/backoff.js.map +1 -0
- package/dist/src/policies/index.d.ts +4 -0
- package/dist/src/policies/index.d.ts.map +1 -0
- package/dist/src/policies/index.js +22 -0
- package/dist/src/policies/index.js.map +1 -0
- package/dist/src/policies/retry-policy.d.ts +13 -0
- package/dist/src/policies/retry-policy.d.ts.map +1 -0
- package/dist/src/policies/retry-policy.js +55 -0
- package/dist/src/policies/retry-policy.js.map +1 -0
- package/dist/src/policies/skip-policy.d.ts +12 -0
- package/dist/src/policies/skip-policy.d.ts.map +1 -0
- package/dist/src/policies/skip-policy.js +44 -0
- package/dist/src/policies/skip-policy.js.map +1 -0
- package/dist/src/registry/index.d.ts +2 -0
- package/dist/src/registry/index.d.ts.map +1 -0
- package/dist/src/registry/index.js +20 -0
- package/dist/src/registry/index.js.map +1 -0
- package/dist/src/registry/job-registry.d.ts +16 -0
- package/dist/src/registry/job-registry.d.ts.map +1 -0
- package/dist/src/registry/job-registry.js +50 -0
- package/dist/src/registry/job-registry.js.map +1 -0
- package/dist/src/repository/id-generator.d.ts +18 -0
- package/dist/src/repository/id-generator.d.ts.map +1 -0
- package/dist/src/repository/id-generator.js +37 -0
- package/dist/src/repository/id-generator.js.map +1 -0
- package/dist/src/repository/in-memory/in-memory-job-repository.d.ts +49 -0
- package/dist/src/repository/in-memory/in-memory-job-repository.d.ts.map +1 -0
- package/dist/src/repository/in-memory/in-memory-job-repository.js +291 -0
- package/dist/src/repository/in-memory/in-memory-job-repository.js.map +1 -0
- package/dist/src/repository/in-memory/index.d.ts +2 -0
- package/dist/src/repository/in-memory/index.d.ts.map +1 -0
- package/dist/src/repository/in-memory/index.js +20 -0
- package/dist/src/repository/in-memory/index.js.map +1 -0
- package/dist/src/repository/index.d.ts +4 -0
- package/dist/src/repository/index.d.ts.map +1 -0
- package/dist/src/repository/index.js +22 -0
- package/dist/src/repository/index.js.map +1 -0
- package/dist/src/repository/uuid-v7.d.ts +20 -0
- package/dist/src/repository/uuid-v7.d.ts.map +1 -0
- package/dist/src/repository/uuid-v7.js +31 -0
- package/dist/src/repository/uuid-v7.js.map +1 -0
- package/dist/src/scheduling/batch-scheduled.d.ts +87 -0
- package/dist/src/scheduling/batch-scheduled.d.ts.map +1 -0
- package/dist/src/scheduling/batch-scheduled.js +170 -0
- package/dist/src/scheduling/batch-scheduled.js.map +1 -0
- package/dist/src/transaction/in-memory-transaction-manager.d.ts +16 -0
- package/dist/src/transaction/in-memory-transaction-manager.d.ts.map +1 -0
- package/dist/src/transaction/in-memory-transaction-manager.js +33 -0
- package/dist/src/transaction/in-memory-transaction-manager.js.map +1 -0
- package/dist/src/transaction/index.d.ts +2 -0
- package/dist/src/transaction/index.d.ts.map +1 -0
- package/dist/src/transaction/index.js +20 -0
- package/dist/src/transaction/index.js.map +1 -0
- package/dist/tests/contracts/index.d.ts +26 -0
- package/dist/tests/contracts/index.d.ts.map +1 -0
- package/dist/tests/contracts/index.js +37 -0
- package/dist/tests/contracts/index.js.map +1 -0
- package/dist/tests/contracts/job-repository.contract.d.ts +46 -0
- package/dist/tests/contracts/job-repository.contract.d.ts.map +1 -0
- package/dist/tests/contracts/job-repository.contract.js +644 -0
- package/dist/tests/contracts/job-repository.contract.js.map +1 -0
- package/package.json +80 -0
- package/src/adapters/in-process.adapter.ts +182 -0
- package/src/adapters/index.ts +17 -0
- package/src/builder/batch-builder.ts +32 -0
- package/src/builder/flow-builder.ts +141 -0
- package/src/builder/index.ts +4 -0
- package/src/builder/job-builder.ts +206 -0
- package/src/builder/step-builder.ts +190 -0
- package/src/compiler/builder-types.ts +27 -0
- package/src/compiler/definition-compiler.ts +325 -0
- package/src/compiler/index.ts +2 -0
- package/src/core/errors.ts +125 -0
- package/src/core/execution-context/index.ts +3 -0
- package/src/core/execution-context/json-value.ts +3 -0
- package/src/core/execution-context/serializer.ts +21 -0
- package/src/core/execution-context/validator.ts +103 -0
- package/src/core/index.ts +7 -0
- package/src/core/ir/decider-definition.ts +25 -0
- package/src/core/ir/index.ts +7 -0
- package/src/core/ir/job-definition.ts +15 -0
- package/src/core/ir/listener-definition.ts +19 -0
- package/src/core/ir/policy-config.ts +19 -0
- package/src/core/ir/refs.ts +42 -0
- package/src/core/ir/step-definition.ts +62 -0
- package/src/core/ir/transition-definition.ts +9 -0
- package/src/core/item/index.ts +1 -0
- package/src/core/item/interfaces.ts +70 -0
- package/src/core/repository/index.ts +2 -0
- package/src/core/repository/job-repository.ts +100 -0
- package/src/core/repository/types.ts +91 -0
- package/src/core/status.ts +31 -0
- package/src/core/transaction/index.ts +1 -0
- package/src/core/transaction/transaction-manager.ts +8 -0
- package/src/core/validation/definition-validator.ts +215 -0
- package/src/core/validation/index.ts +1 -0
- package/src/decorators/constants.ts +9 -0
- package/src/decorators/flow.decorator.ts +31 -0
- package/src/decorators/index.ts +7 -0
- package/src/decorators/item.decorators.ts +51 -0
- package/src/decorators/job.decorator.ts +16 -0
- package/src/decorators/listener.decorators.ts +142 -0
- package/src/decorators/step.decorator.ts +33 -0
- package/src/decorators/tasklet.decorator.ts +14 -0
- package/src/execution/batch-worker-runner.ts +142 -0
- package/src/execution/chunk-step-executor.ts +594 -0
- package/src/execution/execution-strategy.ts +115 -0
- package/src/execution/external-task-execution-strategy.ts +104 -0
- package/src/execution/in-process-execution-strategy.ts +207 -0
- package/src/execution/index.ts +13 -0
- package/src/execution/job-executor.ts +553 -0
- package/src/execution/job-explorer.ts +73 -0
- package/src/execution/job-key.ts +35 -0
- package/src/execution/job-launcher.ts +132 -0
- package/src/execution/job-operator.ts +127 -0
- package/src/execution/listener-invoker.ts +389 -0
- package/src/execution/ref-resolver.ts +64 -0
- package/src/execution/tasklet-step-executor.ts +182 -0
- package/src/explorer/batch-explorer.ts +251 -0
- package/src/explorer/index.ts +1 -0
- package/src/flow/flow-evaluator.ts +89 -0
- package/src/flow/index.ts +1 -0
- package/src/index.ts +24 -0
- package/src/io/checkpoint.ts +47 -0
- package/src/io/database.ts +114 -0
- package/src/io/file-readers.ts +191 -0
- package/src/io/file-writers.ts +99 -0
- package/src/io/index.ts +5 -0
- package/src/io/s3.ts +117 -0
- package/src/listeners/builtin-listeners.ts +151 -0
- package/src/listeners/index.ts +7 -0
- package/src/module/adapter-options.ts +38 -0
- package/src/module/adapter.ts +160 -0
- package/src/module/batch-schedule-registry.ts +0 -0
- package/src/module/index.ts +13 -0
- package/src/module/nest-batch.module.ts +674 -0
- package/src/module/tokens.ts +95 -0
- package/src/observability/event-types.ts +61 -0
- package/src/observability/exporters.ts +96 -0
- package/src/observability/index.ts +2 -0
- package/src/partition-helpers.ts +204 -0
- package/src/policies/backoff.ts +22 -0
- package/src/policies/index.ts +3 -0
- package/src/policies/retry-policy.ts +57 -0
- package/src/policies/skip-policy.ts +51 -0
- package/src/registry/index.ts +1 -0
- package/src/registry/job-registry.ts +42 -0
- package/src/repository/id-generator.ts +25 -0
- package/src/repository/in-memory/in-memory-job-repository.ts +334 -0
- package/src/repository/in-memory/index.ts +1 -0
- package/src/repository/index.ts +3 -0
- package/src/repository/uuid-v7.ts +40 -0
- package/src/scheduling/batch-scheduled.ts +257 -0
- package/src/transaction/in-memory-transaction-manager.ts +23 -0
- package/src/transaction/index.ts +1 -0
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public injection tokens for the `@nest-batch/core` module surface.
|
|
3
|
+
*
|
|
4
|
+
* These tokens are the stable, package-scoped identifiers sibling packages
|
|
5
|
+
* (e.g. `@nest-batch/mikro-orm`, `@nest-batch/typeorm`, `@nest-batch/bullmq`)
|
|
6
|
+
* use to bind their own providers into the core DI graph. They are
|
|
7
|
+
* registered in the global `Symbol.for` registry under stable, package-
|
|
8
|
+
* scoped keys so they are unique across the host process even if the
|
|
9
|
+
* package is loaded multiple times.
|
|
10
|
+
*
|
|
11
|
+
* Why symbols and not string tokens?
|
|
12
|
+
* - Symbols cannot collide with a user string by accident.
|
|
13
|
+
* - `Symbol.for(key)` gives us cross-realm uniqueness without the
|
|
14
|
+
* caller having to thread the token through `import` chains — a
|
|
15
|
+
* host can resolve any of these tokens by reaching into
|
|
16
|
+
* `Symbol.for('...description...')` and getting the same value.
|
|
17
|
+
* - Symbols are erased from emitted JavaScript, so they do not
|
|
18
|
+
* pollute production bundles with debug strings.
|
|
19
|
+
*
|
|
20
|
+
* Why a stable description (not `Symbol(description)`)?
|
|
21
|
+
* - `Symbol.for('k')` only works if the *same* string is passed both
|
|
22
|
+
* times. Hard-coding a `description` lets future sibling packages
|
|
23
|
+
* resolve the token without importing this file (useful for tooling
|
|
24
|
+
* and for ad-hoc cross-package debugging).
|
|
25
|
+
*/
|
|
26
|
+
import { EXECUTION_STRATEGY } from '../execution/execution-strategy';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Injection token for the `JobRepository` implementation.
|
|
30
|
+
*
|
|
31
|
+
* Adapter packages (`@nest-batch/mikro-orm`, `@nest-batch/typeorm`, ...)
|
|
32
|
+
* bind their `JobRepository` subclass to this token. By default the host
|
|
33
|
+
* app is expected to register its own `JobRepository` provider — core
|
|
34
|
+
* does NOT ship a default binding because the choice of persistence
|
|
35
|
+
* backend is the host's decision.
|
|
36
|
+
*/
|
|
37
|
+
export const JOB_REPOSITORY_TOKEN: symbol = Symbol.for(
|
|
38
|
+
'@nest-batch/core/JOB_REPOSITORY',
|
|
39
|
+
);
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Injection token for the `TransactionManager` implementation.
|
|
43
|
+
*
|
|
44
|
+
* Adapter packages bind their transaction manager to this token. The
|
|
45
|
+
* `JobRepository` implementation is expected to participate in the same
|
|
46
|
+
* transaction (e.g. share the same `EntityManager` / `DataSource`).
|
|
47
|
+
*/
|
|
48
|
+
export const TRANSACTION_MANAGER_TOKEN: symbol = Symbol.for(
|
|
49
|
+
'@nest-batch/core/TRANSACTION_MANAGER',
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Injection token for the `BatchScheduleRegistry` provider.
|
|
54
|
+
*
|
|
55
|
+
* The `BatchExplorer` populates this registry with `@BatchScheduled`
|
|
56
|
+
* metadata it discovers on `@Jobable` classes. The future runtime
|
|
57
|
+
* scheduler (the `@nest-batch/bullmq` cron strategy, or a sibling
|
|
58
|
+
* scheduling package) reads from this registry to install the actual
|
|
59
|
+
* timers. Keeping the registry as a stable token means adapters can
|
|
60
|
+
* inject it (for introspection / health checks) without depending on
|
|
61
|
+
* the explorer's internal state.
|
|
62
|
+
*/
|
|
63
|
+
export const BATCH_SCHEDULE_REGISTRY: symbol = Symbol.for(
|
|
64
|
+
'@nest-batch/core/BATCH_SCHEDULE_REGISTRY',
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Injection token for the module's resolved options bag.
|
|
69
|
+
*
|
|
70
|
+
* Backs the post-`useFactory` options read (T2 will wire the async
|
|
71
|
+
* factory provider to write into this slot). Sibling packages and the
|
|
72
|
+
* host app can read the resolved options by injecting this token. The
|
|
73
|
+
* shape is the union of `NestBatchModuleOptions` plus whatever an
|
|
74
|
+
* adapter's own config contributed, so the value is a
|
|
75
|
+
* `Record<string, unknown>` at runtime.
|
|
76
|
+
*
|
|
77
|
+
* The previous `'BATCH_OPTIONS'` string alias was removed in the
|
|
78
|
+
* T1 type-contract refactor — hosts that need the options bag should
|
|
79
|
+
* inject `MODULE_OPTIONS_TOKEN` instead.
|
|
80
|
+
*/
|
|
81
|
+
export const MODULE_OPTIONS_TOKEN: symbol = Symbol.for(
|
|
82
|
+
'@nest-batch/core/MODULE_OPTIONS',
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Polymorphic execution strategy token.
|
|
87
|
+
*
|
|
88
|
+
* Re-exported here from `execution/execution-strategy.ts` so that the
|
|
89
|
+
* module surface is the single import path for downstream packages.
|
|
90
|
+
* Apps that want the default in-process strategy wire up
|
|
91
|
+
* `IN_PROCESS_EXECUTION_STRATEGY_PROVIDER`; sibling packages (e.g.
|
|
92
|
+
* `@nest-batch/bullmq`) provide a custom binding under this same
|
|
93
|
+
* token.
|
|
94
|
+
*/
|
|
95
|
+
export { EXECUTION_STRATEGY };
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { JsonValue } from '../core/execution-context/json-value';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Event type constants emitted by the JobExecutor / step executors.
|
|
5
|
+
*
|
|
6
|
+
* Naming follows `<domain>.<entity>.<verb-past-tense>` so downstream
|
|
7
|
+
* consumers can subscribe via dot-separated filter strings.
|
|
8
|
+
*/
|
|
9
|
+
export const BATCH_EVENT = {
|
|
10
|
+
JOB_STARTED: 'nest-batch.job.started',
|
|
11
|
+
JOB_COMPLETED: 'nest-batch.job.completed',
|
|
12
|
+
JOB_FAILED: 'nest-batch.job.failed',
|
|
13
|
+
STEP_STARTED: 'nest-batch.step.started',
|
|
14
|
+
STEP_COMPLETED: 'nest-batch.step.completed',
|
|
15
|
+
STEP_FAILED: 'nest-batch.step.failed',
|
|
16
|
+
CHUNK_PROCESSED: 'nest-batch.chunk.processed',
|
|
17
|
+
ITEM_SKIPPED: 'nest-batch.item.skipped',
|
|
18
|
+
ITEM_RETRIED: 'nest-batch.item.retried',
|
|
19
|
+
} as const;
|
|
20
|
+
|
|
21
|
+
export type BatchEventType = (typeof BATCH_EVENT)[keyof typeof BATCH_EVENT];
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Single, normalized event payload. `data` is intentionally typed as
|
|
25
|
+
* `JsonValue` so observers can ship events over any transport (HTTP,
|
|
26
|
+
* JSON log line, message queue) without further conversion.
|
|
27
|
+
*
|
|
28
|
+
* - `jobExecutionId` is always present.
|
|
29
|
+
* - `stepExecutionId` is present for STEP_*, CHUNK_*, and ITEM_*
|
|
30
|
+
* events; absent for JOB_* events that fire outside a step context
|
|
31
|
+
* (currently the JOB_STARTED / JOB_COMPLETED / JOB_FAILED trio).
|
|
32
|
+
*/
|
|
33
|
+
export interface BatchEvent {
|
|
34
|
+
type: BatchEventType;
|
|
35
|
+
timestamp: Date;
|
|
36
|
+
jobExecutionId: string;
|
|
37
|
+
stepExecutionId?: string;
|
|
38
|
+
data: JsonValue;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Observer contract for batch lifecycle events. Implementations may
|
|
43
|
+
* emit logs, push to a queue, or aggregate metrics. The executor
|
|
44
|
+
* awaits `onEvent` so a slow observer blocks step transitions — this
|
|
45
|
+
* is intentional: per the plan, exporters are out of scope, and the
|
|
46
|
+
* default observer is a no-op.
|
|
47
|
+
*/
|
|
48
|
+
export interface BatchObserver {
|
|
49
|
+
onEvent(event: BatchEvent): void | Promise<void>;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Default observer used when none is supplied. Discards every event.
|
|
54
|
+
* Useful as a sentinel default that satisfies the interface without
|
|
55
|
+
* doing any I/O.
|
|
56
|
+
*/
|
|
57
|
+
export class NoopBatchObserver implements BatchObserver {
|
|
58
|
+
async onEvent(_event: BatchEvent): Promise<void> {
|
|
59
|
+
// intentional no-op
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { BATCH_EVENT, type BatchEvent, type BatchObserver } from './event-types';
|
|
2
|
+
|
|
3
|
+
export class CompositeBatchObserver implements BatchObserver {
|
|
4
|
+
constructor(private readonly observers: readonly BatchObserver[]) {}
|
|
5
|
+
|
|
6
|
+
async onEvent(event: BatchEvent): Promise<void> {
|
|
7
|
+
await Promise.all(this.observers.map((observer) => observer.onEvent(event)));
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface JsonLogBatchObserverOptions {
|
|
12
|
+
readonly write?: (line: string) => void;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export class JsonLogBatchObserver implements BatchObserver {
|
|
16
|
+
private readonly write: (line: string) => void;
|
|
17
|
+
|
|
18
|
+
constructor(options: JsonLogBatchObserverOptions = {}) {
|
|
19
|
+
this.write = options.write ?? ((line) => console.log(line));
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
onEvent(event: BatchEvent): void {
|
|
23
|
+
this.write(
|
|
24
|
+
JSON.stringify({
|
|
25
|
+
...event,
|
|
26
|
+
timestamp: event.timestamp.toISOString(),
|
|
27
|
+
}),
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface BatchMetricsSnapshot {
|
|
33
|
+
readonly eventsTotal: Readonly<Record<string, number>>;
|
|
34
|
+
readonly jobsStarted: number;
|
|
35
|
+
readonly jobsCompleted: number;
|
|
36
|
+
readonly jobsFailed: number;
|
|
37
|
+
readonly stepsStarted: number;
|
|
38
|
+
readonly stepsCompleted: number;
|
|
39
|
+
readonly stepsFailed: number;
|
|
40
|
+
readonly chunksProcessed: number;
|
|
41
|
+
readonly itemsSkipped: number;
|
|
42
|
+
readonly itemsRetried: number;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export class PrometheusBatchMetricsObserver implements BatchObserver {
|
|
46
|
+
private readonly eventsTotal = new Map<string, number>();
|
|
47
|
+
|
|
48
|
+
onEvent(event: BatchEvent): void {
|
|
49
|
+
this.increment(event.type);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
snapshot(): BatchMetricsSnapshot {
|
|
53
|
+
return {
|
|
54
|
+
eventsTotal: Object.fromEntries(this.eventsTotal.entries()),
|
|
55
|
+
jobsStarted: this.count(BATCH_EVENT.JOB_STARTED),
|
|
56
|
+
jobsCompleted: this.count(BATCH_EVENT.JOB_COMPLETED),
|
|
57
|
+
jobsFailed: this.count(BATCH_EVENT.JOB_FAILED),
|
|
58
|
+
stepsStarted: this.count(BATCH_EVENT.STEP_STARTED),
|
|
59
|
+
stepsCompleted: this.count(BATCH_EVENT.STEP_COMPLETED),
|
|
60
|
+
stepsFailed: this.count(BATCH_EVENT.STEP_FAILED),
|
|
61
|
+
chunksProcessed: this.count(BATCH_EVENT.CHUNK_PROCESSED),
|
|
62
|
+
itemsSkipped: this.count(BATCH_EVENT.ITEM_SKIPPED),
|
|
63
|
+
itemsRetried: this.count(BATCH_EVENT.ITEM_RETRIED),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
renderPrometheus(): string {
|
|
68
|
+
const lines = [
|
|
69
|
+
'# HELP nest_batch_events_total Total batch lifecycle events.',
|
|
70
|
+
'# TYPE nest_batch_events_total counter',
|
|
71
|
+
];
|
|
72
|
+
for (const [type, count] of [...this.eventsTotal.entries()].sort()) {
|
|
73
|
+
lines.push(`nest_batch_events_total{type="${escapeLabel(type)}"} ${count}`);
|
|
74
|
+
}
|
|
75
|
+
const snapshot = this.snapshot();
|
|
76
|
+
lines.push('# HELP nest_batch_jobs_completed_total Completed jobs.');
|
|
77
|
+
lines.push('# TYPE nest_batch_jobs_completed_total counter');
|
|
78
|
+
lines.push(`nest_batch_jobs_completed_total ${snapshot.jobsCompleted}`);
|
|
79
|
+
lines.push('# HELP nest_batch_jobs_failed_total Failed jobs.');
|
|
80
|
+
lines.push('# TYPE nest_batch_jobs_failed_total counter');
|
|
81
|
+
lines.push(`nest_batch_jobs_failed_total ${snapshot.jobsFailed}`);
|
|
82
|
+
return `${lines.join('\n')}\n`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
private increment(type: string): void {
|
|
86
|
+
this.eventsTotal.set(type, this.count(type) + 1);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
private count(type: string): number {
|
|
90
|
+
return this.eventsTotal.get(type) ?? 0;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function escapeLabel(value: string): string {
|
|
95
|
+
return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
|
|
96
|
+
}
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure partition helpers for chunked steps.
|
|
3
|
+
*
|
|
4
|
+
* This module is the single source of truth for partition validation
|
|
5
|
+
* and the default partition-range shape. It is deliberately
|
|
6
|
+
* dependency-light: no `@nest-batch/bullmq`, no `@nest-batch/kafka`,
|
|
7
|
+
* no ORMs, no cron — verified by
|
|
8
|
+
* `packages/core/tests/core/boundary/no-forbidden-imports.test.ts`.
|
|
9
|
+
*
|
|
10
|
+
* The helpers are consumed by:
|
|
11
|
+
* - `packages/core/src/compiler/definition-compiler.ts` (validation
|
|
12
|
+
* at compile time),
|
|
13
|
+
* - `packages/core/src/core/validation/definition-validator.ts`
|
|
14
|
+
* (cross-checks the resolved `JobDefinition`),
|
|
15
|
+
* - `packages/core/src/execution/in-process-execution-strategy.ts`
|
|
16
|
+
* (the in-process adapter's partition guard),
|
|
17
|
+
* - `packages/bullmq/src/bullmq-runtime.service.ts` (the BullMQ
|
|
18
|
+
* strategy's enqueue fan-out + the worker's `partitionIndex`
|
|
19
|
+
* enforcement),
|
|
20
|
+
* - `packages/kafka/src/kafka-runtime.service.ts` (the Kafka
|
|
21
|
+
* mirror — T9).
|
|
22
|
+
*
|
|
23
|
+
* Pinned by:
|
|
24
|
+
* - `docs/RELEASE-0.2.0.md §6` — the v1 partition contract.
|
|
25
|
+
* - `packages/bullmq/tests/partition-invariant.test.ts` — T-AC-3
|
|
26
|
+
* first half (the BullMQ side).
|
|
27
|
+
* - `packages/kafka/tests/partition-invariant.test.ts` — T-AC-3
|
|
28
|
+
* second half (the Kafka side, T9).
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import type { ChunkPartitionConfig } from './core/ir/step-definition';
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Error code returned / thrown by the partition helpers. Stable for
|
|
35
|
+
* callers that want to switch on it (the BullMQ / Kafka workers
|
|
36
|
+
* surface this in their `exitMessage` when a partition payload is
|
|
37
|
+
* out of range).
|
|
38
|
+
*/
|
|
39
|
+
export const INVALID_PARTITION_INDEX = 'INVALID_PARTITION_INDEX';
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Thrown when a partition's `count` is not a positive integer, or
|
|
43
|
+
* when a runtime-resolved `partitionIndex` falls outside `[0, count)`.
|
|
44
|
+
*
|
|
45
|
+
* Distinct from `InvalidFlowGraphError` because this is a *value*
|
|
46
|
+
* problem (the input is out of range) rather than a *graph*
|
|
47
|
+
* problem (the step graph is malformed). The `code` is stable.
|
|
48
|
+
*/
|
|
49
|
+
export class InvalidPartitionsError extends Error {
|
|
50
|
+
readonly code: string;
|
|
51
|
+
constructor(message: string, public readonly details?: unknown) {
|
|
52
|
+
super(message);
|
|
53
|
+
this.name = 'InvalidPartitionsError';
|
|
54
|
+
this.code = 'INVALID_PARTITIONS';
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Validate a `ChunkPartitionConfig` at compile time. Throws
|
|
60
|
+
* `InvalidPartitionsError` when the config is structurally invalid:
|
|
61
|
+
*
|
|
62
|
+
* - `count` is not a finite integer
|
|
63
|
+
* - `count <= 0` (a partition count of zero is meaningless)
|
|
64
|
+
* - `range` is set but is not a function
|
|
65
|
+
*
|
|
66
|
+
* `count === 1` is allowed but the runtime short-circuits it to
|
|
67
|
+
* the non-partitioned path; we still validate it here so a typo
|
|
68
|
+
* (`count: 0` vs `count: 1`) fails loudly at the compiler.
|
|
69
|
+
*/
|
|
70
|
+
export function validatePartitions(partitions: ChunkPartitionConfig | undefined): void {
|
|
71
|
+
if (partitions === undefined) return;
|
|
72
|
+
if (!Number.isInteger(partitions.count) || partitions.count <= 0) {
|
|
73
|
+
throw new InvalidPartitionsError(
|
|
74
|
+
`ChunkStepDefinition.partitions.count must be a positive integer, got ${String(
|
|
75
|
+
partitions.count,
|
|
76
|
+
)}`,
|
|
77
|
+
{ count: partitions.count },
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
if (partitions.range !== undefined && typeof partitions.range !== 'function') {
|
|
81
|
+
throw new InvalidPartitionsError(
|
|
82
|
+
`ChunkStepDefinition.partitions.range must be a function when present, got ${typeof partitions.range}`,
|
|
83
|
+
{ rangeType: typeof partitions.range },
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* The default even-split partition range, given a known total item
|
|
90
|
+
* count. The formula mirrors the v1 contract in
|
|
91
|
+
* `docs/RELEASE-0.2.0.md §6.1`:
|
|
92
|
+
*
|
|
93
|
+
* partition `i` of `n` over `total` items:
|
|
94
|
+
* [floor(i * total / n), floor((i+1) * total / n))
|
|
95
|
+
*
|
|
96
|
+
* Pure function. Exported so hosts that want the "even split"
|
|
97
|
+
* behaviour without re-implementing the math can reuse it:
|
|
98
|
+
*
|
|
99
|
+
* const r = defaultRange(i, n, total);
|
|
100
|
+
* partitions: { count: n, range: (i, n) => defaultRange(i, n, total) }
|
|
101
|
+
*
|
|
102
|
+
* `total` is required because the runtime has no generic way to
|
|
103
|
+
* count the input — only the host's reader knows. The math is
|
|
104
|
+
* robust to `total === 0` (returns `[0, 0)` for every partition).
|
|
105
|
+
*/
|
|
106
|
+
export function defaultRange(
|
|
107
|
+
i: number,
|
|
108
|
+
n: number,
|
|
109
|
+
total: number,
|
|
110
|
+
): readonly [from: number, to: number] {
|
|
111
|
+
if (!Number.isInteger(i) || i < 0 || i >= n) {
|
|
112
|
+
throw new InvalidPartitionsError(
|
|
113
|
+
`defaultRange: partition index ${i} out of range [0, ${n})`,
|
|
114
|
+
{ i, n },
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
if (!Number.isInteger(n) || n <= 0) {
|
|
118
|
+
throw new InvalidPartitionsError(`defaultRange: count ${n} must be a positive integer`, { n });
|
|
119
|
+
}
|
|
120
|
+
if (!Number.isFinite(total) || total < 0) {
|
|
121
|
+
throw new InvalidPartitionsError(
|
|
122
|
+
`defaultRange: total ${total} must be a non-negative finite number`,
|
|
123
|
+
{ total },
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
const from = Math.floor((i * total) / n);
|
|
127
|
+
const to = Math.floor(((i + 1) * total) / n);
|
|
128
|
+
return [from, to] as const;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Runtime check that a `partitionIndex` on a job payload is in
|
|
133
|
+
* range for the configured `count`. Throws
|
|
134
|
+
* `InvalidPartitionsError` when the index is undefined, not an
|
|
135
|
+
* integer, negative, or `>= count`.
|
|
136
|
+
*
|
|
137
|
+
* The BullMQ / Kafka worker's `processJob` calls this with the
|
|
138
|
+
* payload's `partitionIndex` and the step's `partitions.count` so
|
|
139
|
+
* an out-of-range index becomes a hard step failure (the runtime
|
|
140
|
+
* surfaces it as `FAILED` with the invariant violation in the
|
|
141
|
+
* `exitMessage`).
|
|
142
|
+
*/
|
|
143
|
+
export function enforcePartitionIndex(
|
|
144
|
+
partitionIndex: number | undefined,
|
|
145
|
+
count: number,
|
|
146
|
+
): void {
|
|
147
|
+
if (partitionIndex === undefined) {
|
|
148
|
+
throw new InvalidPartitionsError(
|
|
149
|
+
`partitionIndex is required for a partitioned step (count=${count})`,
|
|
150
|
+
{ count },
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
if (
|
|
154
|
+
!Number.isInteger(partitionIndex) ||
|
|
155
|
+
partitionIndex < 0 ||
|
|
156
|
+
partitionIndex >= count
|
|
157
|
+
) {
|
|
158
|
+
throw new InvalidPartitionsError(
|
|
159
|
+
`partitionIndex ${partitionIndex} is out of range [0, ${count})`,
|
|
160
|
+
{ partitionIndex, count },
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Resolved partition info passed through the runtime. Built from
|
|
167
|
+
* the step's `partitions` config and the BullMQ / Kafka payload's
|
|
168
|
+
* `partitionIndex`. The runtime uses this to:
|
|
169
|
+
* - bound the chunk executor's read loop (when the host provided
|
|
170
|
+
* a `range` resolver),
|
|
171
|
+
* - tag the persisted `StepExecution` for diagnostics (a future
|
|
172
|
+
* task; the v1 contract only requires the chunk executor to
|
|
173
|
+
* honour the range).
|
|
174
|
+
*/
|
|
175
|
+
export interface ResolvedPartition {
|
|
176
|
+
readonly count: number;
|
|
177
|
+
readonly index: number;
|
|
178
|
+
/** The partition's resolved `[from, to)` range, or `undefined`
|
|
179
|
+
* when the step did not provide a `range` resolver (the default
|
|
180
|
+
* behaviour is "read until EOF"). */
|
|
181
|
+
readonly range?: readonly [from: number, to: number];
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Resolve a partition's metadata from the step's `partitions`
|
|
186
|
+
* config and the transport's `partitionIndex`. Calls
|
|
187
|
+
* `enforcePartitionIndex` to fail loudly on out-of-range indices.
|
|
188
|
+
*
|
|
189
|
+
* The `range` is computed only when the step provides a resolver;
|
|
190
|
+
* otherwise the returned `ResolvedPartition` is `count` + `index`
|
|
191
|
+
* with no `range`, and the chunk executor reads until EOF.
|
|
192
|
+
*/
|
|
193
|
+
export function resolvePartition(args: {
|
|
194
|
+
partitions: ChunkPartitionConfig;
|
|
195
|
+
partitionIndex: number | undefined;
|
|
196
|
+
}): ResolvedPartition {
|
|
197
|
+
enforcePartitionIndex(args.partitionIndex, args.partitions.count);
|
|
198
|
+
const index = args.partitionIndex as number;
|
|
199
|
+
if (args.partitions.range === undefined) {
|
|
200
|
+
return { count: args.partitions.count, index };
|
|
201
|
+
}
|
|
202
|
+
const range = args.partitions.range(index, args.partitions.count);
|
|
203
|
+
return { count: args.partitions.count, index, range };
|
|
204
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { BackoffConfig } from '../core/ir/policy-config';
|
|
2
|
+
|
|
3
|
+
export function compileBackoff(config: BackoffConfig): (attempt: number) => number {
|
|
4
|
+
switch (config.type) {
|
|
5
|
+
case 'none':
|
|
6
|
+
return () => 0;
|
|
7
|
+
case 'fixed':
|
|
8
|
+
return () => config.ms;
|
|
9
|
+
case 'exponential': {
|
|
10
|
+
const factor = config.factor ?? 2;
|
|
11
|
+
const maxMs = config.maxMs;
|
|
12
|
+
return (attempt: number) => {
|
|
13
|
+
const ms = config.initialMs * Math.pow(factor, Math.max(0, attempt - 1));
|
|
14
|
+
return maxMs !== undefined ? Math.min(ms, maxMs) : ms;
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
default: {
|
|
18
|
+
const _exhaustive: never = config;
|
|
19
|
+
throw new Error(`Unknown backoff type: ${JSON.stringify(_exhaustive)}`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { RetryPolicyConfig, Skippable, BackoffConfig } from '../core/ir/policy-config';
|
|
2
|
+
|
|
3
|
+
export interface RetryContext {
|
|
4
|
+
item: unknown | null;
|
|
5
|
+
phase: 'read' | 'process' | 'write';
|
|
6
|
+
attempt: number; // 1-based
|
|
7
|
+
retryLimit: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface RetryPolicy {
|
|
11
|
+
canRetry(error: unknown, context: RetryContext): boolean;
|
|
12
|
+
backoffMs(attempt: number): number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function matchesSkippable(err: unknown, skippable: Skippable): boolean {
|
|
16
|
+
if (typeof skippable === 'function') {
|
|
17
|
+
if (skippable.length === 1) {
|
|
18
|
+
try {
|
|
19
|
+
return (skippable as (err: unknown) => boolean)(err);
|
|
20
|
+
} catch {
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return err instanceof (skippable as new (...args: unknown[]) => Error);
|
|
25
|
+
}
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function compileRetryPolicy(config: RetryPolicyConfig): RetryPolicy {
|
|
30
|
+
if (config.limit <= 0) {
|
|
31
|
+
throw new Error(`RetryPolicyConfig.limit must be > 0 (got ${config.limit})`);
|
|
32
|
+
}
|
|
33
|
+
const backoff = (cfg: BackoffConfig, attempt: number): number => {
|
|
34
|
+
switch (cfg.type) {
|
|
35
|
+
case 'none':
|
|
36
|
+
return 0;
|
|
37
|
+
case 'fixed':
|
|
38
|
+
return cfg.ms;
|
|
39
|
+
case 'exponential': {
|
|
40
|
+
const factor = cfg.factor ?? 2;
|
|
41
|
+
const ms = cfg.initialMs * Math.pow(factor, Math.max(0, attempt - 1));
|
|
42
|
+
return cfg.maxMs !== undefined ? Math.min(ms, cfg.maxMs) : ms;
|
|
43
|
+
}
|
|
44
|
+
default:
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
return {
|
|
49
|
+
canRetry(error: unknown, context: RetryContext): boolean {
|
|
50
|
+
if (context.attempt > context.retryLimit) return false;
|
|
51
|
+
return config.retryable.some((s) => matchesSkippable(error, s));
|
|
52
|
+
},
|
|
53
|
+
backoffMs(attempt: number): number {
|
|
54
|
+
return backoff(config.backoff, attempt);
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { SkipPolicyConfig, Skippable } from '../core/ir/policy-config';
|
|
2
|
+
import { InvalidFlowGraphError } from '../core/errors';
|
|
3
|
+
|
|
4
|
+
export interface SkipContext {
|
|
5
|
+
item: unknown;
|
|
6
|
+
phase: 'read' | 'process' | 'write';
|
|
7
|
+
skipCount: number;
|
|
8
|
+
skipLimit: number;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface SkipPolicy {
|
|
12
|
+
shouldSkip(error: unknown, context: SkipContext): boolean;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function matchesSkippable(err: unknown, skippable: Skippable): boolean {
|
|
16
|
+
if (typeof skippable === 'function') {
|
|
17
|
+
if (
|
|
18
|
+
skippable.prototype !== undefined &&
|
|
19
|
+
skippable.prototype instanceof Error
|
|
20
|
+
) {
|
|
21
|
+
return err instanceof (skippable as new (...args: unknown[]) => Error);
|
|
22
|
+
}
|
|
23
|
+
try {
|
|
24
|
+
return (skippable as (err: unknown) => boolean)(err);
|
|
25
|
+
} catch {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function compileSkipPolicy(config: SkipPolicyConfig): SkipPolicy {
|
|
33
|
+
if (config.limit <= 0) {
|
|
34
|
+
throw new InvalidFlowGraphError(
|
|
35
|
+
'INVALID_SKIP_LIMIT',
|
|
36
|
+
`SkipPolicyConfig.limit must be > 0 (got ${config.limit})`,
|
|
37
|
+
{ limit: config.limit },
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
shouldSkip(error: unknown, context: SkipContext): boolean {
|
|
42
|
+
if (context.skipCount >= context.skipLimit) {
|
|
43
|
+
// Limit reached; caller (ChunkProcessor) is expected to raise
|
|
44
|
+
// SkipLimitExceededError. From the policy's perspective we simply
|
|
45
|
+
// stop returning `true` for further candidate errors.
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
return config.skippable.some((s) => matchesSkippable(error, s));
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './job-registry';
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { Injectable } from '@nestjs/common';
|
|
2
|
+
import type { JobDefinition } from '../core/ir';
|
|
3
|
+
import { DefinitionValidator } from '../core/validation/definition-validator';
|
|
4
|
+
import {
|
|
5
|
+
JobNotFoundError,
|
|
6
|
+
DuplicateJobDefinitionError,
|
|
7
|
+
InvalidFlowGraphError,
|
|
8
|
+
} from '../core/errors';
|
|
9
|
+
|
|
10
|
+
@Injectable()
|
|
11
|
+
export class JobRegistry {
|
|
12
|
+
private readonly definitions = new Map<string, JobDefinition>();
|
|
13
|
+
private readonly validator = new DefinitionValidator();
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Register a job definition. Validates the graph and throws on duplicates.
|
|
17
|
+
*/
|
|
18
|
+
register(job: JobDefinition): void {
|
|
19
|
+
if (this.definitions.has(job.id)) {
|
|
20
|
+
throw new DuplicateJobDefinitionError(job.id);
|
|
21
|
+
}
|
|
22
|
+
this.validator.validate(job);
|
|
23
|
+
this.definitions.set(job.id, job);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Look up a job definition by ID. Throws JobNotFoundError if missing.
|
|
28
|
+
*/
|
|
29
|
+
get(jobId: string): JobDefinition {
|
|
30
|
+
const def = this.definitions.get(jobId);
|
|
31
|
+
if (!def) throw new JobNotFoundError(jobId);
|
|
32
|
+
return def;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
has(jobId: string): boolean {
|
|
36
|
+
return this.definitions.has(jobId);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
getAll(): JobDefinition[] {
|
|
40
|
+
return Array.from(this.definitions.values());
|
|
41
|
+
}
|
|
42
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { randomUUID } from 'crypto';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Generates unique IDs for JobInstances, JobExecutions, StepExecutions.
|
|
5
|
+
* The default implementation uses crypto.randomUUID() (v4).
|
|
6
|
+
* Tests can use DeterministicIdGenerator for predictable output.
|
|
7
|
+
*/
|
|
8
|
+
export interface IdGenerator {
|
|
9
|
+
next(): string;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export class UuidIdGenerator implements IdGenerator {
|
|
13
|
+
next(): string {
|
|
14
|
+
return randomUUID();
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export class DeterministicIdGenerator implements IdGenerator {
|
|
19
|
+
private counter = 0;
|
|
20
|
+
constructor(private readonly prefix: string = 'id') {}
|
|
21
|
+
next(): string {
|
|
22
|
+
this.counter += 1;
|
|
23
|
+
return `${this.prefix}-${this.counter}`;
|
|
24
|
+
}
|
|
25
|
+
}
|