@workflow/core 5.0.0-beta.4 → 5.0.0-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attribute-changes.d.ts +7 -0
- package/dist/attribute-changes.d.ts.map +1 -0
- package/dist/attribute-changes.js +25 -0
- package/dist/capabilities.d.ts +33 -1
- package/dist/capabilities.d.ts.map +1 -1
- package/dist/capabilities.js +71 -4
- package/dist/capture-stack.d.ts +16 -0
- package/dist/capture-stack.d.ts.map +1 -0
- package/dist/capture-stack.js +21 -0
- package/dist/class-serialization.d.ts +32 -0
- package/dist/class-serialization.d.ts.map +1 -1
- package/dist/class-serialization.js +37 -1
- package/dist/classify-error.d.ts +26 -3
- package/dist/classify-error.d.ts.map +1 -1
- package/dist/classify-error.js +109 -6
- package/dist/context-errors.d.ts +27 -0
- package/dist/context-errors.d.ts.map +1 -0
- package/dist/context-errors.js +101 -0
- package/dist/context-violation-error.d.ts +97 -0
- package/dist/context-violation-error.d.ts.map +1 -0
- package/dist/context-violation-error.js +149 -0
- package/dist/create-hook.d.ts +62 -4
- package/dist/create-hook.d.ts.map +1 -1
- package/dist/create-hook.js +4 -3
- package/dist/define-hook.d.ts.map +1 -1
- package/dist/define-hook.js +20 -5
- package/dist/describe-error.d.ts +70 -0
- package/dist/describe-error.d.ts.map +1 -0
- package/dist/describe-error.js +189 -0
- package/dist/encryption.d.ts +37 -3
- package/dist/encryption.d.ts.map +1 -1
- package/dist/encryption.js +97 -31
- package/dist/events-consumer.d.ts +147 -1
- package/dist/events-consumer.d.ts.map +1 -1
- package/dist/events-consumer.js +444 -43
- package/dist/flushable-stream.d.ts +65 -10
- package/dist/flushable-stream.d.ts.map +1 -1
- package/dist/flushable-stream.js +131 -18
- package/dist/global.d.ts +37 -1
- package/dist/global.d.ts.map +1 -1
- package/dist/global.js +21 -3
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/log-format.d.ts +25 -0
- package/dist/log-format.d.ts.map +1 -0
- package/dist/log-format.js +250 -0
- package/dist/logger.d.ts +29 -30
- package/dist/logger.d.ts.map +1 -1
- package/dist/logger.js +82 -32
- package/dist/private.d.ts +246 -8
- package/dist/private.d.ts.map +1 -1
- package/dist/private.js +345 -9
- package/dist/replay-payload-cache.d.ts +68 -0
- package/dist/replay-payload-cache.d.ts.map +1 -0
- package/dist/replay-payload-cache.js +160 -0
- package/dist/runtime/compute-instance.d.ts +12 -0
- package/dist/runtime/compute-instance.d.ts.map +1 -0
- package/dist/runtime/compute-instance.js +13 -0
- package/dist/runtime/constants.d.ts +251 -0
- package/dist/runtime/constants.d.ts.map +1 -1
- package/dist/runtime/constants.js +425 -16
- package/dist/runtime/count-step-started-events.d.ts +52 -0
- package/dist/runtime/count-step-started-events.d.ts.map +1 -0
- package/dist/runtime/count-step-started-events.js +72 -0
- package/dist/runtime/deployment-guard.d.ts +99 -0
- package/dist/runtime/deployment-guard.d.ts.map +1 -0
- package/dist/runtime/deployment-guard.js +154 -0
- package/dist/runtime/get-port-lazy.d.ts +25 -0
- package/dist/runtime/get-port-lazy.d.ts.map +1 -0
- package/dist/runtime/get-port-lazy.js +92 -0
- package/dist/runtime/get-world-lazy.d.ts +23 -0
- package/dist/runtime/get-world-lazy.d.ts.map +1 -0
- package/dist/runtime/get-world-lazy.js +46 -0
- package/dist/runtime/helpers.d.ts +327 -10
- package/dist/runtime/helpers.d.ts.map +1 -1
- package/dist/runtime/helpers.js +586 -40
- package/dist/runtime/quickjs-assets.generated.d.ts +14 -0
- package/dist/runtime/quickjs-assets.generated.d.ts.map +1 -0
- package/dist/runtime/quickjs-assets.generated.js +30 -0
- package/dist/runtime/quickjs-entrypoint.d.ts +124 -0
- package/dist/runtime/quickjs-entrypoint.d.ts.map +1 -0
- package/dist/runtime/quickjs-entrypoint.js +1437 -0
- package/dist/runtime/quickjs-runtime.d.ts +228 -0
- package/dist/runtime/quickjs-runtime.d.ts.map +1 -0
- package/dist/runtime/quickjs-runtime.js +2372 -0
- package/dist/runtime/quickjs-serde.d.ts +107 -0
- package/dist/runtime/quickjs-serde.d.ts.map +1 -0
- package/dist/runtime/quickjs-serde.js +2098 -0
- package/dist/runtime/replay-budget.d.ts +89 -0
- package/dist/runtime/replay-budget.d.ts.map +1 -0
- package/dist/runtime/replay-budget.js +139 -0
- package/dist/runtime/replay-recovery-reporter.d.ts +36 -0
- package/dist/runtime/replay-recovery-reporter.d.ts.map +1 -0
- package/dist/runtime/replay-recovery-reporter.js +64 -0
- package/dist/runtime/resume-hook.d.ts +27 -4
- package/dist/runtime/resume-hook.d.ts.map +1 -1
- package/dist/runtime/resume-hook.js +452 -76
- package/dist/runtime/run-id-time.d.ts +19 -0
- package/dist/runtime/run-id-time.d.ts.map +1 -0
- package/dist/runtime/run-id-time.js +42 -0
- package/dist/runtime/run.d.ts +8 -2
- package/dist/runtime/run.d.ts.map +1 -1
- package/dist/runtime/run.js +62 -15
- package/dist/runtime/runs.d.ts +54 -3
- package/dist/runtime/runs.d.ts.map +1 -1
- package/dist/runtime/runs.js +119 -13
- package/dist/runtime/start.d.ts +70 -1
- package/dist/runtime/start.d.ts.map +1 -1
- package/dist/runtime/start.js +313 -60
- package/dist/runtime/step-executor.d.ts +177 -0
- package/dist/runtime/step-executor.d.ts.map +1 -0
- package/dist/runtime/step-executor.js +946 -0
- package/dist/runtime/step-latency.d.ts +197 -0
- package/dist/runtime/step-latency.d.ts.map +1 -0
- package/dist/runtime/step-latency.js +207 -0
- package/dist/runtime/step-ownership.d.ts +72 -0
- package/dist/runtime/step-ownership.d.ts.map +1 -0
- package/dist/runtime/step-ownership.js +114 -0
- package/dist/runtime/step-single-flight.d.ts +12 -0
- package/dist/runtime/step-single-flight.d.ts.map +1 -0
- package/dist/runtime/step-single-flight.js +69 -0
- package/dist/runtime/suspension-handler.d.ts +109 -7
- package/dist/runtime/suspension-handler.d.ts.map +1 -1
- package/dist/runtime/suspension-handler.js +486 -140
- package/dist/runtime/vm-mode.d.ts +44 -0
- package/dist/runtime/vm-mode.d.ts.map +1 -0
- package/dist/runtime/vm-mode.js +62 -0
- package/dist/runtime/vm-serde-bundle.generated.d.ts +14 -0
- package/dist/runtime/vm-serde-bundle.generated.d.ts.map +1 -0
- package/dist/runtime/vm-serde-bundle.generated.js +16 -0
- package/dist/runtime/wait-continuation.d.ts +84 -0
- package/dist/runtime/wait-continuation.d.ts.map +1 -0
- package/dist/runtime/wait-continuation.js +105 -0
- package/dist/runtime/wait-until.d.ts +18 -0
- package/dist/runtime/wait-until.d.ts.map +1 -0
- package/dist/runtime/wait-until.js +42 -0
- package/dist/runtime/world-compatibility.d.ts +20 -0
- package/dist/runtime/world-compatibility.d.ts.map +1 -0
- package/dist/runtime/world-compatibility.js +32 -0
- package/dist/runtime/world-init.d.ts +50 -0
- package/dist/runtime/world-init.d.ts.map +1 -0
- package/dist/runtime/world-init.js +50 -0
- package/dist/runtime/world.d.ts +14 -2
- package/dist/runtime/world.d.ts.map +1 -1
- package/dist/runtime/world.js +101 -26
- package/dist/runtime.d.ts +17 -12
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +3135 -328
- package/dist/schemas.d.ts +1 -1
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +1 -1
- package/dist/sealed-box.d.ts +167 -0
- package/dist/sealed-box.d.ts.map +1 -0
- package/dist/sealed-box.js +571 -0
- package/dist/serialization/client.d.ts +17 -0
- package/dist/serialization/client.d.ts.map +1 -0
- package/dist/serialization/client.js +48 -0
- package/dist/serialization/codec-devalue-vm.d.ts +16 -0
- package/dist/serialization/codec-devalue-vm.d.ts.map +1 -0
- package/dist/serialization/codec-devalue-vm.js +148 -0
- package/dist/serialization/codec-devalue.d.ts +14 -0
- package/dist/serialization/codec-devalue.d.ts.map +1 -0
- package/dist/serialization/codec-devalue.js +105 -0
- package/dist/serialization/codec.d.ts +125 -0
- package/dist/serialization/codec.d.ts.map +1 -0
- package/dist/serialization/codec.js +17 -0
- package/dist/serialization/compression.d.ts +104 -0
- package/dist/serialization/compression.d.ts.map +1 -0
- package/dist/serialization/compression.js +260 -0
- package/dist/serialization/encryption.d.ts +134 -0
- package/dist/serialization/encryption.d.ts.map +1 -0
- package/dist/serialization/encryption.js +186 -0
- package/dist/serialization/errors.d.ts +34 -0
- package/dist/serialization/errors.d.ts.map +1 -0
- package/dist/serialization/errors.js +59 -0
- package/dist/serialization/format.d.ts +60 -0
- package/dist/serialization/format.d.ts.map +1 -0
- package/dist/serialization/format.js +97 -0
- package/dist/serialization/hardened.d.ts +155 -0
- package/dist/serialization/hardened.d.ts.map +1 -0
- package/dist/serialization/hardened.js +523 -0
- package/dist/serialization/index.d.ts +20 -0
- package/dist/serialization/index.d.ts.map +1 -0
- package/dist/serialization/index.js +22 -0
- package/dist/serialization/reducers/class-vm.d.ts +20 -0
- package/dist/serialization/reducers/class-vm.d.ts.map +1 -0
- package/dist/serialization/reducers/class-vm.js +77 -0
- package/dist/serialization/reducers/class.d.ts +11 -0
- package/dist/serialization/reducers/class.d.ts.map +1 -0
- package/dist/serialization/reducers/class.js +73 -0
- package/dist/serialization/reducers/common-vm.d.ts +15 -0
- package/dist/serialization/reducers/common-vm.d.ts.map +1 -0
- package/dist/serialization/reducers/common-vm.js +579 -0
- package/dist/serialization/reducers/common.d.ts +16 -0
- package/dist/serialization/reducers/common.d.ts.map +1 -0
- package/dist/serialization/reducers/common.js +478 -0
- package/dist/serialization/reducers/step-function-vm.d.ts +44 -0
- package/dist/serialization/reducers/step-function-vm.d.ts.map +1 -0
- package/dist/serialization/reducers/step-function-vm.js +97 -0
- package/dist/serialization/reducers/step-function.d.ts +35 -0
- package/dist/serialization/reducers/step-function.d.ts.map +1 -0
- package/dist/serialization/reducers/step-function.js +104 -0
- package/dist/serialization/step.d.ts +17 -0
- package/dist/serialization/step.d.ts.map +1 -0
- package/dist/serialization/step.js +48 -0
- package/dist/serialization/types.d.ts +273 -0
- package/dist/serialization/types.d.ts.map +1 -0
- package/dist/serialization/types.js +35 -0
- package/dist/serialization/workflow-vm.d.ts +29 -0
- package/dist/serialization/workflow-vm.d.ts.map +1 -0
- package/dist/serialization/workflow-vm.js +71 -0
- package/dist/serialization/workflow.d.ts +29 -0
- package/dist/serialization/workflow.d.ts.map +1 -0
- package/dist/serialization/workflow.js +54 -0
- package/dist/serialization-format.d.ts +60 -4
- package/dist/serialization-format.d.ts.map +1 -1
- package/dist/serialization-format.js +276 -56
- package/dist/serialization.d.ts +374 -221
- package/dist/serialization.d.ts.map +1 -1
- package/dist/serialization.js +2300 -783
- package/dist/set-attributes.d.ts +13 -0
- package/dist/set-attributes.d.ts.map +1 -0
- package/dist/set-attributes.js +60 -0
- package/dist/sleep.d.ts.map +1 -1
- package/dist/sleep.js +3 -2
- package/dist/source-map.d.ts +25 -0
- package/dist/source-map.d.ts.map +1 -1
- package/dist/source-map.js +148 -10
- package/dist/step/context-storage.d.ts +61 -2
- package/dist/step/context-storage.d.ts.map +1 -1
- package/dist/step/context-storage.js +7 -5
- package/dist/step/get-closure-vars.d.ts.map +1 -1
- package/dist/step/get-closure-vars.js +3 -2
- package/dist/step/get-step-metadata.d.ts.map +1 -1
- package/dist/step/get-step-metadata.js +3 -2
- package/dist/step/get-workflow-metadata.d.ts.map +1 -1
- package/dist/step/get-workflow-metadata.js +3 -2
- package/dist/step/writable-stream.d.ts.map +1 -1
- package/dist/step/writable-stream.js +71 -7
- package/dist/step.d.ts.map +1 -1
- package/dist/step.js +225 -29
- package/dist/symbols.d.ts +54 -0
- package/dist/symbols.d.ts.map +1 -1
- package/dist/symbols.js +55 -1
- package/dist/telemetry/semantic-conventions.d.ts +265 -2
- package/dist/telemetry/semantic-conventions.d.ts.map +1 -1
- package/dist/telemetry/semantic-conventions.js +190 -1
- package/dist/telemetry.d.ts +73 -0
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +153 -16
- package/dist/types.d.ts +6 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +23 -1
- package/dist/util.d.ts +16 -6
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +25 -16
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +2 -2
- package/dist/vm/index.d.ts.map +1 -1
- package/dist/vm/index.js +85 -14
- package/dist/vm/script-cache.d.ts +28 -0
- package/dist/vm/script-cache.d.ts.map +1 -0
- package/dist/vm/script-cache.js +140 -0
- package/dist/workflow/abort-controller.d.ts +65 -0
- package/dist/workflow/abort-controller.d.ts.map +1 -0
- package/dist/workflow/abort-controller.js +312 -0
- package/dist/workflow/attribute-dispatcher.d.ts +6 -0
- package/dist/workflow/attribute-dispatcher.d.ts.map +1 -0
- package/dist/workflow/attribute-dispatcher.js +48 -0
- package/dist/workflow/create-hook.d.ts.map +1 -1
- package/dist/workflow/create-hook.js +25 -3
- package/dist/workflow/define-hook.d.ts +1 -1
- package/dist/workflow/define-hook.d.ts.map +1 -1
- package/dist/workflow/define-hook.js +8 -4
- package/dist/workflow/get-workflow-metadata.d.ts.map +1 -1
- package/dist/workflow/get-workflow-metadata.js +14 -3
- package/dist/workflow/hook.d.ts.map +1 -1
- package/dist/workflow/hook.js +302 -36
- package/dist/workflow/index.d.ts +1 -0
- package/dist/workflow/index.d.ts.map +1 -1
- package/dist/workflow/index.js +5 -3
- package/dist/workflow/set-attributes.d.ts +68 -0
- package/dist/workflow/set-attributes.d.ts.map +1 -0
- package/dist/workflow/set-attributes.js +60 -0
- package/dist/workflow/sleep.d.ts.map +1 -1
- package/dist/workflow/sleep.js +57 -8
- package/dist/workflow/world-init-stub.d.ts +15 -0
- package/dist/workflow/world-init-stub.d.ts.map +1 -0
- package/dist/workflow/world-init-stub.js +15 -0
- package/dist/workflow.d.ts +79 -3
- package/dist/workflow.d.ts.map +1 -1
- package/dist/workflow.js +831 -553
- package/docs/api-reference/create-hook.mdx +79 -0
- package/docs/api-reference/create-webhook.mdx +1 -0
- package/docs/api-reference/define-hook.mdx +26 -24
- package/docs/api-reference/fatal-error.mdx +29 -7
- package/docs/api-reference/fetch.mdx +8 -4
- package/docs/api-reference/index.mdx +3 -0
- package/docs/api-reference/set-attributes.mdx +61 -0
- package/docs/api-reference/sleep.mdx +1 -1
- package/docs/foundations/cancellation.mdx +459 -0
- package/docs/foundations/errors-and-retries.mdx +7 -3
- package/docs/foundations/hooks.mdx +29 -0
- package/docs/foundations/idempotency.mdx +243 -11
- package/docs/foundations/index.mdx +1 -23
- package/docs/foundations/meta.json +3 -1
- package/docs/foundations/serialization.mdx +77 -41
- package/docs/foundations/starting-workflows.mdx +79 -2
- package/docs/foundations/streaming.mdx +14 -23
- package/docs/foundations/versioning.mdx +263 -0
- package/docs/how-it-works/cancellation.mdx +287 -0
- package/docs/how-it-works/code-transform.mdx +21 -17
- package/docs/how-it-works/encryption.mdx +5 -5
- package/docs/how-it-works/event-sourcing.mdx +17 -9
- package/docs/how-it-works/framework-integrations.mdx +96 -337
- package/docs/how-it-works/meta.json +2 -1
- package/package.json +31 -13
- package/dist/runtime/step-handler.d.ts +0 -2
- package/dist/runtime/step-handler.d.ts.map +0 -1
- package/dist/runtime/step-handler.js +0 -678
|
@@ -1,27 +1,436 @@
|
|
|
1
|
+
import { envNumber } from '@workflow/world';
|
|
2
|
+
import { runtimeLogger } from '../logger.js';
|
|
1
3
|
// Maximum number of queue delivery attempts before the handler gives up and
|
|
2
4
|
// gracefully fails the run/step. This must be bounded under the VQS message
|
|
3
5
|
// max visibility window (24 hours) so that our handler-side failure path
|
|
4
6
|
// reliably executes before VQS expires the message.
|
|
5
7
|
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
8
|
+
// The effective wall-clock survival depends on the per-redelivery backoff: the
|
|
9
|
+
// `retry-after` the handler returns (see world-vercel
|
|
10
|
+
// `getHandlerErrorRetryAfterSeconds`) fed through VQS `calculateBackoffDelay`.
|
|
11
|
+
// VQS uses our value for the first 32 attempts (clamped to [5s, 900s]) then
|
|
12
|
+
// applies its own exponential growth — every hop hard-capped at the SQS limit
|
|
13
|
+
// of 900s. With the backoff ramping toward that 900s ceiling (reached by
|
|
14
|
+
// ~delivery 11), 48 attempts span roughly 9–10 hours of wall-clock (~35,000s),
|
|
15
|
+
// comfortably under the 24-hour message-visibility limit so the failure path
|
|
16
|
+
// runs before the message expires. (A flatter, low-capped backoff exhausts the
|
|
17
|
+
// budget in only a few hours, failing otherwise-healthy runs during a transient
|
|
18
|
+
// backend outage; conversely, spanning the full 24h window would require a
|
|
19
|
+
// substantially higher cap here, not a higher per-hop ceiling — VQS clamps
|
|
20
|
+
// every hop at 900s.)
|
|
13
21
|
export const MAX_QUEUE_DELIVERIES = 48;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
22
|
+
/**
|
|
23
|
+
* Effective max queue deliveries. Override via `WORKFLOW_MAX_QUEUE_DELIVERIES`.
|
|
24
|
+
*/
|
|
25
|
+
export function getMaxQueueDeliveries() {
|
|
26
|
+
// Only ever lower the delivery budget. The default is calibrated so the
|
|
27
|
+
// handler-side failure path runs before VQS message-retention expiry (see
|
|
28
|
+
// MAX_QUEUE_DELIVERIES above); a higher value would bypass that invariant and
|
|
29
|
+
// let a bad deployment redeliver until queue expiry instead of recording
|
|
30
|
+
// run_fail. `max` clamps a too-high override back down to the safe default.
|
|
31
|
+
return envNumber('WORKFLOW_MAX_QUEUE_DELIVERIES', MAX_QUEUE_DELIVERIES, {
|
|
32
|
+
integer: true,
|
|
33
|
+
min: 1,
|
|
34
|
+
max: MAX_QUEUE_DELIVERIES,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Default maximum time allowed for the *replay* portion of a single workflow
|
|
39
|
+
* handler invocation (in ms). This budget only covers deterministic-replay
|
|
40
|
+
* and workflow-VM execution between step boundaries — inline step bodies
|
|
41
|
+
* (`"use step"` functions invoked via `executeStep`) do NOT count against
|
|
42
|
+
* it. Step bodies are bounded separately by the platform's function
|
|
43
|
+
* `maxDuration` (e.g. 800s on Vercel Pro Fluid) and `NO_INLINE_REPLAY_AFTER_MS`.
|
|
44
|
+
*
|
|
45
|
+
* If the non-step ("replay") time within a single invocation exceeds this
|
|
46
|
+
* budget, the handler rejects so the queue can retry. After
|
|
47
|
+
* `REPLAY_TIMEOUT_MAX_RETRIES` exhausted attempts the run is failed with
|
|
48
|
+
* `RUN_ERROR_CODES.REPLAY_TIMEOUT`.
|
|
49
|
+
*
|
|
50
|
+
* Note that on Vercel Hobby (standard functions), the platform `maxDuration`
|
|
51
|
+
* is 60s — well below this budget, so the platform SIGTERM will fire first
|
|
52
|
+
* and the queue will re-deliver until the visibility window expires. With
|
|
53
|
+
* Fluid Compute on Hobby the per-function ceiling rises to 300s, still
|
|
54
|
+
* under the default budget.
|
|
55
|
+
*
|
|
56
|
+
* Override via the `WORKFLOW_REPLAY_TIMEOUT_MS` env var (clamped to
|
|
57
|
+
* `MIN_REPLAY_TIMEOUT_MS`..`MAX_REPLAY_TIMEOUT_MS`).
|
|
58
|
+
*/
|
|
21
59
|
export const REPLAY_TIMEOUT_MS = 240_000;
|
|
60
|
+
/** Lower bound for the replay-timeout env var override. */
|
|
61
|
+
export const MIN_REPLAY_TIMEOUT_MS = 30_000;
|
|
62
|
+
/**
|
|
63
|
+
* Upper bound for the replay-timeout env var override. 780s leaves ≥20s of
|
|
64
|
+
* headroom under Vercel Pro Fluid's 800s function ceiling so the handler
|
|
65
|
+
* can write `run_failed` before SIGTERM.
|
|
66
|
+
*/
|
|
67
|
+
export const MAX_REPLAY_TIMEOUT_MS = 780_000;
|
|
68
|
+
// Track which raw env var values we've already warned about so the warning
|
|
69
|
+
// log only fires once per process (the function may be called many times).
|
|
70
|
+
const warnedReplayTimeoutValues = new Set();
|
|
71
|
+
function warnOnce(raw, message, data) {
|
|
72
|
+
if (warnedReplayTimeoutValues.has(raw))
|
|
73
|
+
return;
|
|
74
|
+
warnedReplayTimeoutValues.add(raw);
|
|
75
|
+
runtimeLogger.warn(message, data);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Resolve the effective replay-timeout budget for the current process.
|
|
79
|
+
*
|
|
80
|
+
* Reads `process.env.WORKFLOW_REPLAY_TIMEOUT_MS` lazily so tests and
|
|
81
|
+
* deployments can override per invocation. Invalid / out-of-range values
|
|
82
|
+
* fall back to a safe value (no throw — the env var is an escape hatch,
|
|
83
|
+
* not a hard requirement) and emit a one-time warning so misconfiguration
|
|
84
|
+
* is observable.
|
|
85
|
+
*/
|
|
86
|
+
export function getReplayTimeoutMs() {
|
|
87
|
+
const raw = process.env.WORKFLOW_REPLAY_TIMEOUT_MS;
|
|
88
|
+
if (!raw)
|
|
89
|
+
return REPLAY_TIMEOUT_MS;
|
|
90
|
+
const parsed = Number(raw);
|
|
91
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
92
|
+
warnOnce(raw, 'Ignoring WORKFLOW_REPLAY_TIMEOUT_MS: not a positive finite number; using default', { raw, defaultMs: REPLAY_TIMEOUT_MS });
|
|
93
|
+
return REPLAY_TIMEOUT_MS;
|
|
94
|
+
}
|
|
95
|
+
if (parsed < MIN_REPLAY_TIMEOUT_MS) {
|
|
96
|
+
warnOnce(raw, 'WORKFLOW_REPLAY_TIMEOUT_MS below minimum; clamped', {
|
|
97
|
+
raw,
|
|
98
|
+
clampedMs: MIN_REPLAY_TIMEOUT_MS,
|
|
99
|
+
minMs: MIN_REPLAY_TIMEOUT_MS,
|
|
100
|
+
});
|
|
101
|
+
return MIN_REPLAY_TIMEOUT_MS;
|
|
102
|
+
}
|
|
103
|
+
if (parsed > MAX_REPLAY_TIMEOUT_MS) {
|
|
104
|
+
warnOnce(raw, 'WORKFLOW_REPLAY_TIMEOUT_MS above maximum; clamped', {
|
|
105
|
+
raw,
|
|
106
|
+
clampedMs: MAX_REPLAY_TIMEOUT_MS,
|
|
107
|
+
maxMs: MAX_REPLAY_TIMEOUT_MS,
|
|
108
|
+
});
|
|
109
|
+
return MAX_REPLAY_TIMEOUT_MS;
|
|
110
|
+
}
|
|
111
|
+
return parsed;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Reset the warn-once cache. Test-only — exported so unit tests can
|
|
115
|
+
* exercise the warn path repeatedly without sharing state.
|
|
116
|
+
*
|
|
117
|
+
* @internal
|
|
118
|
+
*/
|
|
119
|
+
export function _resetReplayTimeoutWarnCacheForTests() {
|
|
120
|
+
warnedReplayTimeoutValues.clear();
|
|
121
|
+
}
|
|
22
122
|
// Number of queue delivery attempts to allow before permanently failing a run
|
|
23
123
|
// due to a replay timeout. On attempts 1 through this value, the timeout
|
|
24
|
-
// handler
|
|
124
|
+
// handler rejects without writing run_failed so the queue retries the message.
|
|
25
125
|
// On the next attempt the run is marked as failed.
|
|
26
126
|
export const REPLAY_TIMEOUT_MAX_RETRIES = 3;
|
|
27
|
-
|
|
127
|
+
/**
|
|
128
|
+
* Effective replay-timeout retry budget. Override via
|
|
129
|
+
* `WORKFLOW_REPLAY_TIMEOUT_MAX_RETRIES`.
|
|
130
|
+
*/
|
|
131
|
+
export function getReplayTimeoutMaxRetries() {
|
|
132
|
+
return envNumber('WORKFLOW_REPLAY_TIMEOUT_MAX_RETRIES', REPLAY_TIMEOUT_MAX_RETRIES, { integer: true });
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Default maximum number of steps the owned-inline path runs inline (in
|
|
136
|
+
* parallel) per suspension. The rest are queued to background handlers. Each
|
|
137
|
+
* inline step is created lazily — its `step_created` is folded into the
|
|
138
|
+
* `step_started` that `executeStep` sends — so inlining N steps saves N queue
|
|
139
|
+
* round-trips for a `Promise.all`-style fan-out. `1` reproduces the
|
|
140
|
+
* single-inline-step behavior exactly (useful kill-switch).
|
|
141
|
+
*
|
|
142
|
+
* Override via `WORKFLOW_MAX_INLINE_STEPS` (clamped to
|
|
143
|
+
* `MIN_MAX_INLINE_STEPS`..`MAX_MAX_INLINE_STEPS`).
|
|
144
|
+
*/
|
|
145
|
+
export const MAX_INLINE_STEPS = 3;
|
|
146
|
+
/** Lower bound for the inline-steps env override (1 = single inline step). */
|
|
147
|
+
export const MIN_MAX_INLINE_STEPS = 1;
|
|
148
|
+
/**
|
|
149
|
+
* Upper bound for the inline-steps env override. Inline bodies run in parallel
|
|
150
|
+
* within one function invocation, so this caps memory/CPU fan-out per handler.
|
|
151
|
+
*/
|
|
152
|
+
export const MAX_MAX_INLINE_STEPS = 16;
|
|
153
|
+
// Warn-once cache for WORKFLOW_MAX_INLINE_STEPS, keyed by raw env value.
|
|
154
|
+
const warnedMaxInlineStepsValues = new Set();
|
|
155
|
+
/**
|
|
156
|
+
* Resolve the effective max number of inline steps for the current process.
|
|
157
|
+
*
|
|
158
|
+
* Reads `process.env.WORKFLOW_MAX_INLINE_STEPS` lazily so tests and
|
|
159
|
+
* deployments can override per invocation. Invalid / out-of-range values fall
|
|
160
|
+
* back to a safe value (no throw — the env var is an escape hatch) and emit a
|
|
161
|
+
* one-time warning so misconfiguration is observable.
|
|
162
|
+
*/
|
|
163
|
+
export function getMaxInlineSteps() {
|
|
164
|
+
const raw = process.env.WORKFLOW_MAX_INLINE_STEPS;
|
|
165
|
+
if (!raw)
|
|
166
|
+
return MAX_INLINE_STEPS;
|
|
167
|
+
const parsed = Number(raw);
|
|
168
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
169
|
+
if (!warnedMaxInlineStepsValues.has(raw)) {
|
|
170
|
+
warnedMaxInlineStepsValues.add(raw);
|
|
171
|
+
runtimeLogger.warn('Ignoring WORKFLOW_MAX_INLINE_STEPS: not a positive integer; using default', { raw, defaultValue: MAX_INLINE_STEPS });
|
|
172
|
+
}
|
|
173
|
+
return MAX_INLINE_STEPS;
|
|
174
|
+
}
|
|
175
|
+
if (parsed < MIN_MAX_INLINE_STEPS)
|
|
176
|
+
return MIN_MAX_INLINE_STEPS;
|
|
177
|
+
if (parsed > MAX_MAX_INLINE_STEPS) {
|
|
178
|
+
if (!warnedMaxInlineStepsValues.has(raw)) {
|
|
179
|
+
warnedMaxInlineStepsValues.add(raw);
|
|
180
|
+
runtimeLogger.warn('WORKFLOW_MAX_INLINE_STEPS above maximum; clamped', {
|
|
181
|
+
raw,
|
|
182
|
+
clampedValue: MAX_MAX_INLINE_STEPS,
|
|
183
|
+
maxValue: MAX_MAX_INLINE_STEPS,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
return MAX_MAX_INLINE_STEPS;
|
|
187
|
+
}
|
|
188
|
+
return parsed;
|
|
189
|
+
}
|
|
190
|
+
const warnedMaxEventsValues = new Set();
|
|
191
|
+
/**
|
|
192
|
+
* Optional client-side override for the server-supplied per-run event ceiling.
|
|
193
|
+
* When set to a positive integer, the runtime clamps the server's limit *down*
|
|
194
|
+
* to this value (never raises it) so enforcement can be exercised without a
|
|
195
|
+
* server-side change. `undefined` (unset) ⇒ use the server value as-is.
|
|
196
|
+
*
|
|
197
|
+
* Reads `process.env.WORKFLOW_MAX_EVENTS_OVERRIDE` lazily so tests and
|
|
198
|
+
* deployments can override per invocation. Invalid values fall back to unset
|
|
199
|
+
* (no throw — the env var is an escape hatch) and emit a one-time warning.
|
|
200
|
+
*/
|
|
201
|
+
export function getMaxEventsOverride() {
|
|
202
|
+
const raw = process.env.WORKFLOW_MAX_EVENTS_OVERRIDE;
|
|
203
|
+
if (!raw)
|
|
204
|
+
return undefined;
|
|
205
|
+
const parsed = Number(raw);
|
|
206
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
207
|
+
if (!warnedMaxEventsValues.has(raw)) {
|
|
208
|
+
warnedMaxEventsValues.add(raw);
|
|
209
|
+
runtimeLogger.warn('Ignoring WORKFLOW_MAX_EVENTS_OVERRIDE: not a positive integer; using server limit', { raw });
|
|
210
|
+
}
|
|
211
|
+
return undefined;
|
|
212
|
+
}
|
|
213
|
+
return parsed;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Whether optimistic inline step start is enabled. When on, the owned-inline
|
|
217
|
+
* path begins running a brand-new step's body *before* its lazy `step_started`
|
|
218
|
+
* network call resolves (the input is already known locally), awaiting the
|
|
219
|
+
* `step_started` only before the terminal write.
|
|
220
|
+
*
|
|
221
|
+
* This can run a step body more than once when handlers race for the same
|
|
222
|
+
* step's create-claim — both run the body before one wins. That is unsafe for
|
|
223
|
+
* steps with non-idempotent side effects; in particular, two concurrent runs
|
|
224
|
+
* of a step that writes to the workflow stream (e.g. an AI agent streaming
|
|
225
|
+
* tokens) can interleave and corrupt the stream data. So the optimization is
|
|
226
|
+
* **off by default** and must be explicitly opted into per deployment.
|
|
227
|
+
*
|
|
228
|
+
* Reads `process.env.WORKFLOW_OPTIMISTIC_INLINE_START` lazily. Default OFF;
|
|
229
|
+
* enabled only by an explicit `'1'` / `'true'`.
|
|
230
|
+
*/
|
|
231
|
+
export function isOptimisticInlineStartEnabled() {
|
|
232
|
+
const raw = process.env.WORKFLOW_OPTIMISTIC_INLINE_START;
|
|
233
|
+
if (raw === undefined || raw === '')
|
|
234
|
+
return false;
|
|
235
|
+
return raw === '1' || raw.toLowerCase() === 'true';
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Whether an operator has **explicitly disabled** optimistic inline start via
|
|
239
|
+
* `WORKFLOW_OPTIMISTIC_INLINE_START=0` / `=false`. Distinct from "unset": unset
|
|
240
|
+
* leaves the optimization off by default but lets turbo force it on; an explicit
|
|
241
|
+
* `0`/`false` is an operator opt-out that turbo must honor (turbo's forced
|
|
242
|
+
* optimistic start still runs a step body before `step_started`/`run_started` is
|
|
243
|
+
* confirmed, the property such an operator is opting out of), so
|
|
244
|
+
* `forceOptimisticStart` defers to this. Reads the env var lazily.
|
|
245
|
+
*/
|
|
246
|
+
export function isOptimisticInlineStartExplicitlyDisabled() {
|
|
247
|
+
const raw = process.env.WORKFLOW_OPTIMISTIC_INLINE_START;
|
|
248
|
+
if (raw === undefined || raw === '')
|
|
249
|
+
return false;
|
|
250
|
+
return raw === '0' || raw.toLowerCase() === 'false';
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Whether "turbo mode" is enabled. Turbo mode fast-paths the *first delivery of
|
|
254
|
+
* the first invocation* of a run (detected by the entrypoint via `runInput`
|
|
255
|
+
* presence + `metadata.attempt === 1`): it backgrounds the `run_started` event
|
|
256
|
+
* creation, skips the initial event-log load (nothing has been written yet),
|
|
257
|
+
* and forces optimistic inline step start for that invocation — independent of
|
|
258
|
+
* `WORKFLOW_OPTIMISTIC_INLINE_START`.
|
|
259
|
+
*
|
|
260
|
+
* Forcing optimistic start is safe here because the first delivery has no
|
|
261
|
+
* concurrent peer handler to race the step create-claim, so a step body runs
|
|
262
|
+
* exactly once. That single-handler guarantee ends as soon as the run creates a
|
|
263
|
+
* hook or wait (which introduce resume/parallel invocations), so the runtime
|
|
264
|
+
* exits turbo at that point.
|
|
265
|
+
*
|
|
266
|
+
* Reads `process.env.WORKFLOW_TURBO` lazily. Default **ON**; disabled only by an
|
|
267
|
+
* explicit `'0'` / `'false'` (case-insensitive).
|
|
268
|
+
*/
|
|
269
|
+
export function isTurboEnabled() {
|
|
270
|
+
const raw = process.env.WORKFLOW_TURBO;
|
|
271
|
+
if (raw === undefined || raw === '')
|
|
272
|
+
return true;
|
|
273
|
+
return !(raw === '0' || raw.toLowerCase() === 'false');
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Whether the QuickJS engine's baseline-snapshot startup optimization is
|
|
277
|
+
* enabled (default ON). When on, the engine hydrates a VM with the
|
|
278
|
+
* workflow bundle once per function instance, snapshots it, and starts
|
|
279
|
+
* every invocation by restoring the snapshot instead of re-evaluating
|
|
280
|
+
* the bundle — skipping the dominant share of VM startup (measured
|
|
281
|
+
* ~77ms → ~3ms to first suspension for a 1.3MB bundle). Bundles whose
|
|
282
|
+
* module scope consumes randomness, reads the clock, or replaces a
|
|
283
|
+
* serialization intrinsic are detected at hydrate time and
|
|
284
|
+
* automatically fall back to per-invocation fresh evaluation (see
|
|
285
|
+
* prepareBaselineSnapshot). Set WORKFLOW_QUICKJS_BASELINE_SNAPSHOT=0 to
|
|
286
|
+
* disable.
|
|
287
|
+
*/
|
|
288
|
+
export function isQuickJSBaselineSnapshotEnabled() {
|
|
289
|
+
const raw = process.env.WORKFLOW_QUICKJS_BASELINE_SNAPSHOT;
|
|
290
|
+
if (raw === undefined || raw === '')
|
|
291
|
+
return true;
|
|
292
|
+
return !(raw === '0' || raw.toLowerCase() === 'false');
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Whether the inline loop retains a suspended workflow VM across inline steps
|
|
296
|
+
* within one invocation (default ON). When on, a step-only suspension keeps
|
|
297
|
+
* the live VM, event consumer, and hydrated state alive, and the next loop
|
|
298
|
+
* iteration appends only the newly durable events instead of rebuilding the
|
|
299
|
+
* `vm.Context` and replaying the whole event log. Non-step suspensions and
|
|
300
|
+
* replay divergence always fall back to the ordinary durable replay path.
|
|
301
|
+
*
|
|
302
|
+
* `WORKFLOW_RETAINED_VM=0` (or `false`) is the kill switch: every iteration
|
|
303
|
+
* replays from scratch in a fresh VM, matching the pre-retention behavior.
|
|
304
|
+
*/
|
|
305
|
+
export function isVmRetentionEnabled() {
|
|
306
|
+
const raw = process.env.WORKFLOW_RETAINED_VM;
|
|
307
|
+
if (raw === undefined || raw === '')
|
|
308
|
+
return true;
|
|
309
|
+
return !(raw === '0' || raw.toLowerCase() === 'false');
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Whether inline step ownership is enabled (default ON). When on, the lazy
|
|
313
|
+
* `step_started` that creates an inline step records the owning queue
|
|
314
|
+
* message ID, and wake replays that observe an actively-owned step enqueue a
|
|
315
|
+
* *delayed backstop* message instead of immediately requeueing it — fixing
|
|
316
|
+
* duplicate inline step execution when a hook/wait wakes a run mid-step
|
|
317
|
+
* (vercel/workflow#2780).
|
|
318
|
+
*
|
|
319
|
+
* `WORKFLOW_INLINE_OWNERSHIP=0` (or `false`) is the kill switch: dispatch
|
|
320
|
+
* reverts to the unconditional immediate requeue. Stamping is unaffected —
|
|
321
|
+
* the recorded ownerMessageId is inert data when the switch is off.
|
|
322
|
+
*/
|
|
323
|
+
export function isInlineOwnershipEnabled() {
|
|
324
|
+
const raw = process.env.WORKFLOW_INLINE_OWNERSHIP;
|
|
325
|
+
if (raw === undefined || raw === '')
|
|
326
|
+
return true;
|
|
327
|
+
return !(raw === '0' || raw.toLowerCase() === 'false');
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Default inline-ownership lease, in seconds: how long after a step's latest
|
|
331
|
+
* (stamped) `step_started` a non-owner invocation assumes the owning
|
|
332
|
+
* invocation may still be alive. Within the lease, wake replays enqueue the
|
|
333
|
+
* step's backstop message with `delaySeconds = lease remaining` instead of
|
|
334
|
+
* immediately; past it, they enqueue immediately (today's behavior).
|
|
335
|
+
*
|
|
336
|
+
* Why a fixed 860 and not a value derived from the function's `maxDuration`:
|
|
337
|
+
* neither runtime nor build time can see the resolved value — builders emit
|
|
338
|
+
* `maxDuration: 'max'`, which the platform resolves per-plan at deploy, and
|
|
339
|
+
* no env var or request-context deadline API exposes the result. The bound
|
|
340
|
+
* comes from a platform rule instead: durations above 800s require explicit
|
|
341
|
+
* per-function numeric config, so a builder-emitted `'max'` resolves to at
|
|
342
|
+
* most 800s — 860s therefore dominates any workflow route's invocation
|
|
343
|
+
* lifetime plus scheduling slack. Revisit when that ceiling moves (the
|
|
344
|
+
* 30-minute duration beta becoming reachable via `'max'` would invalidate
|
|
345
|
+
* the bound). Worlds without an invocation kill bound (world-local,
|
|
346
|
+
* self-hosted) get no death proof from any constant — there the in-process
|
|
347
|
+
* single-flight layer (step-single-flight.ts) is what makes a backstop
|
|
348
|
+
* firing mid-step harmless.
|
|
349
|
+
*
|
|
350
|
+
* 860 also stays under the queue's 900s maximum delay (SQS cap), so a
|
|
351
|
+
* backstop's full lease remainder always fits in a single delayed message.
|
|
352
|
+
*
|
|
353
|
+
* Override via `WORKFLOW_INLINE_OWNERSHIP_LEASE_SECONDS` (clamped to
|
|
354
|
+
* 1..`MAX_INLINE_OWNERSHIP_LEASE_SECONDS`).
|
|
355
|
+
*/
|
|
356
|
+
export const INLINE_OWNERSHIP_LEASE_SECONDS = 860;
|
|
357
|
+
/**
|
|
358
|
+
* Upper bound for the lease env override. 900s is the queue's maximum
|
|
359
|
+
* per-message delay (SQS cap) — a longer lease would need delay chaining
|
|
360
|
+
* like long waits use; clamp instead so one delayed message always suffices.
|
|
361
|
+
*/
|
|
362
|
+
export const MAX_INLINE_OWNERSHIP_LEASE_SECONDS = 900;
|
|
363
|
+
/**
|
|
364
|
+
* Effective inline-ownership lease. Override via
|
|
365
|
+
* `WORKFLOW_INLINE_OWNERSHIP_LEASE_SECONDS` — e.g. raise it on self-hosted
|
|
366
|
+
* multi-instance worlds with long-running steps to widen the window in which
|
|
367
|
+
* a live owner is protected from a concurrent backstop execution.
|
|
368
|
+
*/
|
|
369
|
+
export function getInlineOwnershipLeaseSeconds() {
|
|
370
|
+
return envNumber('WORKFLOW_INLINE_OWNERSHIP_LEASE_SECONDS', INLINE_OWNERSHIP_LEASE_SECONDS, { integer: true, min: 1, max: MAX_INLINE_OWNERSHIP_LEASE_SECONDS });
|
|
371
|
+
}
|
|
372
|
+
// A replay-consumer mismatch can be caused by a transient divergent replay
|
|
373
|
+
// rather than an invalid persisted history. Queue bounded recovery replays
|
|
374
|
+
// before recording terminal corruption for a run that cannot replay.
|
|
375
|
+
export const REPLAY_DIVERGENCE_MAX_RETRIES = 3;
|
|
376
|
+
/**
|
|
377
|
+
* Effective replay-divergence recovery budget. Override via
|
|
378
|
+
* `WORKFLOW_REPLAY_DIVERGENCE_MAX_RETRIES`.
|
|
379
|
+
*/
|
|
380
|
+
export function getReplayDivergenceMaxRetries() {
|
|
381
|
+
return envNumber('WORKFLOW_REPLAY_DIVERGENCE_MAX_RETRIES', REPLAY_DIVERGENCE_MAX_RETRIES, { integer: true });
|
|
382
|
+
}
|
|
383
|
+
// A stale-snapshot rejection (412) means the replay's event log was missing an
|
|
384
|
+
// event the World had already recorded, so the replay is re-derived from a
|
|
385
|
+
// corrected log inside the same invocation. Bounded because a persistently
|
|
386
|
+
// rejected write should escalate rather than spin: after this many restarts the
|
|
387
|
+
// run is re-invoked (a new invocation, possibly in a different region), and the
|
|
388
|
+
// run-level budget below then applies.
|
|
389
|
+
export const PRECONDITION_MAX_INPROCESS_RESTARTS = 3;
|
|
390
|
+
/**
|
|
391
|
+
* Effective in-process replay-restart budget for stale-snapshot rejections.
|
|
392
|
+
* Override via `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS`.
|
|
393
|
+
*/
|
|
394
|
+
export function getPreconditionMaxInProcessRestarts() {
|
|
395
|
+
return envNumber('WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS', PRECONDITION_MAX_INPROCESS_RESTARTS, { integer: true });
|
|
396
|
+
}
|
|
397
|
+
// The in-process budget above is per-invocation, and a re-invocation that
|
|
398
|
+
// enqueues a fresh message also restarts the queue's delivery count, so without
|
|
399
|
+
// a counter carried on the message a permanently fenced run has no run-level
|
|
400
|
+
// bound at all. It can stay fenced without any permanent fault — a full reload
|
|
401
|
+
// is not atomic across pages, so a busy run can acquire a new hole on every
|
|
402
|
+
// reload — so the chain is counted and the run fails once this many
|
|
403
|
+
// re-invocations have been spent on stale-snapshot rejections.
|
|
404
|
+
export const PRECONDITION_MAX_REINVOCATIONS = 5;
|
|
405
|
+
/**
|
|
406
|
+
* Effective per-run budget for re-invocations caused by stale-snapshot
|
|
407
|
+
* rejections. Override via `WORKFLOW_PRECONDITION_MAX_REINVOCATIONS`.
|
|
408
|
+
*/
|
|
409
|
+
export function getPreconditionMaxReinvocations() {
|
|
410
|
+
return envNumber('WORKFLOW_PRECONDITION_MAX_REINVOCATIONS', PRECONDITION_MAX_REINVOCATIONS, { integer: true });
|
|
411
|
+
}
|
|
412
|
+
// Backoff before a precondition re-invocation. Unlike the in-process restart
|
|
413
|
+
// (where the point is to re-read immediately), a re-invocation only happens
|
|
414
|
+
// after the in-process budget failed to catch up, so the log is being extended
|
|
415
|
+
// faster than this replay can follow it. Waiting lets the writers quiesce.
|
|
416
|
+
export const PRECONDITION_REINVOKE_DELAY_SECONDS = 2;
|
|
417
|
+
/**
|
|
418
|
+
* Effective delay before a precondition re-invocation. Override via
|
|
419
|
+
* `WORKFLOW_PRECONDITION_REINVOKE_DELAY_SECONDS`.
|
|
420
|
+
*/
|
|
421
|
+
export function getPreconditionReinvokeDelaySeconds() {
|
|
422
|
+
return envNumber('WORKFLOW_PRECONDITION_REINVOKE_DELAY_SECONDS', PRECONDITION_REINVOKE_DELAY_SECONDS, { integer: true });
|
|
423
|
+
}
|
|
424
|
+
// A delivery reaching a deployment the run is not pinned to is not treated as
|
|
425
|
+
// permanent. Re-route the message at the run's own deployment a bounded number
|
|
426
|
+
// of times before failing the run with DEPLOYMENT_MISMATCH.
|
|
427
|
+
export const DEPLOYMENT_MISMATCH_MAX_RETRIES = 3;
|
|
428
|
+
/**
|
|
429
|
+
* Effective deployment-mismatch re-route budget. Override via
|
|
430
|
+
* `WORKFLOW_DEPLOYMENT_MISMATCH_MAX_RETRIES`; `0` fails the run on the first
|
|
431
|
+
* misrouted delivery instead of attempting recovery.
|
|
432
|
+
*/
|
|
433
|
+
export function getDeploymentMismatchMaxRetries() {
|
|
434
|
+
return envNumber('WORKFLOW_DEPLOYMENT_MISMATCH_MAX_RETRIES', DEPLOYMENT_MISMATCH_MAX_RETRIES, { integer: true, min: 0 });
|
|
435
|
+
}
|
|
436
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uc3RhbnRzLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3J1bnRpbWUvY29uc3RhbnRzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxTQUFTLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUM1QyxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0sY0FBYyxDQUFDO0FBRTdDLDRFQUE0RTtBQUM1RSw0RUFBNEU7QUFDNUUseUVBQXlFO0FBQ3pFLG9EQUFvRDtBQUNwRCxFQUFFO0FBQ0YsK0VBQStFO0FBQy9FLHNEQUFzRDtBQUN0RCwrRUFBK0U7QUFDL0UsNEVBQTRFO0FBQzVFLDhFQUE4RTtBQUM5RSx5RUFBeUU7QUFDekUsK0VBQStFO0FBQy9FLDZFQUE2RTtBQUM3RSwrRUFBK0U7QUFDL0UsZ0ZBQWdGO0FBQ2hGLDJFQUEyRTtBQUMzRSwyRUFBMkU7QUFDM0Usc0JBQXNCO0FBQ3RCLE1BQU0sQ0FBQyxNQUFNLG9CQUFvQixHQUFHLEVBQUUsQ0FBQztBQUV2Qzs7R0FFRztBQUNILE1BQU0sVUFBVSxxQkFBcUI7SUFDbkMsd0VBQXdFO0lBQ3hFLDBFQUEwRTtJQUMxRSw4RUFBOEU7SUFDOUUseUVBQXlFO0lBQ3pFLDRFQUE0RTtJQUM1RSxPQUFPLFNBQVMsQ0FBQywrQkFBK0IsRUFBRSxvQkFBb0IsRUFBRTtRQUN0RSxPQUFPLEVBQUUsSUFBSTtRQUNiLEdBQUcsRUFBRSxDQUFDO1FBQ04sR0FBRyxFQUFFLG9CQUFvQjtLQUMxQixDQUFDLENBQUM7QUFDTCxDQUFDO0FBRUQ7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OztHQXFCRztBQUNILE1BQU0sQ0FBQyxNQUFNLGlCQUFpQixHQUFHLE9BQU8sQ0FBQztBQUV6QywyREFBMkQ7QUFDM0QsTUFBTSxDQUFDLE1BQU0scUJBQXFCLEdBQUcsTUFBTSxDQUFDO0FBRTVDOzs7O0dBSUc7QUFDSCxNQUFNLENBQUMsTUFBTSxxQkFBcUIsR0FBRyxPQUFPLENBQUM7QUFFN0MsMkVBQTJFO0FBQzNFLDJFQUEyRTtBQUMzRSxNQUFNLHlCQUF5QixHQUFHLElBQUksR0FBRyxFQUFVLENBQUM7QUFFcEQsU0FBUyxRQUFRLENBQ2YsR0FBVyxFQUNYLE9BQWUsRUFDZixJQUE2QjtJQUU3QixJQUFJLHlCQUF5QixDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUM7UUFBRSxPQUFPO0lBQy9DLHlCQUF5QixDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUNuQyxhQUFhLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxJQUFJLENBQUMsQ0FBQztBQUNwQyxDQUFDO0FBRUQ7Ozs7Ozs7O0dBUUc7QUFDSCxNQUFNLFVBQVUsa0JBQWtCO0lBQ2hDLE1BQU0sR0FBRyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsMEJBQTBCLENBQUM7SUFDbkQsSUFBSSxDQUFDLEdBQUc7UUFBRSxPQUFPLGlCQUFpQixDQUFDO0lBQ25DLE1BQU0sTUFBTSxHQUFHLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUMzQixJQUFJLENBQUMsTUFBTSxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsSUFBSSxNQUFNLElBQUksQ0FBQyxFQUFFLENBQUM7UUFDNUMsUUFBUSxDQUNOLEdBQUcsRUFDSCxrRkFBa0YsRUFDbEYsRUFBRSxHQUFHLEVBQUUsU0FBUyxFQUFFLGlCQUFpQixFQUFFLENBQ3RDLENBQUM7UUFDRixPQUFPLGlCQUFpQixDQUFDO0lBQzNCLENBQUM7SUFDRCxJQUFJLE1BQU0sR0FBRyxxQkFBcUIsRUFBRSxDQUFDO1FBQ25DLFFBQVEsQ0FBQyxHQUFHLEVBQUUsbURBQW1ELEVBQUU7WUFDakUsR0FBRztZQUNILFNBQVMsRUFBRSxxQkFBcUI7WUFDaEMsS0FBSyxFQUFFLHFCQUFxQjtTQUM3QixDQUFDLENBQUM7UUFDSCxPQUFPLHFCQUFxQixDQUFDO0lBQy9CLENBQUM7SUFDRCxJQUFJLE1BQU0sR0FBRyxxQkFBcUIsRUFBRSxDQUFDO1FBQ25DLFFBQVEsQ0FBQyxHQUFHLEVBQUUsbURBQW1ELEVBQUU7WUFDakUsR0FBRztZQUNILFNBQVMsRUFBRSxxQkFBcUI7WUFDaEMsS0FBSyxFQUFFLHFCQUFxQjtTQUM3QixDQUFDLENBQUM7UUFDSCxPQUFPLHFCQUFxQixDQUFDO0lBQy9CLENBQUM7SUFDRCxPQUFPLE1BQU0sQ0FBQztBQUNoQixDQUFDO0FBRUQ7Ozs7O0dBS0c7QUFDSCxNQUFNLFVBQVUsb0NBQW9DO0lBQ2xELHlCQUF5QixDQUFDLEtBQUssRUFBRSxDQUFDO0FBQ3BDLENBQUM7QUFFRCw4RUFBOEU7QUFDOUUseUVBQXlFO0FBQ3pFLCtFQUErRTtBQUMvRSxtREFBbUQ7QUFDbkQsTUFBTSxDQUFDLE1BQU0sMEJBQTBCLEdBQUcsQ0FBQyxDQUFDO0FBRTVDOzs7R0FHRztBQUNILE1BQU0sVUFBVSwwQkFBMEI7SUFDeEMsT0FBTyxTQUFTLENBQ2QscUNBQXFDLEVBQ3JDLDBCQUEwQixFQUMxQixFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsQ0FDbEIsQ0FBQztBQUNKLENBQUM7QUFFRDs7Ozs7Ozs7OztHQVVHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sZ0JBQWdCLEdBQUcsQ0FBQyxDQUFDO0FBRWxDLDhFQUE4RTtBQUM5RSxNQUFNLENBQUMsTUFBTSxvQkFBb0IsR0FBRyxDQUFDLENBQUM7QUFFdEM7OztHQUdHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sb0JBQW9CLEdBQUcsRUFBRSxDQUFDO0FBRXZDLHlFQUF5RTtBQUN6RSxNQUFNLDBCQUEwQixHQUFHLElBQUksR0FBRyxFQUFVLENBQUM7QUFFckQ7Ozs7Ozs7R0FPRztBQUNILE1BQU0sVUFBVSxpQkFBaUI7SUFDL0IsTUFBTSxHQUFHLEdBQUcsT0FBTyxDQUFDLEdBQUcsQ0FBQyx5QkFBeUIsQ0FBQztJQUNsRCxJQUFJLENBQUMsR0FBRztRQUFFLE9BQU8sZ0JBQWdCLENBQUM7SUFDbEMsTUFBTSxNQUFNLEdBQUcsTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQzNCLElBQUksQ0FBQyxNQUFNLENBQUMsU0FBUyxDQUFDLE1BQU0sQ0FBQyxJQUFJLE1BQU0sSUFBSSxDQUFDLEVBQUUsQ0FBQztRQUM3QyxJQUFJLENBQUMsMEJBQTBCLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUM7WUFDekMsMEJBQTBCLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFDO1lBQ3BDLGFBQWEsQ0FBQyxJQUFJLENBQ2hCLDJFQUEyRSxFQUMzRSxFQUFFLEdBQUcsRUFBRSxZQUFZLEVBQUUsZ0JBQWdCLEVBQUUsQ0FDeEMsQ0FBQztRQUNKLENBQUM7UUFDRCxPQUFPLGdCQUFnQixDQUFDO0lBQzFCLENBQUM7SUFDRCxJQUFJLE1BQU0sR0FBRyxvQkFBb0I7UUFBRSxPQUFPLG9CQUFvQixDQUFDO0lBQy9ELElBQUksTUFBTSxHQUFHLG9CQUFvQixFQUFFLENBQUM7UUFDbEMsSUFBSSxDQUFDLDBCQUEwQixDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDO1lBQ3pDLDBCQUEwQixDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsQ0FBQztZQUNwQyxhQUFhLENBQUMsSUFBSSxDQUFDLGtEQUFrRCxFQUFFO2dCQUNyRSxHQUFHO2dCQUNILFlBQVksRUFBRSxvQkFBb0I7Z0JBQ2xDLFFBQVEsRUFBRSxvQkFBb0I7YUFDL0IsQ0FBQyxDQUFDO1FBQ0wsQ0FBQztRQUNELE9BQU8sb0JBQW9CLENBQUM7SUFDOUIsQ0FBQztJQUNELE9BQU8sTUFBTSxDQUFDO0FBQ2hCLENBQUM7QUFFRCxNQUFNLHFCQUFxQixHQUFHLElBQUksR0FBRyxFQUFVLENBQUM7QUFFaEQ7Ozs7Ozs7OztHQVNHO0FBQ0gsTUFBTSxVQUFVLG9CQUFvQjtJQUNsQyxNQUFNLEdBQUcsR0FBRyxPQUFPLENBQUMsR0FBRyxDQUFDLDRCQUE0QixDQUFDO0lBQ3JELElBQUksQ0FBQyxHQUFHO1FBQUUsT0FBTyxTQUFTLENBQUM7SUFDM0IsTUFBTSxNQUFNLEdBQUcsTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQzNCLElBQUksQ0FBQyxNQUFNLENBQUMsU0FBUyxDQUFDLE1BQU0sQ0FBQyxJQUFJLE1BQU0sSUFBSSxDQUFDLEVBQUUsQ0FBQztRQUM3QyxJQUFJLENBQUMscUJBQXFCLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUM7WUFDcEMscUJBQXFCLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFDO1lBQy9CLGFBQWEsQ0FBQyxJQUFJLENBQ2hCLG1GQUFtRixFQUNuRixFQUFFLEdBQUcsRUFBRSxDQUNSLENBQUM7UUFDSixDQUFDO1FBQ0QsT0FBTyxTQUFTLENBQUM7SUFDbkIsQ0FBQztJQUNELE9BQU8sTUFBTSxDQUFDO0FBQ2hCLENBQUM7QUFFRDs7Ozs7Ozs7Ozs7Ozs7O0dBZUc7QUFDSCxNQUFNLFVBQVUsOEJBQThCO0lBQzVDLE1BQU0sR0FBRyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsZ0NBQWdDLENBQUM7SUFDekQsSUFBSSxHQUFHLEtBQUssU0FBUyxJQUFJLEdBQUcsS0FBSyxFQUFFO1FBQUUsT0FBTyxLQUFLLENBQUM7SUFDbEQsT0FBTyxHQUFHLEtBQUssR0FBRyxJQUFJLEdBQUcsQ0FBQyxXQUFXLEVBQUUsS0FBSyxNQUFNLENBQUM7QUFDckQsQ0FBQztBQUVEOzs7Ozs7OztHQVFHO0FBQ0gsTUFBTSxVQUFVLHlDQUF5QztJQUN2RCxNQUFNLEdBQUcsR0FBRyxPQUFPLENBQUMsR0FBRyxDQUFDLGdDQUFnQyxDQUFDO0lBQ3pELElBQUksR0FBRyxLQUFLLFNBQVMsSUFBSSxHQUFHLEtBQUssRUFBRTtRQUFFLE9BQU8sS0FBSyxDQUFDO0lBQ2xELE9BQU8sR0FBRyxLQUFLLEdBQUcsSUFBSSxHQUFHLENBQUMsV0FBVyxFQUFFLEtBQUssT0FBTyxDQUFDO0FBQ3RELENBQUM7QUFFRDs7Ozs7Ozs7Ozs7Ozs7OztHQWdCRztBQUNILE1BQU0sVUFBVSxjQUFjO0lBQzVCLE1BQU0sR0FBRyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsY0FBYyxDQUFDO0lBQ3ZDLElBQUksR0FBRyxLQUFLLFNBQVMsSUFBSSxHQUFHLEtBQUssRUFBRTtRQUFFLE9BQU8sSUFBSSxDQUFDO0lBQ2pELE9BQU8sQ0FBQyxDQUFDLEdBQUcsS0FBSyxHQUFHLElBQUksR0FBRyxDQUFDLFdBQVcsRUFBRSxLQUFLLE9BQU8sQ0FBQyxDQUFDO0FBQ3pELENBQUM7QUFFRDs7Ozs7Ozs7Ozs7O0dBWUc7QUFDSCxNQUFNLFVBQVUsZ0NBQWdDO0lBQzlDLE1BQU0sR0FBRyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsa0NBQWtDLENBQUM7SUFDM0QsSUFBSSxHQUFHLEtBQUssU0FBUyxJQUFJLEdBQUcsS0FBSyxFQUFFO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDakQsT0FBTyxDQUFDLENBQUMsR0FBRyxLQUFLLEdBQUcsSUFBSSxHQUFHLENBQUMsV0FBVyxFQUFFLEtBQUssT0FBTyxDQUFDLENBQUM7QUFDekQsQ0FBQztBQUVEOzs7Ozs7Ozs7O0dBVUc7QUFDSCxNQUFNLFVBQVUsb0JBQW9CO0lBQ2xDLE1BQU0sR0FBRyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsb0JBQW9CLENBQUM7SUFDN0MsSUFBSSxHQUFHLEtBQUssU0FBUyxJQUFJLEdBQUcsS0FBSyxFQUFFO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDakQsT0FBTyxDQUFDLENBQUMsR0FBRyxLQUFLLEdBQUcsSUFBSSxHQUFHLENBQUMsV0FBVyxFQUFFLEtBQUssT0FBTyxDQUFDLENBQUM7QUFDekQsQ0FBQztBQUVEOzs7Ozs7Ozs7OztHQVdHO0FBQ0gsTUFBTSxVQUFVLHdCQUF3QjtJQUN0QyxNQUFNLEdBQUcsR0FBRyxPQUFPLENBQUMsR0FBRyxDQUFDLHlCQUF5QixDQUFDO0lBQ2xELElBQUksR0FBRyxLQUFLLFNBQVMsSUFBSSxHQUFHLEtBQUssRUFBRTtRQUFFLE9BQU8sSUFBSSxDQUFDO0lBQ2pELE9BQU8sQ0FBQyxDQUFDLEdBQUcsS0FBSyxHQUFHLElBQUksR0FBRyxDQUFDLFdBQVcsRUFBRSxLQUFLLE9BQU8sQ0FBQyxDQUFDO0FBQ3pELENBQUM7QUFFRDs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7R0EwQkc7QUFDSCxNQUFNLENBQUMsTUFBTSw4QkFBOEIsR0FBRyxHQUFHLENBQUM7QUFFbEQ7Ozs7R0FJRztBQUNILE1BQU0sQ0FBQyxNQUFNLGtDQUFrQyxHQUFHLEdBQUcsQ0FBQztBQUV0RDs7Ozs7R0FLRztBQUNILE1BQU0sVUFBVSw4QkFBOEI7SUFDNUMsT0FBTyxTQUFTLENBQ2QseUNBQXlDLEVBQ3pDLDhCQUE4QixFQUM5QixFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsR0FBRyxFQUFFLENBQUMsRUFBRSxHQUFHLEVBQUUsa0NBQWtDLEVBQUUsQ0FDbkUsQ0FBQztBQUNKLENBQUM7QUFFRCwyRUFBMkU7QUFDM0UsMkVBQTJFO0FBQzNFLHFFQUFxRTtBQUNyRSxNQUFNLENBQUMsTUFBTSw2QkFBNkIsR0FBRyxDQUFDLENBQUM7QUFFL0M7OztHQUdHO0FBQ0gsTUFBTSxVQUFVLDZCQUE2QjtJQUMzQyxPQUFPLFNBQVMsQ0FDZCx3Q0FBd0MsRUFDeEMsNkJBQTZCLEVBQzdCLEVBQUUsT0FBTyxFQUFFLElBQUksRUFBRSxDQUNsQixDQUFDO0FBQ0osQ0FBQztBQUVELCtFQUErRTtBQUMvRSwyRUFBMkU7QUFDM0UsMkVBQTJFO0FBQzNFLGdGQUFnRjtBQUNoRixnRkFBZ0Y7QUFDaEYsdUNBQXVDO0FBQ3ZDLE1BQU0sQ0FBQyxNQUFNLG1DQUFtQyxHQUFHLENBQUMsQ0FBQztBQUVyRDs7O0dBR0c7QUFDSCxNQUFNLFVBQVUsbUNBQW1DO0lBQ2pELE9BQU8sU0FBUyxDQUNkLDhDQUE4QyxFQUM5QyxtQ0FBbUMsRUFDbkMsRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLENBQ2xCLENBQUM7QUFDSixDQUFDO0FBRUQsMEVBQTBFO0FBQzFFLGdGQUFnRjtBQUNoRiw2RUFBNkU7QUFDN0UsK0VBQStFO0FBQy9FLDRFQUE0RTtBQUM1RSxvRUFBb0U7QUFDcEUsK0RBQStEO0FBQy9ELE1BQU0sQ0FBQyxNQUFNLDhCQUE4QixHQUFHLENBQUMsQ0FBQztBQUVoRDs7O0dBR0c7QUFDSCxNQUFNLFVBQVUsK0JBQStCO0lBQzdDLE9BQU8sU0FBUyxDQUNkLHlDQUF5QyxFQUN6Qyw4QkFBOEIsRUFDOUIsRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLENBQ2xCLENBQUM7QUFDSixDQUFDO0FBRUQsNkVBQTZFO0FBQzdFLDRFQUE0RTtBQUM1RSwrRUFBK0U7QUFDL0UsMkVBQTJFO0FBQzNFLE1BQU0sQ0FBQyxNQUFNLG1DQUFtQyxHQUFHLENBQUMsQ0FBQztBQUVyRDs7O0dBR0c7QUFDSCxNQUFNLFVBQVUsbUNBQW1DO0lBQ2pELE9BQU8sU0FBUyxDQUNkLDhDQUE4QyxFQUM5QyxtQ0FBbUMsRUFDbkMsRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLENBQ2xCLENBQUM7QUFDSixDQUFDO0FBRUQsOEVBQThFO0FBQzlFLCtFQUErRTtBQUMvRSw0REFBNEQ7QUFDNUQsTUFBTSxDQUFDLE1BQU0sK0JBQStCLEdBQUcsQ0FBQyxDQUFDO0FBRWpEOzs7O0dBSUc7QUFDSCxNQUFNLFVBQVUsK0JBQStCO0lBQzdDLE9BQU8sU0FBUyxDQUNkLDBDQUEwQyxFQUMxQywrQkFBK0IsRUFDL0IsRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLEdBQUcsRUFBRSxDQUFDLEVBQUUsQ0FDMUIsQ0FBQztBQUNKLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgeyBlbnZOdW1iZXIgfSBmcm9tICdAd29ya2Zsb3cvd29ybGQnO1xuaW1wb3J0IHsgcnVudGltZUxvZ2dlciB9IGZyb20gJy4uL2xvZ2dlci5qcyc7XG5cbi8vIE1heGltdW0gbnVtYmVyIG9mIHF1ZXVlIGRlbGl2ZXJ5IGF0dGVtcHRzIGJlZm9yZSB0aGUgaGFuZGxlciBnaXZlcyB1cCBhbmRcbi8vIGdyYWNlZnVsbHkgZmFpbHMgdGhlIHJ1bi9zdGVwLiBUaGlzIG11c3QgYmUgYm91bmRlZCB1bmRlciB0aGUgVlFTIG1lc3NhZ2Vcbi8vIG1heCB2aXNpYmlsaXR5IHdpbmRvdyAoMjQgaG91cnMpIHNvIHRoYXQgb3VyIGhhbmRsZXItc2lkZSBmYWlsdXJlIHBhdGhcbi8vIHJlbGlhYmx5IGV4ZWN1dGVzIGJlZm9yZSBWUVMgZXhwaXJlcyB0aGUgbWVzc2FnZS5cbi8vXG4vLyBUaGUgZWZmZWN0aXZlIHdhbGwtY2xvY2sgc3Vydml2YWwgZGVwZW5kcyBvbiB0aGUgcGVyLXJlZGVsaXZlcnkgYmFja29mZjogdGhlXG4vLyBgcmV0cnktYWZ0ZXJgIHRoZSBoYW5kbGVyIHJldHVybnMgKHNlZSB3b3JsZC12ZXJjZWxcbi8vIGBnZXRIYW5kbGVyRXJyb3JSZXRyeUFmdGVyU2Vjb25kc2ApIGZlZCB0aHJvdWdoIFZRUyBgY2FsY3VsYXRlQmFja29mZkRlbGF5YC5cbi8vIFZRUyB1c2VzIG91ciB2YWx1ZSBmb3IgdGhlIGZpcnN0IDMyIGF0dGVtcHRzIChjbGFtcGVkIHRvIFs1cywgOTAwc10pIHRoZW5cbi8vIGFwcGxpZXMgaXRzIG93biBleHBvbmVudGlhbCBncm93dGgg4oCUIGV2ZXJ5IGhvcCBoYXJkLWNhcHBlZCBhdCB0aGUgU1FTIGxpbWl0XG4vLyBvZiA5MDBzLiBXaXRoIHRoZSBiYWNrb2ZmIHJhbXBpbmcgdG93YXJkIHRoYXQgOTAwcyBjZWlsaW5nIChyZWFjaGVkIGJ5XG4vLyB+ZGVsaXZlcnkgMTEpLCA0OCBhdHRlbXB0cyBzcGFuIHJvdWdobHkgOeKAkzEwIGhvdXJzIG9mIHdhbGwtY2xvY2sgKH4zNSwwMDBzKSxcbi8vIGNvbWZvcnRhYmx5IHVuZGVyIHRoZSAyNC1ob3VyIG1lc3NhZ2UtdmlzaWJpbGl0eSBsaW1pdCBzbyB0aGUgZmFpbHVyZSBwYXRoXG4vLyBydW5zIGJlZm9yZSB0aGUgbWVzc2FnZSBleHBpcmVzLiAoQSBmbGF0dGVyLCBsb3ctY2FwcGVkIGJhY2tvZmYgZXhoYXVzdHMgdGhlXG4vLyBidWRnZXQgaW4gb25seSBhIGZldyBob3VycywgZmFpbGluZyBvdGhlcndpc2UtaGVhbHRoeSBydW5zIGR1cmluZyBhIHRyYW5zaWVudFxuLy8gYmFja2VuZCBvdXRhZ2U7IGNvbnZlcnNlbHksIHNwYW5uaW5nIHRoZSBmdWxsIDI0aCB3aW5kb3cgd291bGQgcmVxdWlyZSBhXG4vLyBzdWJzdGFudGlhbGx5IGhpZ2hlciBjYXAgaGVyZSwgbm90IGEgaGlnaGVyIHBlci1ob3AgY2VpbGluZyDigJQgVlFTIGNsYW1wc1xuLy8gZXZlcnkgaG9wIGF0IDkwMHMuKVxuZXhwb3J0IGNvbnN0IE1BWF9RVUVVRV9ERUxJVkVSSUVTID0gNDg7XG5cbi8qKlxuICogRWZmZWN0aXZlIG1heCBxdWV1ZSBkZWxpdmVyaWVzLiBPdmVycmlkZSB2aWEgYFdPUktGTE9XX01BWF9RVUVVRV9ERUxJVkVSSUVTYC5cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGdldE1heFF1ZXVlRGVsaXZlcmllcygpOiBudW1iZXIge1xuICAvLyBPbmx5IGV2ZXIgbG93ZXIgdGhlIGRlbGl2ZXJ5IGJ1ZGdldC4gVGhlIGRlZmF1bHQgaXMgY2FsaWJyYXRlZCBzbyB0aGVcbiAgLy8gaGFuZGxlci1zaWRlIGZhaWx1cmUgcGF0aCBydW5zIGJlZm9yZSBWUVMgbWVzc2FnZS1yZXRlbnRpb24gZXhwaXJ5IChzZWVcbiAgLy8gTUFYX1FVRVVFX0RFTElWRVJJRVMgYWJvdmUpOyBhIGhpZ2hlciB2YWx1ZSB3b3VsZCBieXBhc3MgdGhhdCBpbnZhcmlhbnQgYW5kXG4gIC8vIGxldCBhIGJhZCBkZXBsb3ltZW50IHJlZGVsaXZlciB1bnRpbCBxdWV1ZSBleHBpcnkgaW5zdGVhZCBvZiByZWNvcmRpbmdcbiAgLy8gcnVuX2ZhaWwuIGBtYXhgIGNsYW1wcyBhIHRvby1oaWdoIG92ZXJyaWRlIGJhY2sgZG93biB0byB0aGUgc2FmZSBkZWZhdWx0LlxuICByZXR1cm4gZW52TnVtYmVyKCdXT1JLRkxPV19NQVhfUVVFVUVfREVMSVZFUklFUycsIE1BWF9RVUVVRV9ERUxJVkVSSUVTLCB7XG4gICAgaW50ZWdlcjogdHJ1ZSxcbiAgICBtaW46IDEsXG4gICAgbWF4OiBNQVhfUVVFVUVfREVMSVZFUklFUyxcbiAgfSk7XG59XG5cbi8qKlxuICogRGVmYXVsdCBtYXhpbXVtIHRpbWUgYWxsb3dlZCBmb3IgdGhlICpyZXBsYXkqIHBvcnRpb24gb2YgYSBzaW5nbGUgd29ya2Zsb3dcbiAqIGhhbmRsZXIgaW52b2NhdGlvbiAoaW4gbXMpLiBUaGlzIGJ1ZGdldCBvbmx5IGNvdmVycyBkZXRlcm1pbmlzdGljLXJlcGxheVxuICogYW5kIHdvcmtmbG93LVZNIGV4ZWN1dGlvbiBiZXR3ZWVuIHN0ZXAgYm91bmRhcmllcyDigJQgaW5saW5lIHN0ZXAgYm9kaWVzXG4gKiAoYFwidXNlIHN0ZXBcImAgZnVuY3Rpb25zIGludm9rZWQgdmlhIGBleGVjdXRlU3RlcGApIGRvIE5PVCBjb3VudCBhZ2FpbnN0XG4gKiBpdC4gU3RlcCBib2RpZXMgYXJlIGJvdW5kZWQgc2VwYXJhdGVseSBieSB0aGUgcGxhdGZvcm0ncyBmdW5jdGlvblxuICogYG1heER1cmF0aW9uYCAoZS5nLiA4MDBzIG9uIFZlcmNlbCBQcm8gRmx1aWQpIGFuZCBgTk9fSU5MSU5FX1JFUExBWV9BRlRFUl9NU2AuXG4gKlxuICogSWYgdGhlIG5vbi1zdGVwIChcInJlcGxheVwiKSB0aW1lIHdpdGhpbiBhIHNpbmdsZSBpbnZvY2F0aW9uIGV4Y2VlZHMgdGhpc1xuICogYnVkZ2V0LCB0aGUgaGFuZGxlciByZWplY3RzIHNvIHRoZSBxdWV1ZSBjYW4gcmV0cnkuIEFmdGVyXG4gKiBgUkVQTEFZX1RJTUVPVVRfTUFYX1JFVFJJRVNgIGV4aGF1c3RlZCBhdHRlbXB0cyB0aGUgcnVuIGlzIGZhaWxlZCB3aXRoXG4gKiBgUlVOX0VSUk9SX0NPREVTLlJFUExBWV9USU1FT1VUYC5cbiAqXG4gKiBOb3RlIHRoYXQgb24gVmVyY2VsIEhvYmJ5IChzdGFuZGFyZCBmdW5jdGlvbnMpLCB0aGUgcGxhdGZvcm0gYG1heER1cmF0aW9uYFxuICogaXMgNjBzIOKAlCB3ZWxsIGJlbG93IHRoaXMgYnVkZ2V0LCBzbyB0aGUgcGxhdGZvcm0gU0lHVEVSTSB3aWxsIGZpcmUgZmlyc3RcbiAqIGFuZCB0aGUgcXVldWUgd2lsbCByZS1kZWxpdmVyIHVudGlsIHRoZSB2aXNpYmlsaXR5IHdpbmRvdyBleHBpcmVzLiBXaXRoXG4gKiBGbHVpZCBDb21wdXRlIG9uIEhvYmJ5IHRoZSBwZXItZnVuY3Rpb24gY2VpbGluZyByaXNlcyB0byAzMDBzLCBzdGlsbFxuICogdW5kZXIgdGhlIGRlZmF1bHQgYnVkZ2V0LlxuICpcbiAqIE92ZXJyaWRlIHZpYSB0aGUgYFdPUktGTE9XX1JFUExBWV9USU1FT1VUX01TYCBlbnYgdmFyIChjbGFtcGVkIHRvXG4gKiBgTUlOX1JFUExBWV9USU1FT1VUX01TYC4uYE1BWF9SRVBMQVlfVElNRU9VVF9NU2ApLlxuICovXG5leHBvcnQgY29uc3QgUkVQTEFZX1RJTUVPVVRfTVMgPSAyNDBfMDAwO1xuXG4vKiogTG93ZXIgYm91bmQgZm9yIHRoZSByZXBsYXktdGltZW91dCBlbnYgdmFyIG92ZXJyaWRlLiAqL1xuZXhwb3J0IGNvbnN0IE1JTl9SRVBMQVlfVElNRU9VVF9NUyA9IDMwXzAwMDtcblxuLyoqXG4gKiBVcHBlciBib3VuZCBmb3IgdGhlIHJlcGxheS10aW1lb3V0IGVudiB2YXIgb3ZlcnJpZGUuIDc4MHMgbGVhdmVzIOKJpTIwcyBvZlxuICogaGVhZHJvb20gdW5kZXIgVmVyY2VsIFBybyBGbHVpZCdzIDgwMHMgZnVuY3Rpb24gY2VpbGluZyBzbyB0aGUgaGFuZGxlclxuICogY2FuIHdyaXRlIGBydW5fZmFpbGVkYCBiZWZvcmUgU0lHVEVSTS5cbiAqL1xuZXhwb3J0IGNvbnN0IE1BWF9SRVBMQVlfVElNRU9VVF9NUyA9IDc4MF8wMDA7XG5cbi8vIFRyYWNrIHdoaWNoIHJhdyBlbnYgdmFyIHZhbHVlcyB3ZSd2ZSBhbHJlYWR5IHdhcm5lZCBhYm91dCBzbyB0aGUgd2FybmluZ1xuLy8gbG9nIG9ubHkgZmlyZXMgb25jZSBwZXIgcHJvY2VzcyAodGhlIGZ1bmN0aW9uIG1heSBiZSBjYWxsZWQgbWFueSB0aW1lcykuXG5jb25zdCB3YXJuZWRSZXBsYXlUaW1lb3V0VmFsdWVzID0gbmV3IFNldDxzdHJpbmc+KCk7XG5cbmZ1bmN0aW9uIHdhcm5PbmNlKFxuICByYXc6IHN0cmluZyxcbiAgbWVzc2FnZTogc3RyaW5nLFxuICBkYXRhOiBSZWNvcmQ8c3RyaW5nLCB1bmtub3duPlxuKTogdm9pZCB7XG4gIGlmICh3YXJuZWRSZXBsYXlUaW1lb3V0VmFsdWVzLmhhcyhyYXcpKSByZXR1cm47XG4gIHdhcm5lZFJlcGxheVRpbWVvdXRWYWx1ZXMuYWRkKHJhdyk7XG4gIHJ1bnRpbWVMb2dnZXIud2FybihtZXNzYWdlLCBkYXRhKTtcbn1cblxuLyoqXG4gKiBSZXNvbHZlIHRoZSBlZmZlY3RpdmUgcmVwbGF5LXRpbWVvdXQgYnVkZ2V0IGZvciB0aGUgY3VycmVudCBwcm9jZXNzLlxuICpcbiAqIFJlYWRzIGBwcm9jZXNzLmVudi5XT1JLRkxPV19SRVBMQVlfVElNRU9VVF9NU2AgbGF6aWx5IHNvIHRlc3RzIGFuZFxuICogZGVwbG95bWVudHMgY2FuIG92ZXJyaWRlIHBlciBpbnZvY2F0aW9uLiBJbnZhbGlkIC8gb3V0LW9mLXJhbmdlIHZhbHVlc1xuICogZmFsbCBiYWNrIHRvIGEgc2FmZSB2YWx1ZSAobm8gdGhyb3cg4oCUIHRoZSBlbnYgdmFyIGlzIGFuIGVzY2FwZSBoYXRjaCxcbiAqIG5vdCBhIGhhcmQgcmVxdWlyZW1lbnQpIGFuZCBlbWl0IGEgb25lLXRpbWUgd2FybmluZyBzbyBtaXNjb25maWd1cmF0aW9uXG4gKiBpcyBvYnNlcnZhYmxlLlxuICovXG5leHBvcnQgZnVuY3Rpb24gZ2V0UmVwbGF5VGltZW91dE1zKCk6IG51bWJlciB7XG4gIGNvbnN0IHJhdyA9IHByb2Nlc3MuZW52LldPUktGTE9XX1JFUExBWV9USU1FT1VUX01TO1xuICBpZiAoIXJhdykgcmV0dXJuIFJFUExBWV9USU1FT1VUX01TO1xuICBjb25zdCBwYXJzZWQgPSBOdW1iZXIocmF3KTtcbiAgaWYgKCFOdW1iZXIuaXNGaW5pdGUocGFyc2VkKSB8fCBwYXJzZWQgPD0gMCkge1xuICAgIHdhcm5PbmNlKFxuICAgICAgcmF3LFxuICAgICAgJ0lnbm9yaW5nIFdPUktGTE9XX1JFUExBWV9USU1FT1VUX01TOiBub3QgYSBwb3NpdGl2ZSBmaW5pdGUgbnVtYmVyOyB1c2luZyBkZWZhdWx0JyxcbiAgICAgIHsgcmF3LCBkZWZhdWx0TXM6IFJFUExBWV9USU1FT1VUX01TIH1cbiAgICApO1xuICAgIHJldHVybiBSRVBMQVlfVElNRU9VVF9NUztcbiAgfVxuICBpZiAocGFyc2VkIDwgTUlOX1JFUExBWV9USU1FT1VUX01TKSB7XG4gICAgd2Fybk9uY2UocmF3LCAnV09SS0ZMT1dfUkVQTEFZX1RJTUVPVVRfTVMgYmVsb3cgbWluaW11bTsgY2xhbXBlZCcsIHtcbiAgICAgIHJhdyxcbiAgICAgIGNsYW1wZWRNczogTUlOX1JFUExBWV9USU1FT1VUX01TLFxuICAgICAgbWluTXM6IE1JTl9SRVBMQVlfVElNRU9VVF9NUyxcbiAgICB9KTtcbiAgICByZXR1cm4gTUlOX1JFUExBWV9USU1FT1VUX01TO1xuICB9XG4gIGlmIChwYXJzZWQgPiBNQVhfUkVQTEFZX1RJTUVPVVRfTVMpIHtcbiAgICB3YXJuT25jZShyYXcsICdXT1JLRkxPV19SRVBMQVlfVElNRU9VVF9NUyBhYm92ZSBtYXhpbXVtOyBjbGFtcGVkJywge1xuICAgICAgcmF3LFxuICAgICAgY2xhbXBlZE1zOiBNQVhfUkVQTEFZX1RJTUVPVVRfTVMsXG4gICAgICBtYXhNczogTUFYX1JFUExBWV9USU1FT1VUX01TLFxuICAgIH0pO1xuICAgIHJldHVybiBNQVhfUkVQTEFZX1RJTUVPVVRfTVM7XG4gIH1cbiAgcmV0dXJuIHBhcnNlZDtcbn1cblxuLyoqXG4gKiBSZXNldCB0aGUgd2Fybi1vbmNlIGNhY2hlLiBUZXN0LW9ubHkg4oCUIGV4cG9ydGVkIHNvIHVuaXQgdGVzdHMgY2FuXG4gKiBleGVyY2lzZSB0aGUgd2FybiBwYXRoIHJlcGVhdGVkbHkgd2l0aG91dCBzaGFyaW5nIHN0YXRlLlxuICpcbiAqIEBpbnRlcm5hbFxuICovXG5leHBvcnQgZnVuY3Rpb24gX3Jlc2V0UmVwbGF5VGltZW91dFdhcm5DYWNoZUZvclRlc3RzKCk6IHZvaWQge1xuICB3YXJuZWRSZXBsYXlUaW1lb3V0VmFsdWVzLmNsZWFyKCk7XG59XG5cbi8vIE51bWJlciBvZiBxdWV1ZSBkZWxpdmVyeSBhdHRlbXB0cyB0byBhbGxvdyBiZWZvcmUgcGVybWFuZW50bHkgZmFpbGluZyBhIHJ1blxuLy8gZHVlIHRvIGEgcmVwbGF5IHRpbWVvdXQuIE9uIGF0dGVtcHRzIDEgdGhyb3VnaCB0aGlzIHZhbHVlLCB0aGUgdGltZW91dFxuLy8gaGFuZGxlciByZWplY3RzIHdpdGhvdXQgd3JpdGluZyBydW5fZmFpbGVkIHNvIHRoZSBxdWV1ZSByZXRyaWVzIHRoZSBtZXNzYWdlLlxuLy8gT24gdGhlIG5leHQgYXR0ZW1wdCB0aGUgcnVuIGlzIG1hcmtlZCBhcyBmYWlsZWQuXG5leHBvcnQgY29uc3QgUkVQTEFZX1RJTUVPVVRfTUFYX1JFVFJJRVMgPSAzO1xuXG4vKipcbiAqIEVmZmVjdGl2ZSByZXBsYXktdGltZW91dCByZXRyeSBidWRnZXQuIE92ZXJyaWRlIHZpYVxuICogYFdPUktGTE9XX1JFUExBWV9USU1FT1VUX01BWF9SRVRSSUVTYC5cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGdldFJlcGxheVRpbWVvdXRNYXhSZXRyaWVzKCk6IG51bWJlciB7XG4gIHJldHVybiBlbnZOdW1iZXIoXG4gICAgJ1dPUktGTE9XX1JFUExBWV9USU1FT1VUX01BWF9SRVRSSUVTJyxcbiAgICBSRVBMQVlfVElNRU9VVF9NQVhfUkVUUklFUyxcbiAgICB7IGludGVnZXI6IHRydWUgfVxuICApO1xufVxuXG4vKipcbiAqIERlZmF1bHQgbWF4aW11bSBudW1iZXIgb2Ygc3RlcHMgdGhlIG93bmVkLWlubGluZSBwYXRoIHJ1bnMgaW5saW5lIChpblxuICogcGFyYWxsZWwpIHBlciBzdXNwZW5zaW9uLiBUaGUgcmVzdCBhcmUgcXVldWVkIHRvIGJhY2tncm91bmQgaGFuZGxlcnMuIEVhY2hcbiAqIGlubGluZSBzdGVwIGlzIGNyZWF0ZWQgbGF6aWx5IOKAlCBpdHMgYHN0ZXBfY3JlYXRlZGAgaXMgZm9sZGVkIGludG8gdGhlXG4gKiBgc3RlcF9zdGFydGVkYCB0aGF0IGBleGVjdXRlU3RlcGAgc2VuZHMg4oCUIHNvIGlubGluaW5nIE4gc3RlcHMgc2F2ZXMgTiBxdWV1ZVxuICogcm91bmQtdHJpcHMgZm9yIGEgYFByb21pc2UuYWxsYC1zdHlsZSBmYW4tb3V0LiBgMWAgcmVwcm9kdWNlcyB0aGVcbiAqIHNpbmdsZS1pbmxpbmUtc3RlcCBiZWhhdmlvciBleGFjdGx5ICh1c2VmdWwga2lsbC1zd2l0Y2gpLlxuICpcbiAqIE92ZXJyaWRlIHZpYSBgV09SS0ZMT1dfTUFYX0lOTElORV9TVEVQU2AgKGNsYW1wZWQgdG9cbiAqIGBNSU5fTUFYX0lOTElORV9TVEVQU2AuLmBNQVhfTUFYX0lOTElORV9TVEVQU2ApLlxuICovXG5leHBvcnQgY29uc3QgTUFYX0lOTElORV9TVEVQUyA9IDM7XG5cbi8qKiBMb3dlciBib3VuZCBmb3IgdGhlIGlubGluZS1zdGVwcyBlbnYgb3ZlcnJpZGUgKDEgPSBzaW5nbGUgaW5saW5lIHN0ZXApLiAqL1xuZXhwb3J0IGNvbnN0IE1JTl9NQVhfSU5MSU5FX1NURVBTID0gMTtcblxuLyoqXG4gKiBVcHBlciBib3VuZCBmb3IgdGhlIGlubGluZS1zdGVwcyBlbnYgb3ZlcnJpZGUuIElubGluZSBib2RpZXMgcnVuIGluIHBhcmFsbGVsXG4gKiB3aXRoaW4gb25lIGZ1bmN0aW9uIGludm9jYXRpb24sIHNvIHRoaXMgY2FwcyBtZW1vcnkvQ1BVIGZhbi1vdXQgcGVyIGhhbmRsZXIuXG4gKi9cbmV4cG9ydCBjb25zdCBNQVhfTUFYX0lOTElORV9TVEVQUyA9IDE2O1xuXG4vLyBXYXJuLW9uY2UgY2FjaGUgZm9yIFdPUktGTE9XX01BWF9JTkxJTkVfU1RFUFMsIGtleWVkIGJ5IHJhdyBlbnYgdmFsdWUuXG5jb25zdCB3YXJuZWRNYXhJbmxpbmVTdGVwc1ZhbHVlcyA9IG5ldyBTZXQ8c3RyaW5nPigpO1xuXG4vKipcbiAqIFJlc29sdmUgdGhlIGVmZmVjdGl2ZSBtYXggbnVtYmVyIG9mIGlubGluZSBzdGVwcyBmb3IgdGhlIGN1cnJlbnQgcHJvY2Vzcy5cbiAqXG4gKiBSZWFkcyBgcHJvY2Vzcy5lbnYuV09SS0ZMT1dfTUFYX0lOTElORV9TVEVQU2AgbGF6aWx5IHNvIHRlc3RzIGFuZFxuICogZGVwbG95bWVudHMgY2FuIG92ZXJyaWRlIHBlciBpbnZvY2F0aW9uLiBJbnZhbGlkIC8gb3V0LW9mLXJhbmdlIHZhbHVlcyBmYWxsXG4gKiBiYWNrIHRvIGEgc2FmZSB2YWx1ZSAobm8gdGhyb3cg4oCUIHRoZSBlbnYgdmFyIGlzIGFuIGVzY2FwZSBoYXRjaCkgYW5kIGVtaXQgYVxuICogb25lLXRpbWUgd2FybmluZyBzbyBtaXNjb25maWd1cmF0aW9uIGlzIG9ic2VydmFibGUuXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBnZXRNYXhJbmxpbmVTdGVwcygpOiBudW1iZXIge1xuICBjb25zdCByYXcgPSBwcm9jZXNzLmVudi5XT1JLRkxPV19NQVhfSU5MSU5FX1NURVBTO1xuICBpZiAoIXJhdykgcmV0dXJuIE1BWF9JTkxJTkVfU1RFUFM7XG4gIGNvbnN0IHBhcnNlZCA9IE51bWJlcihyYXcpO1xuICBpZiAoIU51bWJlci5pc0ludGVnZXIocGFyc2VkKSB8fCBwYXJzZWQgPD0gMCkge1xuICAgIGlmICghd2FybmVkTWF4SW5saW5lU3RlcHNWYWx1ZXMuaGFzKHJhdykpIHtcbiAgICAgIHdhcm5lZE1heElubGluZVN0ZXBzVmFsdWVzLmFkZChyYXcpO1xuICAgICAgcnVudGltZUxvZ2dlci53YXJuKFxuICAgICAgICAnSWdub3JpbmcgV09SS0ZMT1dfTUFYX0lOTElORV9TVEVQUzogbm90IGEgcG9zaXRpdmUgaW50ZWdlcjsgdXNpbmcgZGVmYXVsdCcsXG4gICAgICAgIHsgcmF3LCBkZWZhdWx0VmFsdWU6IE1BWF9JTkxJTkVfU1RFUFMgfVxuICAgICAgKTtcbiAgICB9XG4gICAgcmV0dXJuIE1BWF9JTkxJTkVfU1RFUFM7XG4gIH1cbiAgaWYgKHBhcnNlZCA8IE1JTl9NQVhfSU5MSU5FX1NURVBTKSByZXR1cm4gTUlOX01BWF9JTkxJTkVfU1RFUFM7XG4gIGlmIChwYXJzZWQgPiBNQVhfTUFYX0lOTElORV9TVEVQUykge1xuICAgIGlmICghd2FybmVkTWF4SW5saW5lU3RlcHNWYWx1ZXMuaGFzKHJhdykpIHtcbiAgICAgIHdhcm5lZE1heElubGluZVN0ZXBzVmFsdWVzLmFkZChyYXcpO1xuICAgICAgcnVudGltZUxvZ2dlci53YXJuKCdXT1JLRkxPV19NQVhfSU5MSU5FX1NURVBTIGFib3ZlIG1heGltdW07IGNsYW1wZWQnLCB7XG4gICAgICAgIHJhdyxcbiAgICAgICAgY2xhbXBlZFZhbHVlOiBNQVhfTUFYX0lOTElORV9TVEVQUyxcbiAgICAgICAgbWF4VmFsdWU6IE1BWF9NQVhfSU5MSU5FX1NURVBTLFxuICAgICAgfSk7XG4gICAgfVxuICAgIHJldHVybiBNQVhfTUFYX0lOTElORV9TVEVQUztcbiAgfVxuICByZXR1cm4gcGFyc2VkO1xufVxuXG5jb25zdCB3YXJuZWRNYXhFdmVudHNWYWx1ZXMgPSBuZXcgU2V0PHN0cmluZz4oKTtcblxuLyoqXG4gKiBPcHRpb25hbCBjbGllbnQtc2lkZSBvdmVycmlkZSBmb3IgdGhlIHNlcnZlci1zdXBwbGllZCBwZXItcnVuIGV2ZW50IGNlaWxpbmcuXG4gKiBXaGVuIHNldCB0byBhIHBvc2l0aXZlIGludGVnZXIsIHRoZSBydW50aW1lIGNsYW1wcyB0aGUgc2VydmVyJ3MgbGltaXQgKmRvd24qXG4gKiB0byB0aGlzIHZhbHVlIChuZXZlciByYWlzZXMgaXQpIHNvIGVuZm9yY2VtZW50IGNhbiBiZSBleGVyY2lzZWQgd2l0aG91dCBhXG4gKiBzZXJ2ZXItc2lkZSBjaGFuZ2UuIGB1bmRlZmluZWRgICh1bnNldCkg4oeSIHVzZSB0aGUgc2VydmVyIHZhbHVlIGFzLWlzLlxuICpcbiAqIFJlYWRzIGBwcm9jZXNzLmVudi5XT1JLRkxPV19NQVhfRVZFTlRTX09WRVJSSURFYCBsYXppbHkgc28gdGVzdHMgYW5kXG4gKiBkZXBsb3ltZW50cyBjYW4gb3ZlcnJpZGUgcGVyIGludm9jYXRpb24uIEludmFsaWQgdmFsdWVzIGZhbGwgYmFjayB0byB1bnNldFxuICogKG5vIHRocm93IOKAlCB0aGUgZW52IHZhciBpcyBhbiBlc2NhcGUgaGF0Y2gpIGFuZCBlbWl0IGEgb25lLXRpbWUgd2FybmluZy5cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGdldE1heEV2ZW50c092ZXJyaWRlKCk6IG51bWJlciB8IHVuZGVmaW5lZCB7XG4gIGNvbnN0IHJhdyA9IHByb2Nlc3MuZW52LldPUktGTE9XX01BWF9FVkVOVFNfT1ZFUlJJREU7XG4gIGlmICghcmF3KSByZXR1cm4gdW5kZWZpbmVkO1xuICBjb25zdCBwYXJzZWQgPSBOdW1iZXIocmF3KTtcbiAgaWYgKCFOdW1iZXIuaXNJbnRlZ2VyKHBhcnNlZCkgfHwgcGFyc2VkIDw9IDApIHtcbiAgICBpZiAoIXdhcm5lZE1heEV2ZW50c1ZhbHVlcy5oYXMocmF3KSkge1xuICAgICAgd2FybmVkTWF4RXZlbnRzVmFsdWVzLmFkZChyYXcpO1xuICAgICAgcnVudGltZUxvZ2dlci53YXJuKFxuICAgICAgICAnSWdub3JpbmcgV09SS0ZMT1dfTUFYX0VWRU5UU19PVkVSUklERTogbm90IGEgcG9zaXRpdmUgaW50ZWdlcjsgdXNpbmcgc2VydmVyIGxpbWl0JyxcbiAgICAgICAgeyByYXcgfVxuICAgICAgKTtcbiAgICB9XG4gICAgcmV0dXJuIHVuZGVmaW5lZDtcbiAgfVxuICByZXR1cm4gcGFyc2VkO1xufVxuXG4vKipcbiAqIFdoZXRoZXIgb3B0aW1pc3RpYyBpbmxpbmUgc3RlcCBzdGFydCBpcyBlbmFibGVkLiBXaGVuIG9uLCB0aGUgb3duZWQtaW5saW5lXG4gKiBwYXRoIGJlZ2lucyBydW5uaW5nIGEgYnJhbmQtbmV3IHN0ZXAncyBib2R5ICpiZWZvcmUqIGl0cyBsYXp5IGBzdGVwX3N0YXJ0ZWRgXG4gKiBuZXR3b3JrIGNhbGwgcmVzb2x2ZXMgKHRoZSBpbnB1dCBpcyBhbHJlYWR5IGtub3duIGxvY2FsbHkpLCBhd2FpdGluZyB0aGVcbiAqIGBzdGVwX3N0YXJ0ZWRgIG9ubHkgYmVmb3JlIHRoZSB0ZXJtaW5hbCB3cml0ZS5cbiAqXG4gKiBUaGlzIGNhbiBydW4gYSBzdGVwIGJvZHkgbW9yZSB0aGFuIG9uY2Ugd2hlbiBoYW5kbGVycyByYWNlIGZvciB0aGUgc2FtZVxuICogc3RlcCdzIGNyZWF0ZS1jbGFpbSDigJQgYm90aCBydW4gdGhlIGJvZHkgYmVmb3JlIG9uZSB3aW5zLiBUaGF0IGlzIHVuc2FmZSBmb3JcbiAqIHN0ZXBzIHdpdGggbm9uLWlkZW1wb3RlbnQgc2lkZSBlZmZlY3RzOyBpbiBwYXJ0aWN1bGFyLCB0d28gY29uY3VycmVudCBydW5zXG4gKiBvZiBhIHN0ZXAgdGhhdCB3cml0ZXMgdG8gdGhlIHdvcmtmbG93IHN0cmVhbSAoZS5nLiBhbiBBSSBhZ2VudCBzdHJlYW1pbmdcbiAqIHRva2VucykgY2FuIGludGVybGVhdmUgYW5kIGNvcnJ1cHQgdGhlIHN0cmVhbSBkYXRhLiBTbyB0aGUgb3B0aW1pemF0aW9uIGlzXG4gKiAqKm9mZiBieSBkZWZhdWx0KiogYW5kIG11c3QgYmUgZXhwbGljaXRseSBvcHRlZCBpbnRvIHBlciBkZXBsb3ltZW50LlxuICpcbiAqIFJlYWRzIGBwcm9jZXNzLmVudi5XT1JLRkxPV19PUFRJTUlTVElDX0lOTElORV9TVEFSVGAgbGF6aWx5LiBEZWZhdWx0IE9GRjtcbiAqIGVuYWJsZWQgb25seSBieSBhbiBleHBsaWNpdCBgJzEnYCAvIGAndHJ1ZSdgLlxuICovXG5leHBvcnQgZnVuY3Rpb24gaXNPcHRpbWlzdGljSW5saW5lU3RhcnRFbmFibGVkKCk6IGJvb2xlYW4ge1xuICBjb25zdCByYXcgPSBwcm9jZXNzLmVudi5XT1JLRkxPV19PUFRJTUlTVElDX0lOTElORV9TVEFSVDtcbiAgaWYgKHJhdyA9PT0gdW5kZWZpbmVkIHx8IHJhdyA9PT0gJycpIHJldHVybiBmYWxzZTtcbiAgcmV0dXJuIHJhdyA9PT0gJzEnIHx8IHJhdy50b0xvd2VyQ2FzZSgpID09PSAndHJ1ZSc7XG59XG5cbi8qKlxuICogV2hldGhlciBhbiBvcGVyYXRvciBoYXMgKipleHBsaWNpdGx5IGRpc2FibGVkKiogb3B0aW1pc3RpYyBpbmxpbmUgc3RhcnQgdmlhXG4gKiBgV09SS0ZMT1dfT1BUSU1JU1RJQ19JTkxJTkVfU1RBUlQ9MGAgLyBgPWZhbHNlYC4gRGlzdGluY3QgZnJvbSBcInVuc2V0XCI6IHVuc2V0XG4gKiBsZWF2ZXMgdGhlIG9wdGltaXphdGlvbiBvZmYgYnkgZGVmYXVsdCBidXQgbGV0cyB0dXJibyBmb3JjZSBpdCBvbjsgYW4gZXhwbGljaXRcbiAqIGAwYC9gZmFsc2VgIGlzIGFuIG9wZXJhdG9yIG9wdC1vdXQgdGhhdCB0dXJibyBtdXN0IGhvbm9yICh0dXJibydzIGZvcmNlZFxuICogb3B0aW1pc3RpYyBzdGFydCBzdGlsbCBydW5zIGEgc3RlcCBib2R5IGJlZm9yZSBgc3RlcF9zdGFydGVkYC9gcnVuX3N0YXJ0ZWRgIGlzXG4gKiBjb25maXJtZWQsIHRoZSBwcm9wZXJ0eSBzdWNoIGFuIG9wZXJhdG9yIGlzIG9wdGluZyBvdXQgb2YpLCBzb1xuICogYGZvcmNlT3B0aW1pc3RpY1N0YXJ0YCBkZWZlcnMgdG8gdGhpcy4gUmVhZHMgdGhlIGVudiB2YXIgbGF6aWx5LlxuICovXG5leHBvcnQgZnVuY3Rpb24gaXNPcHRpbWlzdGljSW5saW5lU3RhcnRFeHBsaWNpdGx5RGlzYWJsZWQoKTogYm9vbGVhbiB7XG4gIGNvbnN0IHJhdyA9IHByb2Nlc3MuZW52LldPUktGTE9XX09QVElNSVNUSUNfSU5MSU5FX1NUQVJUO1xuICBpZiAocmF3ID09PSB1bmRlZmluZWQgfHwgcmF3ID09PSAnJykgcmV0dXJuIGZhbHNlO1xuICByZXR1cm4gcmF3ID09PSAnMCcgfHwgcmF3LnRvTG93ZXJDYXNlKCkgPT09ICdmYWxzZSc7XG59XG5cbi8qKlxuICogV2hldGhlciBcInR1cmJvIG1vZGVcIiBpcyBlbmFibGVkLiBUdXJibyBtb2RlIGZhc3QtcGF0aHMgdGhlICpmaXJzdCBkZWxpdmVyeSBvZlxuICogdGhlIGZpcnN0IGludm9jYXRpb24qIG9mIGEgcnVuIChkZXRlY3RlZCBieSB0aGUgZW50cnlwb2ludCB2aWEgYHJ1bklucHV0YFxuICogcHJlc2VuY2UgKyBgbWV0YWRhdGEuYXR0ZW1wdCA9PT0gMWApOiBpdCBiYWNrZ3JvdW5kcyB0aGUgYHJ1bl9zdGFydGVkYCBldmVudFxuICogY3JlYXRpb24sIHNraXBzIHRoZSBpbml0aWFsIGV2ZW50LWxvZyBsb2FkIChub3RoaW5nIGhhcyBiZWVuIHdyaXR0ZW4geWV0KSxcbiAqIGFuZCBmb3JjZXMgb3B0aW1pc3RpYyBpbmxpbmUgc3RlcCBzdGFydCBmb3IgdGhhdCBpbnZvY2F0aW9uIOKAlCBpbmRlcGVuZGVudCBvZlxuICogYFdPUktGTE9XX09QVElNSVNUSUNfSU5MSU5FX1NUQVJUYC5cbiAqXG4gKiBGb3JjaW5nIG9wdGltaXN0aWMgc3RhcnQgaXMgc2FmZSBoZXJlIGJlY2F1c2UgdGhlIGZpcnN0IGRlbGl2ZXJ5IGhhcyBub1xuICogY29uY3VycmVudCBwZWVyIGhhbmRsZXIgdG8gcmFjZSB0aGUgc3RlcCBjcmVhdGUtY2xhaW0sIHNvIGEgc3RlcCBib2R5IHJ1bnNcbiAqIGV4YWN0bHkgb25jZS4gVGhhdCBzaW5nbGUtaGFuZGxlciBndWFyYW50ZWUgZW5kcyBhcyBzb29uIGFzIHRoZSBydW4gY3JlYXRlcyBhXG4gKiBob29rIG9yIHdhaXQgKHdoaWNoIGludHJvZHVjZSByZXN1bWUvcGFyYWxsZWwgaW52b2NhdGlvbnMpLCBzbyB0aGUgcnVudGltZVxuICogZXhpdHMgdHVyYm8gYXQgdGhhdCBwb2ludC5cbiAqXG4gKiBSZWFkcyBgcHJvY2Vzcy5lbnYuV09SS0ZMT1dfVFVSQk9gIGxhemlseS4gRGVmYXVsdCAqKk9OKio7IGRpc2FibGVkIG9ubHkgYnkgYW5cbiAqIGV4cGxpY2l0IGAnMCdgIC8gYCdmYWxzZSdgIChjYXNlLWluc2Vuc2l0aXZlKS5cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGlzVHVyYm9FbmFibGVkKCk6IGJvb2xlYW4ge1xuICBjb25zdCByYXcgPSBwcm9jZXNzLmVudi5XT1JLRkxPV19UVVJCTztcbiAgaWYgKHJhdyA9PT0gdW5kZWZpbmVkIHx8IHJhdyA9PT0gJycpIHJldHVybiB0cnVlO1xuICByZXR1cm4gIShyYXcgPT09ICcwJyB8fCByYXcudG9Mb3dlckNhc2UoKSA9PT0gJ2ZhbHNlJyk7XG59XG5cbi8qKlxuICogV2hldGhlciB0aGUgUXVpY2tKUyBlbmdpbmUncyBiYXNlbGluZS1zbmFwc2hvdCBzdGFydHVwIG9wdGltaXphdGlvbiBpc1xuICogZW5hYmxlZCAoZGVmYXVsdCBPTikuIFdoZW4gb24sIHRoZSBlbmdpbmUgaHlkcmF0ZXMgYSBWTSB3aXRoIHRoZVxuICogd29ya2Zsb3cgYnVuZGxlIG9uY2UgcGVyIGZ1bmN0aW9uIGluc3RhbmNlLCBzbmFwc2hvdHMgaXQsIGFuZCBzdGFydHNcbiAqIGV2ZXJ5IGludm9jYXRpb24gYnkgcmVzdG9yaW5nIHRoZSBzbmFwc2hvdCBpbnN0ZWFkIG9mIHJlLWV2YWx1YXRpbmdcbiAqIHRoZSBidW5kbGUg4oCUIHNraXBwaW5nIHRoZSBkb21pbmFudCBzaGFyZSBvZiBWTSBzdGFydHVwIChtZWFzdXJlZFxuICogfjc3bXMg4oaSIH4zbXMgdG8gZmlyc3Qgc3VzcGVuc2lvbiBmb3IgYSAxLjNNQiBidW5kbGUpLiBCdW5kbGVzIHdob3NlXG4gKiBtb2R1bGUgc2NvcGUgY29uc3VtZXMgcmFuZG9tbmVzcywgcmVhZHMgdGhlIGNsb2NrLCBvciByZXBsYWNlcyBhXG4gKiBzZXJpYWxpemF0aW9uIGludHJpbnNpYyBhcmUgZGV0ZWN0ZWQgYXQgaHlkcmF0ZSB0aW1lIGFuZFxuICogYXV0b21hdGljYWxseSBmYWxsIGJhY2sgdG8gcGVyLWludm9jYXRpb24gZnJlc2ggZXZhbHVhdGlvbiAoc2VlXG4gKiBwcmVwYXJlQmFzZWxpbmVTbmFwc2hvdCkuIFNldCBXT1JLRkxPV19RVUlDS0pTX0JBU0VMSU5FX1NOQVBTSE9UPTAgdG9cbiAqIGRpc2FibGUuXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBpc1F1aWNrSlNCYXNlbGluZVNuYXBzaG90RW5hYmxlZCgpOiBib29sZWFuIHtcbiAgY29uc3QgcmF3ID0gcHJvY2Vzcy5lbnYuV09SS0ZMT1dfUVVJQ0tKU19CQVNFTElORV9TTkFQU0hPVDtcbiAgaWYgKHJhdyA9PT0gdW5kZWZpbmVkIHx8IHJhdyA9PT0gJycpIHJldHVybiB0cnVlO1xuICByZXR1cm4gIShyYXcgPT09ICcwJyB8fCByYXcudG9Mb3dlckNhc2UoKSA9PT0gJ2ZhbHNlJyk7XG59XG5cbi8qKlxuICogV2hldGhlciB0aGUgaW5saW5lIGxvb3AgcmV0YWlucyBhIHN1c3BlbmRlZCB3b3JrZmxvdyBWTSBhY3Jvc3MgaW5saW5lIHN0ZXBzXG4gKiB3aXRoaW4gb25lIGludm9jYXRpb24gKGRlZmF1bHQgT04pLiBXaGVuIG9uLCBhIHN0ZXAtb25seSBzdXNwZW5zaW9uIGtlZXBzXG4gKiB0aGUgbGl2ZSBWTSwgZXZlbnQgY29uc3VtZXIsIGFuZCBoeWRyYXRlZCBzdGF0ZSBhbGl2ZSwgYW5kIHRoZSBuZXh0IGxvb3BcbiAqIGl0ZXJhdGlvbiBhcHBlbmRzIG9ubHkgdGhlIG5ld2x5IGR1cmFibGUgZXZlbnRzIGluc3RlYWQgb2YgcmVidWlsZGluZyB0aGVcbiAqIGB2bS5Db250ZXh0YCBhbmQgcmVwbGF5aW5nIHRoZSB3aG9sZSBldmVudCBsb2cuIE5vbi1zdGVwIHN1c3BlbnNpb25zIGFuZFxuICogcmVwbGF5IGRpdmVyZ2VuY2UgYWx3YXlzIGZhbGwgYmFjayB0byB0aGUgb3JkaW5hcnkgZHVyYWJsZSByZXBsYXkgcGF0aC5cbiAqXG4gKiBgV09SS0ZMT1dfUkVUQUlORURfVk09MGAgKG9yIGBmYWxzZWApIGlzIHRoZSBraWxsIHN3aXRjaDogZXZlcnkgaXRlcmF0aW9uXG4gKiByZXBsYXlzIGZyb20gc2NyYXRjaCBpbiBhIGZyZXNoIFZNLCBtYXRjaGluZyB0aGUgcHJlLXJldGVudGlvbiBiZWhhdmlvci5cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGlzVm1SZXRlbnRpb25FbmFibGVkKCk6IGJvb2xlYW4ge1xuICBjb25zdCByYXcgPSBwcm9jZXNzLmVudi5XT1JLRkxPV19SRVRBSU5FRF9WTTtcbiAgaWYgKHJhdyA9PT0gdW5kZWZpbmVkIHx8IHJhdyA9PT0gJycpIHJldHVybiB0cnVlO1xuICByZXR1cm4gIShyYXcgPT09ICcwJyB8fCByYXcudG9Mb3dlckNhc2UoKSA9PT0gJ2ZhbHNlJyk7XG59XG5cbi8qKlxuICogV2hldGhlciBpbmxpbmUgc3RlcCBvd25lcnNoaXAgaXMgZW5hYmxlZCAoZGVmYXVsdCBPTikuIFdoZW4gb24sIHRoZSBsYXp5XG4gKiBgc3RlcF9zdGFydGVkYCB0aGF0IGNyZWF0ZXMgYW4gaW5saW5lIHN0ZXAgcmVjb3JkcyB0aGUgb3duaW5nIHF1ZXVlXG4gKiBtZXNzYWdlIElELCBhbmQgd2FrZSByZXBsYXlzIHRoYXQgb2JzZXJ2ZSBhbiBhY3RpdmVseS1vd25lZCBzdGVwIGVucXVldWUgYVxuICogKmRlbGF5ZWQgYmFja3N0b3AqIG1lc3NhZ2UgaW5zdGVhZCBvZiBpbW1lZGlhdGVseSByZXF1ZXVlaW5nIGl0IOKAlCBmaXhpbmdcbiAqIGR1cGxpY2F0ZSBpbmxpbmUgc3RlcCBleGVjdXRpb24gd2hlbiBhIGhvb2svd2FpdCB3YWtlcyBhIHJ1biBtaWQtc3RlcFxuICogKHZlcmNlbC93b3JrZmxvdyMyNzgwKS5cbiAqXG4gKiBgV09SS0ZMT1dfSU5MSU5FX09XTkVSU0hJUD0wYCAob3IgYGZhbHNlYCkgaXMgdGhlIGtpbGwgc3dpdGNoOiBkaXNwYXRjaFxuICogcmV2ZXJ0cyB0byB0aGUgdW5jb25kaXRpb25hbCBpbW1lZGlhdGUgcmVxdWV1ZS4gU3RhbXBpbmcgaXMgdW5hZmZlY3RlZCDigJRcbiAqIHRoZSByZWNvcmRlZCBvd25lck1lc3NhZ2VJZCBpcyBpbmVydCBkYXRhIHdoZW4gdGhlIHN3aXRjaCBpcyBvZmYuXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBpc0lubGluZU93bmVyc2hpcEVuYWJsZWQoKTogYm9vbGVhbiB7XG4gIGNvbnN0IHJhdyA9IHByb2Nlc3MuZW52LldPUktGTE9XX0lOTElORV9PV05FUlNISVA7XG4gIGlmIChyYXcgPT09IHVuZGVmaW5lZCB8fCByYXcgPT09ICcnKSByZXR1cm4gdHJ1ZTtcbiAgcmV0dXJuICEocmF3ID09PSAnMCcgfHwgcmF3LnRvTG93ZXJDYXNlKCkgPT09ICdmYWxzZScpO1xufVxuXG4vKipcbiAqIERlZmF1bHQgaW5saW5lLW93bmVyc2hpcCBsZWFzZSwgaW4gc2Vjb25kczogaG93IGxvbmcgYWZ0ZXIgYSBzdGVwJ3MgbGF0ZXN0XG4gKiAoc3RhbXBlZCkgYHN0ZXBfc3RhcnRlZGAgYSBub24tb3duZXIgaW52b2NhdGlvbiBhc3N1bWVzIHRoZSBvd25pbmdcbiAqIGludm9jYXRpb24gbWF5IHN0aWxsIGJlIGFsaXZlLiBXaXRoaW4gdGhlIGxlYXNlLCB3YWtlIHJlcGxheXMgZW5xdWV1ZSB0aGVcbiAqIHN0ZXAncyBiYWNrc3RvcCBtZXNzYWdlIHdpdGggYGRlbGF5U2Vjb25kcyA9IGxlYXNlIHJlbWFpbmluZ2AgaW5zdGVhZCBvZlxuICogaW1tZWRpYXRlbHk7IHBhc3QgaXQsIHRoZXkgZW5xdWV1ZSBpbW1lZGlhdGVseSAodG9kYXkncyBiZWhhdmlvcikuXG4gKlxuICogV2h5IGEgZml4ZWQgODYwIGFuZCBub3QgYSB2YWx1ZSBkZXJpdmVkIGZyb20gdGhlIGZ1bmN0aW9uJ3MgYG1heER1cmF0aW9uYDpcbiAqIG5laXRoZXIgcnVudGltZSBub3IgYnVpbGQgdGltZSBjYW4gc2VlIHRoZSByZXNvbHZlZCB2YWx1ZSDigJQgYnVpbGRlcnMgZW1pdFxuICogYG1heER1cmF0aW9uOiAnbWF4J2AsIHdoaWNoIHRoZSBwbGF0Zm9ybSByZXNvbHZlcyBwZXItcGxhbiBhdCBkZXBsb3ksIGFuZFxuICogbm8gZW52IHZhciBvciByZXF1ZXN0LWNvbnRleHQgZGVhZGxpbmUgQVBJIGV4cG9zZXMgdGhlIHJlc3VsdC4gVGhlIGJvdW5kXG4gKiBjb21lcyBmcm9tIGEgcGxhdGZvcm0gcnVsZSBpbnN0ZWFkOiBkdXJhdGlvbnMgYWJvdmUgODAwcyByZXF1aXJlIGV4cGxpY2l0XG4gKiBwZXItZnVuY3Rpb24gbnVtZXJpYyBjb25maWcsIHNvIGEgYnVpbGRlci1lbWl0dGVkIGAnbWF4J2AgcmVzb2x2ZXMgdG8gYXRcbiAqIG1vc3QgODAwcyDigJQgODYwcyB0aGVyZWZvcmUgZG9taW5hdGVzIGFueSB3b3JrZmxvdyByb3V0ZSdzIGludm9jYXRpb25cbiAqIGxpZmV0aW1lIHBsdXMgc2NoZWR1bGluZyBzbGFjay4gUmV2aXNpdCB3aGVuIHRoYXQgY2VpbGluZyBtb3ZlcyAodGhlXG4gKiAzMC1taW51dGUgZHVyYXRpb24gYmV0YSBiZWNvbWluZyByZWFjaGFibGUgdmlhIGAnbWF4J2Agd291bGQgaW52YWxpZGF0ZVxuICogdGhlIGJvdW5kKS4gV29ybGRzIHdpdGhvdXQgYW4gaW52b2NhdGlvbiBraWxsIGJvdW5kICh3b3JsZC1sb2NhbCxcbiAqIHNlbGYtaG9zdGVkKSBnZXQgbm8gZGVhdGggcHJvb2YgZnJvbSBhbnkgY29uc3RhbnQg4oCUIHRoZXJlIHRoZSBpbi1wcm9jZXNzXG4gKiBzaW5nbGUtZmxpZ2h0IGxheWVyIChzdGVwLXNpbmdsZS1mbGlnaHQudHMpIGlzIHdoYXQgbWFrZXMgYSBiYWNrc3RvcFxuICogZmlyaW5nIG1pZC1zdGVwIGhhcm1sZXNzLlxuICpcbiAqIDg2MCBhbHNvIHN0YXlzIHVuZGVyIHRoZSBxdWV1ZSdzIDkwMHMgbWF4aW11bSBkZWxheSAoU1FTIGNhcCksIHNvIGFcbiAqIGJhY2tzdG9wJ3MgZnVsbCBsZWFzZSByZW1haW5kZXIgYWx3YXlzIGZpdHMgaW4gYSBzaW5nbGUgZGVsYXllZCBtZXNzYWdlLlxuICpcbiAqIE92ZXJyaWRlIHZpYSBgV09SS0ZMT1dfSU5MSU5FX09XTkVSU0hJUF9MRUFTRV9TRUNPTkRTYCAoY2xhbXBlZCB0b1xuICogMS4uYE1BWF9JTkxJTkVfT1dORVJTSElQX0xFQVNFX1NFQ09ORFNgKS5cbiAqL1xuZXhwb3J0IGNvbnN0IElOTElORV9PV05FUlNISVBfTEVBU0VfU0VDT05EUyA9IDg2MDtcblxuLyoqXG4gKiBVcHBlciBib3VuZCBmb3IgdGhlIGxlYXNlIGVudiBvdmVycmlkZS4gOTAwcyBpcyB0aGUgcXVldWUncyBtYXhpbXVtXG4gKiBwZXItbWVzc2FnZSBkZWxheSAoU1FTIGNhcCkg4oCUIGEgbG9uZ2VyIGxlYXNlIHdvdWxkIG5lZWQgZGVsYXkgY2hhaW5pbmdcbiAqIGxpa2UgbG9uZyB3YWl0cyB1c2U7IGNsYW1wIGluc3RlYWQgc28gb25lIGRlbGF5ZWQgbWVzc2FnZSBhbHdheXMgc3VmZmljZXMuXG4gKi9cbmV4cG9ydCBjb25zdCBNQVhfSU5MSU5FX09XTkVSU0hJUF9MRUFTRV9TRUNPTkRTID0gOTAwO1xuXG4vKipcbiAqIEVmZmVjdGl2ZSBpbmxpbmUtb3duZXJzaGlwIGxlYXNlLiBPdmVycmlkZSB2aWFcbiAqIGBXT1JLRkxPV19JTkxJTkVfT1dORVJTSElQX0xFQVNFX1NFQ09ORFNgIOKAlCBlLmcuIHJhaXNlIGl0IG9uIHNlbGYtaG9zdGVkXG4gKiBtdWx0aS1pbnN0YW5jZSB3b3JsZHMgd2l0aCBsb25nLXJ1bm5pbmcgc3RlcHMgdG8gd2lkZW4gdGhlIHdpbmRvdyBpbiB3aGljaFxuICogYSBsaXZlIG93bmVyIGlzIHByb3RlY3RlZCBmcm9tIGEgY29uY3VycmVudCBiYWNrc3RvcCBleGVjdXRpb24uXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBnZXRJbmxpbmVPd25lcnNoaXBMZWFzZVNlY29uZHMoKTogbnVtYmVyIHtcbiAgcmV0dXJuIGVudk51bWJlcihcbiAgICAnV09SS0ZMT1dfSU5MSU5FX09XTkVSU0hJUF9MRUFTRV9TRUNPTkRTJyxcbiAgICBJTkxJTkVfT1dORVJTSElQX0xFQVNFX1NFQ09ORFMsXG4gICAgeyBpbnRlZ2VyOiB0cnVlLCBtaW46IDEsIG1heDogTUFYX0lOTElORV9PV05FUlNISVBfTEVBU0VfU0VDT05EUyB9XG4gICk7XG59XG5cbi8vIEEgcmVwbGF5LWNvbnN1bWVyIG1pc21hdGNoIGNhbiBiZSBjYXVzZWQgYnkgYSB0cmFuc2llbnQgZGl2ZXJnZW50IHJlcGxheVxuLy8gcmF0aGVyIHRoYW4gYW4gaW52YWxpZCBwZXJzaXN0ZWQgaGlzdG9yeS4gUXVldWUgYm91bmRlZCByZWNvdmVyeSByZXBsYXlzXG4vLyBiZWZvcmUgcmVjb3JkaW5nIHRlcm1pbmFsIGNvcnJ1cHRpb24gZm9yIGEgcnVuIHRoYXQgY2Fubm90IHJlcGxheS5cbmV4cG9ydCBjb25zdCBSRVBMQVlfRElWRVJHRU5DRV9NQVhfUkVUUklFUyA9IDM7XG5cbi8qKlxuICogRWZmZWN0aXZlIHJlcGxheS1kaXZlcmdlbmNlIHJlY292ZXJ5IGJ1ZGdldC4gT3ZlcnJpZGUgdmlhXG4gKiBgV09SS0ZMT1dfUkVQTEFZX0RJVkVSR0VOQ0VfTUFYX1JFVFJJRVNgLlxuICovXG5leHBvcnQgZnVuY3Rpb24gZ2V0UmVwbGF5RGl2ZXJnZW5jZU1heFJldHJpZXMoKTogbnVtYmVyIHtcbiAgcmV0dXJuIGVudk51bWJlcihcbiAgICAnV09SS0ZMT1dfUkVQTEFZX0RJVkVSR0VOQ0VfTUFYX1JFVFJJRVMnLFxuICAgIFJFUExBWV9ESVZFUkdFTkNFX01BWF9SRVRSSUVTLFxuICAgIHsgaW50ZWdlcjogdHJ1ZSB9XG4gICk7XG59XG5cbi8vIEEgc3RhbGUtc25hcHNob3QgcmVqZWN0aW9uICg0MTIpIG1lYW5zIHRoZSByZXBsYXkncyBldmVudCBsb2cgd2FzIG1pc3NpbmcgYW5cbi8vIGV2ZW50IHRoZSBXb3JsZCBoYWQgYWxyZWFkeSByZWNvcmRlZCwgc28gdGhlIHJlcGxheSBpcyByZS1kZXJpdmVkIGZyb20gYVxuLy8gY29ycmVjdGVkIGxvZyBpbnNpZGUgdGhlIHNhbWUgaW52b2NhdGlvbi4gQm91bmRlZCBiZWNhdXNlIGEgcGVyc2lzdGVudGx5XG4vLyByZWplY3RlZCB3cml0ZSBzaG91bGQgZXNjYWxhdGUgcmF0aGVyIHRoYW4gc3BpbjogYWZ0ZXIgdGhpcyBtYW55IHJlc3RhcnRzIHRoZVxuLy8gcnVuIGlzIHJlLWludm9rZWQgKGEgbmV3IGludm9jYXRpb24sIHBvc3NpYmx5IGluIGEgZGlmZmVyZW50IHJlZ2lvbiksIGFuZCB0aGVcbi8vIHJ1bi1sZXZlbCBidWRnZXQgYmVsb3cgdGhlbiBhcHBsaWVzLlxuZXhwb3J0IGNvbnN0IFBSRUNPTkRJVElPTl9NQVhfSU5QUk9DRVNTX1JFU1RBUlRTID0gMztcblxuLyoqXG4gKiBFZmZlY3RpdmUgaW4tcHJvY2VzcyByZXBsYXktcmVzdGFydCBidWRnZXQgZm9yIHN0YWxlLXNuYXBzaG90IHJlamVjdGlvbnMuXG4gKiBPdmVycmlkZSB2aWEgYFdPUktGTE9XX1BSRUNPTkRJVElPTl9NQVhfSU5QUk9DRVNTX1JFU1RBUlRTYC5cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGdldFByZWNvbmRpdGlvbk1heEluUHJvY2Vzc1Jlc3RhcnRzKCk6IG51bWJlciB7XG4gIHJldHVybiBlbnZOdW1iZXIoXG4gICAgJ1dPUktGTE9XX1BSRUNPTkRJVElPTl9NQVhfSU5QUk9DRVNTX1JFU1RBUlRTJyxcbiAgICBQUkVDT05ESVRJT05fTUFYX0lOUFJPQ0VTU19SRVNUQVJUUyxcbiAgICB7IGludGVnZXI6IHRydWUgfVxuICApO1xufVxuXG4vLyBUaGUgaW4tcHJvY2VzcyBidWRnZXQgYWJvdmUgaXMgcGVyLWludm9jYXRpb24sIGFuZCBhIHJlLWludm9jYXRpb24gdGhhdFxuLy8gZW5xdWV1ZXMgYSBmcmVzaCBtZXNzYWdlIGFsc28gcmVzdGFydHMgdGhlIHF1ZXVlJ3MgZGVsaXZlcnkgY291bnQsIHNvIHdpdGhvdXRcbi8vIGEgY291bnRlciBjYXJyaWVkIG9uIHRoZSBtZXNzYWdlIGEgcGVybWFuZW50bHkgZmVuY2VkIHJ1biBoYXMgbm8gcnVuLWxldmVsXG4vLyBib3VuZCBhdCBhbGwuIEl0IGNhbiBzdGF5IGZlbmNlZCB3aXRob3V0IGFueSBwZXJtYW5lbnQgZmF1bHQg4oCUIGEgZnVsbCByZWxvYWRcbi8vIGlzIG5vdCBhdG9taWMgYWNyb3NzIHBhZ2VzLCBzbyBhIGJ1c3kgcnVuIGNhbiBhY3F1aXJlIGEgbmV3IGhvbGUgb24gZXZlcnlcbi8vIHJlbG9hZCDigJQgc28gdGhlIGNoYWluIGlzIGNvdW50ZWQgYW5kIHRoZSBydW4gZmFpbHMgb25jZSB0aGlzIG1hbnlcbi8vIHJlLWludm9jYXRpb25zIGhhdmUgYmVlbiBzcGVudCBvbiBzdGFsZS1zbmFwc2hvdCByZWplY3Rpb25zLlxuZXhwb3J0IGNvbnN0IFBSRUNPTkRJVElPTl9NQVhfUkVJTlZPQ0FUSU9OUyA9IDU7XG5cbi8qKlxuICogRWZmZWN0aXZlIHBlci1ydW4gYnVkZ2V0IGZvciByZS1pbnZvY2F0aW9ucyBjYXVzZWQgYnkgc3RhbGUtc25hcHNob3RcbiAqIHJlamVjdGlvbnMuIE92ZXJyaWRlIHZpYSBgV09SS0ZMT1dfUFJFQ09ORElUSU9OX01BWF9SRUlOVk9DQVRJT05TYC5cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGdldFByZWNvbmRpdGlvbk1heFJlaW52b2NhdGlvbnMoKTogbnVtYmVyIHtcbiAgcmV0dXJuIGVudk51bWJlcihcbiAgICAnV09SS0ZMT1dfUFJFQ09ORElUSU9OX01BWF9SRUlOVk9DQVRJT05TJyxcbiAgICBQUkVDT05ESVRJT05fTUFYX1JFSU5WT0NBVElPTlMsXG4gICAgeyBpbnRlZ2VyOiB0cnVlIH1cbiAgKTtcbn1cblxuLy8gQmFja29mZiBiZWZvcmUgYSBwcmVjb25kaXRpb24gcmUtaW52b2NhdGlvbi4gVW5saWtlIHRoZSBpbi1wcm9jZXNzIHJlc3RhcnRcbi8vICh3aGVyZSB0aGUgcG9pbnQgaXMgdG8gcmUtcmVhZCBpbW1lZGlhdGVseSksIGEgcmUtaW52b2NhdGlvbiBvbmx5IGhhcHBlbnNcbi8vIGFmdGVyIHRoZSBpbi1wcm9jZXNzIGJ1ZGdldCBmYWlsZWQgdG8gY2F0Y2ggdXAsIHNvIHRoZSBsb2cgaXMgYmVpbmcgZXh0ZW5kZWRcbi8vIGZhc3RlciB0aGFuIHRoaXMgcmVwbGF5IGNhbiBmb2xsb3cgaXQuIFdhaXRpbmcgbGV0cyB0aGUgd3JpdGVycyBxdWllc2NlLlxuZXhwb3J0IGNvbnN0IFBSRUNPTkRJVElPTl9SRUlOVk9LRV9ERUxBWV9TRUNPTkRTID0gMjtcblxuLyoqXG4gKiBFZmZlY3RpdmUgZGVsYXkgYmVmb3JlIGEgcHJlY29uZGl0aW9uIHJlLWludm9jYXRpb24uIE92ZXJyaWRlIHZpYVxuICogYFdPUktGTE9XX1BSRUNPTkRJVElPTl9SRUlOVk9LRV9ERUxBWV9TRUNPTkRTYC5cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGdldFByZWNvbmRpdGlvblJlaW52b2tlRGVsYXlTZWNvbmRzKCk6IG51bWJlciB7XG4gIHJldHVybiBlbnZOdW1iZXIoXG4gICAgJ1dPUktGTE9XX1BSRUNPTkRJVElPTl9SRUlOVk9LRV9ERUxBWV9TRUNPTkRTJyxcbiAgICBQUkVDT05ESVRJT05fUkVJTlZPS0VfREVMQVlfU0VDT05EUyxcbiAgICB7IGludGVnZXI6IHRydWUgfVxuICApO1xufVxuXG4vLyBBIGRlbGl2ZXJ5IHJlYWNoaW5nIGEgZGVwbG95bWVudCB0aGUgcnVuIGlzIG5vdCBwaW5uZWQgdG8gaXMgbm90IHRyZWF0ZWQgYXNcbi8vIHBlcm1hbmVudC4gUmUtcm91dGUgdGhlIG1lc3NhZ2UgYXQgdGhlIHJ1bidzIG93biBkZXBsb3ltZW50IGEgYm91bmRlZCBudW1iZXJcbi8vIG9mIHRpbWVzIGJlZm9yZSBmYWlsaW5nIHRoZSBydW4gd2l0aCBERVBMT1lNRU5UX01JU01BVENILlxuZXhwb3J0IGNvbnN0IERFUExPWU1FTlRfTUlTTUFUQ0hfTUFYX1JFVFJJRVMgPSAzO1xuXG4vKipcbiAqIEVmZmVjdGl2ZSBkZXBsb3ltZW50LW1pc21hdGNoIHJlLXJvdXRlIGJ1ZGdldC4gT3ZlcnJpZGUgdmlhXG4gKiBgV09SS0ZMT1dfREVQTE9ZTUVOVF9NSVNNQVRDSF9NQVhfUkVUUklFU2A7IGAwYCBmYWlscyB0aGUgcnVuIG9uIHRoZSBmaXJzdFxuICogbWlzcm91dGVkIGRlbGl2ZXJ5IGluc3RlYWQgb2YgYXR0ZW1wdGluZyByZWNvdmVyeS5cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGdldERlcGxveW1lbnRNaXNtYXRjaE1heFJldHJpZXMoKTogbnVtYmVyIHtcbiAgcmV0dXJuIGVudk51bWJlcihcbiAgICAnV09SS0ZMT1dfREVQTE9ZTUVOVF9NSVNNQVRDSF9NQVhfUkVUUklFUycsXG4gICAgREVQTE9ZTUVOVF9NSVNNQVRDSF9NQVhfUkVUUklFUyxcbiAgICB7IGludGVnZXI6IHRydWUsIG1pbjogMCB9XG4gICk7XG59XG4iXX0=
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { Event } from '@workflow/world';
|
|
2
|
+
/**
|
|
3
|
+
* Scope for {@link countStepStartedEvents}:
|
|
4
|
+
* - `{ type: 'ownedBy', messageId }` counts only starts whose
|
|
5
|
+
* `eventData.ownerMessageId` matches — i.e. attempts performed by that
|
|
6
|
+
* owning queue message (inline lazy starts and owned-recovery re-stamps).
|
|
7
|
+
* - `{ type: 'totalAttempts' }` estimates the step's genuine attempt total
|
|
8
|
+
* across its whole lifecycle: unstamped (bare) starts — which only real
|
|
9
|
+
* queue-dispatched background deliveries write — plus the starts of the
|
|
10
|
+
* single most-started owner. A genuine lifecycle has at most one inline
|
|
11
|
+
* owner phase (ownership is claimed atomically at step creation and
|
|
12
|
+
* lapses permanently at `step_retrying` or on any bare start), so taking
|
|
13
|
+
* the max over owners counts that phase's real attempts while invocations
|
|
14
|
+
* racing on the same batch — which stamp their own one-off message IDs —
|
|
15
|
+
* contribute at most their single largest count instead of accumulating.
|
|
16
|
+
*/
|
|
17
|
+
export type StepStartScope = {
|
|
18
|
+
type: 'ownedBy';
|
|
19
|
+
messageId: string;
|
|
20
|
+
} | {
|
|
21
|
+
type: 'totalAttempts';
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Number of `step_started` events already recorded for a step, used as the
|
|
25
|
+
* authoritative attempt count for the maxRetries ceiling (the only bound for
|
|
26
|
+
* steps that time out without writing a `step_failed`).
|
|
27
|
+
*
|
|
28
|
+
* IMPORTANT: the raw (unscoped) count is NOT a reliable attempt number.
|
|
29
|
+
* Concurrent invocations racing on the same pending batch (stale replays,
|
|
30
|
+
* wake replays, a step message dispatched by a replay that lost the
|
|
31
|
+
* create-claim race, ...) can each write a `step_started` for the same
|
|
32
|
+
* logical attempt — worlds without an atomic start guard (world-local) let
|
|
33
|
+
* all of them through. Counting those duplicates as "attempts" made the
|
|
34
|
+
* maxRetries ceiling fire on healthy runs and fail them with a false
|
|
35
|
+
* "exceeded max retries" (see workflow#3069).
|
|
36
|
+
*
|
|
37
|
+
* Callers therefore scope the count to the starts their ceiling is actually
|
|
38
|
+
* about via `scope`:
|
|
39
|
+
* - the inline owned-recovery ceiling counts only THIS message's starts
|
|
40
|
+
* (`ownedBy`) — each real (re)delivery of the owning message stamps its
|
|
41
|
+
* `ownerMessageId` on the start it writes, while racing invocations stamp
|
|
42
|
+
* their own IDs (or none), so they no longer inflate the count;
|
|
43
|
+
* - the background-step ceiling counts the lifecycle attempt total
|
|
44
|
+
* (`totalAttempts`: bare starts plus the largest single owner's starts) —
|
|
45
|
+
* so a step that exhausted part of its budget under inline ownership
|
|
46
|
+
* before transitioning to queued/bare retries still trips the combined
|
|
47
|
+
* ceiling, while racers' one-off stamped duplicates don't accumulate, and
|
|
48
|
+
* throttle/too-early redeliveries (which write no start at all) stay
|
|
49
|
+
* excluded as before.
|
|
50
|
+
*/
|
|
51
|
+
export declare function countStepStartedEvents(events: Event[] | null | undefined, stepId: string, scope?: StepStartScope): number;
|
|
52
|
+
//# sourceMappingURL=count-step-started-events.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"count-step-started-events.d.ts","sourceRoot":"","sources":["../../src/runtime/count-step-started-events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAE7C;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,cAAc,GACtB;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACtC;IAAE,IAAI,EAAE,eAAe,CAAA;CAAE,CAAC;AAU9B;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,GAAG,SAAS,EAClC,MAAM,EAAE,MAAM,EACd,KAAK,CAAC,EAAE,cAAc,GACrB,MAAM,CAmCR"}
|