@apifuse/provider-sdk 2.2.0-beta.4 → 2.2.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +501 -13
- package/CHANGELOG.md +165 -1
- package/README.md +72 -14
- package/SUBMISSION.md +1 -1
- package/bin/apifuse-check.ts +88 -4
- package/bin/apifuse-dev.ts +38 -5
- package/bin/apifuse-pack-check.ts +22 -2
- package/bin/apifuse-pack-smoke.ts +57 -2
- package/bin/apifuse-pack-types.ts +356 -38
- package/bin/apifuse-perf.ts +14 -13
- package/bin/apifuse-record.ts +691 -68
- package/bin/apifuse-submit-check.ts +518 -37
- package/bin/apifuse-sync-assets.ts +117 -0
- package/bin/submit-check-delimited-text.ts +50 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth-turn/index.js +1 -1
- package/dist/auth.d.ts +14 -0
- package/dist/auth.js +67 -0
- package/dist/ceremonies/index.d.ts +16 -0
- package/dist/ceremonies/index.js +141 -36
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +8 -0
- package/dist/cli/create.d.ts +3 -0
- package/dist/cli/create.js +34 -35
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/cli/templates/provider/Dockerfile.tpl +1 -1
- package/dist/cli/templates/provider/README.md.tpl +4 -4
- package/dist/cli/templates/provider/index.ts.tpl +6 -3
- package/dist/cli/templates/provider/operations/ping.ts.tpl +2 -1
- package/dist/config/loader.d.ts +177 -16
- package/dist/config/loader.js +424 -127
- package/dist/contract-serialization.js +4 -8
- package/dist/contract-types.d.ts +1 -0
- package/dist/contract.js +2 -0
- package/dist/declaration-validation.d.ts +32 -0
- package/dist/declaration-validation.js +207 -0
- package/dist/define.d.ts +51 -25
- package/dist/define.js +752 -38
- package/dist/error-resolution.d.ts +4 -0
- package/dist/error-resolution.js +122 -0
- package/dist/errors.d.ts +18 -0
- package/dist/errors.js +40 -0
- package/dist/fixture-sanitization.d.ts +28 -0
- package/dist/fixture-sanitization.js +217 -0
- package/dist/health-scenario.d.ts +1842 -0
- package/dist/health-scenario.js +624 -0
- package/dist/index.d.ts +19 -9
- package/dist/index.js +10 -6
- package/dist/lint.d.ts +6 -1
- package/dist/lint.js +362 -3
- package/dist/native-address.d.ts +43 -0
- package/dist/native-address.js +281 -0
- package/dist/native-egress-policy.d.ts +31 -0
- package/dist/native-egress-policy.js +288 -0
- package/dist/observability.d.ts +5 -2
- package/dist/observability.js +48 -1
- package/dist/provider.d.ts +8 -2
- package/dist/provider.js +3 -1
- package/dist/runtime/auth-flow.d.ts +5 -1
- package/dist/runtime/auth-flow.js +6 -0
- package/dist/runtime/browser.d.ts +1 -0
- package/dist/runtime/browser.js +492 -49
- package/dist/runtime/cache.d.ts +1 -0
- package/dist/runtime/cache.js +169 -15
- package/dist/runtime/choice-wordlist.d.ts +9 -0
- package/dist/runtime/choice-wordlist.js +138 -0
- package/dist/runtime/choice.d.ts +13 -1
- package/dist/runtime/choice.js +490 -102
- package/dist/runtime/executor.d.ts +2 -2
- package/dist/runtime/executor.js +26 -2
- package/dist/runtime/http.d.ts +1 -0
- package/dist/runtime/http.js +515 -53
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +366 -8
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +96 -0
- package/dist/runtime/native-network.js +1232 -0
- package/dist/runtime/ocr.d.ts +29 -0
- package/dist/runtime/ocr.js +440 -0
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +56 -0
- package/dist/runtime/proxy-nodemaven.js +146 -0
- package/dist/runtime/proxy-telemetry.d.ts +80 -1
- package/dist/runtime/proxy-telemetry.js +154 -47
- package/dist/runtime/redirects.d.ts +29 -0
- package/dist/runtime/redirects.js +36 -0
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/request-options.d.ts +68 -1
- package/dist/runtime/request-options.js +548 -0
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-public.d.ts +1 -0
- package/dist/runtime/resolver-public.js +1 -0
- package/dist/runtime/resolver-shared.d.ts +3 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +48 -0
- package/dist/runtime/resolver-vendors/bindings.js +40 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +22 -0
- package/dist/runtime/resolver-vendors/browser.js +377 -0
- package/dist/runtime/resolver-vendors/capsolver.d.ts +22 -0
- package/dist/runtime/resolver-vendors/capsolver.js +526 -0
- package/dist/runtime/resolver-vendors/hosts.d.ts +2 -0
- package/dist/runtime/resolver-vendors/hosts.js +33 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +24 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +407 -0
- package/dist/runtime/resolver-vendors/types.d.ts +94 -0
- package/dist/runtime/resolver-vendors/types.js +96 -0
- package/dist/runtime/resolver.d.ts +60 -0
- package/dist/runtime/resolver.js +737 -0
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/state.d.ts +3 -0
- package/dist/runtime/state.js +277 -71
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.d.ts +28 -3
- package/dist/runtime/stealth.js +519 -255
- package/dist/runtime/stt.js +1 -12
- package/dist/runtime/timeout.d.ts +5 -0
- package/dist/runtime/timeout.js +12 -0
- package/dist/runtime/trace-config.d.ts +12 -0
- package/dist/runtime/trace-config.js +61 -0
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +5 -3
- package/dist/server/index.js +3 -3
- package/dist/server/self-test-input-tokens.d.ts +2 -1
- package/dist/server/self-test-input-tokens.js +18 -14
- package/dist/server/self-test.d.ts +114 -0
- package/dist/server/self-test.js +784 -148
- package/dist/server/serve-implementation.d.ts +213 -0
- package/dist/server/serve-implementation.js +2173 -0
- package/dist/server/serve.d.ts +1 -70
- package/dist/server/serve.js +1 -1130
- package/dist/server/trace-output.d.ts +4 -0
- package/dist/server/trace-output.js +20 -0
- package/dist/server/types.d.ts +30 -5
- package/dist/server/types.js +13 -1
- package/dist/stateful/errors.d.ts +19 -0
- package/dist/stateful/errors.js +24 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +215 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +32 -0
- package/dist/stateful/stateful-provider-runtime-context.js +60 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +67 -0
- package/dist/stateful/stateful-provider-session-routing.js +345 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.js +16 -7
- package/dist/stream-evidence.d.ts +74 -0
- package/dist/stream-evidence.js +785 -0
- package/dist/stream.js +7 -1
- package/dist/testing/index.d.ts +2 -1
- package/dist/testing/index.js +2 -1
- package/dist/testing/run.d.ts +32 -2
- package/dist/testing/run.js +489 -21
- package/dist/trace-sanitization.d.ts +5 -0
- package/dist/trace-sanitization.js +45 -0
- package/dist/types.d.ts +545 -23
- package/dist/types.js +1 -0
- package/package.json +44 -5
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +118 -0
- package/src/ceremonies/index.ts +189 -46
- package/src/cli/commands.ts +10 -0
- package/src/cli/create.ts +42 -35
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/cli/templates/provider/Dockerfile.tpl +1 -1
- package/src/cli/templates/provider/README.md.tpl +4 -4
- package/src/cli/templates/provider/index.ts.tpl +6 -3
- package/src/cli/templates/provider/operations/ping.ts.tpl +2 -1
- package/src/config/loader.ts +665 -163
- package/src/contract-serialization.ts +5 -7
- package/src/contract-types.ts +1 -0
- package/src/contract.ts +2 -0
- package/src/declaration-validation.ts +266 -0
- package/src/define.ts +970 -87
- package/src/error-resolution.ts +127 -0
- package/src/errors.ts +52 -0
- package/src/fixture-sanitization.ts +248 -0
- package/src/health-scenario.ts +875 -0
- package/src/index.ts +204 -8
- package/src/lint.ts +408 -4
- package/src/native-address.ts +340 -0
- package/src/native-egress-policy.ts +358 -0
- package/src/observability.ts +51 -1
- package/src/provider.ts +133 -0
- package/src/runtime/auth-flow.ts +12 -0
- package/src/runtime/browser.ts +661 -63
- package/src/runtime/cache.ts +189 -14
- package/src/runtime/choice-wordlist.ts +145 -0
- package/src/runtime/choice.ts +631 -120
- package/src/runtime/executor.ts +40 -7
- package/src/runtime/http.ts +641 -61
- package/src/runtime/instrumentation.ts +520 -15
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +1605 -0
- package/src/runtime/ocr.ts +523 -0
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +221 -0
- package/src/runtime/proxy-telemetry.ts +244 -75
- package/src/runtime/redirects.ts +66 -0
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/request-options.ts +680 -1
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-public.ts +20 -0
- package/src/runtime/resolver-shared.ts +17 -0
- package/src/runtime/resolver-vendors/bindings.ts +56 -0
- package/src/runtime/resolver-vendors/browser.ts +533 -0
- package/src/runtime/resolver-vendors/capsolver.ts +700 -0
- package/src/runtime/resolver-vendors/hosts.ts +38 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +539 -0
- package/src/runtime/resolver-vendors/types.ts +212 -0
- package/src/runtime/resolver.ts +1103 -0
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/state.ts +394 -77
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +675 -289
- package/src/runtime/stt.ts +1 -19
- package/src/runtime/timeout.ts +18 -0
- package/src/runtime/trace-config.ts +77 -0
- package/src/serve.ts +6 -1
- package/src/server/index.ts +37 -2
- package/src/server/self-test-input-tokens.ts +29 -14
- package/src/server/self-test.ts +1025 -175
- package/src/server/serve-implementation.ts +3250 -0
- package/src/server/serve.ts +1 -1626
- package/src/server/trace-output.ts +32 -0
- package/src/server/types.ts +13 -1
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +35 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +287 -0
- package/src/stateful/stateful-provider-runtime-context.ts +92 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +546 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +17 -7
- package/src/stream-evidence.ts +988 -0
- package/src/stream.ts +8 -1
- package/src/testing/index.ts +10 -1
- package/src/testing/run.ts +658 -15
- package/src/trace-sanitization.ts +63 -0
- package/src/types.ts +660 -38
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/src/server/self-test.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { readFileSync } from "node:fs";
|
|
3
3
|
|
|
4
|
+
import { TURN_KINDS } from "../auth-turn/index.js";
|
|
5
|
+
|
|
4
6
|
import { type Context, Hono } from "hono";
|
|
5
7
|
import { z } from "zod";
|
|
6
8
|
import type {
|
|
@@ -75,26 +77,108 @@ export interface SelfTestResponse {
|
|
|
75
77
|
results: SelfTestCaseResult[];
|
|
76
78
|
}
|
|
77
79
|
|
|
80
|
+
export interface SelfTestCancellationLogEvent {
|
|
81
|
+
level: "info";
|
|
82
|
+
event: "self_test_run_cancelled";
|
|
83
|
+
providerId: string;
|
|
84
|
+
requestId: string;
|
|
85
|
+
operationId: string;
|
|
86
|
+
caseName: string;
|
|
87
|
+
reason: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
78
90
|
export type SelfTestOperationInvoke = (args: {
|
|
79
91
|
operationId: string;
|
|
80
92
|
input: unknown;
|
|
81
93
|
connection?: OperationConnection;
|
|
82
94
|
requestId: string;
|
|
95
|
+
signal?: AbortSignal;
|
|
83
96
|
}) => Promise<{
|
|
84
97
|
status: number;
|
|
85
98
|
data: unknown;
|
|
86
99
|
meta?: Record<string, unknown>;
|
|
87
100
|
}>;
|
|
88
101
|
|
|
102
|
+
export type SelfTestAuthFlowRoute = "start" | "continue";
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* In-process driver for the tenant app's /auth pipeline. Self-test uses it to
|
|
106
|
+
* materialize `requiresConnection` credentials through the provider's declared
|
|
107
|
+
* auth flow — the exact path production connections take — instead of
|
|
108
|
+
* injecting raw credential inputs as connection secrets.
|
|
109
|
+
*/
|
|
110
|
+
export type SelfTestAuthFlowInvoke = (args: {
|
|
111
|
+
route: SelfTestAuthFlowRoute;
|
|
112
|
+
requestId: string;
|
|
113
|
+
flowId: string;
|
|
114
|
+
/** Stable per-credential connection id — keeps login on the probe's affinity. */
|
|
115
|
+
connectionId?: string;
|
|
116
|
+
/** The probe connection's externalRef — flows reading ctx.externalRef see the same identity. */
|
|
117
|
+
externalRef?: string;
|
|
118
|
+
input?: Record<string, unknown>;
|
|
119
|
+
context?: Record<string, unknown>;
|
|
120
|
+
signal?: AbortSignal;
|
|
121
|
+
}) => Promise<{
|
|
122
|
+
status: number;
|
|
123
|
+
body: unknown;
|
|
124
|
+
}>;
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Skip reason reported when a declared auth flow does not complete in a single
|
|
128
|
+
* continue (OTP, retry loop). Cross-repo contract: the health-monitor maps
|
|
129
|
+
* this exact string to `self_test_incapable`; never vary it.
|
|
130
|
+
*/
|
|
131
|
+
export const SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON = "auth_flow_multi_turn";
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* A `retry` turn after credential submission: the flow REJECTED the
|
|
135
|
+
* configured inputs (bad password, exchange failure). Distinct from the
|
|
136
|
+
* multi-turn gap so monitoring surfaces it as a real credential outage, and
|
|
137
|
+
* memoized like multi-turn so the probe does not re-submit rejected
|
|
138
|
+
* credentials every cycle (lockout safety).
|
|
139
|
+
*/
|
|
140
|
+
export const SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON = "auth_flow_rejected";
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Known interactive turn kinds that justify the memoized multi-turn skip —
|
|
144
|
+
* they mean a human must participate (OTP, challenge, redirect, …).
|
|
145
|
+
* `retry` is deliberately excluded: after a credential submission it means
|
|
146
|
+
* rejection, not interaction (see SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON).
|
|
147
|
+
* Kinds outside TURN_KINDS entirely are treated as flow errors.
|
|
148
|
+
*/
|
|
149
|
+
/**
|
|
150
|
+
* Post-submission /auth/continue statuses that mean the flow REJECTED the
|
|
151
|
+
* credentials (thrown AuthError -> 401, forbidden -> 403): memoized as
|
|
152
|
+
* `auth_flow_rejected`. Deliberately NOT 400 — the auth route maps generic
|
|
153
|
+
* ProviderErrors and Zod request errors there, which are often transient or
|
|
154
|
+
* fixable and must stay uncached retries (like 408/429/5xx).
|
|
155
|
+
*/
|
|
156
|
+
const AUTH_REJECTION_HTTP_STATUSES: ReadonlySet<number> = new Set([401, 403]);
|
|
157
|
+
|
|
158
|
+
const INTERACTIVE_TURN_KIND_SET: ReadonlySet<string> = new Set(
|
|
159
|
+
TURN_KINDS.filter(
|
|
160
|
+
(descriptor) => descriptor.rendering !== "terminal" && descriptor.kind !== "retry",
|
|
161
|
+
).map((descriptor) => descriptor.kind),
|
|
162
|
+
);
|
|
163
|
+
|
|
89
164
|
export interface SelfTestAppOptions {
|
|
90
165
|
/** Derived-token verification secrets; without them every self-test route 404s. */
|
|
91
166
|
secrets?: SelfTestMasterSecrets;
|
|
92
167
|
/** In-process invoke bound to the tenant-facing app's /v1 pipeline. */
|
|
93
168
|
invoke: SelfTestOperationInvoke;
|
|
169
|
+
/**
|
|
170
|
+
* In-process auth-flow driver bound to the tenant-facing app's /auth
|
|
171
|
+
* pipeline. Required for providers that declare `auth.mode: "credentials"`
|
|
172
|
+
* with a flow; without it their requiresConnection cases report a visible
|
|
173
|
+
* auth_flow_unavailable error instead of probing with raw inputs.
|
|
174
|
+
*/
|
|
175
|
+
authFlow?: SelfTestAuthFlowInvoke;
|
|
94
176
|
/** Overall request budget; defaults to env / 120s. */
|
|
95
177
|
requestBudgetMs?: number;
|
|
96
178
|
/** Env override for secret collection + budget resolution (tests). */
|
|
97
179
|
env?: Readonly<Record<string, string | undefined>>;
|
|
180
|
+
/** Structured server logger; defaults to the same JSON console shape as the provider server. */
|
|
181
|
+
logger?: (event: SelfTestCancellationLogEvent) => void;
|
|
98
182
|
}
|
|
99
183
|
|
|
100
184
|
function resolveSdkVersion(): string {
|
|
@@ -178,16 +262,25 @@ export function isSelfTestReadOnlyOperation(operation: OperationDefinition): boo
|
|
|
178
262
|
export function createSelfTestInvoke(app: {
|
|
179
263
|
request: (input: string, requestInit?: RequestInit) => Response | Promise<Response>;
|
|
180
264
|
}): SelfTestOperationInvoke {
|
|
181
|
-
return async ({ operationId, input, connection, requestId }) => {
|
|
182
|
-
const
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
265
|
+
return async ({ operationId, input, connection, requestId, signal }) => {
|
|
266
|
+
const responsePromise = Promise.resolve(
|
|
267
|
+
app.request(`/v1/${encodeURIComponent(operationId)}`, {
|
|
268
|
+
method: "POST",
|
|
269
|
+
headers: { "content-type": "application/json" },
|
|
270
|
+
body: JSON.stringify({
|
|
271
|
+
requestId,
|
|
272
|
+
input: input ?? {},
|
|
273
|
+
...(connection ? { connection } : {}),
|
|
274
|
+
}),
|
|
275
|
+
...(signal ? { signal } : {}),
|
|
189
276
|
}),
|
|
190
|
-
|
|
277
|
+
);
|
|
278
|
+
const response = await responsePromise;
|
|
279
|
+
if (signal?.aborted) {
|
|
280
|
+
await cancelSelfTestResponse(response);
|
|
281
|
+
signal.throwIfAborted();
|
|
282
|
+
}
|
|
283
|
+
// An abort after body consumption starts may let response.text() finish.
|
|
191
284
|
const text = await response.text();
|
|
192
285
|
let body: unknown = text;
|
|
193
286
|
try {
|
|
@@ -207,6 +300,82 @@ export function createSelfTestInvoke(app: {
|
|
|
207
300
|
};
|
|
208
301
|
}
|
|
209
302
|
|
|
303
|
+
async function cancelSelfTestResponse(response: Response): Promise<void> {
|
|
304
|
+
await response.body?.cancel().catch(() => undefined);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function selfTestAbortReason(reason: unknown, sensitiveValues: readonly string[]): string {
|
|
308
|
+
const text =
|
|
309
|
+
reason instanceof Error
|
|
310
|
+
? reason.message || reason.name
|
|
311
|
+
: reason === undefined
|
|
312
|
+
? "aborted"
|
|
313
|
+
: String(reason);
|
|
314
|
+
return redactSelfTestText(text, sensitiveValues);
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
function logSelfTestCancellation(
|
|
318
|
+
provider: ProviderDefinition,
|
|
319
|
+
options: SelfTestAppOptions,
|
|
320
|
+
requestId: string,
|
|
321
|
+
selected: SelectedCase,
|
|
322
|
+
signal: AbortSignal,
|
|
323
|
+
sensitiveValues: readonly string[],
|
|
324
|
+
): void {
|
|
325
|
+
const event: SelfTestCancellationLogEvent = {
|
|
326
|
+
level: "info",
|
|
327
|
+
event: "self_test_run_cancelled",
|
|
328
|
+
providerId: provider.id,
|
|
329
|
+
requestId,
|
|
330
|
+
operationId: selected.operationId,
|
|
331
|
+
caseName: selected.healthCase.name,
|
|
332
|
+
reason: selfTestAbortReason(signal.reason, sensitiveValues),
|
|
333
|
+
};
|
|
334
|
+
if (options.logger) {
|
|
335
|
+
options.logger(event);
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
console.log(JSON.stringify(event));
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** Binds the self-test auth-flow driver to a tenant app's /auth pipeline in-process. */
|
|
342
|
+
export function createSelfTestAuthFlowInvoke(app: {
|
|
343
|
+
request: (input: string, requestInit?: RequestInit) => Response | Promise<Response>;
|
|
344
|
+
}): SelfTestAuthFlowInvoke {
|
|
345
|
+
return async ({
|
|
346
|
+
route,
|
|
347
|
+
requestId,
|
|
348
|
+
flowId,
|
|
349
|
+
connectionId,
|
|
350
|
+
externalRef,
|
|
351
|
+
input,
|
|
352
|
+
context,
|
|
353
|
+
signal,
|
|
354
|
+
}) => {
|
|
355
|
+
const response = await app.request(`/auth/${route}`, {
|
|
356
|
+
method: "POST",
|
|
357
|
+
headers: { "content-type": "application/json" },
|
|
358
|
+
body: JSON.stringify({
|
|
359
|
+
requestId,
|
|
360
|
+
flowId,
|
|
361
|
+
...(connectionId ? { connectionId } : {}),
|
|
362
|
+
...(externalRef ? { externalRef } : {}),
|
|
363
|
+
...(input ? { input } : {}),
|
|
364
|
+
...(context ? { context } : {}),
|
|
365
|
+
}),
|
|
366
|
+
...(signal ? { signal } : {}),
|
|
367
|
+
});
|
|
368
|
+
const text = await response.text();
|
|
369
|
+
let body: unknown = text;
|
|
370
|
+
try {
|
|
371
|
+
body = text.length > 0 ? JSON.parse(text) : undefined;
|
|
372
|
+
} catch {
|
|
373
|
+
// non-JSON transports keep the raw text as body
|
|
374
|
+
}
|
|
375
|
+
return { status: response.status, body };
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
210
379
|
class SelfTestCaseTimeoutError extends Error {
|
|
211
380
|
constructor(timeoutMs: number) {
|
|
212
381
|
super(`Self-test case timed out after ${timeoutMs}ms`);
|
|
@@ -214,17 +383,32 @@ class SelfTestCaseTimeoutError extends Error {
|
|
|
214
383
|
}
|
|
215
384
|
}
|
|
216
385
|
|
|
217
|
-
async function withCaseTimeout<T>(
|
|
386
|
+
async function withCaseTimeout<T>(
|
|
387
|
+
run: () => Promise<T>,
|
|
388
|
+
timeoutMs: number,
|
|
389
|
+
controller: AbortController,
|
|
390
|
+
): Promise<T> {
|
|
218
391
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
392
|
+
let onAbort: (() => void) | undefined;
|
|
219
393
|
try {
|
|
220
394
|
return await Promise.race([
|
|
221
395
|
run(),
|
|
222
396
|
new Promise<never>((_, reject) => {
|
|
223
|
-
timer = setTimeout(() =>
|
|
397
|
+
timer = setTimeout(() => {
|
|
398
|
+
const error = new SelfTestCaseTimeoutError(timeoutMs);
|
|
399
|
+
controller.abort(error);
|
|
400
|
+
reject(error);
|
|
401
|
+
}, timeoutMs);
|
|
402
|
+
}),
|
|
403
|
+
new Promise<never>((_, reject) => {
|
|
404
|
+
onAbort = () => reject(controller.signal.reason);
|
|
405
|
+
controller.signal.addEventListener("abort", onAbort, { once: true });
|
|
406
|
+
if (controller.signal.aborted) onAbort();
|
|
224
407
|
}),
|
|
225
408
|
]);
|
|
226
409
|
} finally {
|
|
227
410
|
if (timer !== undefined) clearTimeout(timer);
|
|
411
|
+
if (onAbort) controller.signal.removeEventListener("abort", onAbort);
|
|
228
412
|
}
|
|
229
413
|
}
|
|
230
414
|
|
|
@@ -250,13 +434,69 @@ function upstreamErrorMessage(body: unknown): string | undefined {
|
|
|
250
434
|
return typeof message === "string" ? message : undefined;
|
|
251
435
|
}
|
|
252
436
|
|
|
437
|
+
/**
|
|
438
|
+
* How long a memoized multi-turn flow outcome suppresses re-driving the auth
|
|
439
|
+
* flow. Generous on purpose: a multi-turn ceremony (OTP, device approval) is a
|
|
440
|
+
* provider property that changes on the timescale of releases, not probe
|
|
441
|
+
* cycles, and every re-drive is a REAL upstream login submission. The cache is
|
|
442
|
+
* in-process, so a pod restart also clears the entry.
|
|
443
|
+
*/
|
|
444
|
+
export const SELF_TEST_MULTI_TURN_RETRY_AFTER_MS = 24 * 60 * 60 * 1000;
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* Age bound for POSITIVE cached credentials. Expiry modes that never produce
|
|
448
|
+
* a 401/403 (a 200 login page, an assertion failure) would otherwise replay
|
|
449
|
+
* the same stale session until pod restart — one re-login per day is the
|
|
450
|
+
* upstream-safe recovery for them.
|
|
451
|
+
*/
|
|
452
|
+
export const SELF_TEST_CREDENTIAL_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
453
|
+
|
|
454
|
+
export type SelfTestCredentialSessionEntry =
|
|
455
|
+
/** Flow-materialized credential reused across probe cycles. */
|
|
456
|
+
| { kind: "credential"; credential: Record<string, string>; cachedAtMs: number }
|
|
457
|
+
/**
|
|
458
|
+
* Negative entry: the flow did not complete in a single continue turn
|
|
459
|
+
* (`auth_flow_multi_turn`). Memoized so subsequent cycles report the skip
|
|
460
|
+
* WITHOUT contacting the upstream again — the first attempt already
|
|
461
|
+
* submitted real credentials (and may have triggered an OTP send).
|
|
462
|
+
*/
|
|
463
|
+
| { kind: "multi_turn"; cachedAtMs: number }
|
|
464
|
+
/**
|
|
465
|
+
* Negative entry: the flow REJECTED the submitted credential inputs
|
|
466
|
+
* (`retry` turn after continue — bad password, exchange failure).
|
|
467
|
+
* Memoized so the probe does not re-submit rejected credentials every
|
|
468
|
+
* cycle; a new entry is attempted when the inputs rotate (new hash),
|
|
469
|
+
* the TTL lapses, or the process restarts.
|
|
470
|
+
*/
|
|
471
|
+
| { kind: "rejected"; cachedAtMs: number };
|
|
472
|
+
|
|
473
|
+
/**
|
|
474
|
+
* In-process cache of per-(providerId + stable hash of credentialInputs) auth
|
|
475
|
+
* flow outcomes, so consecutive probe cycles reuse the session — or the
|
|
476
|
+
* memoized multi-turn skip — instead of logging in every cycle (upstream
|
|
477
|
+
* account safety, DR-7). Credential entries are invalidated on a probe auth
|
|
478
|
+
* failure, at most once; multi-turn entries expire after
|
|
479
|
+
* `SELF_TEST_MULTI_TURN_RETRY_AFTER_MS` or on process restart. Flow ERRORS
|
|
480
|
+
* (transport/protocol failures, thrown start/continue) are deliberately NEVER
|
|
481
|
+
* cached: they are typically transient, and retrying a failed request next
|
|
482
|
+
* cycle is not a repeated login submission.
|
|
483
|
+
*/
|
|
484
|
+
export type SelfTestCredentialSessionCache = Map<string, SelfTestCredentialSessionEntry>;
|
|
485
|
+
|
|
253
486
|
interface SelfTestExecutionContext {
|
|
254
487
|
provider: ProviderDefinition;
|
|
255
488
|
invoke: SelfTestOperationInvoke;
|
|
489
|
+
authFlow?: SelfTestAuthFlowInvoke;
|
|
256
490
|
requestId: string;
|
|
257
491
|
credentials?: Readonly<Record<string, string>>;
|
|
258
492
|
requestTimeoutMs?: number;
|
|
259
|
-
|
|
493
|
+
/**
|
|
494
|
+
* Mutable on purpose: every secret value materialized by an auth flow is
|
|
495
|
+
* appended here BEFORE any probe output is built, so redactSelfTestText
|
|
496
|
+
* scrubs flow-issued cookies/tokens exactly like request-supplied inputs.
|
|
497
|
+
*/
|
|
498
|
+
sensitiveValues: string[];
|
|
499
|
+
sessionCache: SelfTestCredentialSessionCache;
|
|
260
500
|
}
|
|
261
501
|
|
|
262
502
|
function resolveCaseTimeoutMs(
|
|
@@ -275,188 +515,762 @@ function resolveCaseTimeoutMs(
|
|
|
275
515
|
);
|
|
276
516
|
}
|
|
277
517
|
|
|
278
|
-
function
|
|
518
|
+
function credentialSessionCacheKey(
|
|
519
|
+
providerId: string,
|
|
520
|
+
inputs: Readonly<Record<string, string>>,
|
|
521
|
+
): string {
|
|
522
|
+
const canonical = JSON.stringify(
|
|
523
|
+
Object.keys(inputs)
|
|
524
|
+
.sort()
|
|
525
|
+
.map((key) => [key, inputs[key]]),
|
|
526
|
+
);
|
|
527
|
+
return `${providerId}:${createHash("sha256").update(canonical).digest("hex")}`;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
function registerSensitiveValues(
|
|
531
|
+
execution: SelfTestExecutionContext,
|
|
532
|
+
values: Iterable<string>,
|
|
533
|
+
): void {
|
|
534
|
+
for (const value of values) {
|
|
535
|
+
if (
|
|
536
|
+
typeof value === "string" &&
|
|
537
|
+
value.length > 0 &&
|
|
538
|
+
!execution.sensitiveValues.includes(value)
|
|
539
|
+
) {
|
|
540
|
+
execution.sensitiveValues.push(value);
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
type SelfTestConnectionResolution =
|
|
546
|
+
| {
|
|
547
|
+
kind: "connection";
|
|
548
|
+
connection?: OperationConnection;
|
|
549
|
+
credentialSource?: "inputs" | "flow" | "cache";
|
|
550
|
+
cacheKey?: string;
|
|
551
|
+
}
|
|
552
|
+
| { kind: "skip"; skipReason: string }
|
|
553
|
+
| { kind: "flow_error"; code: string; message: string };
|
|
554
|
+
|
|
555
|
+
type ParsedAuthFlowTurn =
|
|
556
|
+
| {
|
|
557
|
+
ok: true;
|
|
558
|
+
turn: { kind: string; data?: unknown; expectedInput?: unknown };
|
|
559
|
+
contextPatch?: Record<string, unknown>;
|
|
560
|
+
}
|
|
561
|
+
| { ok: false; code: string; message: string; httpStatus: number };
|
|
562
|
+
|
|
563
|
+
function parseAuthFlowResponse(result: { status: number; body: unknown }): ParsedAuthFlowTurn {
|
|
564
|
+
const errorEnvelope = objectProperty(result.body, "error");
|
|
565
|
+
if (result.status < 200 || result.status >= 300 || errorEnvelope !== undefined) {
|
|
566
|
+
const message = objectProperty(errorEnvelope, "message");
|
|
567
|
+
return {
|
|
568
|
+
ok: false,
|
|
569
|
+
code: "auth_flow_failed",
|
|
570
|
+
message:
|
|
571
|
+
typeof message === "string"
|
|
572
|
+
? message
|
|
573
|
+
: `Auth flow request failed with status ${result.status}`,
|
|
574
|
+
httpStatus: result.status,
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
const turnValue = objectProperty(result.body, "data");
|
|
578
|
+
const turnKind = objectProperty(turnValue, "kind");
|
|
579
|
+
if (typeof turnKind !== "string") {
|
|
580
|
+
return {
|
|
581
|
+
ok: false,
|
|
582
|
+
code: "auth_flow_failed",
|
|
583
|
+
message: "Auth flow returned an unrecognized turn.",
|
|
584
|
+
httpStatus: result.status,
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
const contextPatch = objectProperty(result.body, "contextPatch");
|
|
588
|
+
return {
|
|
589
|
+
ok: true,
|
|
590
|
+
turn: {
|
|
591
|
+
kind: turnKind,
|
|
592
|
+
data: objectProperty(turnValue, "data"),
|
|
593
|
+
expectedInput: objectProperty(turnValue, "expectedInput"),
|
|
594
|
+
},
|
|
595
|
+
...(contextPatch && typeof contextPatch === "object" && !Array.isArray(contextPatch)
|
|
596
|
+
? { contextPatch: contextPatch as Record<string, unknown> }
|
|
597
|
+
: {}),
|
|
598
|
+
};
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
function applyAuthFlowContextPatch(
|
|
602
|
+
base: Record<string, unknown>,
|
|
603
|
+
patch: Record<string, unknown> | undefined,
|
|
604
|
+
): Record<string, unknown> {
|
|
605
|
+
if (!patch) return base;
|
|
606
|
+
const next = { ...base };
|
|
607
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
608
|
+
if (value === null) {
|
|
609
|
+
delete next[key];
|
|
610
|
+
} else {
|
|
611
|
+
next[key] = value;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
return next;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Extracts the completed credential from a complete turn's data payload — the
|
|
619
|
+
* same `data.credential` record the gateway persists as connection secrets in
|
|
620
|
+
* production (`persistCredential` → credential-service `UpdateCredential`).
|
|
621
|
+
*/
|
|
622
|
+
function completedCredentialFromTurn(turnData: unknown): Record<string, string> | undefined {
|
|
623
|
+
const credential = objectProperty(turnData, "credential");
|
|
624
|
+
if (!credential || typeof credential !== "object" || Array.isArray(credential)) {
|
|
625
|
+
return undefined;
|
|
626
|
+
}
|
|
627
|
+
const secrets: Record<string, string> = {};
|
|
628
|
+
for (const [key, value] of Object.entries(credential)) {
|
|
629
|
+
if (typeof value === "string") secrets[key] = value;
|
|
630
|
+
}
|
|
631
|
+
return Object.keys(secrets).length > 0 ? secrets : undefined;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* Drives the provider's declared auth flow exactly like production does:
|
|
636
|
+
* `flow.start()` then a single `flow.continue(credentialInputs)`. Anything
|
|
637
|
+
* other than a complete turn is a visible multi-turn gap, never a fabricated
|
|
638
|
+
* probe failure.
|
|
639
|
+
*/
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* Fields an input-prompt turn actually requests. The canonical auth-turn
|
|
643
|
+
* shape carries the JSON schema DIRECTLY on `expectedInput` (`ctx.auth
|
|
644
|
+
* .nextForm`/`defineCredentialsAuth`, the committed fixtures); some providers
|
|
645
|
+
* nest it as `expectedInput.schema`. Both are honored. `null` when the turn
|
|
646
|
+
* declares no schema (legacy/loose flows keep full-input semantics).
|
|
647
|
+
*/
|
|
648
|
+
function turnRequestedFields(
|
|
649
|
+
turn: { expectedInput?: unknown },
|
|
650
|
+
): { properties: string[]; required: string[] } | null {
|
|
651
|
+
const expectedInput = turn.expectedInput;
|
|
652
|
+
if (!expectedInput || typeof expectedInput !== "object" || Array.isArray(expectedInput)) {
|
|
653
|
+
return null;
|
|
654
|
+
}
|
|
655
|
+
const schemaOf = (candidate: unknown): { properties: string[]; required: string[] } | null => {
|
|
656
|
+
const properties =
|
|
657
|
+
candidate && typeof candidate === "object"
|
|
658
|
+
? (candidate as { properties?: unknown }).properties
|
|
659
|
+
: undefined;
|
|
660
|
+
if (!properties || typeof properties !== "object" || Array.isArray(properties)) {
|
|
661
|
+
return null;
|
|
662
|
+
}
|
|
663
|
+
const requiredRaw =
|
|
664
|
+
candidate && typeof candidate === "object"
|
|
665
|
+
? (candidate as { required?: unknown }).required
|
|
666
|
+
: undefined;
|
|
667
|
+
const required = Array.isArray(requiredRaw)
|
|
668
|
+
? requiredRaw.filter((field): field is string => typeof field === "string")
|
|
669
|
+
: [];
|
|
670
|
+
return { properties: Object.keys(properties), required };
|
|
671
|
+
};
|
|
672
|
+
return (
|
|
673
|
+
schemaOf(expectedInput) ?? schemaOf((expectedInput as { schema?: unknown }).schema)
|
|
674
|
+
);
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
async function materializeFlowCredential(
|
|
678
|
+
execution: SelfTestExecutionContext,
|
|
679
|
+
inputs: Readonly<Record<string, string>>,
|
|
680
|
+
options: {
|
|
681
|
+
isAbandoned?: () => boolean;
|
|
682
|
+
connectionId?: string;
|
|
683
|
+
externalRef?: string;
|
|
684
|
+
signal?: AbortSignal;
|
|
685
|
+
} = {},
|
|
686
|
+
): Promise<
|
|
687
|
+
| { credential: Record<string, string> }
|
|
688
|
+
| Exclude<SelfTestConnectionResolution, { kind: "connection" }>
|
|
689
|
+
> {
|
|
690
|
+
const authFlow = execution.authFlow;
|
|
691
|
+
if (!authFlow) {
|
|
692
|
+
return {
|
|
693
|
+
kind: "flow_error",
|
|
694
|
+
code: "auth_flow_unavailable",
|
|
695
|
+
message:
|
|
696
|
+
"Provider declares a credentials auth flow but the self-test host has no auth-flow driver.",
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
const flowId = `self-test-${randomUUID()}`;
|
|
700
|
+
// The login must ride the SAME proxy/connection affinity the probe will
|
|
701
|
+
// use (createAuthFlowContext keys affinity on connectionId) — otherwise
|
|
702
|
+
// IP/session-bound upstreams see the cookie arrive from a different
|
|
703
|
+
// session and reject it.
|
|
704
|
+
const started = parseAuthFlowResponse(
|
|
705
|
+
await authFlow({
|
|
706
|
+
route: "start",
|
|
707
|
+
requestId: `${execution.requestId}-auth-start-${randomUUID()}`,
|
|
708
|
+
flowId,
|
|
709
|
+
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
|
710
|
+
...(options.externalRef ? { externalRef: options.externalRef } : {}),
|
|
711
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
712
|
+
}),
|
|
713
|
+
);
|
|
714
|
+
if (!started.ok) {
|
|
715
|
+
return { kind: "flow_error", code: started.code, message: started.message };
|
|
716
|
+
}
|
|
717
|
+
let turn = started.turn;
|
|
718
|
+
const flowContext = applyAuthFlowContextPatch({}, started.contextPatch);
|
|
719
|
+
if (turn.kind === "abort") {
|
|
720
|
+
// Terminal turn: continuing after an abort would replay credentials into
|
|
721
|
+
// a flow that already refused to proceed. Not memoized (flow errors are
|
|
722
|
+
// never cached) — an abort can be transient upstream maintenance.
|
|
723
|
+
return {
|
|
724
|
+
kind: "flow_error",
|
|
725
|
+
code: "auth_flow_aborted",
|
|
726
|
+
message: "Auth flow aborted before requesting input.",
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
if (turn.kind !== "complete") {
|
|
730
|
+
// Validate the start turn BEFORE submitting credentials: an unknown
|
|
731
|
+
// kind may be a provider typo or a stage that must not receive the
|
|
732
|
+
// probe inputs. `retry` counts as an input prompt at this stage.
|
|
733
|
+
if (turn.kind !== "retry" && !INTERACTIVE_TURN_KIND_SET.has(turn.kind)) {
|
|
734
|
+
return {
|
|
735
|
+
kind: "flow_error",
|
|
736
|
+
code: "auth_flow_unexpected_turn",
|
|
737
|
+
message: `Auth flow start returned an unrecognized turn kind "${turn.kind}".`,
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
// Auto-continue ONLY into input prompts (form/retry). Other known
|
|
741
|
+
// interactive stages (redirect, poll, pending, challenge, message,
|
|
742
|
+
// multi_choice) are valid flows that are NOT asking for the credential
|
|
743
|
+
// inputs — posting the password there submits it to the wrong stage.
|
|
744
|
+
// They are a genuine headless gap: the memoized multi-turn skip.
|
|
745
|
+
if (turn.kind !== "form" && turn.kind !== "retry") {
|
|
746
|
+
return { kind: "skip", skipReason: SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON };
|
|
747
|
+
}
|
|
748
|
+
// Submit ONLY what the turn asks for: a first stage of a multi-step
|
|
749
|
+
// login may request a subset (or different fields entirely) — posting
|
|
750
|
+
// the full inputs would send secrets to the wrong stage. Only the
|
|
751
|
+
// schema's REQUIRED fields are mandatory (defineCredentialsAuth
|
|
752
|
+
// encodes optional fields by omitting them from `required`); a turn
|
|
753
|
+
// whose required fields we do not hold is a headless gap (multi-turn).
|
|
754
|
+
// A turn with no declared schema keeps full-input semantics.
|
|
755
|
+
const requestedFields = turnRequestedFields(turn);
|
|
756
|
+
let submitInputs: Record<string, string> = { ...inputs };
|
|
757
|
+
if (requestedFields !== null) {
|
|
758
|
+
if (requestedFields.required.some((field) => inputs[field] === undefined)) {
|
|
759
|
+
return { kind: "skip", skipReason: SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON };
|
|
760
|
+
}
|
|
761
|
+
submitInputs = Object.fromEntries(
|
|
762
|
+
requestedFields.properties
|
|
763
|
+
.filter((field) => inputs[field] !== undefined)
|
|
764
|
+
.map((field) => [field, inputs[field] as string]),
|
|
765
|
+
);
|
|
766
|
+
}
|
|
767
|
+
// The case deadline may have fired while start() was still running.
|
|
768
|
+
// Never submit real credentials into a flow whose case already
|
|
769
|
+
// reported self_test_timeout — a late continue is a real upstream
|
|
770
|
+
// login/OTP attempt nobody is waiting for.
|
|
771
|
+
if (options.isAbandoned?.() === true) {
|
|
772
|
+
return {
|
|
773
|
+
kind: "flow_error",
|
|
774
|
+
code: "self_test_timeout",
|
|
775
|
+
message: "Case deadline passed before credential submission; flow abandoned.",
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
const continued = parseAuthFlowResponse(
|
|
779
|
+
await authFlow({
|
|
780
|
+
route: "continue",
|
|
781
|
+
requestId: `${execution.requestId}-auth-continue-${randomUUID()}`,
|
|
782
|
+
flowId,
|
|
783
|
+
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
|
784
|
+
...(options.externalRef ? { externalRef: options.externalRef } : {}),
|
|
785
|
+
input: submitInputs,
|
|
786
|
+
...(Object.keys(flowContext).length > 0 ? { context: flowContext } : {}),
|
|
787
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
788
|
+
}),
|
|
789
|
+
);
|
|
790
|
+
if (!continued.ok) {
|
|
791
|
+
// Providers built with defineCredentialsAuth cannot return a retry
|
|
792
|
+
// turn — a rejected password THROWS and /auth/continue answers with
|
|
793
|
+
// an auth-shaped 401/403. That is a credential REJECTION (memoized,
|
|
794
|
+
// so the probe never hammers a locked-out login); every other
|
|
795
|
+
// status stays an uncached transient retry.
|
|
796
|
+
if (AUTH_REJECTION_HTTP_STATUSES.has(continued.httpStatus)) {
|
|
797
|
+
return { kind: "skip", skipReason: SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON };
|
|
798
|
+
}
|
|
799
|
+
return { kind: "flow_error", code: continued.code, message: continued.message };
|
|
800
|
+
}
|
|
801
|
+
turn = continued.turn;
|
|
802
|
+
}
|
|
803
|
+
if (turn.kind === "abort") {
|
|
804
|
+
return {
|
|
805
|
+
kind: "flow_error",
|
|
806
|
+
code: "auth_flow_aborted",
|
|
807
|
+
message: "Auth flow aborted after credential submission.",
|
|
808
|
+
};
|
|
809
|
+
}
|
|
810
|
+
if (turn.kind === "retry") {
|
|
811
|
+
// A retry turn AFTER submission is a credential rejection, not an
|
|
812
|
+
// interactive gap — surfaced distinctly so monitoring can treat it as
|
|
813
|
+
// a real outage, and memoized by the caller (lockout safety).
|
|
814
|
+
return { kind: "skip", skipReason: SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON };
|
|
815
|
+
}
|
|
816
|
+
if (turn.kind !== "complete") {
|
|
817
|
+
// Only KNOWN interactive kinds are a genuine "cannot complete headless"
|
|
818
|
+
// multi-turn gap (memoized by the caller). An unknown kind is ambiguous
|
|
819
|
+
// — it may encode a transient provider failure — so it reports as a
|
|
820
|
+
// flow error, which is never memoized, instead of freezing the signal.
|
|
821
|
+
if (!INTERACTIVE_TURN_KIND_SET.has(turn.kind)) {
|
|
822
|
+
return {
|
|
823
|
+
kind: "flow_error",
|
|
824
|
+
code: "auth_flow_unexpected_turn",
|
|
825
|
+
message: `Auth flow returned an unrecognized turn kind "${turn.kind}".`,
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
return { kind: "skip", skipReason: SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON };
|
|
829
|
+
}
|
|
830
|
+
const credential = completedCredentialFromTurn(turn.data);
|
|
831
|
+
if (!credential) {
|
|
832
|
+
return {
|
|
833
|
+
kind: "flow_error",
|
|
834
|
+
code: "auth_flow_invalid_credential",
|
|
835
|
+
message: "Auth flow completed without a string-valued credential payload.",
|
|
836
|
+
};
|
|
837
|
+
}
|
|
838
|
+
// Redaction contract: flow-issued secrets are registered BEFORE any probe
|
|
839
|
+
// output can be built from them.
|
|
840
|
+
registerSensitiveValues(execution, Object.values(credential));
|
|
841
|
+
return { credential };
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
async function resolveSelfTestConnection(
|
|
279
845
|
execution: SelfTestExecutionContext,
|
|
280
846
|
operationId: string,
|
|
281
847
|
suite: AnyHealthCheckSuite,
|
|
282
|
-
|
|
283
|
-
|
|
848
|
+
options: { forceLogin?: boolean; isAbandoned?: () => boolean; signal?: AbortSignal } = {},
|
|
849
|
+
): Promise<SelfTestConnectionResolution> {
|
|
850
|
+
if (!suite.requiresConnection) return { kind: "connection" };
|
|
284
851
|
const inputs = execution.credentials ?? {};
|
|
285
852
|
const declaredFields = Object.keys(
|
|
286
853
|
(execution.provider.healthProbe ?? execution.provider.healthMonitor)?.credentialInputs ?? {},
|
|
287
854
|
);
|
|
288
855
|
for (const field of declaredFields) {
|
|
289
856
|
if (!inputs[field]) {
|
|
290
|
-
return { skipReason: `credential_missing:${field}` };
|
|
857
|
+
return { kind: "skip", skipReason: `credential_missing:${field}` };
|
|
291
858
|
}
|
|
292
859
|
}
|
|
293
860
|
if (declaredFields.length === 0 && Object.keys(inputs).length === 0) {
|
|
294
|
-
return { skipReason: "credential_missing:credentials" };
|
|
861
|
+
return { kind: "skip", skipReason: "credential_missing:credentials" };
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
// The connection id seeds proxy/connection affinity in the provider
|
|
865
|
+
// context, so it must be STABLE per (provider, credentialInputs): a cached
|
|
866
|
+
// session replayed under a per-request id would ride a different proxy/IP
|
|
867
|
+
// each cycle and upstreams would treat the cookie as stale or suspicious.
|
|
868
|
+
// The id carries only a hash of the inputs, never the inputs themselves.
|
|
869
|
+
//
|
|
870
|
+
// Providers declaring `proxy.session.affinity: "operation"` pin the PROBE's
|
|
871
|
+
// proxy to `${providerId}/${operationId}` regardless of connection id — so
|
|
872
|
+
// the login must ride that exact key, and the session cache splits per
|
|
873
|
+
// operation (one shared cookie would otherwise hop between per-operation
|
|
874
|
+
// proxies).
|
|
875
|
+
const operationAffinity =
|
|
876
|
+
typeof execution.provider.proxy === "object" &&
|
|
877
|
+
execution.provider.proxy?.session?.affinity === "operation";
|
|
878
|
+
const credentialKey = credentialSessionCacheKey(execution.provider.id, inputs);
|
|
879
|
+
const affinityKey = operationAffinity ? `${credentialKey}:${operationId}` : credentialKey;
|
|
880
|
+
// ONE id for the auth flow AND the probe connection: providers may bind
|
|
881
|
+
// the issued credential to FlowContext.connectionId and later compare it
|
|
882
|
+
// against ctx.request.connectionId. Operation-affinity providers use the
|
|
883
|
+
// probe's exact proxy key (providerId/operationId); everyone else uses the
|
|
884
|
+
// stable per-credential hash.
|
|
885
|
+
const connectionId = operationAffinity
|
|
886
|
+
? `${execution.provider.id}/${operationId}`
|
|
887
|
+
: `self-test-${createHash("sha256").update(affinityKey).digest("hex").slice(0, 22)}`;
|
|
888
|
+
const buildConnection = (secrets: Readonly<Record<string, string>>): OperationConnection => ({
|
|
889
|
+
id: connectionId,
|
|
890
|
+
mode: "credentials",
|
|
891
|
+
secrets: { ...secrets },
|
|
892
|
+
metadata: { purpose: "provider-self-test", operationId },
|
|
893
|
+
externalRef: `${execution.provider.id}-${operationId}-self-test`,
|
|
894
|
+
});
|
|
895
|
+
|
|
896
|
+
const auth = execution.provider.auth;
|
|
897
|
+
if (auth?.mode !== "credentials" || !auth.flow) {
|
|
898
|
+
// Providers without a declared credentials flow keep raw-input semantics
|
|
899
|
+
// — and the pre-existing per-request connection id: there is no session
|
|
900
|
+
// to keep on one affinity, and a stable id would silently pin every
|
|
901
|
+
// cycle of a connection-affinity proxy to the same upstream session.
|
|
902
|
+
return {
|
|
903
|
+
kind: "connection",
|
|
904
|
+
connection: {
|
|
905
|
+
...buildConnection(inputs),
|
|
906
|
+
id: `self-test-${execution.requestId}`,
|
|
907
|
+
},
|
|
908
|
+
credentialSource: "inputs",
|
|
909
|
+
};
|
|
295
910
|
}
|
|
911
|
+
|
|
912
|
+
const cacheKey = affinityKey;
|
|
913
|
+
const cached = execution.sessionCache.get(cacheKey);
|
|
914
|
+
// DR-7 upstream-account safety: a memoized multi-turn outcome
|
|
915
|
+
// short-circuits to the auth_flow_multi_turn skip WITHOUT re-driving
|
|
916
|
+
// flow.start()/flow.continue() — every re-drive is a real upstream login
|
|
917
|
+
// submission (OTP sends, lockout risk), and the probe scheduler would
|
|
918
|
+
// otherwise repeat it every cycle forever. Changed credentialInputs hash
|
|
919
|
+
// to a different key and re-attempt immediately; otherwise the entry
|
|
920
|
+
// expires after a generous TTL (or process restart) so a provider whose
|
|
921
|
+
// flow becomes single-turn again is eventually re-probed.
|
|
922
|
+
if (cached?.kind === "multi_turn" || cached?.kind === "rejected") {
|
|
923
|
+
if (Date.now() - cached.cachedAtMs < SELF_TEST_MULTI_TURN_RETRY_AFTER_MS) {
|
|
924
|
+
return {
|
|
925
|
+
kind: "skip",
|
|
926
|
+
skipReason:
|
|
927
|
+
cached.kind === "rejected"
|
|
928
|
+
? SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON
|
|
929
|
+
: SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON,
|
|
930
|
+
};
|
|
931
|
+
}
|
|
932
|
+
execution.sessionCache.delete(cacheKey);
|
|
933
|
+
}
|
|
934
|
+
if (options.forceLogin !== true && cached?.kind === "credential") {
|
|
935
|
+
if (Date.now() - cached.cachedAtMs >= SELF_TEST_CREDENTIAL_MAX_AGE_MS) {
|
|
936
|
+
// Age-bounded: expiry modes that never 401 (login-page 200s,
|
|
937
|
+
// assertion failures) must not replay one stale session forever.
|
|
938
|
+
execution.sessionCache.delete(cacheKey);
|
|
939
|
+
} else {
|
|
940
|
+
registerSensitiveValues(execution, Object.values(cached.credential));
|
|
941
|
+
return {
|
|
942
|
+
kind: "connection",
|
|
943
|
+
connection: buildConnection(cached.credential),
|
|
944
|
+
credentialSource: "cache",
|
|
945
|
+
cacheKey,
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
const materialized = await materializeFlowCredential(execution, inputs, {
|
|
950
|
+
...(options.isAbandoned !== undefined ? { isAbandoned: options.isAbandoned } : {}),
|
|
951
|
+
connectionId,
|
|
952
|
+
externalRef: `${execution.provider.id}-${operationId}-self-test`,
|
|
953
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
954
|
+
});
|
|
955
|
+
if (!("credential" in materialized)) {
|
|
956
|
+
// Only the multi-turn SKIP is negative-cached. Flow ERRORS
|
|
957
|
+
// (auth_flow_unavailable / auth_flow_failed / invalid credential
|
|
958
|
+
// payloads, or a thrown start/continue) are never memoized: they are
|
|
959
|
+
// typically transient upstream or host failures, so each cycle may
|
|
960
|
+
// retry — permanently caching an error would silently freeze the
|
|
961
|
+
// signal on a blip, while retrying a FAILED request is not a repeated
|
|
962
|
+
// successful login submission.
|
|
963
|
+
if (materialized.kind === "skip" && options.isAbandoned?.() !== true) {
|
|
964
|
+
if (materialized.skipReason === SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON) {
|
|
965
|
+
execution.sessionCache.set(cacheKey, { kind: "multi_turn", cachedAtMs: Date.now() });
|
|
966
|
+
} else if (materialized.skipReason === SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON) {
|
|
967
|
+
execution.sessionCache.set(cacheKey, { kind: "rejected", cachedAtMs: Date.now() });
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
return materialized;
|
|
971
|
+
}
|
|
972
|
+
// Abort is cooperative: a flow may still complete after the case deadline
|
|
973
|
+
// if provider code ignores the signal. Never cache that late credential or
|
|
974
|
+
// let the next probe hide the timed-out login.
|
|
975
|
+
if (options.isAbandoned?.() === true) {
|
|
976
|
+
return {
|
|
977
|
+
kind: "flow_error",
|
|
978
|
+
code: "self_test_timeout",
|
|
979
|
+
message: "Auth flow completed after the case deadline; credential discarded.",
|
|
980
|
+
};
|
|
981
|
+
}
|
|
982
|
+
execution.sessionCache.set(cacheKey, {
|
|
983
|
+
kind: "credential",
|
|
984
|
+
credential: materialized.credential,
|
|
985
|
+
cachedAtMs: Date.now(),
|
|
986
|
+
});
|
|
296
987
|
return {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
metadata: { purpose: "provider-self-test", operationId },
|
|
302
|
-
externalRef: `${execution.provider.id}-${operationId}-self-test`,
|
|
303
|
-
},
|
|
988
|
+
kind: "connection",
|
|
989
|
+
connection: buildConnection(materialized.credential),
|
|
990
|
+
credentialSource: "flow",
|
|
991
|
+
cacheKey,
|
|
304
992
|
};
|
|
305
993
|
}
|
|
306
994
|
|
|
995
|
+
/** A failed probe whose HTTP status is auth-shaped invalidates a cached session once. */
|
|
996
|
+
function isAuthFailureCaseResult(result: SelfTestCaseResult): boolean {
|
|
997
|
+
return result.status === "failed" && (result.httpStatus === 401 || result.httpStatus === 403);
|
|
998
|
+
}
|
|
999
|
+
|
|
307
1000
|
async function executeSelfTestCase(
|
|
308
1001
|
execution: SelfTestExecutionContext,
|
|
309
1002
|
operationId: string,
|
|
310
1003
|
suite: AnyHealthCheckSuite,
|
|
311
1004
|
healthCase: AnyHealthCheckCase,
|
|
1005
|
+
caseController: AbortController,
|
|
312
1006
|
): Promise<SelfTestCaseResult> {
|
|
313
|
-
const { provider, invoke
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
const
|
|
317
|
-
const finish = (
|
|
318
|
-
partial: Omit<
|
|
319
|
-
SelfTestCaseResult,
|
|
320
|
-
"operationId" | "caseName" | "startedAt" | "finishedAt" | "responseTimeMs"
|
|
321
|
-
> & { responseTimeMs?: number },
|
|
322
|
-
): SelfTestCaseResult => ({
|
|
323
|
-
operationId,
|
|
324
|
-
caseName: healthCase.name,
|
|
325
|
-
startedAt,
|
|
326
|
-
finishedAt: new Date().toISOString(),
|
|
327
|
-
responseTimeMs:
|
|
328
|
-
partial.responseTimeMs ?? Math.max(0, Math.round(performance.now() - startedAtMs)),
|
|
329
|
-
...partial,
|
|
330
|
-
});
|
|
1007
|
+
const { provider, invoke } = execution;
|
|
1008
|
+
// execution.sensitiveValues may grow while the case runs (flow-issued
|
|
1009
|
+
// secrets); redact always reads the live array.
|
|
1010
|
+
const redact = (text: string) => redactSelfTestText(text, execution.sensitiveValues);
|
|
331
1011
|
const defaultLabel = redact(healthCase.description ?? healthCase.name);
|
|
1012
|
+
const timeoutMs = resolveCaseTimeoutMs(execution, suite, healthCase);
|
|
1013
|
+
// One deadline for the WHOLE case: connection materialization (auth flow),
|
|
1014
|
+
// the probe, and the one-shot auth retry all draw from the same budget —
|
|
1015
|
+
// a 30s case must never take ~4×30s across its stages.
|
|
1016
|
+
const caseDeadlineAtMs = performance.now() + timeoutMs;
|
|
1017
|
+
const remainingCaseTimeoutMs = () =>
|
|
1018
|
+
Math.max(1, Math.ceil(caseDeadlineAtMs - performance.now()));
|
|
1019
|
+
const runWithCaseTimeout = <T>(run: () => Promise<T>, remainingMs: number) =>
|
|
1020
|
+
withCaseTimeout(run, remainingMs, caseController);
|
|
1021
|
+
|
|
1022
|
+
const beginCase = () => {
|
|
1023
|
+
const startedAt = new Date().toISOString();
|
|
1024
|
+
const startedAtMs = performance.now();
|
|
1025
|
+
return {
|
|
1026
|
+
startedAtMs,
|
|
1027
|
+
finish: (
|
|
1028
|
+
partial: Omit<
|
|
1029
|
+
SelfTestCaseResult,
|
|
1030
|
+
"operationId" | "caseName" | "startedAt" | "finishedAt" | "responseTimeMs"
|
|
1031
|
+
> & { responseTimeMs?: number },
|
|
1032
|
+
): SelfTestCaseResult => ({
|
|
1033
|
+
operationId,
|
|
1034
|
+
caseName: healthCase.name,
|
|
1035
|
+
startedAt,
|
|
1036
|
+
finishedAt: new Date().toISOString(),
|
|
1037
|
+
responseTimeMs:
|
|
1038
|
+
partial.responseTimeMs ?? Math.max(0, Math.round(performance.now() - startedAtMs)),
|
|
1039
|
+
...partial,
|
|
1040
|
+
}),
|
|
1041
|
+
};
|
|
1042
|
+
};
|
|
1043
|
+
const caseScope = beginCase();
|
|
332
1044
|
|
|
333
1045
|
if (healthCase.enabled && healthCase.enabled() === false) {
|
|
334
|
-
return finish({
|
|
1046
|
+
return caseScope.finish({
|
|
335
1047
|
status: "skipped",
|
|
336
1048
|
label: defaultLabel,
|
|
337
1049
|
skipReason: "disabled",
|
|
338
1050
|
});
|
|
339
1051
|
}
|
|
340
1052
|
|
|
341
|
-
const
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
1053
|
+
const resolveConnection = async (forceLogin: boolean): Promise<SelfTestConnectionResolution> => {
|
|
1054
|
+
// The deadline aborts the in-process flow request. Keep the abandonment
|
|
1055
|
+
// guard as a backstop for provider code that ignores cancellation so a
|
|
1056
|
+
// late completion still cannot write the session cache.
|
|
1057
|
+
let abandoned = false;
|
|
1058
|
+
try {
|
|
1059
|
+
return await runWithCaseTimeout(
|
|
1060
|
+
() =>
|
|
1061
|
+
resolveSelfTestConnection(execution, operationId, suite, {
|
|
1062
|
+
forceLogin,
|
|
1063
|
+
isAbandoned: () => abandoned,
|
|
1064
|
+
signal: caseController.signal,
|
|
1065
|
+
}),
|
|
1066
|
+
remainingCaseTimeoutMs(),
|
|
1067
|
+
);
|
|
1068
|
+
} catch (error) {
|
|
1069
|
+
abandoned = true;
|
|
1070
|
+
return {
|
|
1071
|
+
kind: "flow_error",
|
|
1072
|
+
code: error instanceof SelfTestCaseTimeoutError ? "self_test_timeout" : "auth_flow_failed",
|
|
1073
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1074
|
+
};
|
|
1075
|
+
}
|
|
1076
|
+
};
|
|
1077
|
+
|
|
1078
|
+
const nonConnectionResult = (
|
|
1079
|
+
resolution: Exclude<SelfTestConnectionResolution, { kind: "connection" }>,
|
|
1080
|
+
): SelfTestCaseResult => {
|
|
1081
|
+
if (resolution.kind === "skip") {
|
|
1082
|
+
return caseScope.finish({
|
|
1083
|
+
status: "skipped",
|
|
1084
|
+
label: defaultLabel,
|
|
1085
|
+
skipReason: resolution.skipReason,
|
|
1086
|
+
});
|
|
1087
|
+
}
|
|
1088
|
+
return caseScope.finish({
|
|
1089
|
+
status: "error",
|
|
345
1090
|
label: defaultLabel,
|
|
346
|
-
|
|
1091
|
+
error: { code: resolution.code, message: redact(resolution.message) },
|
|
347
1092
|
});
|
|
348
|
-
}
|
|
349
|
-
const connection = connectionResolution.connection;
|
|
1093
|
+
};
|
|
350
1094
|
|
|
351
|
-
const
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
1095
|
+
const runProbeAttempt = async (
|
|
1096
|
+
connection: OperationConnection | undefined,
|
|
1097
|
+
): Promise<SelfTestCaseResult> => {
|
|
1098
|
+
// gateway.execute keeps its contract — EVERY helper status is returned
|
|
1099
|
+
// to the prepareInput hook (it may branch on 401 itself). The last
|
|
1100
|
+
// auth-shaped helper status is only RECORDED: if the hook then throws,
|
|
1101
|
+
// the case fails WITH that status so stale-session recovery triggers.
|
|
1102
|
+
let prepareAuthStatus: number | null = null;
|
|
1103
|
+
// Share the OUTER case scope: startedAt/responseTimeMs must cover the
|
|
1104
|
+
// WHOLE case — auth-flow materialization included — not just the final
|
|
1105
|
+
// operation attempt, or a slow login reads as a fast healthy case.
|
|
1106
|
+
const { startedAtMs, finish } = caseScope;
|
|
1107
|
+
try {
|
|
1108
|
+
return await runWithCaseTimeout(async () => {
|
|
1109
|
+
const resolvedInput = resolveHealthCheckInputDateTokens(healthCase.input);
|
|
1110
|
+
const preparedInput = healthCase.prepareInput
|
|
1111
|
+
? await healthCase.prepareInput({
|
|
1112
|
+
providerId: provider.id,
|
|
1113
|
+
operationId,
|
|
1114
|
+
input: resolvedInput,
|
|
1115
|
+
...(connection ? { connectionId: connection.id } : {}),
|
|
1116
|
+
gateway: {
|
|
1117
|
+
execute: async (foreignProviderId, gatewayOperationId, gatewayInput) => {
|
|
1118
|
+
if (foreignProviderId !== provider.id) {
|
|
1119
|
+
throw new Error(
|
|
1120
|
+
`Self-test prepareInput may only invoke provider "${provider.id}" operations (requested "${foreignProviderId}").`,
|
|
1121
|
+
);
|
|
1122
|
+
}
|
|
1123
|
+
const startedGatewayMs = performance.now();
|
|
1124
|
+
const executed = await invoke({
|
|
1125
|
+
operationId: gatewayOperationId,
|
|
1126
|
+
input: gatewayInput,
|
|
1127
|
+
connection,
|
|
1128
|
+
requestId: `${execution.requestId}-prepare-${randomUUID()}`,
|
|
1129
|
+
signal: caseController.signal,
|
|
1130
|
+
});
|
|
1131
|
+
if (executed.status === 401 || executed.status === 403) {
|
|
1132
|
+
prepareAuthStatus = executed.status;
|
|
1133
|
+
}
|
|
1134
|
+
return {
|
|
1135
|
+
status: executed.status,
|
|
1136
|
+
duration: performance.now() - startedGatewayMs,
|
|
1137
|
+
data: executed.data,
|
|
1138
|
+
meta: executed.meta,
|
|
1139
|
+
};
|
|
1140
|
+
},
|
|
381
1141
|
},
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
: resolvedInput;
|
|
1142
|
+
})
|
|
1143
|
+
: resolvedInput;
|
|
385
1144
|
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
1145
|
+
const executed = await invoke({
|
|
1146
|
+
operationId,
|
|
1147
|
+
input: preparedInput,
|
|
1148
|
+
connection,
|
|
1149
|
+
requestId: `${execution.requestId}-${randomUUID()}`,
|
|
1150
|
+
signal: caseController.signal,
|
|
1151
|
+
});
|
|
1152
|
+
const durationMs = performance.now() - startedAtMs;
|
|
1153
|
+
|
|
1154
|
+
if (executed.status < 200 || executed.status >= 300) {
|
|
1155
|
+
return finish({
|
|
1156
|
+
status: "failed",
|
|
1157
|
+
label: defaultLabel,
|
|
1158
|
+
httpStatus: executed.status,
|
|
1159
|
+
error: {
|
|
1160
|
+
code: upstreamErrorCode(executed.data) ?? "operation_failed",
|
|
1161
|
+
message: redact(
|
|
1162
|
+
upstreamErrorMessage(executed.data) ??
|
|
1163
|
+
`Operation invocation failed with status ${executed.status}`,
|
|
1164
|
+
),
|
|
1165
|
+
},
|
|
1166
|
+
});
|
|
1167
|
+
}
|
|
393
1168
|
|
|
394
|
-
|
|
1169
|
+
const assertionContext: HealthCheckAssertionContext = {
|
|
1170
|
+
status: executed.status,
|
|
1171
|
+
data: executed.data,
|
|
1172
|
+
durationMs,
|
|
1173
|
+
...(executed.meta ? { meta: executed.meta } : {}),
|
|
1174
|
+
};
|
|
1175
|
+
let assertionResult: unknown;
|
|
1176
|
+
try {
|
|
1177
|
+
assertionResult = await healthCase.assertions(assertionContext);
|
|
1178
|
+
} catch (assertionError) {
|
|
1179
|
+
return finish({
|
|
1180
|
+
status: "failed",
|
|
1181
|
+
label: defaultLabel,
|
|
1182
|
+
httpStatus: executed.status,
|
|
1183
|
+
assertion: {
|
|
1184
|
+
passed: false,
|
|
1185
|
+
message: redact(
|
|
1186
|
+
assertionError instanceof Error ? assertionError.message : String(assertionError),
|
|
1187
|
+
),
|
|
1188
|
+
},
|
|
1189
|
+
});
|
|
1190
|
+
}
|
|
1191
|
+
const statusValue = objectProperty(assertionResult, "status");
|
|
1192
|
+
const overrideStatus =
|
|
1193
|
+
statusValue === "ok" || statusValue === "degraded" ? statusValue : undefined;
|
|
1194
|
+
const labelValue = objectProperty(assertionResult, "label");
|
|
1195
|
+
const overrideLabel = typeof labelValue === "string" ? redact(labelValue) : undefined;
|
|
395
1196
|
return finish({
|
|
396
|
-
status: "
|
|
397
|
-
label: defaultLabel,
|
|
1197
|
+
status: overrideStatus ?? "ok",
|
|
1198
|
+
label: overrideLabel ?? defaultLabel,
|
|
398
1199
|
httpStatus: executed.status,
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
1200
|
+
assertion: { passed: true },
|
|
1201
|
+
});
|
|
1202
|
+
}, remainingCaseTimeoutMs());
|
|
1203
|
+
} catch (error) {
|
|
1204
|
+
if (error instanceof SelfTestCaseTimeoutError) {
|
|
1205
|
+
return finish({
|
|
1206
|
+
status: "error",
|
|
1207
|
+
label: defaultLabel,
|
|
1208
|
+
error: { code: "self_test_timeout", message: redact(error.message) },
|
|
406
1209
|
});
|
|
407
1210
|
}
|
|
408
|
-
|
|
409
|
-
const assertionContext: HealthCheckAssertionContext = {
|
|
410
|
-
status: executed.status,
|
|
411
|
-
data: executed.data,
|
|
412
|
-
durationMs,
|
|
413
|
-
...(executed.meta ? { meta: executed.meta } : {}),
|
|
414
|
-
};
|
|
415
|
-
let assertionResult: unknown;
|
|
416
|
-
try {
|
|
417
|
-
assertionResult = await healthCase.assertions(assertionContext);
|
|
418
|
-
} catch (assertionError) {
|
|
1211
|
+
if (prepareAuthStatus !== null) {
|
|
419
1212
|
return finish({
|
|
420
1213
|
status: "failed",
|
|
421
1214
|
label: defaultLabel,
|
|
422
|
-
httpStatus:
|
|
1215
|
+
httpStatus: prepareAuthStatus,
|
|
423
1216
|
assertion: {
|
|
424
1217
|
passed: false,
|
|
425
|
-
message: redact(
|
|
426
|
-
assertionError instanceof Error ? assertionError.message : String(assertionError),
|
|
427
|
-
),
|
|
1218
|
+
message: redact(error instanceof Error ? error.message : String(error)),
|
|
428
1219
|
},
|
|
429
1220
|
});
|
|
430
1221
|
}
|
|
431
|
-
const statusValue = objectProperty(assertionResult, "status");
|
|
432
|
-
const overrideStatus =
|
|
433
|
-
statusValue === "ok" || statusValue === "degraded" ? statusValue : undefined;
|
|
434
|
-
const labelValue = objectProperty(assertionResult, "label");
|
|
435
|
-
const overrideLabel = typeof labelValue === "string" ? redact(labelValue) : undefined;
|
|
436
|
-
return finish({
|
|
437
|
-
status: overrideStatus ?? "ok",
|
|
438
|
-
label: overrideLabel ?? defaultLabel,
|
|
439
|
-
httpStatus: executed.status,
|
|
440
|
-
assertion: { passed: true },
|
|
441
|
-
});
|
|
442
|
-
}, timeoutMs);
|
|
443
|
-
} catch (error) {
|
|
444
|
-
if (error instanceof SelfTestCaseTimeoutError) {
|
|
445
1222
|
return finish({
|
|
446
1223
|
status: "error",
|
|
447
1224
|
label: defaultLabel,
|
|
448
|
-
error: {
|
|
1225
|
+
error: {
|
|
1226
|
+
code: "self_test_execution_error",
|
|
1227
|
+
message: redact(error instanceof Error ? error.message : String(error)),
|
|
1228
|
+
},
|
|
449
1229
|
});
|
|
450
1230
|
}
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
message: redact(error instanceof Error ? error.message : String(error)),
|
|
457
|
-
},
|
|
458
|
-
});
|
|
1231
|
+
};
|
|
1232
|
+
|
|
1233
|
+
const resolution = await resolveConnection(false);
|
|
1234
|
+
if (resolution.kind !== "connection") {
|
|
1235
|
+
return nonConnectionResult(resolution);
|
|
459
1236
|
}
|
|
1237
|
+
|
|
1238
|
+
let result = await runProbeAttempt(resolution.connection);
|
|
1239
|
+
|
|
1240
|
+
// One-shot session recovery: a cached credential that fails the probe with
|
|
1241
|
+
// an auth-shaped status is invalidated, the flow re-runs ONCE, and the
|
|
1242
|
+
// probe retries once. Fresh (just-materialized) credentials never retry.
|
|
1243
|
+
if (
|
|
1244
|
+
resolution.credentialSource === "cache" &&
|
|
1245
|
+
resolution.cacheKey !== undefined &&
|
|
1246
|
+
isAuthFailureCaseResult(result)
|
|
1247
|
+
) {
|
|
1248
|
+
execution.sessionCache.delete(resolution.cacheKey);
|
|
1249
|
+
const retryResolution = await resolveConnection(true);
|
|
1250
|
+
if (retryResolution.kind !== "connection") {
|
|
1251
|
+
return nonConnectionResult(retryResolution);
|
|
1252
|
+
}
|
|
1253
|
+
result = await runProbeAttempt(retryResolution.connection);
|
|
1254
|
+
// The retry's fresh credential is subject to the same eviction rule
|
|
1255
|
+
// as a first-attempt fresh credential (below).
|
|
1256
|
+
if (retryResolution.cacheKey !== undefined && isAuthFailureCaseResult(result)) {
|
|
1257
|
+
execution.sessionCache.delete(retryResolution.cacheKey);
|
|
1258
|
+
}
|
|
1259
|
+
return result;
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
// A FRESH credential the probe just rejected is known-bad: evict it so the
|
|
1263
|
+
// next cycle logs in anew instead of replaying a guaranteed-stale session
|
|
1264
|
+
// once before recovering. (No retry here — fresh credentials never retry.)
|
|
1265
|
+
if (
|
|
1266
|
+
resolution.credentialSource === "flow" &&
|
|
1267
|
+
resolution.cacheKey !== undefined &&
|
|
1268
|
+
isAuthFailureCaseResult(result)
|
|
1269
|
+
) {
|
|
1270
|
+
execution.sessionCache.delete(resolution.cacheKey);
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
return result;
|
|
460
1274
|
}
|
|
461
1275
|
|
|
462
1276
|
interface SelectedCase {
|
|
@@ -557,6 +1371,10 @@ export function createSelfTestApp(provider: ProviderDefinition, options: SelfTes
|
|
|
557
1371
|
const app = new Hono();
|
|
558
1372
|
const planDigest = computeSelfTestPlanDigest(provider);
|
|
559
1373
|
const requestBudgetMs = resolveRequestBudgetMs(options);
|
|
1374
|
+
// In-process flow-credential session cache (providerId + credentialInputs
|
|
1375
|
+
// hash → materialized credential). Lives as long as the app so consecutive
|
|
1376
|
+
// probe cycles never log in to the upstream more than once per session.
|
|
1377
|
+
const sessionCache: SelfTestCredentialSessionCache = new Map();
|
|
560
1378
|
let busy = false;
|
|
561
1379
|
|
|
562
1380
|
app.notFound((c) => c.json({ error: { code: "not_found", message: "Not found" } }, 404));
|
|
@@ -648,6 +1466,7 @@ export function createSelfTestApp(provider: ProviderDefinition, options: SelfTes
|
|
|
648
1466
|
const execution: SelfTestExecutionContext = {
|
|
649
1467
|
provider,
|
|
650
1468
|
invoke: options.invoke,
|
|
1469
|
+
...(options.authFlow ? { authFlow: options.authFlow } : {}),
|
|
651
1470
|
requestId: request.requestId,
|
|
652
1471
|
credentials: request.credentials?.inputs,
|
|
653
1472
|
requestTimeoutMs: request.timeoutMs,
|
|
@@ -655,50 +1474,81 @@ export function createSelfTestApp(provider: ProviderDefinition, options: SelfTes
|
|
|
655
1474
|
env: options.env,
|
|
656
1475
|
credentialInputs: request.credentials?.inputs,
|
|
657
1476
|
}),
|
|
1477
|
+
sessionCache,
|
|
658
1478
|
};
|
|
659
1479
|
const deadline = performance.now() + requestBudgetMs;
|
|
660
1480
|
const results: SelfTestCaseResult[] = [];
|
|
1481
|
+
const runSignal = c.req.raw.signal;
|
|
1482
|
+
let interruptedCase: SelectedCase | undefined;
|
|
661
1483
|
// Sequential execution (parallelism 1): self-tests run on serving pods
|
|
662
1484
|
// and must never compete with themselves for upstream quota.
|
|
663
1485
|
for (const selected of selection.cases) {
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
1486
|
+
const caseController = new AbortController();
|
|
1487
|
+
const abortFromRun = () => caseController.abort(runSignal.reason);
|
|
1488
|
+
runSignal.addEventListener("abort", abortFromRun, { once: true });
|
|
1489
|
+
if (runSignal.aborted) abortFromRun();
|
|
1490
|
+
try {
|
|
1491
|
+
if (runSignal.aborted) {
|
|
1492
|
+
interruptedCase = selected;
|
|
1493
|
+
break;
|
|
1494
|
+
}
|
|
1495
|
+
if (performance.now() >= deadline) {
|
|
1496
|
+
const now = new Date().toISOString();
|
|
1497
|
+
results.push({
|
|
1498
|
+
operationId: selected.operationId,
|
|
1499
|
+
caseName: selected.healthCase.name,
|
|
1500
|
+
status: "skipped",
|
|
1501
|
+
label: selected.healthCase.name,
|
|
1502
|
+
responseTimeMs: 0,
|
|
1503
|
+
skipReason: "budget_exhausted",
|
|
1504
|
+
startedAt: now,
|
|
1505
|
+
finishedAt: now,
|
|
1506
|
+
});
|
|
1507
|
+
continue;
|
|
1508
|
+
}
|
|
1509
|
+
if (!isSelfTestReadOnlyOperation(selected.operation)) {
|
|
1510
|
+
const now = new Date().toISOString();
|
|
1511
|
+
results.push({
|
|
1512
|
+
operationId: selected.operationId,
|
|
1513
|
+
caseName: selected.healthCase.name,
|
|
1514
|
+
status: "error",
|
|
1515
|
+
label: selected.healthCase.name,
|
|
1516
|
+
responseTimeMs: 0,
|
|
1517
|
+
error: {
|
|
1518
|
+
code: "operation_not_read_only",
|
|
1519
|
+
message: `Operation "${selected.operationId}" is not classified read-only; self-test refuses to execute it.`,
|
|
1520
|
+
},
|
|
1521
|
+
startedAt: now,
|
|
1522
|
+
finishedAt: now,
|
|
1523
|
+
});
|
|
1524
|
+
continue;
|
|
1525
|
+
}
|
|
1526
|
+
results.push(
|
|
1527
|
+
await executeSelfTestCase(
|
|
1528
|
+
execution,
|
|
1529
|
+
selected.operationId,
|
|
1530
|
+
selected.suite,
|
|
1531
|
+
selected.healthCase,
|
|
1532
|
+
caseController,
|
|
1533
|
+
),
|
|
1534
|
+
);
|
|
1535
|
+
if (runSignal.aborted) {
|
|
1536
|
+
interruptedCase = selected;
|
|
1537
|
+
break;
|
|
1538
|
+
}
|
|
1539
|
+
} finally {
|
|
1540
|
+
runSignal.removeEventListener("abort", abortFromRun);
|
|
1541
|
+
caseController.abort();
|
|
694
1542
|
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
1543
|
+
}
|
|
1544
|
+
if (interruptedCase) {
|
|
1545
|
+
logSelfTestCancellation(
|
|
1546
|
+
provider,
|
|
1547
|
+
options,
|
|
1548
|
+
request.requestId,
|
|
1549
|
+
interruptedCase,
|
|
1550
|
+
runSignal,
|
|
1551
|
+
execution.sensitiveValues,
|
|
702
1552
|
);
|
|
703
1553
|
}
|
|
704
1554
|
const singleCase = request.operationId !== undefined && request.caseName !== undefined;
|