deepline 0.3.16 → 0.3.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +7 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/integrations/bettercontact-execution-policy.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +19 -3
- package/dist/bundling-sources/shared_libs/play-runtime/bettercontact-batching.ts +726 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +45 -24
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +254 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +6 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +28 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +5 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +344 -26
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +50 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts +378 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-reliability-policy.ts +391 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-traffic-policy.ts +125 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +36 -39
- package/dist/cli/index.js +265 -1
- package/dist/cli/index.mjs +265 -1
- package/dist/index.js +265 -1
- package/dist/index.mjs +265 -1
- package/dist/install-integrity.json +6 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -783,7 +783,7 @@ var SDK_RELEASE = {
|
|
|
783
783
|
// 0.3.0 introduces raw-v2: complete scrubbed provider responses are
|
|
784
784
|
// available at toolResponse.rawV2 while toolResponse.raw and all declared
|
|
785
785
|
// getters keep their established compatibility behavior.
|
|
786
|
-
version: "0.3.
|
|
786
|
+
version: "0.3.18",
|
|
787
787
|
updateSummary: "New raw-v2 tool responses preserve complete scrubbed provider envelopes at toolResponse.rawV2, including JSON:API included resources, links, cursors, and totals. Existing toolResponse.raw and declared getters keep working unchanged.",
|
|
788
788
|
contracts: {
|
|
789
789
|
api: {
|
|
@@ -981,8 +981,259 @@ var RUNTIME_ENVIRONMENT_TOKEN_HEADER = "x-deepline-runtime-environment-token";
|
|
|
981
981
|
var ABSURD_RELEASE_OVERRIDE_HEADER = "x-deepline-absurd-release";
|
|
982
982
|
var SYNTHETIC_RUN_HEADER = "x-deepline-synthetic-run";
|
|
983
983
|
|
|
984
|
+
// ../shared_libs/play-runtime/runtime-incident-drills.ts
|
|
985
|
+
var RUNTIME_TEST_FAULT_IDS = [
|
|
986
|
+
"receipt_complete_write_fail",
|
|
987
|
+
"receipt_fail_write_fail",
|
|
988
|
+
"worker_receipt_complete_write_fail",
|
|
989
|
+
"invocation_response_delivery_abort",
|
|
990
|
+
"receipt_gateway_hold_ms",
|
|
991
|
+
"receipt_claim_query_hold_once_ms",
|
|
992
|
+
"receipt_claim_response_timeout",
|
|
993
|
+
"modal_sandbox_create_resource_exhausted",
|
|
994
|
+
"runtime_sheet_page_tail_hold_ms"
|
|
995
|
+
];
|
|
996
|
+
var LOCAL_TOPOLOGY = ["local_daytona"];
|
|
997
|
+
var LOCAL_AND_PREVIEW_TOPOLOGIES = [
|
|
998
|
+
"local_daytona",
|
|
999
|
+
"preview_fly",
|
|
1000
|
+
"preview_direct_modal"
|
|
1001
|
+
];
|
|
1002
|
+
var MODAL_PREVIEW_TOPOLOGY = ["preview_direct_modal"];
|
|
1003
|
+
var RUNTIME_INCIDENT_DRILL_CATALOG = [
|
|
1004
|
+
{
|
|
1005
|
+
id: "local_worker_process_outage",
|
|
1006
|
+
description: "Interrupt the scheduler worker in the selected non-production runtime topology.",
|
|
1007
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1008
|
+
control: {
|
|
1009
|
+
kind: "process",
|
|
1010
|
+
component: "worker",
|
|
1011
|
+
modes: ["pause", "crash"]
|
|
1012
|
+
},
|
|
1013
|
+
expected: {
|
|
1014
|
+
outcome: "The same logical run stays durable through the worker outage and completes after the selected topology recovers.",
|
|
1015
|
+
safetyFacts: ["same_logical_run", "no_false_success"]
|
|
1016
|
+
},
|
|
1017
|
+
verification: {
|
|
1018
|
+
kind: "local_black_box",
|
|
1019
|
+
target: "local-component-outage-recovery-e2e"
|
|
1020
|
+
}
|
|
1021
|
+
},
|
|
1022
|
+
{
|
|
1023
|
+
id: "local_gateway_process_outage",
|
|
1024
|
+
description: "Interrupt the receipt gateway in the selected non-production runtime topology.",
|
|
1025
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1026
|
+
control: {
|
|
1027
|
+
kind: "process",
|
|
1028
|
+
component: "gateway",
|
|
1029
|
+
modes: ["pause", "crash"]
|
|
1030
|
+
},
|
|
1031
|
+
expected: {
|
|
1032
|
+
outcome: "In-flight work recovers only at a safe receipt boundary and the same logical run completes after the gateway recovers.",
|
|
1033
|
+
safetyFacts: [
|
|
1034
|
+
"same_logical_run",
|
|
1035
|
+
"no_false_success",
|
|
1036
|
+
"bounded_provider_execution"
|
|
1037
|
+
]
|
|
1038
|
+
},
|
|
1039
|
+
verification: {
|
|
1040
|
+
kind: "local_black_box",
|
|
1041
|
+
target: "local-component-outage-recovery-e2e"
|
|
1042
|
+
}
|
|
1043
|
+
},
|
|
1044
|
+
{
|
|
1045
|
+
id: "local_runtime_process_outage",
|
|
1046
|
+
description: "Interrupt both runtime components in the selected non-production topology.",
|
|
1047
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1048
|
+
control: { kind: "process", component: "all", modes: ["pause", "crash"] },
|
|
1049
|
+
expected: {
|
|
1050
|
+
outcome: "Durable work waits through the combined outage; it never becomes a false success or creates a second logical run.",
|
|
1051
|
+
safetyFacts: ["same_logical_run", "no_false_success"]
|
|
1052
|
+
},
|
|
1053
|
+
verification: {
|
|
1054
|
+
kind: "local_black_box",
|
|
1055
|
+
target: "local-component-outage-recovery-e2e"
|
|
1056
|
+
}
|
|
1057
|
+
},
|
|
1058
|
+
{
|
|
1059
|
+
id: "receipt_complete_write_fail",
|
|
1060
|
+
description: "Fail the next durable receipt-completion write.",
|
|
1061
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1062
|
+
control: {
|
|
1063
|
+
kind: "runtime_test_fault",
|
|
1064
|
+
defaultValue: 1,
|
|
1065
|
+
valueMeaning: "occurrence"
|
|
1066
|
+
},
|
|
1067
|
+
expected: {
|
|
1068
|
+
outcome: "The failed run is loud, and a later bounded recovery may repeat the affected provider operation if its prior completion was not durable.",
|
|
1069
|
+
safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
|
|
1070
|
+
},
|
|
1071
|
+
verification: {
|
|
1072
|
+
kind: "local_black_box",
|
|
1073
|
+
target: "receipt-persist-failure-run-fatal-e2e"
|
|
1074
|
+
}
|
|
1075
|
+
},
|
|
1076
|
+
{
|
|
1077
|
+
id: "receipt_fail_write_fail",
|
|
1078
|
+
description: "Fail the next durable receipt-failure write.",
|
|
1079
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1080
|
+
control: {
|
|
1081
|
+
kind: "runtime_test_fault",
|
|
1082
|
+
defaultValue: 1,
|
|
1083
|
+
valueMeaning: "occurrence"
|
|
1084
|
+
},
|
|
1085
|
+
expected: {
|
|
1086
|
+
outcome: "The failure is loud; recovery remains bounded and may repeat an operation whose prior outcome was not durably recorded.",
|
|
1087
|
+
safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
|
|
1088
|
+
},
|
|
1089
|
+
verification: {
|
|
1090
|
+
kind: "focused_regression",
|
|
1091
|
+
target: "tests/lib/plays/runtime-test-fault-registry.test.ts"
|
|
1092
|
+
}
|
|
1093
|
+
},
|
|
1094
|
+
{
|
|
1095
|
+
id: "worker_receipt_complete_write_fail",
|
|
1096
|
+
description: "Fail the worker-side receipt-completion write once.",
|
|
1097
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1098
|
+
control: {
|
|
1099
|
+
kind: "runtime_test_fault",
|
|
1100
|
+
defaultValue: 1,
|
|
1101
|
+
valueMeaning: "occurrence"
|
|
1102
|
+
},
|
|
1103
|
+
expected: {
|
|
1104
|
+
outcome: "The worker reports the persistence failure loudly; bounded recovery may repeat work whose completion was not durable.",
|
|
1105
|
+
safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
|
|
1106
|
+
},
|
|
1107
|
+
verification: {
|
|
1108
|
+
kind: "focused_regression",
|
|
1109
|
+
target: "tests/lib/plays/runtime-test-fault-registry.test.ts"
|
|
1110
|
+
}
|
|
1111
|
+
},
|
|
1112
|
+
{
|
|
1113
|
+
id: "invocation_response_delivery_abort",
|
|
1114
|
+
description: "Abort one receipt-gateway response after invocation delivery.",
|
|
1115
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1116
|
+
control: {
|
|
1117
|
+
kind: "runtime_test_fault",
|
|
1118
|
+
defaultValue: 1,
|
|
1119
|
+
valueMeaning: "occurrence"
|
|
1120
|
+
},
|
|
1121
|
+
expected: {
|
|
1122
|
+
outcome: "The same logical run completes through receipt replay; recovery remains bounded if the provider outcome must be retried.",
|
|
1123
|
+
safetyFacts: ["same_logical_run", "bounded_provider_execution"]
|
|
1124
|
+
},
|
|
1125
|
+
verification: {
|
|
1126
|
+
kind: "local_black_box",
|
|
1127
|
+
target: "invocation-delivery-replay-e2e"
|
|
1128
|
+
}
|
|
1129
|
+
},
|
|
1130
|
+
{
|
|
1131
|
+
id: "receipt_gateway_hold_ms",
|
|
1132
|
+
description: "Hold one checked-out receipt-gateway scheduler client.",
|
|
1133
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1134
|
+
control: {
|
|
1135
|
+
kind: "runtime_test_fault",
|
|
1136
|
+
defaultValue: 500,
|
|
1137
|
+
valueMeaning: "milliseconds"
|
|
1138
|
+
},
|
|
1139
|
+
expected: {
|
|
1140
|
+
outcome: "The request is delayed inside the real gateway pool and either completes within its deadline or follows bounded transport recovery.",
|
|
1141
|
+
safetyFacts: ["bounded_provider_execution"]
|
|
1142
|
+
},
|
|
1143
|
+
verification: {
|
|
1144
|
+
kind: "local_black_box",
|
|
1145
|
+
target: "agent-heavy-gateway-load-e2e"
|
|
1146
|
+
}
|
|
1147
|
+
},
|
|
1148
|
+
{
|
|
1149
|
+
id: "receipt_claim_query_hold_once_ms",
|
|
1150
|
+
description: "Hold one physical receipt-claim query.",
|
|
1151
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1152
|
+
control: {
|
|
1153
|
+
kind: "runtime_test_fault",
|
|
1154
|
+
defaultValue: 500,
|
|
1155
|
+
valueMeaning: "milliseconds"
|
|
1156
|
+
},
|
|
1157
|
+
expected: {
|
|
1158
|
+
outcome: "The checked-out claim either completes or times out through bounded recovery; an ambiguous outcome may be retried within its budget.",
|
|
1159
|
+
safetyFacts: ["bounded_provider_execution"]
|
|
1160
|
+
},
|
|
1161
|
+
verification: {
|
|
1162
|
+
kind: "local_black_box",
|
|
1163
|
+
target: "receipt-claim-deadline-recovery-e2e"
|
|
1164
|
+
}
|
|
1165
|
+
},
|
|
1166
|
+
{
|
|
1167
|
+
id: "receipt_claim_response_timeout",
|
|
1168
|
+
description: "Report one fully received receipt-claim response as timed out.",
|
|
1169
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1170
|
+
control: {
|
|
1171
|
+
kind: "runtime_test_fault",
|
|
1172
|
+
defaultValue: 1,
|
|
1173
|
+
valueMeaning: "occurrence"
|
|
1174
|
+
},
|
|
1175
|
+
expected: {
|
|
1176
|
+
outcome: "The same logical run completes by replaying the committed receipt response or uses bounded recovery if that response is unavailable.",
|
|
1177
|
+
safetyFacts: ["same_logical_run", "bounded_provider_execution"]
|
|
1178
|
+
},
|
|
1179
|
+
verification: {
|
|
1180
|
+
kind: "local_black_box",
|
|
1181
|
+
target: "receipt-ambiguous-response-recovery-e2e"
|
|
1182
|
+
}
|
|
1183
|
+
},
|
|
1184
|
+
{
|
|
1185
|
+
id: "modal_sandbox_create_resource_exhausted",
|
|
1186
|
+
description: "Make Modal sandbox creation return RESOURCE_EXHAUSTED before user code.",
|
|
1187
|
+
topologies: MODAL_PREVIEW_TOPOLOGY,
|
|
1188
|
+
control: {
|
|
1189
|
+
kind: "runtime_test_fault",
|
|
1190
|
+
defaultValue: 1,
|
|
1191
|
+
valueMeaning: "occurrence"
|
|
1192
|
+
},
|
|
1193
|
+
expected: {
|
|
1194
|
+
outcome: "The same logical run enters bounded pre-code defer/recovery and later retries sandbox startup; no user code or provider call precedes the retry.",
|
|
1195
|
+
safetyFacts: [
|
|
1196
|
+
"same_logical_run",
|
|
1197
|
+
"no_user_code_before_recovery",
|
|
1198
|
+
"bounded_provider_execution"
|
|
1199
|
+
]
|
|
1200
|
+
},
|
|
1201
|
+
verification: {
|
|
1202
|
+
kind: "preview_black_box",
|
|
1203
|
+
target: "modal-sandbox-recovery-e2e"
|
|
1204
|
+
}
|
|
1205
|
+
},
|
|
1206
|
+
{
|
|
1207
|
+
id: "runtime_sheet_page_tail_hold_ms",
|
|
1208
|
+
description: "Hold one runtime-sheet tail-page operation.",
|
|
1209
|
+
topologies: LOCAL_TOPOLOGY,
|
|
1210
|
+
control: {
|
|
1211
|
+
kind: "runtime_test_fault",
|
|
1212
|
+
defaultValue: 500,
|
|
1213
|
+
valueMeaning: "milliseconds"
|
|
1214
|
+
},
|
|
1215
|
+
expected: {
|
|
1216
|
+
outcome: "Page-tail work stays pending during the bounded hold, then completes without losing or duplicating rows.",
|
|
1217
|
+
safetyFacts: ["all_rows_settle_once"]
|
|
1218
|
+
},
|
|
1219
|
+
verification: {
|
|
1220
|
+
kind: "local_black_box",
|
|
1221
|
+
target: "page-tail-sheet-write-hold-e2e"
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
];
|
|
1225
|
+
function isRuntimeTestFaultDrill(drill) {
|
|
1226
|
+
return drill.control.kind === "runtime_test_fault" && RUNTIME_TEST_FAULT_IDS.some((faultId) => faultId === drill.id);
|
|
1227
|
+
}
|
|
1228
|
+
function runtimeTestFaultDrills() {
|
|
1229
|
+
return RUNTIME_INCIDENT_DRILL_CATALOG.filter(isRuntimeTestFaultDrill);
|
|
1230
|
+
}
|
|
1231
|
+
|
|
984
1232
|
// ../shared_libs/play-runtime/test-runtime-seams.ts
|
|
985
1233
|
var PLAY_RUNTIME_TEST_FAULT_HEADER = "x-deepline-test-fault";
|
|
1234
|
+
var SUPPORTED_RUNTIME_TEST_FAULTS = new Set(
|
|
1235
|
+
runtimeTestFaultDrills().map(({ id }) => id)
|
|
1236
|
+
);
|
|
986
1237
|
var MAX_RUNTIME_TEST_POLICY_MS = 10 * 6e4;
|
|
987
1238
|
|
|
988
1239
|
// src/http.ts
|
|
@@ -3483,6 +3734,16 @@ function resolveTheirstackClientTimeoutMs(endpointId, payload) {
|
|
|
3483
3734
|
return usesExtendedTheirstackJobSearchBudget(endpointId, payload) ? THEIRSTACK_SLOW_JOB_SEARCH_CLIENT_TIMEOUT_MS : null;
|
|
3484
3735
|
}
|
|
3485
3736
|
|
|
3737
|
+
// ../shared_libs/integrations/bettercontact-execution-policy.ts
|
|
3738
|
+
var BETTERCONTACT_LAUNCHER_CLIENT_TIMEOUT_MS = 12e4;
|
|
3739
|
+
var BETTERCONTACT_LAUNCHER_TOOL_IDS = /* @__PURE__ */ new Set([
|
|
3740
|
+
"bettercontact_enrich",
|
|
3741
|
+
"bettercontact_bulk_enrich"
|
|
3742
|
+
]);
|
|
3743
|
+
function usesExtendedBetterContactLauncherBudget(toolId) {
|
|
3744
|
+
return BETTERCONTACT_LAUNCHER_TOOL_IDS.has(toolId.trim().toLowerCase());
|
|
3745
|
+
}
|
|
3746
|
+
|
|
3486
3747
|
// ../shared_libs/play-runtime/backend.ts
|
|
3487
3748
|
var PLAY_RUNTIME_BACKENDS = {
|
|
3488
3749
|
localProcess: "local_process",
|
|
@@ -3763,6 +4024,9 @@ function resolveToolExecuteTimeoutMs(toolId, input) {
|
|
|
3763
4024
|
input
|
|
3764
4025
|
);
|
|
3765
4026
|
if (theirstackTimeoutMs !== null) return theirstackTimeoutMs;
|
|
4027
|
+
if (usesExtendedBetterContactLauncherBudget(normalized)) {
|
|
4028
|
+
return BETTERCONTACT_LAUNCHER_CLIENT_TIMEOUT_MS;
|
|
4029
|
+
}
|
|
3766
4030
|
return normalized === "deeplineagent" || normalized === "deeplineagent_deeplineagent" || normalized === "ai_inference" || normalized === "deeplineagent_ai_inference" || normalized === "aiinference" ? DEEPLINEAGENT_EXECUTE_TIMEOUT_MS : void 0;
|
|
3767
4031
|
}
|
|
3768
4032
|
var RUNS_FAILED_LOG_LIMIT = 20;
|
package/dist/index.mjs
CHANGED
|
@@ -706,7 +706,7 @@ var SDK_RELEASE = {
|
|
|
706
706
|
// 0.3.0 introduces raw-v2: complete scrubbed provider responses are
|
|
707
707
|
// available at toolResponse.rawV2 while toolResponse.raw and all declared
|
|
708
708
|
// getters keep their established compatibility behavior.
|
|
709
|
-
version: "0.3.
|
|
709
|
+
version: "0.3.18",
|
|
710
710
|
updateSummary: "New raw-v2 tool responses preserve complete scrubbed provider envelopes at toolResponse.rawV2, including JSON:API included resources, links, cursors, and totals. Existing toolResponse.raw and declared getters keep working unchanged.",
|
|
711
711
|
contracts: {
|
|
712
712
|
api: {
|
|
@@ -904,8 +904,259 @@ var RUNTIME_ENVIRONMENT_TOKEN_HEADER = "x-deepline-runtime-environment-token";
|
|
|
904
904
|
var ABSURD_RELEASE_OVERRIDE_HEADER = "x-deepline-absurd-release";
|
|
905
905
|
var SYNTHETIC_RUN_HEADER = "x-deepline-synthetic-run";
|
|
906
906
|
|
|
907
|
+
// ../shared_libs/play-runtime/runtime-incident-drills.ts
|
|
908
|
+
var RUNTIME_TEST_FAULT_IDS = [
|
|
909
|
+
"receipt_complete_write_fail",
|
|
910
|
+
"receipt_fail_write_fail",
|
|
911
|
+
"worker_receipt_complete_write_fail",
|
|
912
|
+
"invocation_response_delivery_abort",
|
|
913
|
+
"receipt_gateway_hold_ms",
|
|
914
|
+
"receipt_claim_query_hold_once_ms",
|
|
915
|
+
"receipt_claim_response_timeout",
|
|
916
|
+
"modal_sandbox_create_resource_exhausted",
|
|
917
|
+
"runtime_sheet_page_tail_hold_ms"
|
|
918
|
+
];
|
|
919
|
+
var LOCAL_TOPOLOGY = ["local_daytona"];
|
|
920
|
+
var LOCAL_AND_PREVIEW_TOPOLOGIES = [
|
|
921
|
+
"local_daytona",
|
|
922
|
+
"preview_fly",
|
|
923
|
+
"preview_direct_modal"
|
|
924
|
+
];
|
|
925
|
+
var MODAL_PREVIEW_TOPOLOGY = ["preview_direct_modal"];
|
|
926
|
+
var RUNTIME_INCIDENT_DRILL_CATALOG = [
|
|
927
|
+
{
|
|
928
|
+
id: "local_worker_process_outage",
|
|
929
|
+
description: "Interrupt the scheduler worker in the selected non-production runtime topology.",
|
|
930
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
931
|
+
control: {
|
|
932
|
+
kind: "process",
|
|
933
|
+
component: "worker",
|
|
934
|
+
modes: ["pause", "crash"]
|
|
935
|
+
},
|
|
936
|
+
expected: {
|
|
937
|
+
outcome: "The same logical run stays durable through the worker outage and completes after the selected topology recovers.",
|
|
938
|
+
safetyFacts: ["same_logical_run", "no_false_success"]
|
|
939
|
+
},
|
|
940
|
+
verification: {
|
|
941
|
+
kind: "local_black_box",
|
|
942
|
+
target: "local-component-outage-recovery-e2e"
|
|
943
|
+
}
|
|
944
|
+
},
|
|
945
|
+
{
|
|
946
|
+
id: "local_gateway_process_outage",
|
|
947
|
+
description: "Interrupt the receipt gateway in the selected non-production runtime topology.",
|
|
948
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
949
|
+
control: {
|
|
950
|
+
kind: "process",
|
|
951
|
+
component: "gateway",
|
|
952
|
+
modes: ["pause", "crash"]
|
|
953
|
+
},
|
|
954
|
+
expected: {
|
|
955
|
+
outcome: "In-flight work recovers only at a safe receipt boundary and the same logical run completes after the gateway recovers.",
|
|
956
|
+
safetyFacts: [
|
|
957
|
+
"same_logical_run",
|
|
958
|
+
"no_false_success",
|
|
959
|
+
"bounded_provider_execution"
|
|
960
|
+
]
|
|
961
|
+
},
|
|
962
|
+
verification: {
|
|
963
|
+
kind: "local_black_box",
|
|
964
|
+
target: "local-component-outage-recovery-e2e"
|
|
965
|
+
}
|
|
966
|
+
},
|
|
967
|
+
{
|
|
968
|
+
id: "local_runtime_process_outage",
|
|
969
|
+
description: "Interrupt both runtime components in the selected non-production topology.",
|
|
970
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
971
|
+
control: { kind: "process", component: "all", modes: ["pause", "crash"] },
|
|
972
|
+
expected: {
|
|
973
|
+
outcome: "Durable work waits through the combined outage; it never becomes a false success or creates a second logical run.",
|
|
974
|
+
safetyFacts: ["same_logical_run", "no_false_success"]
|
|
975
|
+
},
|
|
976
|
+
verification: {
|
|
977
|
+
kind: "local_black_box",
|
|
978
|
+
target: "local-component-outage-recovery-e2e"
|
|
979
|
+
}
|
|
980
|
+
},
|
|
981
|
+
{
|
|
982
|
+
id: "receipt_complete_write_fail",
|
|
983
|
+
description: "Fail the next durable receipt-completion write.",
|
|
984
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
985
|
+
control: {
|
|
986
|
+
kind: "runtime_test_fault",
|
|
987
|
+
defaultValue: 1,
|
|
988
|
+
valueMeaning: "occurrence"
|
|
989
|
+
},
|
|
990
|
+
expected: {
|
|
991
|
+
outcome: "The failed run is loud, and a later bounded recovery may repeat the affected provider operation if its prior completion was not durable.",
|
|
992
|
+
safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
|
|
993
|
+
},
|
|
994
|
+
verification: {
|
|
995
|
+
kind: "local_black_box",
|
|
996
|
+
target: "receipt-persist-failure-run-fatal-e2e"
|
|
997
|
+
}
|
|
998
|
+
},
|
|
999
|
+
{
|
|
1000
|
+
id: "receipt_fail_write_fail",
|
|
1001
|
+
description: "Fail the next durable receipt-failure write.",
|
|
1002
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1003
|
+
control: {
|
|
1004
|
+
kind: "runtime_test_fault",
|
|
1005
|
+
defaultValue: 1,
|
|
1006
|
+
valueMeaning: "occurrence"
|
|
1007
|
+
},
|
|
1008
|
+
expected: {
|
|
1009
|
+
outcome: "The failure is loud; recovery remains bounded and may repeat an operation whose prior outcome was not durably recorded.",
|
|
1010
|
+
safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
|
|
1011
|
+
},
|
|
1012
|
+
verification: {
|
|
1013
|
+
kind: "focused_regression",
|
|
1014
|
+
target: "tests/lib/plays/runtime-test-fault-registry.test.ts"
|
|
1015
|
+
}
|
|
1016
|
+
},
|
|
1017
|
+
{
|
|
1018
|
+
id: "worker_receipt_complete_write_fail",
|
|
1019
|
+
description: "Fail the worker-side receipt-completion write once.",
|
|
1020
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1021
|
+
control: {
|
|
1022
|
+
kind: "runtime_test_fault",
|
|
1023
|
+
defaultValue: 1,
|
|
1024
|
+
valueMeaning: "occurrence"
|
|
1025
|
+
},
|
|
1026
|
+
expected: {
|
|
1027
|
+
outcome: "The worker reports the persistence failure loudly; bounded recovery may repeat work whose completion was not durable.",
|
|
1028
|
+
safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
|
|
1029
|
+
},
|
|
1030
|
+
verification: {
|
|
1031
|
+
kind: "focused_regression",
|
|
1032
|
+
target: "tests/lib/plays/runtime-test-fault-registry.test.ts"
|
|
1033
|
+
}
|
|
1034
|
+
},
|
|
1035
|
+
{
|
|
1036
|
+
id: "invocation_response_delivery_abort",
|
|
1037
|
+
description: "Abort one receipt-gateway response after invocation delivery.",
|
|
1038
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1039
|
+
control: {
|
|
1040
|
+
kind: "runtime_test_fault",
|
|
1041
|
+
defaultValue: 1,
|
|
1042
|
+
valueMeaning: "occurrence"
|
|
1043
|
+
},
|
|
1044
|
+
expected: {
|
|
1045
|
+
outcome: "The same logical run completes through receipt replay; recovery remains bounded if the provider outcome must be retried.",
|
|
1046
|
+
safetyFacts: ["same_logical_run", "bounded_provider_execution"]
|
|
1047
|
+
},
|
|
1048
|
+
verification: {
|
|
1049
|
+
kind: "local_black_box",
|
|
1050
|
+
target: "invocation-delivery-replay-e2e"
|
|
1051
|
+
}
|
|
1052
|
+
},
|
|
1053
|
+
{
|
|
1054
|
+
id: "receipt_gateway_hold_ms",
|
|
1055
|
+
description: "Hold one checked-out receipt-gateway scheduler client.",
|
|
1056
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1057
|
+
control: {
|
|
1058
|
+
kind: "runtime_test_fault",
|
|
1059
|
+
defaultValue: 500,
|
|
1060
|
+
valueMeaning: "milliseconds"
|
|
1061
|
+
},
|
|
1062
|
+
expected: {
|
|
1063
|
+
outcome: "The request is delayed inside the real gateway pool and either completes within its deadline or follows bounded transport recovery.",
|
|
1064
|
+
safetyFacts: ["bounded_provider_execution"]
|
|
1065
|
+
},
|
|
1066
|
+
verification: {
|
|
1067
|
+
kind: "local_black_box",
|
|
1068
|
+
target: "agent-heavy-gateway-load-e2e"
|
|
1069
|
+
}
|
|
1070
|
+
},
|
|
1071
|
+
{
|
|
1072
|
+
id: "receipt_claim_query_hold_once_ms",
|
|
1073
|
+
description: "Hold one physical receipt-claim query.",
|
|
1074
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1075
|
+
control: {
|
|
1076
|
+
kind: "runtime_test_fault",
|
|
1077
|
+
defaultValue: 500,
|
|
1078
|
+
valueMeaning: "milliseconds"
|
|
1079
|
+
},
|
|
1080
|
+
expected: {
|
|
1081
|
+
outcome: "The checked-out claim either completes or times out through bounded recovery; an ambiguous outcome may be retried within its budget.",
|
|
1082
|
+
safetyFacts: ["bounded_provider_execution"]
|
|
1083
|
+
},
|
|
1084
|
+
verification: {
|
|
1085
|
+
kind: "local_black_box",
|
|
1086
|
+
target: "receipt-claim-deadline-recovery-e2e"
|
|
1087
|
+
}
|
|
1088
|
+
},
|
|
1089
|
+
{
|
|
1090
|
+
id: "receipt_claim_response_timeout",
|
|
1091
|
+
description: "Report one fully received receipt-claim response as timed out.",
|
|
1092
|
+
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
1093
|
+
control: {
|
|
1094
|
+
kind: "runtime_test_fault",
|
|
1095
|
+
defaultValue: 1,
|
|
1096
|
+
valueMeaning: "occurrence"
|
|
1097
|
+
},
|
|
1098
|
+
expected: {
|
|
1099
|
+
outcome: "The same logical run completes by replaying the committed receipt response or uses bounded recovery if that response is unavailable.",
|
|
1100
|
+
safetyFacts: ["same_logical_run", "bounded_provider_execution"]
|
|
1101
|
+
},
|
|
1102
|
+
verification: {
|
|
1103
|
+
kind: "local_black_box",
|
|
1104
|
+
target: "receipt-ambiguous-response-recovery-e2e"
|
|
1105
|
+
}
|
|
1106
|
+
},
|
|
1107
|
+
{
|
|
1108
|
+
id: "modal_sandbox_create_resource_exhausted",
|
|
1109
|
+
description: "Make Modal sandbox creation return RESOURCE_EXHAUSTED before user code.",
|
|
1110
|
+
topologies: MODAL_PREVIEW_TOPOLOGY,
|
|
1111
|
+
control: {
|
|
1112
|
+
kind: "runtime_test_fault",
|
|
1113
|
+
defaultValue: 1,
|
|
1114
|
+
valueMeaning: "occurrence"
|
|
1115
|
+
},
|
|
1116
|
+
expected: {
|
|
1117
|
+
outcome: "The same logical run enters bounded pre-code defer/recovery and later retries sandbox startup; no user code or provider call precedes the retry.",
|
|
1118
|
+
safetyFacts: [
|
|
1119
|
+
"same_logical_run",
|
|
1120
|
+
"no_user_code_before_recovery",
|
|
1121
|
+
"bounded_provider_execution"
|
|
1122
|
+
]
|
|
1123
|
+
},
|
|
1124
|
+
verification: {
|
|
1125
|
+
kind: "preview_black_box",
|
|
1126
|
+
target: "modal-sandbox-recovery-e2e"
|
|
1127
|
+
}
|
|
1128
|
+
},
|
|
1129
|
+
{
|
|
1130
|
+
id: "runtime_sheet_page_tail_hold_ms",
|
|
1131
|
+
description: "Hold one runtime-sheet tail-page operation.",
|
|
1132
|
+
topologies: LOCAL_TOPOLOGY,
|
|
1133
|
+
control: {
|
|
1134
|
+
kind: "runtime_test_fault",
|
|
1135
|
+
defaultValue: 500,
|
|
1136
|
+
valueMeaning: "milliseconds"
|
|
1137
|
+
},
|
|
1138
|
+
expected: {
|
|
1139
|
+
outcome: "Page-tail work stays pending during the bounded hold, then completes without losing or duplicating rows.",
|
|
1140
|
+
safetyFacts: ["all_rows_settle_once"]
|
|
1141
|
+
},
|
|
1142
|
+
verification: {
|
|
1143
|
+
kind: "local_black_box",
|
|
1144
|
+
target: "page-tail-sheet-write-hold-e2e"
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
];
|
|
1148
|
+
function isRuntimeTestFaultDrill(drill) {
|
|
1149
|
+
return drill.control.kind === "runtime_test_fault" && RUNTIME_TEST_FAULT_IDS.some((faultId) => faultId === drill.id);
|
|
1150
|
+
}
|
|
1151
|
+
function runtimeTestFaultDrills() {
|
|
1152
|
+
return RUNTIME_INCIDENT_DRILL_CATALOG.filter(isRuntimeTestFaultDrill);
|
|
1153
|
+
}
|
|
1154
|
+
|
|
907
1155
|
// ../shared_libs/play-runtime/test-runtime-seams.ts
|
|
908
1156
|
var PLAY_RUNTIME_TEST_FAULT_HEADER = "x-deepline-test-fault";
|
|
1157
|
+
var SUPPORTED_RUNTIME_TEST_FAULTS = new Set(
|
|
1158
|
+
runtimeTestFaultDrills().map(({ id }) => id)
|
|
1159
|
+
);
|
|
909
1160
|
var MAX_RUNTIME_TEST_POLICY_MS = 10 * 6e4;
|
|
910
1161
|
|
|
911
1162
|
// src/http.ts
|
|
@@ -3406,6 +3657,16 @@ function resolveTheirstackClientTimeoutMs(endpointId, payload) {
|
|
|
3406
3657
|
return usesExtendedTheirstackJobSearchBudget(endpointId, payload) ? THEIRSTACK_SLOW_JOB_SEARCH_CLIENT_TIMEOUT_MS : null;
|
|
3407
3658
|
}
|
|
3408
3659
|
|
|
3660
|
+
// ../shared_libs/integrations/bettercontact-execution-policy.ts
|
|
3661
|
+
var BETTERCONTACT_LAUNCHER_CLIENT_TIMEOUT_MS = 12e4;
|
|
3662
|
+
var BETTERCONTACT_LAUNCHER_TOOL_IDS = /* @__PURE__ */ new Set([
|
|
3663
|
+
"bettercontact_enrich",
|
|
3664
|
+
"bettercontact_bulk_enrich"
|
|
3665
|
+
]);
|
|
3666
|
+
function usesExtendedBetterContactLauncherBudget(toolId) {
|
|
3667
|
+
return BETTERCONTACT_LAUNCHER_TOOL_IDS.has(toolId.trim().toLowerCase());
|
|
3668
|
+
}
|
|
3669
|
+
|
|
3409
3670
|
// ../shared_libs/play-runtime/backend.ts
|
|
3410
3671
|
var PLAY_RUNTIME_BACKENDS = {
|
|
3411
3672
|
localProcess: "local_process",
|
|
@@ -3686,6 +3947,9 @@ function resolveToolExecuteTimeoutMs(toolId, input) {
|
|
|
3686
3947
|
input
|
|
3687
3948
|
);
|
|
3688
3949
|
if (theirstackTimeoutMs !== null) return theirstackTimeoutMs;
|
|
3950
|
+
if (usesExtendedBetterContactLauncherBudget(normalized)) {
|
|
3951
|
+
return BETTERCONTACT_LAUNCHER_CLIENT_TIMEOUT_MS;
|
|
3952
|
+
}
|
|
3689
3953
|
return normalized === "deeplineagent" || normalized === "deeplineagent_deeplineagent" || normalized === "ai_inference" || normalized === "deeplineagent_ai_inference" || normalized === "aiinference" ? DEEPLINEAGENT_EXECUTE_TIMEOUT_MS : void 0;
|
|
3690
3954
|
}
|
|
3691
3955
|
var RUNS_FAILED_LOG_LIMIT = 20;
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"dist/bundling-sources/sdk/src/tool-output.ts",
|
|
21
21
|
"dist/bundling-sources/sdk/src/types.ts",
|
|
22
22
|
"dist/bundling-sources/sdk/src/version.ts",
|
|
23
|
+
"dist/bundling-sources/shared_libs/integrations/bettercontact-execution-policy.ts",
|
|
23
24
|
"dist/bundling-sources/shared_libs/integrations/theirstack-execution-policy.ts",
|
|
24
25
|
"dist/bundling-sources/shared_libs/observability/node-tracing.ts",
|
|
25
26
|
"dist/bundling-sources/shared_libs/observability/redaction.ts",
|
|
@@ -36,6 +37,7 @@
|
|
|
36
37
|
"dist/bundling-sources/shared_libs/play-runtime/backend.ts",
|
|
37
38
|
"dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts",
|
|
38
39
|
"dist/bundling-sources/shared_libs/play-runtime/batching-types.ts",
|
|
40
|
+
"dist/bundling-sources/shared_libs/play-runtime/bettercontact-batching.ts",
|
|
39
41
|
"dist/bundling-sources/shared_libs/play-runtime/bounded-dispatch.ts",
|
|
40
42
|
"dist/bundling-sources/shared_libs/play-runtime/builtin-pacing.ts",
|
|
41
43
|
"dist/bundling-sources/shared_libs/play-runtime/cell-provenance.ts",
|
|
@@ -98,6 +100,7 @@
|
|
|
98
100
|
"dist/bundling-sources/shared_libs/play-runtime/play-input.ts",
|
|
99
101
|
"dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts",
|
|
100
102
|
"dist/bundling-sources/shared_libs/play-runtime/play-node-scope.ts",
|
|
103
|
+
"dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts",
|
|
101
104
|
"dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts",
|
|
102
105
|
"dist/bundling-sources/shared_libs/play-runtime/postgres-json.ts",
|
|
103
106
|
"dist/bundling-sources/shared_libs/play-runtime/profiles.ts",
|
|
@@ -141,12 +144,14 @@
|
|
|
141
144
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-constraints.ts",
|
|
142
145
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts",
|
|
143
146
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts",
|
|
147
|
+
"dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts",
|
|
144
148
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-infra-mode.ts",
|
|
145
149
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts",
|
|
146
150
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts",
|
|
147
151
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts",
|
|
148
152
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts",
|
|
149
153
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts",
|
|
154
|
+
"dist/bundling-sources/shared_libs/play-runtime/runtime-reliability-policy.ts",
|
|
150
155
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts",
|
|
151
156
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts",
|
|
152
157
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts",
|
|
@@ -154,6 +159,7 @@
|
|
|
154
159
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts",
|
|
155
160
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts",
|
|
156
161
|
"dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-session.ts",
|
|
162
|
+
"dist/bundling-sources/shared_libs/play-runtime/runtime-traffic-policy.ts",
|
|
157
163
|
"dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts",
|
|
158
164
|
"dist/bundling-sources/shared_libs/play-runtime/sandbox-runtime-limits.ts",
|
|
159
165
|
"dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts",
|