gencow 0.1.243 → 0.1.244
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/index.js +1 -0
- package/lib/app-delete-diagnostic.mjs +359 -5
- package/lib/app-delete-operation.mjs +24 -1
- package/lib/app-response-error.mjs +34 -0
- package/lib/jobs-command.mjs +22 -1
- package/package.json +3 -3
- package/runtime/server.mjs +3982 -3197
- package/runtime/tooling.mjs +1 -0
package/core/index.js
CHANGED
|
@@ -3636,6 +3636,7 @@ var RESERVED_TENANT_RUNTIME_ENV_KEYS = [
|
|
|
3636
3636
|
"GENCOW_RUNTIME_NODE_MODULES",
|
|
3637
3637
|
"GENCOW_INTERNAL_TOKEN",
|
|
3638
3638
|
"GENCOW_CRON_TOKEN",
|
|
3639
|
+
"GENCOW_SCHEDULED_JOB_PUBLIC_KEY",
|
|
3639
3640
|
"GENCOW_DEPLOY_ID",
|
|
3640
3641
|
"GENCOW_AI_PROXY_URL",
|
|
3641
3642
|
"GENCOW_AI_PROXY_URL_ALT",
|
|
@@ -3,6 +3,99 @@ export const APP_DELETE_OPERATION_PATTERN =
|
|
|
3
3
|
|
|
4
4
|
const APP_ID_PATTERN = /^(?=.{3,63}$)[a-z][a-z0-9]*(?:-[a-z0-9]+){2,7}$/u;
|
|
5
5
|
const CORRELATION_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/u;
|
|
6
|
+
const SAFE_DELETE_STATES = new Set([
|
|
7
|
+
"accepted",
|
|
8
|
+
"stopping_children",
|
|
9
|
+
"draining_runtime",
|
|
10
|
+
"stop_dispatched",
|
|
11
|
+
"runtime_absent",
|
|
12
|
+
"tenant_db_drop_pending",
|
|
13
|
+
"catalog_delete_pending",
|
|
14
|
+
"blocked_ownership_ambiguous",
|
|
15
|
+
"blocked_active_work",
|
|
16
|
+
"blocked_runtime_outcome_unknown",
|
|
17
|
+
"blocked_catalog_outcome_unknown",
|
|
18
|
+
"failed_tenant_db_cleanup",
|
|
19
|
+
"failed_catalog_cleanup",
|
|
20
|
+
"failed_delete_recovery_required",
|
|
21
|
+
"restoring",
|
|
22
|
+
"restored",
|
|
23
|
+
]);
|
|
24
|
+
const RESTORE_ROUTE_ACTION_PRECONDITIONS = new Map([
|
|
25
|
+
[
|
|
26
|
+
"APP_DELETE_RESTORE_ROUTE_RECOVERY_DISABLED",
|
|
27
|
+
"Platform must enable serving-route publication for this app and environment before owner wake can proceed.",
|
|
28
|
+
],
|
|
29
|
+
[
|
|
30
|
+
"APP_DELETE_RESTORE_APP_NOT_IN_ROUTE_ALLOWLIST",
|
|
31
|
+
"Platform must include this app in the serving-route rollout allowlist before owner wake can proceed.",
|
|
32
|
+
],
|
|
33
|
+
[
|
|
34
|
+
"APP_DELETE_RESTORE_ROUTE_CONTROL_UNAVAILABLE",
|
|
35
|
+
"Platform must restore a valid serving-route rollout configuration before owner wake can proceed.",
|
|
36
|
+
],
|
|
37
|
+
]);
|
|
38
|
+
const RESTORE_ADMISSION_ACTIONS = new Map([
|
|
39
|
+
[
|
|
40
|
+
"APP_DELETE_RESTORE_TENANT_DATABASE_ABSENT",
|
|
41
|
+
{
|
|
42
|
+
retryable: false,
|
|
43
|
+
retryClass: "owner-action",
|
|
44
|
+
userAction: "RETRY_DELETE",
|
|
45
|
+
nextAction: "RETRY_DELETE",
|
|
46
|
+
retryPrecondition:
|
|
47
|
+
"The tenant database is absent. Wake cannot restore the app; retry the app deletion so catalog cleanup can finish.",
|
|
48
|
+
},
|
|
49
|
+
],
|
|
50
|
+
[
|
|
51
|
+
"APP_DELETE_RESTORE_ACTIVE_DEPLOYMENT_ATTEMPT",
|
|
52
|
+
{
|
|
53
|
+
retryable: true,
|
|
54
|
+
retryClass: "platform-action",
|
|
55
|
+
userAction: "WAIT_THEN_RETRY_WAKE",
|
|
56
|
+
nextAction: "WAIT_FOR_PLATFORM_RECONCILIATION",
|
|
57
|
+
retryPrecondition:
|
|
58
|
+
"Wait until the active app deployment attempt finishes and its lease clears; then retry the same owner wake.",
|
|
59
|
+
},
|
|
60
|
+
],
|
|
61
|
+
...[
|
|
62
|
+
"APP_DELETE_RESTORE_ACTIVE_LEASE",
|
|
63
|
+
"APP_DELETE_RESTORE_ACTIVE_WORK",
|
|
64
|
+
"APP_DELETE_RESTORE_RUNTIME_NOT_ABSENT",
|
|
65
|
+
"APP_DELETE_RESTORE_TOMBSTONE_NOT_ACKNOWLEDGED",
|
|
66
|
+
"APP_DELETE_RESTORE_TENANT_DATABASE_AUTHORITY_UNAVAILABLE",
|
|
67
|
+
].map((code) => [
|
|
68
|
+
code,
|
|
69
|
+
{
|
|
70
|
+
retryable: true,
|
|
71
|
+
retryClass: "platform-action",
|
|
72
|
+
userAction: "WAIT_THEN_RETRY_WAKE",
|
|
73
|
+
nextAction: "WAIT_FOR_PLATFORM_RECONCILIATION",
|
|
74
|
+
retryPrecondition:
|
|
75
|
+
"Wait until the stated runtime or serving-route condition is clear, then retry the same owner wake/resume operation.",
|
|
76
|
+
},
|
|
77
|
+
]),
|
|
78
|
+
...[
|
|
79
|
+
"APP_DELETE_RESTORE_ADMISSION_FAILED",
|
|
80
|
+
"APP_DELETE_RESTORE_CATALOG_MUTATED",
|
|
81
|
+
"APP_DELETE_RESTORE_CHILD_ACTIVE",
|
|
82
|
+
"APP_DELETE_RESTORE_HANDOFF_MISMATCH",
|
|
83
|
+
"APP_DELETE_RESTORE_IDENTITY_MISMATCH",
|
|
84
|
+
"APP_DELETE_RESTORE_RUNTIME_WAKE_MISSING",
|
|
85
|
+
"APP_DELETE_RESTORE_STATE_CONFLICT",
|
|
86
|
+
"APP_DELETE_RESTORE_TOMBSTONE_MISMATCH",
|
|
87
|
+
].map((code) => [
|
|
88
|
+
code,
|
|
89
|
+
{
|
|
90
|
+
retryable: false,
|
|
91
|
+
retryClass: "platform-action",
|
|
92
|
+
userAction: "VIEW_DELETE_STATUS",
|
|
93
|
+
nextAction: "VIEW_DELETE_STATUS",
|
|
94
|
+
retryPrecondition:
|
|
95
|
+
"The failed delete receipt and serving-route identity require platform reconciliation before wake can proceed.",
|
|
96
|
+
},
|
|
97
|
+
]),
|
|
98
|
+
]);
|
|
6
99
|
|
|
7
100
|
export function isAppDeleteDiagnosticAppId(value) {
|
|
8
101
|
return APP_ID_PATTERN.test(value ?? "");
|
|
@@ -33,6 +126,158 @@ function isCanonicalTimestamp(value) {
|
|
|
33
126
|
return Number.isFinite(date.getTime()) && date.toISOString() === value;
|
|
34
127
|
}
|
|
35
128
|
|
|
129
|
+
export function readRestoreRouteActionAppDeleteDiagnostic(value, expectedAppId) {
|
|
130
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
131
|
+
const retryPrecondition = RESTORE_ROUTE_ACTION_PRECONDITIONS.get(value.code);
|
|
132
|
+
if (
|
|
133
|
+
!retryPrecondition ||
|
|
134
|
+
!isAppDeleteDiagnosticAppId(expectedAppId) ||
|
|
135
|
+
!APP_DELETE_OPERATION_PATTERN.test(value.operationId ?? "") ||
|
|
136
|
+
!CORRELATION_ID_PATTERN.test(value.correlationId ?? "") ||
|
|
137
|
+
!SAFE_DELETE_STATES.has(value.state) ||
|
|
138
|
+
value.stage !== "restore-admission" ||
|
|
139
|
+
!["runtime-resume", "runtime-reconcile"].includes(value.capability) ||
|
|
140
|
+
![
|
|
141
|
+
"The failed delete receipt remains unchanged; the app runtime was not started.",
|
|
142
|
+
"The same restore receipt remains active; no replacement delete or restore operation was created.",
|
|
143
|
+
].includes(value.stateImpact) ||
|
|
144
|
+
value.retryable !== false ||
|
|
145
|
+
value.retryPrecondition !== retryPrecondition ||
|
|
146
|
+
value.retryClass !== "platform-action" ||
|
|
147
|
+
value.userAction !== "WAIT_FOR_PLATFORM_RECONCILIATION" ||
|
|
148
|
+
value.nextAction !== "WAIT_FOR_PLATFORM_RECONCILIATION"
|
|
149
|
+
) {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
return {
|
|
153
|
+
code: value.code,
|
|
154
|
+
operationId: value.operationId,
|
|
155
|
+
correlationId: value.correlationId,
|
|
156
|
+
state: value.state,
|
|
157
|
+
stateImpact: value.stateImpact,
|
|
158
|
+
retryPrecondition,
|
|
159
|
+
statusCommand: `gencow app status ${expectedAppId}`,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function isRestoreRouteActionAppDeleteDiagnosticCandidate(value) {
|
|
164
|
+
return Boolean(
|
|
165
|
+
value &&
|
|
166
|
+
typeof value === "object" &&
|
|
167
|
+
!Array.isArray(value) &&
|
|
168
|
+
RESTORE_ROUTE_ACTION_PRECONDITIONS.has(value.code),
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function readRestoreAdmissionAppDeleteDiagnostic(value, expectedAppId) {
|
|
173
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
174
|
+
const action = RESTORE_ADMISSION_ACTIONS.get(value.code);
|
|
175
|
+
const stateImpactByState = {
|
|
176
|
+
failed_delete_recovery_required:
|
|
177
|
+
"The failed delete receipt remains unchanged; the app runtime was not started.",
|
|
178
|
+
restoring:
|
|
179
|
+
"The same restore receipt remains active; no replacement delete or restore operation was created.",
|
|
180
|
+
};
|
|
181
|
+
const stateImpact = stateImpactByState[value.state];
|
|
182
|
+
if (
|
|
183
|
+
!action ||
|
|
184
|
+
!isAppDeleteDiagnosticAppId(expectedAppId) ||
|
|
185
|
+
value.status !== 409 ||
|
|
186
|
+
!APP_DELETE_OPERATION_PATTERN.test(value.operationId ?? "") ||
|
|
187
|
+
!CORRELATION_ID_PATTERN.test(value.correlationId ?? "") ||
|
|
188
|
+
!stateImpact ||
|
|
189
|
+
value.stage !== "restore-admission" ||
|
|
190
|
+
!["runtime-resume", "runtime-reconcile"].includes(value.capability) ||
|
|
191
|
+
value.stateImpact !== stateImpact ||
|
|
192
|
+
value.retryable !== action.retryable ||
|
|
193
|
+
value.retryClass !== action.retryClass ||
|
|
194
|
+
value.userAction !== action.userAction ||
|
|
195
|
+
value.nextAction !== action.nextAction ||
|
|
196
|
+
value.retryPrecondition !== action.retryPrecondition
|
|
197
|
+
) {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
return {
|
|
201
|
+
code: value.code,
|
|
202
|
+
operationId: value.operationId,
|
|
203
|
+
correlationId: value.correlationId,
|
|
204
|
+
state: value.state,
|
|
205
|
+
stateImpact,
|
|
206
|
+
retryable: action.retryable,
|
|
207
|
+
retryPrecondition: action.retryPrecondition,
|
|
208
|
+
nextAction: action.nextAction,
|
|
209
|
+
statusCommand: `gencow app status ${expectedAppId}`,
|
|
210
|
+
nextCommand: action.userAction === "RETRY_DELETE" ? `gencow app delete ${expectedAppId}` : null,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
export function isRestoreAdmissionAppDeleteDiagnosticCandidate(value) {
|
|
215
|
+
return Boolean(
|
|
216
|
+
value && typeof value === "object" && !Array.isArray(value) && RESTORE_ADMISSION_ACTIONS.has(value.code),
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
export function formatRestoreAdmissionAppDeleteDiagnosticLines(diagnostic) {
|
|
221
|
+
const lines = [
|
|
222
|
+
"App wake is blocked while Gencow checks the failed delete recovery.",
|
|
223
|
+
`Code: ${diagnostic.code}`,
|
|
224
|
+
`Operation ID: ${diagnostic.operationId}`,
|
|
225
|
+
`Correlation ID: ${diagnostic.correlationId}`,
|
|
226
|
+
`State: ${diagnostic.state}`,
|
|
227
|
+
`State impact: ${diagnostic.stateImpact}`,
|
|
228
|
+
`Retryable: ${diagnostic.retryable ? "yes" : "no"}`,
|
|
229
|
+
`Retry precondition: ${diagnostic.retryPrecondition}`,
|
|
230
|
+
`Next action: ${diagnostic.nextAction}`,
|
|
231
|
+
];
|
|
232
|
+
lines.push(`Next command: ${diagnostic.nextCommand ?? diagnostic.statusCommand}`);
|
|
233
|
+
return lines;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export function formatInvalidRestoreAdmissionAppDeleteDiagnosticLines(value, expectedAppId) {
|
|
237
|
+
const lines = ["App wake is blocked; the failed delete recovery diagnosis could not be verified safely."];
|
|
238
|
+
if (RESTORE_ADMISSION_ACTIONS.has(value.code)) lines.push(`Code: ${value.code}`);
|
|
239
|
+
if (APP_DELETE_OPERATION_PATTERN.test(value.operationId ?? "")) {
|
|
240
|
+
lines.push(`Operation ID: ${value.operationId}`);
|
|
241
|
+
}
|
|
242
|
+
if (CORRELATION_ID_PATTERN.test(value.correlationId ?? "")) {
|
|
243
|
+
lines.push(`Correlation ID: ${value.correlationId}`);
|
|
244
|
+
}
|
|
245
|
+
if (isAppDeleteDiagnosticAppId(expectedAppId)) {
|
|
246
|
+
lines.push(`Next command: gencow app status ${expectedAppId}`);
|
|
247
|
+
}
|
|
248
|
+
return lines;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export function formatRestoreRouteActionAppDeleteDiagnosticLines(diagnostic) {
|
|
252
|
+
return [
|
|
253
|
+
"App wake is blocked by Platform serving-route configuration.",
|
|
254
|
+
`Code: ${diagnostic.code}`,
|
|
255
|
+
`Operation ID: ${diagnostic.operationId}`,
|
|
256
|
+
`Correlation ID: ${diagnostic.correlationId}`,
|
|
257
|
+
`State: ${diagnostic.state}`,
|
|
258
|
+
`State impact: ${diagnostic.stateImpact}`,
|
|
259
|
+
"Retryable: no",
|
|
260
|
+
`Retry precondition: ${diagnostic.retryPrecondition}`,
|
|
261
|
+
"Next action: WAIT_FOR_PLATFORM_RECONCILIATION",
|
|
262
|
+
`Next command: ${diagnostic.statusCommand}`,
|
|
263
|
+
];
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
export function formatInvalidRestoreRouteActionAppDeleteDiagnosticLines(value, expectedAppId) {
|
|
267
|
+
const lines = [
|
|
268
|
+
"App wake is blocked; the Platform serving-route diagnosis could not be verified safely.",
|
|
269
|
+
`Next command: gencow app status ${expectedAppId}`,
|
|
270
|
+
];
|
|
271
|
+
if (RESTORE_ROUTE_ACTION_PRECONDITIONS.has(value.code)) lines.splice(1, 0, `Code: ${value.code}`);
|
|
272
|
+
if (APP_DELETE_OPERATION_PATTERN.test(value.operationId ?? "")) {
|
|
273
|
+
lines.splice(lines.length - 1, 0, `Operation ID: ${value.operationId}`);
|
|
274
|
+
}
|
|
275
|
+
if (CORRELATION_ID_PATTERN.test(value.correlationId ?? "")) {
|
|
276
|
+
lines.splice(lines.length - 1, 0, `Correlation ID: ${value.correlationId}`);
|
|
277
|
+
}
|
|
278
|
+
return lines;
|
|
279
|
+
}
|
|
280
|
+
|
|
36
281
|
// The Platform's blocked receipt is an untrusted transport value until all of
|
|
37
282
|
// its identity, app scope, and safe-action fields agree. Keep that boundary in
|
|
38
283
|
// one module so polling and terminal rendering cannot drift apart.
|
|
@@ -64,14 +309,123 @@ export function readBlockedAppDeleteDiagnostic(value, expectedAppId) {
|
|
|
64
309
|
};
|
|
65
310
|
}
|
|
66
311
|
|
|
312
|
+
export function readRestoreInProgressAppDeleteDiagnostic(value, expectedAppId) {
|
|
313
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
314
|
+
const immediateRpc =
|
|
315
|
+
value.success === undefined &&
|
|
316
|
+
value.error === "APP_DELETE_RESTORE_IN_PROGRESS" &&
|
|
317
|
+
value.stateImpact ===
|
|
318
|
+
"The same restore receipt remains active; no replacement delete or restore operation was created." &&
|
|
319
|
+
value.userAction === "WAIT_FOR_RESTORE" &&
|
|
320
|
+
value.nextAction === "WAIT_FOR_RESTORE";
|
|
321
|
+
const statusReceipt =
|
|
322
|
+
value.success === false &&
|
|
323
|
+
value.deleted === expectedAppId &&
|
|
324
|
+
typeof value.statusPath === "string" &&
|
|
325
|
+
isAllowedAppDeleteStatusPath(value.statusPath, value.operationId, expectedAppId);
|
|
326
|
+
if (
|
|
327
|
+
!isAppDeleteDiagnosticAppId(expectedAppId) ||
|
|
328
|
+
!APP_DELETE_OPERATION_PATTERN.test(value.operationId ?? "") ||
|
|
329
|
+
!CORRELATION_ID_PATTERN.test(value.correlationId ?? "") ||
|
|
330
|
+
value.state !== "restoring" ||
|
|
331
|
+
value.code !== "APP_DELETE_RESTORE_IN_PROGRESS" ||
|
|
332
|
+
value.retryable !== false ||
|
|
333
|
+
value.retryPrecondition !==
|
|
334
|
+
"Wait until this restore receipt leaves restoring; then check the app status." ||
|
|
335
|
+
(!immediateRpc && !statusReceipt)
|
|
336
|
+
) {
|
|
337
|
+
return null;
|
|
338
|
+
}
|
|
339
|
+
return {
|
|
340
|
+
code: value.code,
|
|
341
|
+
operationId: value.operationId,
|
|
342
|
+
correlationId: value.correlationId,
|
|
343
|
+
state: value.state,
|
|
344
|
+
stateImpact:
|
|
345
|
+
"The same restore receipt remains active; no replacement delete or restore operation was created.",
|
|
346
|
+
retryPrecondition: "Wait until this restore receipt leaves restoring; then check the app status.",
|
|
347
|
+
statusCommand: `gencow app status ${expectedAppId}`,
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
export function formatRestoreInProgressAppDeleteDiagnosticLines(diagnostic) {
|
|
352
|
+
return [
|
|
353
|
+
"App restoration is still in progress; this delete did not complete.",
|
|
354
|
+
`Code: ${diagnostic.code}`,
|
|
355
|
+
`Operation ID: ${diagnostic.operationId}`,
|
|
356
|
+
`Correlation ID: ${diagnostic.correlationId}`,
|
|
357
|
+
`State: ${diagnostic.state}`,
|
|
358
|
+
`State impact: ${diagnostic.stateImpact}`,
|
|
359
|
+
"Retryable: no",
|
|
360
|
+
`Retry precondition: ${diagnostic.retryPrecondition}`,
|
|
361
|
+
"Next action: WAIT_FOR_RESTORE",
|
|
362
|
+
`Next command: ${diagnostic.statusCommand}`,
|
|
363
|
+
];
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
export function readRestoredAppDeleteDiagnostic(value, expectedAppId) {
|
|
367
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
368
|
+
const immediateRpc =
|
|
369
|
+
value.success === undefined &&
|
|
370
|
+
value.error === "APP_DELETE_RESTORE_COMPLETED_NEW_DELETE_REQUIRED" &&
|
|
371
|
+
value.stateImpact ===
|
|
372
|
+
"The previous restore is complete; this request did not create a delete operation." &&
|
|
373
|
+
value.retryPrecondition ===
|
|
374
|
+
"The old delete request is terminal. Start a new delete command to create a fresh operation." &&
|
|
375
|
+
value.userAction === "RETRY_DELETE_WITH_NEW_REQUEST" &&
|
|
376
|
+
value.nextAction === "RETRY_DELETE_WITH_NEW_REQUEST";
|
|
377
|
+
const statusReceipt =
|
|
378
|
+
value.success === false &&
|
|
379
|
+
value.deleted === expectedAppId &&
|
|
380
|
+
value.failureClass === "OWNER_ACTION_REQUIRED" &&
|
|
381
|
+
value.action === "RETRY_DELETE_WITH_NEW_REQUEST" &&
|
|
382
|
+
isAllowedAppDeleteStatusPath(value.statusPath, value.operationId, expectedAppId);
|
|
383
|
+
if (
|
|
384
|
+
!isAppDeleteDiagnosticAppId(expectedAppId) ||
|
|
385
|
+
!APP_DELETE_OPERATION_PATTERN.test(value.operationId ?? "") ||
|
|
386
|
+
!CORRELATION_ID_PATTERN.test(value.correlationId ?? "") ||
|
|
387
|
+
value.state !== "restored" ||
|
|
388
|
+
value.code !== "APP_DELETE_RESTORE_COMPLETED_NEW_DELETE_REQUIRED" ||
|
|
389
|
+
value.retryable !== false ||
|
|
390
|
+
(!immediateRpc && !statusReceipt)
|
|
391
|
+
) {
|
|
392
|
+
return null;
|
|
393
|
+
}
|
|
394
|
+
return {
|
|
395
|
+
code: value.code,
|
|
396
|
+
operationId: value.operationId,
|
|
397
|
+
correlationId: value.correlationId,
|
|
398
|
+
state: value.state,
|
|
399
|
+
stateImpact: "The previous restore is complete; this request did not create a delete operation.",
|
|
400
|
+
retryPrecondition:
|
|
401
|
+
"The old delete request is terminal. Start a new delete command to create a fresh operation.",
|
|
402
|
+
deleteCommand: `gencow app delete ${expectedAppId}`,
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
export function formatRestoredAppDeleteDiagnosticLines(diagnostic) {
|
|
407
|
+
return [
|
|
408
|
+
"The app was restored; the previous delete request did not delete it.",
|
|
409
|
+
`Code: ${diagnostic.code}`,
|
|
410
|
+
`Operation ID: ${diagnostic.operationId}`,
|
|
411
|
+
`Correlation ID: ${diagnostic.correlationId}`,
|
|
412
|
+
`State: ${diagnostic.state}`,
|
|
413
|
+
`State impact: ${diagnostic.stateImpact}`,
|
|
414
|
+
"Retryable: no",
|
|
415
|
+
`Retry precondition: ${diagnostic.retryPrecondition}`,
|
|
416
|
+
"Next action: RETRY_DELETE_WITH_NEW_REQUEST",
|
|
417
|
+
`Next command: ${diagnostic.deleteCommand}`,
|
|
418
|
+
];
|
|
419
|
+
}
|
|
420
|
+
|
|
67
421
|
export function isBlockedAppDeleteDiagnosticCandidate(value) {
|
|
68
422
|
return Boolean(
|
|
69
423
|
value &&
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
424
|
+
typeof value === "object" &&
|
|
425
|
+
!Array.isArray(value) &&
|
|
426
|
+
(value.code === "PLATFORM_ACTION_REQUIRED" ||
|
|
427
|
+
value.failureClass === "PLATFORM_ACTION_REQUIRED" ||
|
|
428
|
+
value.action === "VIEW_DELETE_STATUS"),
|
|
75
429
|
);
|
|
76
430
|
}
|
|
77
431
|
|
|
@@ -3,6 +3,8 @@ import {
|
|
|
3
3
|
APP_DELETE_OPERATION_PATTERN,
|
|
4
4
|
isAllowedAppDeleteStatusPath,
|
|
5
5
|
readBlockedAppDeleteDiagnostic,
|
|
6
|
+
readRestoredAppDeleteDiagnostic,
|
|
7
|
+
readRestoreInProgressAppDeleteDiagnostic,
|
|
6
8
|
} from "./app-delete-diagnostic.mjs";
|
|
7
9
|
import { readJsonObjectResponse } from "./http-response-json.mjs";
|
|
8
10
|
|
|
@@ -21,8 +23,9 @@ const APP_DELETE_ACTIVE_STATES = new Set([
|
|
|
21
23
|
"failed_tenant_db_cleanup",
|
|
22
24
|
"failed_catalog_cleanup",
|
|
23
25
|
"failed_delete_recovery_required",
|
|
26
|
+
"restoring",
|
|
24
27
|
]);
|
|
25
|
-
const APP_DELETE_TERMINAL_FAILURE_STATES = new Set();
|
|
28
|
+
const APP_DELETE_TERMINAL_FAILURE_STATES = new Set(["restored"]);
|
|
26
29
|
const APP_DELETE_RESPONSE_STATES = new Set([
|
|
27
30
|
...APP_DELETE_ACTIVE_STATES,
|
|
28
31
|
...APP_DELETE_TERMINAL_FAILURE_STATES,
|
|
@@ -98,6 +101,12 @@ export async function pollAppDeleteOperation({
|
|
|
98
101
|
rpcQueryImpl,
|
|
99
102
|
sleepImpl,
|
|
100
103
|
}) {
|
|
104
|
+
if (
|
|
105
|
+
readRestoreInProgressAppDeleteDiagnostic(initialBody, name) ||
|
|
106
|
+
readRestoredAppDeleteDiagnostic(initialBody, name)
|
|
107
|
+
) {
|
|
108
|
+
return initialBody;
|
|
109
|
+
}
|
|
101
110
|
const accepted = parseAppDeleteOperationEnvelope(initialBody, name);
|
|
102
111
|
if (!accepted) return null;
|
|
103
112
|
if (isMatchingTerminalFailureResponse(initialBody, accepted, name)) {
|
|
@@ -147,6 +156,20 @@ export async function pollAppDeleteOperation({
|
|
|
147
156
|
}
|
|
148
157
|
|
|
149
158
|
const pollingUnavailable = successfulPollResponses === 0 && transportFailures > 0;
|
|
159
|
+
if (latest.state === "restoring") {
|
|
160
|
+
return {
|
|
161
|
+
success: false,
|
|
162
|
+
deleted: name,
|
|
163
|
+
error: "App restoration is still in progress; this delete did not complete.",
|
|
164
|
+
code: "APP_DELETE_RESTORE_IN_PROGRESS",
|
|
165
|
+
operationId: accepted.operationId,
|
|
166
|
+
correlationId: latest.correlationId ?? accepted.correlationId,
|
|
167
|
+
statusPath: `/api/apps/${encodeURIComponent(name)}/delete/${encodeURIComponent(accepted.operationId)}`,
|
|
168
|
+
state: "restoring",
|
|
169
|
+
retryable: false,
|
|
170
|
+
retryPrecondition: "Wait until this restore receipt leaves restoring; then check the app status.",
|
|
171
|
+
};
|
|
172
|
+
}
|
|
150
173
|
return {
|
|
151
174
|
error: pollingUnavailable
|
|
152
175
|
? "App deletion status is temporarily unavailable. Retry the command with the same app."
|
|
@@ -4,8 +4,20 @@ import {
|
|
|
4
4
|
APP_DELETE_OPERATION_PATTERN,
|
|
5
5
|
formatBlockedAppDeleteDiagnosticLines,
|
|
6
6
|
formatInvalidBlockedAppDeleteDiagnosticLines,
|
|
7
|
+
formatRestoreInProgressAppDeleteDiagnosticLines,
|
|
8
|
+
formatRestoreRouteActionAppDeleteDiagnosticLines,
|
|
9
|
+
formatInvalidRestoreRouteActionAppDeleteDiagnosticLines,
|
|
10
|
+
formatRestoreAdmissionAppDeleteDiagnosticLines,
|
|
11
|
+
formatInvalidRestoreAdmissionAppDeleteDiagnosticLines,
|
|
12
|
+
formatRestoredAppDeleteDiagnosticLines,
|
|
13
|
+
isRestoreAdmissionAppDeleteDiagnosticCandidate,
|
|
14
|
+
isRestoreRouteActionAppDeleteDiagnosticCandidate,
|
|
7
15
|
isBlockedAppDeleteDiagnosticCandidate,
|
|
8
16
|
readBlockedAppDeleteDiagnostic,
|
|
17
|
+
readRestoreInProgressAppDeleteDiagnostic,
|
|
18
|
+
readRestoreRouteActionAppDeleteDiagnostic,
|
|
19
|
+
readRestoreAdmissionAppDeleteDiagnostic,
|
|
20
|
+
readRestoredAppDeleteDiagnostic,
|
|
9
21
|
} from "./app-delete-diagnostic.mjs";
|
|
10
22
|
|
|
11
23
|
export { APP_DELETE_OPERATION_PATTERN } from "./app-delete-diagnostic.mjs";
|
|
@@ -35,6 +47,28 @@ export function appResponseError(body, fallback, response = null, options = {})
|
|
|
35
47
|
if (blockedDeleteDiagnostic) {
|
|
36
48
|
return formatBlockedAppDeleteDiagnosticLines(blockedDeleteDiagnostic).join("\n");
|
|
37
49
|
}
|
|
50
|
+
const restoreInProgressDiagnostic = readRestoreInProgressAppDeleteDiagnostic(body, options.expectedAppId);
|
|
51
|
+
if (restoreInProgressDiagnostic) {
|
|
52
|
+
return formatRestoreInProgressAppDeleteDiagnosticLines(restoreInProgressDiagnostic).join("\n");
|
|
53
|
+
}
|
|
54
|
+
const restoreRouteActionDiagnostic = readRestoreRouteActionAppDeleteDiagnostic(body, options.expectedAppId);
|
|
55
|
+
if (restoreRouteActionDiagnostic) {
|
|
56
|
+
return formatRestoreRouteActionAppDeleteDiagnosticLines(restoreRouteActionDiagnostic).join("\n");
|
|
57
|
+
}
|
|
58
|
+
if (isRestoreRouteActionAppDeleteDiagnosticCandidate(body)) {
|
|
59
|
+
return formatInvalidRestoreRouteActionAppDeleteDiagnosticLines(body, options.expectedAppId).join("\n");
|
|
60
|
+
}
|
|
61
|
+
const restoredDeleteDiagnostic = readRestoredAppDeleteDiagnostic(body, options.expectedAppId);
|
|
62
|
+
if (restoredDeleteDiagnostic) {
|
|
63
|
+
return formatRestoredAppDeleteDiagnosticLines(restoredDeleteDiagnostic).join("\n");
|
|
64
|
+
}
|
|
65
|
+
const restoreAdmissionDiagnostic = readRestoreAdmissionAppDeleteDiagnostic(body, options.expectedAppId);
|
|
66
|
+
if (restoreAdmissionDiagnostic) {
|
|
67
|
+
return formatRestoreAdmissionAppDeleteDiagnosticLines(restoreAdmissionDiagnostic).join("\n");
|
|
68
|
+
}
|
|
69
|
+
if (isRestoreAdmissionAppDeleteDiagnosticCandidate(body)) {
|
|
70
|
+
return formatInvalidRestoreAdmissionAppDeleteDiagnosticLines(body, options.expectedAppId).join("\n");
|
|
71
|
+
}
|
|
38
72
|
if (isBlockedAppDeleteDiagnosticCandidate(body)) {
|
|
39
73
|
return formatInvalidBlockedAppDeleteDiagnosticLines(body, options.expectedAppId).join("\n");
|
|
40
74
|
}
|
package/lib/jobs-command.mjs
CHANGED
|
@@ -13,6 +13,27 @@ export function formatCronFailureDetail(errorMessage, failureDomain, maxLength)
|
|
|
13
13
|
return `${domainPrefix}${String(errorMessage).slice(0, maxLength)}`;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
+
export function formatScheduledJobFailure(errorMessage) {
|
|
17
|
+
try {
|
|
18
|
+
const diagnostic = JSON.parse(String(errorMessage));
|
|
19
|
+
if (diagnostic.code === "SCHEDULED_ACTION_NOT_REGISTERED") {
|
|
20
|
+
const state = diagnostic.stateChanged === false ? "app data unchanged" : "state impact unknown";
|
|
21
|
+
const retry = diagnostic.retryable
|
|
22
|
+
? `Retry condition: ${diagnostic.retryCondition}`
|
|
23
|
+
: "This failure is not retryable.";
|
|
24
|
+
return [
|
|
25
|
+
`${diagnostic.code} at ${diagnostic.stage}; ${state}.`,
|
|
26
|
+
retry,
|
|
27
|
+
`Next: ${diagnostic.nextAction}`,
|
|
28
|
+
`Job ID: ${diagnostic.correlationId}`,
|
|
29
|
+
].join("\n ");
|
|
30
|
+
}
|
|
31
|
+
} catch {
|
|
32
|
+
// Legacy job errors remain plain text.
|
|
33
|
+
}
|
|
34
|
+
return String(errorMessage).slice(0, 70);
|
|
35
|
+
}
|
|
36
|
+
|
|
16
37
|
export function parseJobsCommandArgs(jobsArgs) {
|
|
17
38
|
let appId = null;
|
|
18
39
|
let isProd = false;
|
|
@@ -503,7 +524,7 @@ async function handleJobsList(params) {
|
|
|
503
524
|
log(
|
|
504
525
|
` ${RED}✗${RESET} ${failure.action} ${DIM}(${failure.retryCount}/${failure.maxRetries} retries)${RESET}`,
|
|
505
526
|
);
|
|
506
|
-
if (failure.error) log(` ${RED}${failure.error
|
|
527
|
+
if (failure.error) log(` ${RED}${formatScheduledJobFailure(failure.error)}${RESET}`);
|
|
507
528
|
}
|
|
508
529
|
log("");
|
|
509
530
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gencow",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.244",
|
|
4
4
|
"description": "Gencow — AI Backend Engine",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"dashboard/"
|
|
21
21
|
],
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@gencow/migration-contract": "0.1.
|
|
23
|
+
"@gencow/migration-contract": "0.1.21",
|
|
24
24
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
25
25
|
"drizzle-kit": "1.0.0-rc.4",
|
|
26
26
|
"drizzle-orm": "1.0.0-rc.4",
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"ai": "^6.0.233",
|
|
38
38
|
"@types/node": "^25.9.5",
|
|
39
39
|
"better-auth": "^1.6.23",
|
|
40
|
-
"@gencow/client": "0.2.7",
|
|
41
40
|
"@gencow/core": "0.1.45",
|
|
41
|
+
"@gencow/client": "0.2.7",
|
|
42
42
|
"@gencow/react": "0.2.7"
|
|
43
43
|
},
|
|
44
44
|
"scripts": {
|