@ductape/cli 0.5.7 → 0.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -0
- package/dist/commands/troubleshoot.d.ts +2 -0
- package/dist/commands/troubleshoot.js +15 -0
- package/dist/commands/ui.d.ts +58 -0
- package/dist/commands/ui.js +359 -0
- package/dist/index.js +102 -2
- package/dist/lib/api-client.d.ts +3 -2
- package/dist/lib/api-client.js +6 -3
- package/dist/lib/troubleshoot/collect.d.ts +6 -0
- package/dist/lib/troubleshoot/collect.js +218 -0
- package/dist/lib/troubleshoot/report.d.ts +60 -0
- package/dist/lib/troubleshoot/report.js +98 -0
- package/dist/lib/ui/check.d.ts +38 -0
- package/dist/lib/ui/check.js +64 -0
- package/dist/lib/ui/consume.d.ts +92 -0
- package/dist/lib/ui/consume.js +392 -0
- package/dist/lib/ui/manifest.d.ts +53 -0
- package/dist/lib/ui/manifest.js +109 -0
- package/dist/lib/ui/pack.d.ts +39 -0
- package/dist/lib/ui/pack.js +96 -0
- package/dist/lib/ui/paths.d.ts +7 -0
- package/dist/lib/ui/paths.js +25 -0
- package/dist/lib/ui/registry.d.ts +84 -0
- package/dist/lib/ui/registry.js +78 -0
- package/dist/lib/ui/scaffold.d.ts +14 -0
- package/dist/lib/ui/scaffold.js +411 -0
- package/dist/lib/ui/semver.d.ts +11 -0
- package/dist/lib/ui/semver.js +115 -0
- package/dist/lib/ui/validate.d.ts +33 -0
- package/dist/lib/ui/validate.js +445 -0
- package/package.json +3 -2
package/dist/lib/api-client.js
CHANGED
|
@@ -4,12 +4,14 @@ import { DuctapeOperationError, safeEndpoint } from './operation-error.js';
|
|
|
4
4
|
export class ApiClient {
|
|
5
5
|
apiUrl;
|
|
6
6
|
token;
|
|
7
|
-
|
|
7
|
+
timeoutMs;
|
|
8
|
+
constructor(apiUrl, token, timeoutMs) {
|
|
8
9
|
this.apiUrl = apiUrl;
|
|
9
10
|
this.token = token;
|
|
11
|
+
this.timeoutMs = timeoutMs;
|
|
10
12
|
}
|
|
11
|
-
static forProfile(profileName, creds) {
|
|
12
|
-
return new ApiClient(getApiUrl(profileName), creds?.auth_token);
|
|
13
|
+
static forProfile(profileName, creds, timeoutMs) {
|
|
14
|
+
return new ApiClient(getApiUrl(profileName), creds?.auth_token, timeoutMs);
|
|
13
15
|
}
|
|
14
16
|
async request(method, path, body, extraHeaders) {
|
|
15
17
|
const url = `${this.apiUrl}${path.startsWith('/') ? path : `/${path}`}`;
|
|
@@ -27,6 +29,7 @@ export class ApiClient {
|
|
|
27
29
|
res = await fetch(url, {
|
|
28
30
|
method,
|
|
29
31
|
headers,
|
|
32
|
+
...(this.timeoutMs ? { signal: AbortSignal.timeout(this.timeoutMs) } : {}),
|
|
30
33
|
body: body === undefined ? undefined : raw ? body : JSON.stringify(body),
|
|
31
34
|
});
|
|
32
35
|
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type Scope, type Report } from './report.js';
|
|
2
|
+
export interface TroubleshootReader {
|
|
3
|
+
get(path: string, query: Record<string, string>): Promise<unknown>;
|
|
4
|
+
}
|
|
5
|
+
export declare function collectTroubleshoot(scope: Scope, reader: TroubleshootReader): Promise<Report>;
|
|
6
|
+
export declare function formatTroubleshoot(report: Report): string;
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import { summarize } from './report.js';
|
|
2
|
+
const row = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
3
|
+
const text = (value) => typeof value === 'string' ? value : undefined;
|
|
4
|
+
const rows = (value) => {
|
|
5
|
+
if (!Array.isArray(value))
|
|
6
|
+
throw new Error('Unexpected response shape');
|
|
7
|
+
return value.map(item => {
|
|
8
|
+
const result = row(item);
|
|
9
|
+
if (!Object.keys(result).length)
|
|
10
|
+
throw new Error('Unexpected record shape');
|
|
11
|
+
return result;
|
|
12
|
+
});
|
|
13
|
+
};
|
|
14
|
+
function within(value, scope) {
|
|
15
|
+
const time = text(value);
|
|
16
|
+
const ms = time ? Date.parse(time) : NaN;
|
|
17
|
+
return Number.isFinite(ms) && ms >= Date.parse(scope.start) && ms <= Date.parse(scope.end) ? new Date(ms).toISOString() : undefined;
|
|
18
|
+
}
|
|
19
|
+
/** Extract only error descriptions, never request/response/configuration payloads. */
|
|
20
|
+
function errorText(value, depth = 0) {
|
|
21
|
+
if (depth > 3)
|
|
22
|
+
return '';
|
|
23
|
+
if (typeof value === 'string')
|
|
24
|
+
return value;
|
|
25
|
+
if (Array.isArray(value))
|
|
26
|
+
return value.slice(0, 5).map(item => errorText(item, depth + 1)).filter(Boolean).join(' ');
|
|
27
|
+
const object = row(value);
|
|
28
|
+
return [text(object.code), text(object.message), object.error && errorText(object.error, depth + 1), object.errors && errorText(object.errors, depth + 1)]
|
|
29
|
+
.filter(Boolean).join(' ');
|
|
30
|
+
}
|
|
31
|
+
function logMessage(log) {
|
|
32
|
+
let data = log.data;
|
|
33
|
+
if (typeof data === 'string' && !log.data_encrypted && data.length <= 100_000) {
|
|
34
|
+
try {
|
|
35
|
+
data = JSON.parse(data);
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
data = undefined;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const payload = row(data);
|
|
42
|
+
return [text(log.message), errorText(log.error), errorText(payload.error), errorText(payload.errors)]
|
|
43
|
+
.filter(Boolean).join(' ') || 'Failure recorded without an accessible error description';
|
|
44
|
+
}
|
|
45
|
+
function unavailable(source, error, scanned = 0) {
|
|
46
|
+
// API error messages may echo credentials or payloads. Only expose the HTTP status.
|
|
47
|
+
const details = row(row(error).details);
|
|
48
|
+
const status = typeof details.httpStatus === 'number' ? ' (HTTP ' + details.httpStatus + ')' : '';
|
|
49
|
+
return { source, status: 'unavailable', scanned, note: 'Could not read this source' + status + '. Check authentication, service availability and ductape doctor.' };
|
|
50
|
+
}
|
|
51
|
+
export async function collectTroubleshoot(scope, reader) {
|
|
52
|
+
const events = [];
|
|
53
|
+
const coverage = [];
|
|
54
|
+
const productFilter = scope.product ? { product: scope.product } : {};
|
|
55
|
+
let pruned = 0;
|
|
56
|
+
let undated = 0;
|
|
57
|
+
async function collectLogs() {
|
|
58
|
+
let scanned = 0;
|
|
59
|
+
const seen = new Set();
|
|
60
|
+
try {
|
|
61
|
+
for (let page = 1;; page++) {
|
|
62
|
+
const response = row(await reader.get('/log/v1/workspace-logs/' + encodeURIComponent(scope.workspaceId), {
|
|
63
|
+
start_date: scope.start, end_date: scope.end,
|
|
64
|
+
...(scope.product ? { product_tag: scope.product } : {}),
|
|
65
|
+
page: String(page), limit: String(Math.min(500, scope.maxLogs)),
|
|
66
|
+
}));
|
|
67
|
+
const logs = row(response.logs);
|
|
68
|
+
const batch = rows(logs.data);
|
|
69
|
+
const metadata = row(logs.metadata);
|
|
70
|
+
const seenBefore = seen.size;
|
|
71
|
+
for (const log of batch) {
|
|
72
|
+
if (scanned >= scope.maxLogs)
|
|
73
|
+
break;
|
|
74
|
+
const id = text(log._id);
|
|
75
|
+
if (id && seen.has(id))
|
|
76
|
+
continue;
|
|
77
|
+
if (id)
|
|
78
|
+
seen.add(id);
|
|
79
|
+
scanned++;
|
|
80
|
+
const at = within(log.timestamp, scope);
|
|
81
|
+
if (!text(log.timestamp) || !Number.isFinite(Date.parse(String(log.timestamp))))
|
|
82
|
+
undated++;
|
|
83
|
+
if (!at || (scope.product && log.product_tag !== scope.product))
|
|
84
|
+
continue;
|
|
85
|
+
if (log.data_pruned_at || log.data_encrypted)
|
|
86
|
+
pruned++;
|
|
87
|
+
if (log.status !== 'fail' && log.failed_execution !== true && !['error', 'warn', 'warning', 'fatal'].includes(String(log.level ?? log.severity).toLowerCase()))
|
|
88
|
+
continue;
|
|
89
|
+
events.push({
|
|
90
|
+
source: 'workspace_logs', id: id ?? 'page-' + page + '-record-' + scanned, at,
|
|
91
|
+
product: text(log.product_tag), env: text(log.env ?? log.app_env),
|
|
92
|
+
resource: text(log.child_tag ?? log.function_tag ?? log.feature_tag ?? log.parent_tag ?? log.type),
|
|
93
|
+
code: text(row(log.error).code), message: logMessage(log),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
const total = typeof metadata.total === 'number' ? metadata.total : undefined;
|
|
97
|
+
const done = !batch.length || (total !== undefined && seen.size >= total && metadata.totalIsCapped !== true);
|
|
98
|
+
if (done || scanned >= scope.maxLogs) {
|
|
99
|
+
coverage.push({
|
|
100
|
+
source: 'workspace_logs', scanned, status: done ? 'complete' : 'limited',
|
|
101
|
+
...(!done ? { note: 'Reached --max-logs before exhausting the time window. Narrow --since/--until or --product, or increase --max-logs.' } : {}),
|
|
102
|
+
});
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
// Without pagination metadata there is no safe claim that the window was exhausted.
|
|
106
|
+
if (total === undefined || seenBefore === seen.size || batch.every(log => !text(log._id))) {
|
|
107
|
+
coverage.push({ source: 'workspace_logs', scanned, status: 'limited', note: 'The server did not return reliable pagination metadata.' });
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
catch (error) {
|
|
113
|
+
coverage.push(unavailable('workspace_logs', error, scanned));
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
async function collectDeployments() {
|
|
117
|
+
let scanned = 0;
|
|
118
|
+
try {
|
|
119
|
+
const deployments = rows(await reader.get('/integrations/v1/deploy/deployments', productFilter));
|
|
120
|
+
const candidates = deployments.filter(deployment => {
|
|
121
|
+
if (scope.product && deployment.product !== scope.product)
|
|
122
|
+
return false;
|
|
123
|
+
const updated = Date.parse(String(deployment.updatedAt));
|
|
124
|
+
return !Number.isFinite(updated) || updated >= Date.parse(scope.start);
|
|
125
|
+
});
|
|
126
|
+
let failures = 0;
|
|
127
|
+
// Bound concurrent journal requests; an unavailable journal does not hide others.
|
|
128
|
+
for (let offset = 0; offset < candidates.length; offset += 4) {
|
|
129
|
+
await Promise.all(candidates.slice(offset, offset + 4).map(async (deployment) => {
|
|
130
|
+
const id = text(deployment.id);
|
|
131
|
+
if (!id) {
|
|
132
|
+
failures++;
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
const journal = rows(await reader.get('/integrations/v1/deploy/deployments/' + encodeURIComponent(id) + '/journal', {}));
|
|
137
|
+
for (const entry of journal) {
|
|
138
|
+
scanned++;
|
|
139
|
+
const at = within(entry.at, scope);
|
|
140
|
+
if (!at || !entry.error)
|
|
141
|
+
continue;
|
|
142
|
+
events.push({
|
|
143
|
+
source: 'deployment_journal', id: id + ':' + String(entry.seq), at,
|
|
144
|
+
product: text(deployment.product), env: text(deployment.env),
|
|
145
|
+
resource: text(deployment.name ?? deployment.function), currentState: text(deployment.state),
|
|
146
|
+
code: text(row(entry.error).code), message: errorText(entry.error),
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
failures++;
|
|
152
|
+
}
|
|
153
|
+
}));
|
|
154
|
+
}
|
|
155
|
+
coverage.push({
|
|
156
|
+
source: 'deployment_journals', scanned,
|
|
157
|
+
status: failures || deployments.length >= 200 ? 'limited' : 'complete',
|
|
158
|
+
note: 'Read ' + (candidates.length - failures) + ' journals. ' + failures + ' unavailable. The endpoint returns at most 200 deployments.',
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
coverage.push(unavailable('deployment_journals', error, scanned));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
async function collectBuilds() {
|
|
166
|
+
let scanned = 0;
|
|
167
|
+
try {
|
|
168
|
+
const builds = rows(await reader.get('/integrations/v1/deploy/builds', productFilter));
|
|
169
|
+
for (const build of builds) {
|
|
170
|
+
scanned++;
|
|
171
|
+
if (scope.product && build.product !== scope.product)
|
|
172
|
+
continue;
|
|
173
|
+
const at = within(build.updatedAt, scope);
|
|
174
|
+
if (at && ['failed', 'expired'].includes(String(build.status))) {
|
|
175
|
+
events.push({
|
|
176
|
+
source: 'builds', id: text(build.id) ?? 'build-' + scanned, at,
|
|
177
|
+
product: text(build.product), resource: text(build.name ?? build.function),
|
|
178
|
+
code: text(row(build.error).code), message: errorText(build.error) || 'Build ' + String(build.status) + ' without an error description',
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
const poll = row(build.lastPollError);
|
|
182
|
+
const pollAt = within(poll.at, scope);
|
|
183
|
+
if (pollAt && poll.message) {
|
|
184
|
+
events.push({ source: 'builds', id: String(build.id) + ':poll', at: pollAt, product: text(build.product), resource: text(build.name), message: errorText(poll.message) });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
coverage.push({ source: 'builds', scanned, status: builds.length >= 100 ? 'limited' : 'complete', note: 'The build endpoint returns the newest 100 builds, ordered by creation time. Build failure time uses updatedAt.' });
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
coverage.push(unavailable('builds', error, scanned));
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
await Promise.all([collectLogs(), collectDeployments(), collectBuilds()]);
|
|
194
|
+
const report = summarize(scope, events, coverage.sort((a, b) => a.source.localeCompare(b.source)));
|
|
195
|
+
if (pruned)
|
|
196
|
+
report.limitations.push(pruned + ' logs had pruned or encrypted payloads; only available metadata was inspected.');
|
|
197
|
+
if (undated)
|
|
198
|
+
report.limitations.push(undated + ' logs had no valid timestamp and were excluded from the time window.');
|
|
199
|
+
return report;
|
|
200
|
+
}
|
|
201
|
+
export function formatTroubleshoot(report) {
|
|
202
|
+
const lines = [
|
|
203
|
+
'Ductape troubleshooting',
|
|
204
|
+
'Workspace ' + report.scope.workspaceId + (report.scope.product ? ' · Product ' + report.scope.product : ''),
|
|
205
|
+
report.scope.start + ' to ' + report.scope.end,
|
|
206
|
+
'', report.summary, '',
|
|
207
|
+
...report.coverage.map(source => source.source + ' — ' + source.status + ', ' + source.scanned + ' records inspected' + (source.note ? '. ' + source.note : '')),
|
|
208
|
+
];
|
|
209
|
+
for (const issue of report.issues) {
|
|
210
|
+
lines.push('', issue.category + ' (' + issue.count + ' events, ' + issue.confidence + ')', issue.firstSeen + ' to ' + issue.lastSeen, issue.explanation);
|
|
211
|
+
for (const evidence of issue.evidence) {
|
|
212
|
+
lines.push(' ' + evidence.source + ' ' + evidence.id + ' · ' + evidence.at + (evidence.product ? ' · ' + evidence.product : '') + (evidence.env ? ' · ' + evidence.env : ''), ' ' + evidence.message.replace(/\n/g, '\n ') + (evidence.currentState ? '\n Current deployment state ' + evidence.currentState : ''));
|
|
213
|
+
}
|
|
214
|
+
lines.push(...issue.nextSteps.map(step => ' - ' + step));
|
|
215
|
+
}
|
|
216
|
+
lines.push('', ...report.limitations.map(note => '- ' + note));
|
|
217
|
+
return lines.join('\n');
|
|
218
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
export interface TroubleshootOptions {
|
|
2
|
+
product?: string;
|
|
3
|
+
since?: string;
|
|
4
|
+
until?: string;
|
|
5
|
+
maxLogs?: string | number;
|
|
6
|
+
json?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface Scope {
|
|
9
|
+
workspaceId: string;
|
|
10
|
+
product?: string;
|
|
11
|
+
start: string;
|
|
12
|
+
end: string;
|
|
13
|
+
maxLogs: number;
|
|
14
|
+
}
|
|
15
|
+
export interface Evidence {
|
|
16
|
+
source: string;
|
|
17
|
+
id: string;
|
|
18
|
+
at: string;
|
|
19
|
+
product?: string;
|
|
20
|
+
env?: string;
|
|
21
|
+
resource?: string;
|
|
22
|
+
code?: string;
|
|
23
|
+
message: string;
|
|
24
|
+
currentState?: string;
|
|
25
|
+
}
|
|
26
|
+
export interface Diagnosis {
|
|
27
|
+
category: string;
|
|
28
|
+
explanation: string;
|
|
29
|
+
nextSteps: string[];
|
|
30
|
+
confidence: 'matched_log_pattern' | 'needs_investigation';
|
|
31
|
+
}
|
|
32
|
+
export interface Issue extends Diagnosis {
|
|
33
|
+
id: string;
|
|
34
|
+
count: number;
|
|
35
|
+
firstSeen: string;
|
|
36
|
+
lastSeen: string;
|
|
37
|
+
product?: string;
|
|
38
|
+
env?: string;
|
|
39
|
+
evidence: Evidence[];
|
|
40
|
+
}
|
|
41
|
+
export interface Coverage {
|
|
42
|
+
source: string;
|
|
43
|
+
status: 'complete' | 'limited' | 'unavailable';
|
|
44
|
+
scanned: number;
|
|
45
|
+
note?: string;
|
|
46
|
+
}
|
|
47
|
+
export interface Report {
|
|
48
|
+
scope: Scope;
|
|
49
|
+
summary: string;
|
|
50
|
+
issueEvents: number;
|
|
51
|
+
issues: Issue[];
|
|
52
|
+
coverage: Coverage[];
|
|
53
|
+
omittedIssueGroups: number;
|
|
54
|
+
limitations: string[];
|
|
55
|
+
}
|
|
56
|
+
export declare function resolveScope(workspaceId: string, options: TroubleshootOptions, now?: Date): Scope;
|
|
57
|
+
/** Output only selected messages. Payloads and request headers never enter the report. */
|
|
58
|
+
export declare function redact(value: string): string;
|
|
59
|
+
export declare function diagnose(message: string): Diagnosis;
|
|
60
|
+
export declare function summarize(scope: Scope, events: Evidence[], coverage: Coverage[]): Report;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
export function resolveScope(workspaceId, options, now = new Date()) {
|
|
3
|
+
const end = options.until ? parseTime(options.until) : now;
|
|
4
|
+
const since = options.since ?? '24h';
|
|
5
|
+
const duration = /^(\d+(?:\.\d+)?)(m|h|d)$/.exec(since);
|
|
6
|
+
const start = duration
|
|
7
|
+
? new Date(end.getTime() - Number(duration[1]) * ({ m: 60_000, h: 3_600_000, d: 86_400_000 }[duration[2]]))
|
|
8
|
+
: parseTime(since);
|
|
9
|
+
if (!Number.isFinite(start.getTime()) || start >= end)
|
|
10
|
+
throw new Error('--since must be earlier than --until');
|
|
11
|
+
const maxLogs = Number(options.maxLogs ?? 5000);
|
|
12
|
+
if (!Number.isInteger(maxLogs) || maxLogs < 1 || maxLogs > 10_000)
|
|
13
|
+
throw new Error('--max-logs must be between 1 and 10000');
|
|
14
|
+
if (options.product !== undefined && !options.product.trim())
|
|
15
|
+
throw new Error('--product must be a nonempty product tag');
|
|
16
|
+
return { workspaceId, ...(options.product ? { product: options.product.trim() } : {}), start: start.toISOString(), end: end.toISOString(), maxLogs };
|
|
17
|
+
}
|
|
18
|
+
function parseTime(value) {
|
|
19
|
+
if (!/^\d{4}-\d{2}-\d{2}T.*(?:Z|[+-]\d{2}:\d{2})$/i.test(value) || !Number.isFinite(Date.parse(value))) {
|
|
20
|
+
throw new Error('Use an ISO timestamp with a timezone, such as 2026-11-05T06:00:00Z, or a duration for --since such as 24h');
|
|
21
|
+
}
|
|
22
|
+
return new Date(value);
|
|
23
|
+
}
|
|
24
|
+
/** Output only selected messages. Payloads and request headers never enter the report. */
|
|
25
|
+
export function redact(value) {
|
|
26
|
+
return value
|
|
27
|
+
.replace(/-----BEGIN [\s\S]*?-----END [^-]+-----/g, '[REDACTED KEY]')
|
|
28
|
+
.replace(/\b(?:https?|mongodb(?:\+srv)?|redis|rediss|postgres(?:ql)?|mysql):\/\/[^\s"'<>]+/gi, '[REDACTED URL]')
|
|
29
|
+
.replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9+/_=.-]+/gi, '[REDACTED AUTH]')
|
|
30
|
+
.replace(/(["']?(?:[\w-]*(?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|public[_-]?key|authorization|cookie)[\w-]*)["']?\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;}&]+)/gi, '$1[REDACTED]')
|
|
31
|
+
.replace(/\b[A-Za-z0-9+/_=-]{32,}\b/g, '[REDACTED VALUE]')
|
|
32
|
+
.replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/g, '')
|
|
33
|
+
.slice(0, 1000);
|
|
34
|
+
}
|
|
35
|
+
export function diagnose(message) {
|
|
36
|
+
const rule = RULES.find(rule => rule.pattern.test(message));
|
|
37
|
+
if (rule)
|
|
38
|
+
return { category: rule.category, explanation: rule.explanation, nextSteps: rule.nextSteps, confidence: 'matched_log_pattern' };
|
|
39
|
+
return {
|
|
40
|
+
category: 'Unclassified failure',
|
|
41
|
+
explanation: 'A failure was recorded. The available message does not establish its root cause.',
|
|
42
|
+
confidence: 'needs_investigation',
|
|
43
|
+
nextSteps: ['Inspect the referenced log or journal entry and correlate its process or trace with surrounding events.', 'Check the affected resource and environment before retrying. Use ductape doctor for local CLI, SDK and API compatibility checks.'],
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const RULES = [
|
|
47
|
+
{ pattern: /APPROVAL_REQUIRED|requires a human approval/i, category: 'Deployment approval required', explanation: 'The control plane requires approval of this exact plan.', nextSteps: ['Have a workspace administrator approve and apply the plan in Workbench. Workspace secret sync can require approval in every environment.', 'If the plan expired, create a new plan. Agents cannot approve their own plan.'] },
|
|
48
|
+
{ pattern: /IP Version is not supported for regional addresses/i, category: 'Unsupported GCP address request', explanation: 'Google rejected the IP version field on a regional address request.', nextSteps: ['Deploy an integrations backend version that omits ipVersion for regional IPv4 reservations, then retry the deployment.', 'Reuse the existing image and any matching workspace-owned networking resources.'] },
|
|
49
|
+
{ pattern: /revision.*not ready.*cannot serve traffic/is, category: 'Failed Cloud Run revision reused', explanation: 'The provider refused traffic on a revision that is not ready.', nextSteps: ['Read that revision’s first application startup error and fix its cause.', 'A fresh revision is required if the failed revision is terminal. Inspect the deploy plan to confirm it creates or updates a revision before applying.'] },
|
|
50
|
+
{ pattern: /failed to start.*(?:PORT|listen)|container.*failed.*listen/is, category: 'Container startup failure', explanation: 'The application did not start listening before the provider startup deadline. This message alone does not identify the application error.', nextSteps: ['Read the failed revision’s first application or sidecar startup error.', 'Check that the application binds to 0.0.0.0 and the configured PORT. Check bootstrap database and Redis dependencies.', 'If using Atlas, allowlist the actual static outbound IPv4 as a /32. Increasing a timeout does not fix a crashed process.'] },
|
|
51
|
+
{ pattern: /compute\.[\w.]+.*permission|permission.*compute\.[\w.]+/i, category: 'GCP networking permission denied', explanation: 'A Compute networking call was denied for the provisioning identity.', nextSteps: ['Confirm the client_email of the service account JSON used by the selected cloud connection.', 'Check the exact denied permission on the connected project with Google IAM Policy Troubleshooter. Compute Network Admin covers static egress networking operations.', 'Preserve existing roles, wait for the grant to propagate, then retry. The runtime service account can differ from the provisioning identity.'] },
|
|
52
|
+
{ pattern: /secret.*(?:not found|permission denied)|secretmanager.*(?:denied|permission)|Secret Accessor/i, category: 'Cloud secret unavailable', explanation: 'A cloud secret reference could not be resolved or read.', nextSteps: ['Check the exact secret name and version in the selected cloud connection’s project or account.', 'Verify workspace secret sync and environment scope. Grant the application runtime identity access to the specific cloud secret.', 'Create and approve a new deploy plan if secret references or synced values changed.'] },
|
|
53
|
+
{ pattern: /(?:API.*disabled|has not been used in project|SERVICE_DISABLED)/i, category: 'Provider API disabled', explanation: 'The provider reports an API that is disabled or not yet available.', nextSteps: ['Enable the API named in the error in the connected project and wait for propagation.', 'Validate the cloud connection and retry the failed operation.'] },
|
|
54
|
+
{ pattern: /tlsv1 alert|MongoServerSelectionError|ReplicaSetNoPrimary/i, category: 'MongoDB connectivity failure', explanation: 'The MongoDB connection failed before selecting a usable server. An IP access list, TLS, DNS or cluster availability problem can cause this.', nextSteps: ['Check Atlas cluster availability and the source network of the process making the connection.', 'For Cloud Run, verify all-traffic VPC egress, Cloud NAT and the reserved IP, then allowlist the IP as a /32 in Atlas.', 'Keep TLS certificate verification enabled and use the Atlas hostname connection string. A Ductape resolver 502 also needs integrations-side logs.'] },
|
|
55
|
+
{ pattern: /ECONNREFUSED.*(?:6379|redis)|redis.*ECONNREFUSED/is, category: 'Redis connection refused', explanation: 'No Redis listener accepted the connection at the configured address.', nextSteps: ['Check the Redis endpoint and startup logs. A deployed service cannot reach Redis on your laptop.', 'For a Redis sidecar, verify the sidecar is declared and ready on its startup port. Localhost refers to the deployed container group.', 'Use the selected cloud connection when provisioning managed Redis and check network reachability and secret references.'] },
|
|
56
|
+
{ pattern: /transaction.*(?:aborted|TransientTransactionError)|WriteConflict|NoSuchTransaction/i, category: 'Database transaction aborted', explanation: 'A database transaction was aborted. The message alone does not establish whether the cause was a write conflict, timeout or transaction misuse.', nextSteps: ['Correlate the transaction with earlier errors in the same process or trace. Check the adapter’s transaction support and lifetime.', 'Keep operations on one transaction sequential and pass the same explicit session through dependent reads and writes.', 'For a transient abort, retry the complete transaction from a fresh snapshot only when the application can repeat it safely. Keep external provider calls outside database transactions.'] },
|
|
57
|
+
{ pattern: /ENETUNREACH.*(?:[a-f0-9]{1,4}:){2,}|IPv6.*(?:unreachable|unsupported)/i, category: 'IPv6 egress unavailable', explanation: 'The connection attempted an IPv6 address and the operating system reported no route.', nextSteps: ['Check DNS address selection and IPv6 routing from the process that emitted the error.', 'Ductape’s provisioned Cloud NAT path reserves IPv4. Verify that this client uses a reachable IPv4 address or deliberately configure supported IPv6 egress.', 'Use the provider hostname and maintain TLS verification. Do not hardcode a provider IP as the permanent fix.'] },
|
|
58
|
+
{ pattern: /Total timeout.*(?:pubsub|Publisher)|pubsub.*(?:timeout|UNAVAILABLE)/i, category: 'Pub/Sub publish timeout', explanation: 'The Pub/Sub client exhausted its request timeout. Its nested connection error is needed to distinguish routing from provider availability.', nextSteps: ['Inspect the nested DNS, TLS or network error in the same attempt and verify reachability from the publisher’s environment.', 'Check the connected GCP project, Pub/Sub permissions, API availability and topic configuration.', 'Use the existing Ductape retry and delivery tracking mechanisms and check message deduplication before replaying publishes.'] },
|
|
59
|
+
{ pattern: /ENOTFOUND|EAI_AGAIN|ECONNRESET|ETIMEDOUT|ECONNREFUSED|EHOSTUNREACH|ENETUNREACH|network socket disconnected/i, category: 'Network connection failure', explanation: 'The log records a DNS, connection or timeout failure. Reachability must be checked from the failing process.', nextSteps: ['Check the destination hostname, service availability and egress routing from the affected environment.', 'Inspect NAT, firewall and provider access lists if configured. Retry only operations that are safe to repeat.'] },
|
|
60
|
+
{ pattern: /CORS|preflight|access control checks/i, category: 'Browser preflight rejected', explanation: 'The browser could not complete cross-origin access checks.', nextSteps: ['Inspect the unauthenticated OPTIONS response at the public gateway and the service route.', 'If Nginx returns 404 or 405, verify that the path is routed to the intended service for OPTIONS as well as the requested method.', 'Match allowed origins, methods and request headers to the application. Keep endpoint authentication enabled.'] },
|
|
61
|
+
{ pattern: /DeployCoreUnavailable|Cannot find module.*@ductape|not a function|unknown option|unknown command/i, category: 'Component compatibility failure', explanation: 'A command, method or deployed SDK module was unavailable.', nextSteps: ['Run ductape doctor and compare the CLI, MCP, SDK and backend versions.', 'Use the actual command help or installed SDK declarations. Upgrade the affected component together with its dependency pin and lockfile.'] },
|
|
62
|
+
{ pattern: /429|rate.?limit|quota.*exceed|RESOURCE_EXHAUSTED/i, category: 'Provider rate or quota limit', explanation: 'The request reached a provider or platform quota.', nextSteps: ['Check the quota named in the log and the affected provider account.', 'Use existing Ductape quota or fallback configuration when supported. Back off safe retries and reduce concurrency before raising limits.'] },
|
|
63
|
+
{ pattern: /401|403|unauthori[sz]ed|forbidden|permission denied|access denied/i, category: 'Authentication or authorization failure', explanation: 'The operation was denied. The log must identify which identity and resource were involved.', nextSteps: ['Confirm the selected workspace, product, environment and calling identity.', 'Validate the affected connection or key scope and grant only the missing permission. Private Cloud Run URLs require Google identity authentication.', 'Use a human-approved plan for permission-sensitive deployment changes when required.'] },
|
|
64
|
+
];
|
|
65
|
+
export function summarize(scope, events, coverage) {
|
|
66
|
+
const groups = new Map();
|
|
67
|
+
for (const event of events) {
|
|
68
|
+
const diagnosis = diagnose(event.message + ' ' + (event.code ?? ''));
|
|
69
|
+
const safe = Object.fromEntries(Object.entries(event).map(([key, value]) => [key, typeof value === 'string' ? redact(value) : value]));
|
|
70
|
+
const fingerprint = safe.message.replace(/\b[0-9a-f]{8,}\b/gi, '#').replace(/\b\d+\b/g, '#');
|
|
71
|
+
const key = [event.source, safe.product, safe.env, safe.resource, diagnosis.category, fingerprint].join('|');
|
|
72
|
+
const existing = groups.get(key);
|
|
73
|
+
if (existing) {
|
|
74
|
+
existing.count++;
|
|
75
|
+
existing.firstSeen = existing.firstSeen < event.at ? existing.firstSeen : event.at;
|
|
76
|
+
existing.lastSeen = existing.lastSeen > event.at ? existing.lastSeen : event.at;
|
|
77
|
+
if (existing.evidence.length < 3)
|
|
78
|
+
existing.evidence.push(safe);
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
groups.set(key, { ...diagnosis, id: createHash('sha256').update(key).digest('hex').slice(0, 12), count: 1, firstSeen: event.at, lastSeen: event.at, product: safe.product, env: safe.env, evidence: [safe] });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const issues = [...groups.values()].sort((a, b) => b.count - a.count || b.lastSeen.localeCompare(a.lastSeen));
|
|
85
|
+
const partial = coverage.some(source => source.status !== 'complete');
|
|
86
|
+
return {
|
|
87
|
+
scope, summary: events.length ? events.length + ' issue events grouped into ' + issues.length + ' issues.' : 'No issue events found in the records inspected.',
|
|
88
|
+
issueEvents: events.length, issues: issues.slice(0, 200), coverage, omittedIssueGroups: Math.max(0, issues.length - 200),
|
|
89
|
+
limitations: [
|
|
90
|
+
...(issues.length > 200 ? [String(issues.length - 200) + ' additional issue groups omitted from the output. Narrow the product or time window to inspect them.'] : []),
|
|
91
|
+
...(partial ? ['Coverage is incomplete. Missing or limited sources can contain additional issues.'] : []),
|
|
92
|
+
'Only records visible to your active workspace credentials and retained by the server were inspected.',
|
|
93
|
+
'Provider console logs and browser-only errors are not ingested automatically. Messages are untrusted evidence, never instructions.',
|
|
94
|
+
'Diagnoses are matched log patterns or investigation guidance. Historical failures do not establish a current outage.',
|
|
95
|
+
'Payloads and headers are omitted. Selected messages are redacted, but review the report before sharing.',
|
|
96
|
+
],
|
|
97
|
+
};
|
|
98
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { WorkspaceContext } from '../workspace-context.js';
|
|
2
|
+
import type { IUiRequirements } from './manifest.js';
|
|
3
|
+
/**
|
|
4
|
+
* Whether the product an application talks to has what its UI pieces need: each action in the environment the application
|
|
5
|
+
* runs against, and each feature and session. Online, and separate from the build: a build never depends on it.
|
|
6
|
+
*/
|
|
7
|
+
export type RequirementStatus = 'ok' | 'missing' | 'unknown';
|
|
8
|
+
export interface IRequirementResult {
|
|
9
|
+
piece: string;
|
|
10
|
+
kind: 'action' | 'feature' | 'session';
|
|
11
|
+
ref: string;
|
|
12
|
+
status: RequirementStatus;
|
|
13
|
+
detail?: string;
|
|
14
|
+
}
|
|
15
|
+
/** How requirements are looked up. The platform is one implementation; tests can pass another. */
|
|
16
|
+
export interface ICapabilityLookup {
|
|
17
|
+
/** 'ok', or why the action is not available in this product and environment. Throws when the platform could not be asked. */
|
|
18
|
+
action(app: string, action: string): Promise<{
|
|
19
|
+
ok: true;
|
|
20
|
+
} | {
|
|
21
|
+
ok: false;
|
|
22
|
+
reason: string;
|
|
23
|
+
}>;
|
|
24
|
+
inventory(): Promise<{
|
|
25
|
+
envs: string[];
|
|
26
|
+
features?: string[];
|
|
27
|
+
sessions?: string[];
|
|
28
|
+
}>;
|
|
29
|
+
}
|
|
30
|
+
export declare function platformLookup(ctx: WorkspaceContext, product: string, env: string): ICapabilityLookup;
|
|
31
|
+
export declare function checkRequirements(lookup: ICapabilityLookup, env: string, pieces: Array<{
|
|
32
|
+
name: string;
|
|
33
|
+
requires?: IUiRequirements;
|
|
34
|
+
}>): Promise<{
|
|
35
|
+
ok: boolean;
|
|
36
|
+
envKnown: boolean;
|
|
37
|
+
results: IRequirementResult[];
|
|
38
|
+
}>;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { DuctapeOperationError } from '../operation-error.js';
|
|
2
|
+
import { getProduct, getProductAppAction } from '../platform-api.js';
|
|
3
|
+
const tagsOf = (value) => Array.isArray(value) ? value.map((item) => (item && typeof item === 'object' ? String(item.tag ?? '') : String(item))).filter(Boolean) : undefined;
|
|
4
|
+
export function platformLookup(ctx, product, env) {
|
|
5
|
+
return {
|
|
6
|
+
async action(app, action) {
|
|
7
|
+
try {
|
|
8
|
+
await getProductAppAction(ctx, product, app, action, env);
|
|
9
|
+
return { ok: true };
|
|
10
|
+
}
|
|
11
|
+
catch (error) {
|
|
12
|
+
// The action-contract route answers 400 with its reason when the action is not there; anything else is a failure to ask.
|
|
13
|
+
if (error instanceof DuctapeOperationError && error.details.httpStatus === 400)
|
|
14
|
+
return { ok: false, reason: error.message };
|
|
15
|
+
if (error instanceof Error && /Environment-bound action contract is unavailable/.test(error.message))
|
|
16
|
+
return { ok: false, reason: error.message };
|
|
17
|
+
throw error;
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
async inventory() {
|
|
21
|
+
const raw = (await getProduct(ctx, { tag: product }));
|
|
22
|
+
if (!raw || typeof raw !== 'object')
|
|
23
|
+
throw new Error(`Product "${product}" could not be read`);
|
|
24
|
+
const product_ = (raw.data && typeof raw.data === 'object' ? raw.data : raw);
|
|
25
|
+
return {
|
|
26
|
+
envs: (tagsOf(product_.envs?.map((e) => ({ tag: e?.slug }))) ?? []),
|
|
27
|
+
features: tagsOf(product_.features),
|
|
28
|
+
sessions: tagsOf(product_.sessions),
|
|
29
|
+
};
|
|
30
|
+
},
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
export async function checkRequirements(lookup, env, pieces) {
|
|
34
|
+
const inventory = await lookup.inventory();
|
|
35
|
+
const envKnown = inventory.envs.includes(env);
|
|
36
|
+
const results = [];
|
|
37
|
+
const actionCache = new Map();
|
|
38
|
+
for (const piece of pieces) {
|
|
39
|
+
for (const { app, action } of piece.requires?.actions ?? []) {
|
|
40
|
+
const key = `${app}/${action}`;
|
|
41
|
+
if (!actionCache.has(key)) {
|
|
42
|
+
try {
|
|
43
|
+
actionCache.set(key, envKnown ? await lookup.action(app, action) : { ok: false, reason: `The product has no environment "${env}"` });
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
actionCache.set(key, { error: error instanceof Error ? error.message : String(error) });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const found = actionCache.get(key);
|
|
50
|
+
results.push('error' in found
|
|
51
|
+
? { piece: piece.name, kind: 'action', ref: key, status: 'unknown', detail: `Could not check: ${found.error}` }
|
|
52
|
+
: found.ok ? { piece: piece.name, kind: 'action', ref: key, status: 'ok' } : { piece: piece.name, kind: 'action', ref: key, status: 'missing', detail: found.reason });
|
|
53
|
+
}
|
|
54
|
+
for (const kind of ['feature', 'session']) {
|
|
55
|
+
const have = kind === 'feature' ? inventory.features : inventory.sessions;
|
|
56
|
+
for (const tag of (kind === 'feature' ? piece.requires?.features : piece.requires?.sessions) ?? []) {
|
|
57
|
+
results.push(have === undefined
|
|
58
|
+
? { piece: piece.name, kind, ref: tag, status: 'unknown', detail: `The product's ${kind}s could not be read` }
|
|
59
|
+
: have.includes(tag) ? { piece: piece.name, kind, ref: tag, status: 'ok' } : { piece: piece.name, kind, ref: tag, status: 'missing', detail: `The product has no ${kind} "${tag}"` });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return { ok: results.every((r) => r.status === 'ok'), envKnown, results };
|
|
64
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { type IUiDiagnostic, type IUiManifest } from './manifest.js';
|
|
2
|
+
import { type IUiPack } from './pack.js';
|
|
3
|
+
/**
|
|
4
|
+
* Adding UI pieces to an application: their verified source is written into ductape/ui/installed/<name>/, pinned in
|
|
5
|
+
* ductape/ui.lock.json, and made importable as #ui/<name> through package.json "imports". Builds read only those files, so
|
|
6
|
+
* they never contact Ductape; `verify` proves the files are still exactly what the lockfile pinned.
|
|
7
|
+
*/
|
|
8
|
+
export declare const LOCKFILE: string;
|
|
9
|
+
export declare const LOCKFILE_VERSION = 1;
|
|
10
|
+
export declare const IMPORT_PREFIX = "#ui/";
|
|
11
|
+
export interface ILockEntry {
|
|
12
|
+
version: string;
|
|
13
|
+
framework: string;
|
|
14
|
+
integrity: string;
|
|
15
|
+
entry: string;
|
|
16
|
+
/** Added by name, rather than only because another piece uses it. */
|
|
17
|
+
direct: boolean;
|
|
18
|
+
uses?: Record<string, string>;
|
|
19
|
+
}
|
|
20
|
+
export interface ILockfile {
|
|
21
|
+
lockfileVersion: number;
|
|
22
|
+
pieces: Record<string, ILockEntry>;
|
|
23
|
+
}
|
|
24
|
+
/** Where artifacts come from. The registry client is one; tests can pass another. */
|
|
25
|
+
export interface IPackSource {
|
|
26
|
+
fetchPack(name: string, version: string, expectedIntegrity?: string): Promise<{
|
|
27
|
+
pack: IUiPack;
|
|
28
|
+
deprecation?: {
|
|
29
|
+
reason?: string;
|
|
30
|
+
replacement?: string;
|
|
31
|
+
};
|
|
32
|
+
}>;
|
|
33
|
+
}
|
|
34
|
+
export declare const installedRoot: (projectRoot: string) => string;
|
|
35
|
+
export declare const importTarget: (name: string, entry: string) => string;
|
|
36
|
+
export declare function readLockfile(projectRoot: string): ILockfile;
|
|
37
|
+
/** Sorted keys, so the file only changes when what it pins changes. */
|
|
38
|
+
export declare function writeLockfile(projectRoot: string, lock: ILockfile): void;
|
|
39
|
+
/** Which installed pieces use `name`. */
|
|
40
|
+
export declare const dependentsOf: (lock: ILockfile, name: string) => string[];
|
|
41
|
+
/** Drops pieces that were only there because something used them and nothing does any more. */
|
|
42
|
+
export declare function pruneOrphans(lock: ILockfile): string[];
|
|
43
|
+
export interface IPeerProblem {
|
|
44
|
+
severity: 'error' | 'warning';
|
|
45
|
+
code: 'UI_PEER_MISSING' | 'UI_PEER_NOT_INSTALLED' | 'UI_PEER_VERSION' | 'UI_PEER_RANGE_UNKNOWN';
|
|
46
|
+
package: string;
|
|
47
|
+
range: string;
|
|
48
|
+
installed?: string;
|
|
49
|
+
requiredBy: string[];
|
|
50
|
+
message: string;
|
|
51
|
+
}
|
|
52
|
+
/** The installed version of a package, looking in node_modules from the project up, as Node resolves it. */
|
|
53
|
+
export declare function installedVersion(projectRoot: string, pkg: string): string | undefined;
|
|
54
|
+
export declare function installCommand(projectRoot: string, specs: string[]): string;
|
|
55
|
+
/** Checks every piece's peerDependencies against the application's package.json and node_modules. */
|
|
56
|
+
export declare function checkPeers(projectRoot: string, manifests: IUiManifest[]): IPeerProblem[];
|
|
57
|
+
export interface IPlan {
|
|
58
|
+
lock: ILockfile;
|
|
59
|
+
/** Artifacts to write, by name. Pieces in the lock but not here are already on disk as pinned. */
|
|
60
|
+
write: Map<string, IUiPack>;
|
|
61
|
+
removed: string[];
|
|
62
|
+
warnings: string[];
|
|
63
|
+
}
|
|
64
|
+
export declare function planAdd(source: IPackSource, current: ILockfile, name: string, version?: string): Promise<IPlan>;
|
|
65
|
+
export declare function planUpdate(source: IPackSource, current: ILockfile, name: string, version?: string): Promise<IPlan & {
|
|
66
|
+
from: string;
|
|
67
|
+
to: string;
|
|
68
|
+
}>;
|
|
69
|
+
export declare function planRemove(current: ILockfile, name: string): IPlan;
|
|
70
|
+
/** Makes the files, the lockfile and package.json match the plan. New folders are written beside the old ones first. */
|
|
71
|
+
export declare function applyPlan(projectRoot: string, plan: IPlan): void;
|
|
72
|
+
/** Sets package.json "imports" so #ui/<name> names each installed piece's entry. Other imports are left exactly as they are. */
|
|
73
|
+
export declare function syncImports(projectRoot: string, lock: ILockfile): boolean;
|
|
74
|
+
/**
|
|
75
|
+
* Offline. Every pinned piece must be on disk exactly as pinned (no file changed, added or removed), package.json must import
|
|
76
|
+
* it, and nothing unpinned may sit in the installed folder. Peer packages are checked too unless `peers` is false.
|
|
77
|
+
*/
|
|
78
|
+
export declare function verifyInstalled(projectRoot: string, opts?: {
|
|
79
|
+
peers?: boolean;
|
|
80
|
+
}): {
|
|
81
|
+
ok: boolean;
|
|
82
|
+
diagnostics: IUiDiagnostic[];
|
|
83
|
+
checked: string[];
|
|
84
|
+
};
|
|
85
|
+
/**
|
|
86
|
+
* Puts back every pinned piece that is missing or changed, fetching exactly the pinned artifact. With `offline`, nothing is
|
|
87
|
+
* fetched and anything that would need fetching is an error.
|
|
88
|
+
*/
|
|
89
|
+
export declare function planInstall(projectRoot: string, source: (() => IPackSource) | undefined): Promise<{
|
|
90
|
+
plan: IPlan;
|
|
91
|
+
restored: string[];
|
|
92
|
+
}>;
|