@ductape/cli 0.5.6 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/dist/commands/troubleshoot.d.ts +2 -0
- package/dist/commands/troubleshoot.js +15 -0
- package/dist/index.js +12 -2
- package/dist/lib/api-client.d.ts +3 -2
- package/dist/lib/api-client.js +6 -3
- package/dist/lib/cloud-connection.js +4 -1
- package/dist/lib/troubleshoot/collect.d.ts +6 -0
- package/dist/lib/troubleshoot/collect.js +218 -0
- package/dist/lib/troubleshoot/report.d.ts +60 -0
- package/dist/lib/troubleshoot/report.js +98 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -43,6 +43,24 @@ ductape --help
|
|
|
43
43
|
|
|
44
44
|
Do not use `sudo npm link`; it mixes root-owned globals with your normal user.
|
|
45
45
|
|
|
46
|
+
## Troubleshooting workspace issues
|
|
47
|
+
|
|
48
|
+
Run the command in your logged-in CLI context. It uses the active workspace automatically.
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
ductape troubleshoot
|
|
52
|
+
ductape troubleshoot --product ductape:statecraft --since 2h
|
|
53
|
+
ductape troubleshoot --since 2026-11-04T00:00:00Z --until 2026-11-05T00:00:00Z --json
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The default window is the last 24 hours. The report groups failures from retained workspace logs,
|
|
57
|
+
deployment journals and build history, with evidence references, possible causes and supported next steps.
|
|
58
|
+
It reports missing sources and scan limits. Use `--max-logs 10000` or narrow the product/time window
|
|
59
|
+
when the default 5000-record limit is reached. Reports show up to 200 issue groups and disclose omissions.
|
|
60
|
+
Provider console logs and browser-only failures require separate evidence. Payloads and headers are
|
|
61
|
+
omitted and selected messages are redacted. Review reports before sharing them.
|
|
62
|
+
The command performs reads only. If every source is unavailable it prints a report and exits with code 1.
|
|
63
|
+
|
|
46
64
|
## Quick start
|
|
47
65
|
|
|
48
66
|
```bash
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { ApiClient } from '../lib/api-client.js';
|
|
2
|
+
import { requireWorkspaceContext, workspaceAuthQuery } from '../lib/workspace-context.js';
|
|
3
|
+
import { resolveScope } from '../lib/troubleshoot/report.js';
|
|
4
|
+
import { collectTroubleshoot, formatTroubleshoot } from '../lib/troubleshoot/collect.js';
|
|
5
|
+
export async function runTroubleshoot(options) {
|
|
6
|
+
const context = requireWorkspaceContext();
|
|
7
|
+
const scope = resolveScope(context.workspaceId, options);
|
|
8
|
+
const api = ApiClient.forProfile(context.profile, context.credentials, 15_000);
|
|
9
|
+
const report = await collectTroubleshoot(scope, {
|
|
10
|
+
get: (path, query) => api.getPath(path, workspaceAuthQuery(context, query)),
|
|
11
|
+
});
|
|
12
|
+
console.log(options.json ? JSON.stringify(report, null, 2) : formatTroubleshoot(report));
|
|
13
|
+
if (report.coverage.every(source => source.status === 'unavailable'))
|
|
14
|
+
process.exitCode = 1;
|
|
15
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,7 @@ import path from 'node:path';
|
|
|
6
6
|
import { runLogin } from './commands/login.js';
|
|
7
7
|
import { runLogout } from './commands/logout.js';
|
|
8
8
|
import { runWhoami } from './commands/whoami.js';
|
|
9
|
+
import { runTroubleshoot } from './commands/troubleshoot.js';
|
|
9
10
|
import { runDoctor } from './commands/doctor.js';
|
|
10
11
|
import { runProfilesList, runProfilesUse } from './commands/profiles.js';
|
|
11
12
|
import { runLink } from './commands/link.js';
|
|
@@ -127,7 +128,7 @@ const codeTarget = (c) => c
|
|
|
127
128
|
.option('--product <tag>').option('--function <name>').option('--fn-version <version>').option('--env <slug>');
|
|
128
129
|
remote('step-up', 'start', 'Email yourself a six-digit code for one change (a person only)', codeTarget).action(op('stepup.start'));
|
|
129
130
|
remote('step-up', 'verify', 'Check the code and print a grant, good for ten minutes and for that one change; put it in an environment variable and pass its name with --grant-env', (c) => codeTarget(c).requiredOption('--code <code>', 'the six-digit code from your email')).action(op('stepup.verify'));
|
|
130
|
-
remote('plans', 'create', 'Create a plan from a JSON file; nothing is changed', (c) => c.requiredOption('--file <file>', 'Plan request JSON')).action(op('plans.create'));
|
|
131
|
+
remote('plans', 'create', 'Create a plan from a JSON file; nothing is changed', (c) => c.requiredOption('--file <file>', 'Plan request JSON (static egress uses runtime.vpcEgress on Cloud Run with optional provision: true or runtime.awsVpcEgress on Lambda or ECS Fargate)')).action(op('plans.create'));
|
|
131
132
|
remote('plans', 'approve <planId>', 'Approve a plan (a person only; refused for agents)', grantOption).action(op('plans.approve'));
|
|
132
133
|
remote('plans', 'list', 'List plans with where each stands: blocked, pending_approval, approved, ready, applied or expired', (c) => c.option('--product <tag>').option('--env <env>').option('--stage <stage>')).action(op('plans.list'));
|
|
133
134
|
remote('plans', 'get <planId>', 'Show one plan: what it would change, its warnings and the state of the cloud it was made from').action(op('plans.get'));
|
|
@@ -136,7 +137,7 @@ remote('deployments', 'list', 'List deployments (functions and services)', (c) =
|
|
|
136
137
|
remote('deployments', 'get <deploymentId>', 'Show one deployment').action(op('deployments.get'));
|
|
137
138
|
remote('deployments', 'journal <deploymentId>', 'Show a deployment\'s journal').action(op('deployments.journal'));
|
|
138
139
|
const planFromDeployment = (c) => c
|
|
139
|
-
.option('--runtime-file <file>', 'Runtime settings JSON that replace the deployment\'s own (cpu, memory, minInstances, maxInstances
|
|
140
|
+
.option('--runtime-file <file>', 'Runtime settings JSON that replace the deployment\'s own (cpu, memory, minInstances, maxInstances, or Cloud Run vpcEgress with network and subnet names)')
|
|
140
141
|
.option('--observed-file <file>', 'What the provider holds; only while the control plane does not read the provider itself');
|
|
141
142
|
remote('deployments', 'redeploy <deploymentId>', 'Plan running the deployment\'s current release again (retry a failed one); nothing changes until the plan is approved and applied', planFromDeployment).action(op('deployments.redeploy'));
|
|
142
143
|
remote('deployments', 'rollback <deploymentId>', 'Plan putting an earlier release back at full traffic (default: the last one that held all traffic)', (c) => planFromDeployment(c).option('--release-hash <hash>', 'A release this deployment has run')).action(op('deployments.rollback'));
|
|
@@ -193,6 +194,15 @@ program
|
|
|
193
194
|
skipWorkspaceSelect: Boolean(opts.skipWorkspaceSelect),
|
|
194
195
|
})));
|
|
195
196
|
program.command('logout').description('Clear local credentials').action(wrap(runLogout));
|
|
197
|
+
program
|
|
198
|
+
.command('troubleshoot')
|
|
199
|
+
.description('Summarize issues from logs, deployment journals and builds in the active workspace')
|
|
200
|
+
.option('--product <tag>', 'Only this product')
|
|
201
|
+
.option('--since <time>', 'Duration such as 24h or ISO timestamp with timezone', '24h')
|
|
202
|
+
.option('--until <time>', 'ISO timestamp with timezone; defaults to now')
|
|
203
|
+
.option('--max-logs <count>', 'Maximum workspace log records to inspect, 1 to 10000', '5000')
|
|
204
|
+
.option('--json', 'Structured report with evidence and coverage')
|
|
205
|
+
.action(wrap(runTroubleshoot));
|
|
196
206
|
program
|
|
197
207
|
.command('doctor')
|
|
198
208
|
.description('Report CLI, SDK, MCP, API/schema, authentication, and project compatibility facts')
|
package/dist/lib/api-client.d.ts
CHANGED
|
@@ -7,8 +7,9 @@ export interface ApiEnvelope<T> {
|
|
|
7
7
|
export declare class ApiClient {
|
|
8
8
|
private readonly apiUrl;
|
|
9
9
|
private readonly token?;
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
private readonly timeoutMs?;
|
|
11
|
+
constructor(apiUrl: string, token?: string | undefined, timeoutMs?: number | undefined);
|
|
12
|
+
static forProfile(profileName?: string, creds?: Credentials | null, timeoutMs?: number): ApiClient;
|
|
12
13
|
request<T>(method: string, path: string, body?: unknown, extraHeaders?: Record<string, string>): Promise<T>;
|
|
13
14
|
post<T>(path: string, body?: unknown, headers?: Record<string, string>): Promise<T>;
|
|
14
15
|
put<T>(path: string, body?: unknown, headers?: Record<string, string>): Promise<T>;
|
package/dist/lib/api-client.js
CHANGED
|
@@ -4,12 +4,14 @@ import { DuctapeOperationError, safeEndpoint } from './operation-error.js';
|
|
|
4
4
|
export class ApiClient {
|
|
5
5
|
apiUrl;
|
|
6
6
|
token;
|
|
7
|
-
|
|
7
|
+
timeoutMs;
|
|
8
|
+
constructor(apiUrl, token, timeoutMs) {
|
|
8
9
|
this.apiUrl = apiUrl;
|
|
9
10
|
this.token = token;
|
|
11
|
+
this.timeoutMs = timeoutMs;
|
|
10
12
|
}
|
|
11
|
-
static forProfile(profileName, creds) {
|
|
12
|
-
return new ApiClient(getApiUrl(profileName), creds?.auth_token);
|
|
13
|
+
static forProfile(profileName, creds, timeoutMs) {
|
|
14
|
+
return new ApiClient(getApiUrl(profileName), creds?.auth_token, timeoutMs);
|
|
13
15
|
}
|
|
14
16
|
async request(method, path, body, extraHeaders) {
|
|
15
17
|
const url = `${this.apiUrl}${path.startsWith('/') ? path : `/${path}`}`;
|
|
@@ -27,6 +29,7 @@ export class ApiClient {
|
|
|
27
29
|
res = await fetch(url, {
|
|
28
30
|
method,
|
|
29
31
|
headers,
|
|
32
|
+
...(this.timeoutMs ? { signal: AbortSignal.timeout(this.timeoutMs) } : {}),
|
|
30
33
|
body: body === undefined ? undefined : raw ? body : JSON.stringify(body),
|
|
31
34
|
});
|
|
32
35
|
}
|
|
@@ -309,6 +309,9 @@ export const GCP_DEPLOY_CREATES = [
|
|
|
309
309
|
];
|
|
310
310
|
export const GCP_DEPLOY_SAFETY = 'Ductape only adds resources it labels as its own (ductape_managed), never changes or reuses one it did not create, and never deletes anything, including what it created. Removing deployed resources is always your own action. Use a dedicated project if you can, so what Ductape adds is easy to see.';
|
|
311
311
|
export const GCP_DEPLOY_TROUBLESHOOTING = [
|
|
312
|
+
{ problem: 'Static egress creation fails with a Compute API or IAM error', fix: 'Enable compute.googleapis.com in the connected project. Grant the connection service account the Compute permissions named in the error, then retry the deployment. The journal records the outbound IPv4 once the network is ready.' },
|
|
313
|
+
{ problem: 'Cloud Run cannot attach the VPC subnet', fix: 'Check that the subnet belongs to the selected VPC and is in the deployment region and connected project. If Google Cloud reports an IAM denial, check the principal and permission named in the error.' },
|
|
314
|
+
{ problem: 'Cloud Run starts but Atlas connections time out or fail during TLS setup', fix: 'Check that Cloud NAT serves the selected subnet and uses a reserved IPv4. Verify the actual outbound IP from the revision and allowlist it as a /32 in Atlas. If the Ductape resolver returns 502, check the integrations logs for that request separately.' },
|
|
312
315
|
{ problem: '"<API> has not been used in project <PROJECT_ID> before or it is disabled"', fix: 'Enable that API (step 1), wait a minute or two for it to take effect, and try again.' },
|
|
313
316
|
{ problem: 'Validation lists missing permissions and the roles that grant them', fix: 'Grant exactly those roles to the Ductape service account on the project (step 3) and validate again. Role changes can take a minute to apply.' },
|
|
314
317
|
{ problem: '"caller does not have permission to act as service account ..." when a build or deploy starts', fix: 'Grant roles/iam.serviceAccountUser to the Ductape service account on the project. Cloud Build and Cloud Run run as a service account, and the Ductape account must be allowed to act as it.' },
|
|
@@ -595,7 +598,7 @@ export function connectionGuide(providerInput, purposeInput = 'resources') {
|
|
|
595
598
|
},
|
|
596
599
|
{
|
|
597
600
|
step: 4, who: 'you', title: 'Deploy the template into the resource group',
|
|
598
|
-
commands: ['az deployment group create --resource-group ductape-deploy --template-file ductape-deploy.json --query properties.outputs'],
|
|
601
|
+
commands: ['az deployment group create --resource-group ductape-deploy --template-file ductape-deploy.json --query properties.outputs', '# Static egress: add --parameters environmentName=ductape-env-nat provisionStaticEgress=true (creates a billable NAT gateway and static IP). To reuse a delegated NAT subnet, set infrastructureSubnetId instead.'],
|
|
599
602
|
notes: ['It creates a user assigned identity for your apps to run as, a container registry, a log workspace and a container apps environment, and lets the identity pull from the registry. Read it first. The role assignment in it needs you to hold Owner or User Access Administrator on the group.', 'The outputs give the registry name, environment name and identity id used in step 6.'],
|
|
600
603
|
},
|
|
601
604
|
{
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type Scope, type Report } from './report.js';
|
|
2
|
+
export interface TroubleshootReader {
|
|
3
|
+
get(path: string, query: Record<string, string>): Promise<unknown>;
|
|
4
|
+
}
|
|
5
|
+
export declare function collectTroubleshoot(scope: Scope, reader: TroubleshootReader): Promise<Report>;
|
|
6
|
+
export declare function formatTroubleshoot(report: Report): string;
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import { summarize } from './report.js';
|
|
2
|
+
const row = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
3
|
+
const text = (value) => typeof value === 'string' ? value : undefined;
|
|
4
|
+
const rows = (value) => {
|
|
5
|
+
if (!Array.isArray(value))
|
|
6
|
+
throw new Error('Unexpected response shape');
|
|
7
|
+
return value.map(item => {
|
|
8
|
+
const result = row(item);
|
|
9
|
+
if (!Object.keys(result).length)
|
|
10
|
+
throw new Error('Unexpected record shape');
|
|
11
|
+
return result;
|
|
12
|
+
});
|
|
13
|
+
};
|
|
14
|
+
function within(value, scope) {
|
|
15
|
+
const time = text(value);
|
|
16
|
+
const ms = time ? Date.parse(time) : NaN;
|
|
17
|
+
return Number.isFinite(ms) && ms >= Date.parse(scope.start) && ms <= Date.parse(scope.end) ? new Date(ms).toISOString() : undefined;
|
|
18
|
+
}
|
|
19
|
+
/** Extract only error descriptions, never request/response/configuration payloads. */
|
|
20
|
+
function errorText(value, depth = 0) {
|
|
21
|
+
if (depth > 3)
|
|
22
|
+
return '';
|
|
23
|
+
if (typeof value === 'string')
|
|
24
|
+
return value;
|
|
25
|
+
if (Array.isArray(value))
|
|
26
|
+
return value.slice(0, 5).map(item => errorText(item, depth + 1)).filter(Boolean).join(' ');
|
|
27
|
+
const object = row(value);
|
|
28
|
+
return [text(object.code), text(object.message), object.error && errorText(object.error, depth + 1), object.errors && errorText(object.errors, depth + 1)]
|
|
29
|
+
.filter(Boolean).join(' ');
|
|
30
|
+
}
|
|
31
|
+
function logMessage(log) {
|
|
32
|
+
let data = log.data;
|
|
33
|
+
if (typeof data === 'string' && !log.data_encrypted && data.length <= 100_000) {
|
|
34
|
+
try {
|
|
35
|
+
data = JSON.parse(data);
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
data = undefined;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const payload = row(data);
|
|
42
|
+
return [text(log.message), errorText(log.error), errorText(payload.error), errorText(payload.errors)]
|
|
43
|
+
.filter(Boolean).join(' ') || 'Failure recorded without an accessible error description';
|
|
44
|
+
}
|
|
45
|
+
function unavailable(source, error, scanned = 0) {
|
|
46
|
+
// API error messages may echo credentials or payloads. Only expose the HTTP status.
|
|
47
|
+
const details = row(row(error).details);
|
|
48
|
+
const status = typeof details.httpStatus === 'number' ? ' (HTTP ' + details.httpStatus + ')' : '';
|
|
49
|
+
return { source, status: 'unavailable', scanned, note: 'Could not read this source' + status + '. Check authentication, service availability and ductape doctor.' };
|
|
50
|
+
}
|
|
51
|
+
export async function collectTroubleshoot(scope, reader) {
|
|
52
|
+
const events = [];
|
|
53
|
+
const coverage = [];
|
|
54
|
+
const productFilter = scope.product ? { product: scope.product } : {};
|
|
55
|
+
let pruned = 0;
|
|
56
|
+
let undated = 0;
|
|
57
|
+
async function collectLogs() {
|
|
58
|
+
let scanned = 0;
|
|
59
|
+
const seen = new Set();
|
|
60
|
+
try {
|
|
61
|
+
for (let page = 1;; page++) {
|
|
62
|
+
const response = row(await reader.get('/log/v1/workspace-logs/' + encodeURIComponent(scope.workspaceId), {
|
|
63
|
+
start_date: scope.start, end_date: scope.end,
|
|
64
|
+
...(scope.product ? { product_tag: scope.product } : {}),
|
|
65
|
+
page: String(page), limit: String(Math.min(500, scope.maxLogs)),
|
|
66
|
+
}));
|
|
67
|
+
const logs = row(response.logs);
|
|
68
|
+
const batch = rows(logs.data);
|
|
69
|
+
const metadata = row(logs.metadata);
|
|
70
|
+
const seenBefore = seen.size;
|
|
71
|
+
for (const log of batch) {
|
|
72
|
+
if (scanned >= scope.maxLogs)
|
|
73
|
+
break;
|
|
74
|
+
const id = text(log._id);
|
|
75
|
+
if (id && seen.has(id))
|
|
76
|
+
continue;
|
|
77
|
+
if (id)
|
|
78
|
+
seen.add(id);
|
|
79
|
+
scanned++;
|
|
80
|
+
const at = within(log.timestamp, scope);
|
|
81
|
+
if (!text(log.timestamp) || !Number.isFinite(Date.parse(String(log.timestamp))))
|
|
82
|
+
undated++;
|
|
83
|
+
if (!at || (scope.product && log.product_tag !== scope.product))
|
|
84
|
+
continue;
|
|
85
|
+
if (log.data_pruned_at || log.data_encrypted)
|
|
86
|
+
pruned++;
|
|
87
|
+
if (log.status !== 'fail' && log.failed_execution !== true && !['error', 'warn', 'warning', 'fatal'].includes(String(log.level ?? log.severity).toLowerCase()))
|
|
88
|
+
continue;
|
|
89
|
+
events.push({
|
|
90
|
+
source: 'workspace_logs', id: id ?? 'page-' + page + '-record-' + scanned, at,
|
|
91
|
+
product: text(log.product_tag), env: text(log.env ?? log.app_env),
|
|
92
|
+
resource: text(log.child_tag ?? log.function_tag ?? log.feature_tag ?? log.parent_tag ?? log.type),
|
|
93
|
+
code: text(row(log.error).code), message: logMessage(log),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
const total = typeof metadata.total === 'number' ? metadata.total : undefined;
|
|
97
|
+
const done = !batch.length || (total !== undefined && seen.size >= total && metadata.totalIsCapped !== true);
|
|
98
|
+
if (done || scanned >= scope.maxLogs) {
|
|
99
|
+
coverage.push({
|
|
100
|
+
source: 'workspace_logs', scanned, status: done ? 'complete' : 'limited',
|
|
101
|
+
...(!done ? { note: 'Reached --max-logs before exhausting the time window. Narrow --since/--until or --product, or increase --max-logs.' } : {}),
|
|
102
|
+
});
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
// Without pagination metadata there is no safe claim that the window was exhausted.
|
|
106
|
+
if (total === undefined || seenBefore === seen.size || batch.every(log => !text(log._id))) {
|
|
107
|
+
coverage.push({ source: 'workspace_logs', scanned, status: 'limited', note: 'The server did not return reliable pagination metadata.' });
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
catch (error) {
|
|
113
|
+
coverage.push(unavailable('workspace_logs', error, scanned));
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
async function collectDeployments() {
|
|
117
|
+
let scanned = 0;
|
|
118
|
+
try {
|
|
119
|
+
const deployments = rows(await reader.get('/integrations/v1/deploy/deployments', productFilter));
|
|
120
|
+
const candidates = deployments.filter(deployment => {
|
|
121
|
+
if (scope.product && deployment.product !== scope.product)
|
|
122
|
+
return false;
|
|
123
|
+
const updated = Date.parse(String(deployment.updatedAt));
|
|
124
|
+
return !Number.isFinite(updated) || updated >= Date.parse(scope.start);
|
|
125
|
+
});
|
|
126
|
+
let failures = 0;
|
|
127
|
+
// Bound concurrent journal requests; an unavailable journal does not hide others.
|
|
128
|
+
for (let offset = 0; offset < candidates.length; offset += 4) {
|
|
129
|
+
await Promise.all(candidates.slice(offset, offset + 4).map(async (deployment) => {
|
|
130
|
+
const id = text(deployment.id);
|
|
131
|
+
if (!id) {
|
|
132
|
+
failures++;
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
const journal = rows(await reader.get('/integrations/v1/deploy/deployments/' + encodeURIComponent(id) + '/journal', {}));
|
|
137
|
+
for (const entry of journal) {
|
|
138
|
+
scanned++;
|
|
139
|
+
const at = within(entry.at, scope);
|
|
140
|
+
if (!at || !entry.error)
|
|
141
|
+
continue;
|
|
142
|
+
events.push({
|
|
143
|
+
source: 'deployment_journal', id: id + ':' + String(entry.seq), at,
|
|
144
|
+
product: text(deployment.product), env: text(deployment.env),
|
|
145
|
+
resource: text(deployment.name ?? deployment.function), currentState: text(deployment.state),
|
|
146
|
+
code: text(row(entry.error).code), message: errorText(entry.error),
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
failures++;
|
|
152
|
+
}
|
|
153
|
+
}));
|
|
154
|
+
}
|
|
155
|
+
coverage.push({
|
|
156
|
+
source: 'deployment_journals', scanned,
|
|
157
|
+
status: failures || deployments.length >= 200 ? 'limited' : 'complete',
|
|
158
|
+
note: 'Read ' + (candidates.length - failures) + ' journals. ' + failures + ' unavailable. The endpoint returns at most 200 deployments.',
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
coverage.push(unavailable('deployment_journals', error, scanned));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
async function collectBuilds() {
|
|
166
|
+
let scanned = 0;
|
|
167
|
+
try {
|
|
168
|
+
const builds = rows(await reader.get('/integrations/v1/deploy/builds', productFilter));
|
|
169
|
+
for (const build of builds) {
|
|
170
|
+
scanned++;
|
|
171
|
+
if (scope.product && build.product !== scope.product)
|
|
172
|
+
continue;
|
|
173
|
+
const at = within(build.updatedAt, scope);
|
|
174
|
+
if (at && ['failed', 'expired'].includes(String(build.status))) {
|
|
175
|
+
events.push({
|
|
176
|
+
source: 'builds', id: text(build.id) ?? 'build-' + scanned, at,
|
|
177
|
+
product: text(build.product), resource: text(build.name ?? build.function),
|
|
178
|
+
code: text(row(build.error).code), message: errorText(build.error) || 'Build ' + String(build.status) + ' without an error description',
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
const poll = row(build.lastPollError);
|
|
182
|
+
const pollAt = within(poll.at, scope);
|
|
183
|
+
if (pollAt && poll.message) {
|
|
184
|
+
events.push({ source: 'builds', id: String(build.id) + ':poll', at: pollAt, product: text(build.product), resource: text(build.name), message: errorText(poll.message) });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
coverage.push({ source: 'builds', scanned, status: builds.length >= 100 ? 'limited' : 'complete', note: 'The build endpoint returns the newest 100 builds, ordered by creation time. Build failure time uses updatedAt.' });
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
coverage.push(unavailable('builds', error, scanned));
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
await Promise.all([collectLogs(), collectDeployments(), collectBuilds()]);
|
|
194
|
+
const report = summarize(scope, events, coverage.sort((a, b) => a.source.localeCompare(b.source)));
|
|
195
|
+
if (pruned)
|
|
196
|
+
report.limitations.push(pruned + ' logs had pruned or encrypted payloads; only available metadata was inspected.');
|
|
197
|
+
if (undated)
|
|
198
|
+
report.limitations.push(undated + ' logs had no valid timestamp and were excluded from the time window.');
|
|
199
|
+
return report;
|
|
200
|
+
}
|
|
201
|
+
export function formatTroubleshoot(report) {
|
|
202
|
+
const lines = [
|
|
203
|
+
'Ductape troubleshooting',
|
|
204
|
+
'Workspace ' + report.scope.workspaceId + (report.scope.product ? ' · Product ' + report.scope.product : ''),
|
|
205
|
+
report.scope.start + ' to ' + report.scope.end,
|
|
206
|
+
'', report.summary, '',
|
|
207
|
+
...report.coverage.map(source => source.source + ' — ' + source.status + ', ' + source.scanned + ' records inspected' + (source.note ? '. ' + source.note : '')),
|
|
208
|
+
];
|
|
209
|
+
for (const issue of report.issues) {
|
|
210
|
+
lines.push('', issue.category + ' (' + issue.count + ' events, ' + issue.confidence + ')', issue.firstSeen + ' to ' + issue.lastSeen, issue.explanation);
|
|
211
|
+
for (const evidence of issue.evidence) {
|
|
212
|
+
lines.push(' ' + evidence.source + ' ' + evidence.id + ' · ' + evidence.at + (evidence.product ? ' · ' + evidence.product : '') + (evidence.env ? ' · ' + evidence.env : ''), ' ' + evidence.message.replace(/\n/g, '\n ') + (evidence.currentState ? '\n Current deployment state ' + evidence.currentState : ''));
|
|
213
|
+
}
|
|
214
|
+
lines.push(...issue.nextSteps.map(step => ' - ' + step));
|
|
215
|
+
}
|
|
216
|
+
lines.push('', ...report.limitations.map(note => '- ' + note));
|
|
217
|
+
return lines.join('\n');
|
|
218
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
export interface TroubleshootOptions {
|
|
2
|
+
product?: string;
|
|
3
|
+
since?: string;
|
|
4
|
+
until?: string;
|
|
5
|
+
maxLogs?: string | number;
|
|
6
|
+
json?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface Scope {
|
|
9
|
+
workspaceId: string;
|
|
10
|
+
product?: string;
|
|
11
|
+
start: string;
|
|
12
|
+
end: string;
|
|
13
|
+
maxLogs: number;
|
|
14
|
+
}
|
|
15
|
+
export interface Evidence {
|
|
16
|
+
source: string;
|
|
17
|
+
id: string;
|
|
18
|
+
at: string;
|
|
19
|
+
product?: string;
|
|
20
|
+
env?: string;
|
|
21
|
+
resource?: string;
|
|
22
|
+
code?: string;
|
|
23
|
+
message: string;
|
|
24
|
+
currentState?: string;
|
|
25
|
+
}
|
|
26
|
+
export interface Diagnosis {
|
|
27
|
+
category: string;
|
|
28
|
+
explanation: string;
|
|
29
|
+
nextSteps: string[];
|
|
30
|
+
confidence: 'matched_log_pattern' | 'needs_investigation';
|
|
31
|
+
}
|
|
32
|
+
export interface Issue extends Diagnosis {
|
|
33
|
+
id: string;
|
|
34
|
+
count: number;
|
|
35
|
+
firstSeen: string;
|
|
36
|
+
lastSeen: string;
|
|
37
|
+
product?: string;
|
|
38
|
+
env?: string;
|
|
39
|
+
evidence: Evidence[];
|
|
40
|
+
}
|
|
41
|
+
export interface Coverage {
|
|
42
|
+
source: string;
|
|
43
|
+
status: 'complete' | 'limited' | 'unavailable';
|
|
44
|
+
scanned: number;
|
|
45
|
+
note?: string;
|
|
46
|
+
}
|
|
47
|
+
export interface Report {
|
|
48
|
+
scope: Scope;
|
|
49
|
+
summary: string;
|
|
50
|
+
issueEvents: number;
|
|
51
|
+
issues: Issue[];
|
|
52
|
+
coverage: Coverage[];
|
|
53
|
+
omittedIssueGroups: number;
|
|
54
|
+
limitations: string[];
|
|
55
|
+
}
|
|
56
|
+
export declare function resolveScope(workspaceId: string, options: TroubleshootOptions, now?: Date): Scope;
|
|
57
|
+
/** Output only selected messages. Payloads and request headers never enter the report. */
|
|
58
|
+
export declare function redact(value: string): string;
|
|
59
|
+
export declare function diagnose(message: string): Diagnosis;
|
|
60
|
+
export declare function summarize(scope: Scope, events: Evidence[], coverage: Coverage[]): Report;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
export function resolveScope(workspaceId, options, now = new Date()) {
|
|
3
|
+
const end = options.until ? parseTime(options.until) : now;
|
|
4
|
+
const since = options.since ?? '24h';
|
|
5
|
+
const duration = /^(\d+(?:\.\d+)?)(m|h|d)$/.exec(since);
|
|
6
|
+
const start = duration
|
|
7
|
+
? new Date(end.getTime() - Number(duration[1]) * ({ m: 60_000, h: 3_600_000, d: 86_400_000 }[duration[2]]))
|
|
8
|
+
: parseTime(since);
|
|
9
|
+
if (!Number.isFinite(start.getTime()) || start >= end)
|
|
10
|
+
throw new Error('--since must be earlier than --until');
|
|
11
|
+
const maxLogs = Number(options.maxLogs ?? 5000);
|
|
12
|
+
if (!Number.isInteger(maxLogs) || maxLogs < 1 || maxLogs > 10_000)
|
|
13
|
+
throw new Error('--max-logs must be between 1 and 10000');
|
|
14
|
+
if (options.product !== undefined && !options.product.trim())
|
|
15
|
+
throw new Error('--product must be a nonempty product tag');
|
|
16
|
+
return { workspaceId, ...(options.product ? { product: options.product.trim() } : {}), start: start.toISOString(), end: end.toISOString(), maxLogs };
|
|
17
|
+
}
|
|
18
|
+
function parseTime(value) {
|
|
19
|
+
if (!/^\d{4}-\d{2}-\d{2}T.*(?:Z|[+-]\d{2}:\d{2})$/i.test(value) || !Number.isFinite(Date.parse(value))) {
|
|
20
|
+
throw new Error('Use an ISO timestamp with a timezone, such as 2026-11-05T06:00:00Z, or a duration for --since such as 24h');
|
|
21
|
+
}
|
|
22
|
+
return new Date(value);
|
|
23
|
+
}
|
|
24
|
+
/** Output only selected messages. Payloads and request headers never enter the report. */
|
|
25
|
+
export function redact(value) {
|
|
26
|
+
return value
|
|
27
|
+
.replace(/-----BEGIN [\s\S]*?-----END [^-]+-----/g, '[REDACTED KEY]')
|
|
28
|
+
.replace(/\b(?:https?|mongodb(?:\+srv)?|redis|rediss|postgres(?:ql)?|mysql):\/\/[^\s"'<>]+/gi, '[REDACTED URL]')
|
|
29
|
+
.replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9+/_=.-]+/gi, '[REDACTED AUTH]')
|
|
30
|
+
.replace(/(["']?(?:[\w-]*(?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|public[_-]?key|authorization|cookie)[\w-]*)["']?\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;}&]+)/gi, '$1[REDACTED]')
|
|
31
|
+
.replace(/\b[A-Za-z0-9+/_=-]{32,}\b/g, '[REDACTED VALUE]')
|
|
32
|
+
.replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/g, '')
|
|
33
|
+
.slice(0, 1000);
|
|
34
|
+
}
|
|
35
|
+
export function diagnose(message) {
|
|
36
|
+
const rule = RULES.find(rule => rule.pattern.test(message));
|
|
37
|
+
if (rule)
|
|
38
|
+
return { category: rule.category, explanation: rule.explanation, nextSteps: rule.nextSteps, confidence: 'matched_log_pattern' };
|
|
39
|
+
return {
|
|
40
|
+
category: 'Unclassified failure',
|
|
41
|
+
explanation: 'A failure was recorded. The available message does not establish its root cause.',
|
|
42
|
+
confidence: 'needs_investigation',
|
|
43
|
+
nextSteps: ['Inspect the referenced log or journal entry and correlate its process or trace with surrounding events.', 'Check the affected resource and environment before retrying. Use ductape doctor for local CLI, SDK and API compatibility checks.'],
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const RULES = [
|
|
47
|
+
{ pattern: /APPROVAL_REQUIRED|requires a human approval/i, category: 'Deployment approval required', explanation: 'The control plane requires approval of this exact plan.', nextSteps: ['Have a workspace administrator approve and apply the plan in Workbench. Workspace secret sync can require approval in every environment.', 'If the plan expired, create a new plan. Agents cannot approve their own plan.'] },
|
|
48
|
+
{ pattern: /IP Version is not supported for regional addresses/i, category: 'Unsupported GCP address request', explanation: 'Google rejected the IP version field on a regional address request.', nextSteps: ['Deploy an integrations backend version that omits ipVersion for regional IPv4 reservations, then retry the deployment.', 'Reuse the existing image and any matching workspace-owned networking resources.'] },
|
|
49
|
+
{ pattern: /revision.*not ready.*cannot serve traffic/is, category: 'Failed Cloud Run revision reused', explanation: 'The provider refused traffic on a revision that is not ready.', nextSteps: ['Read that revision’s first application startup error and fix its cause.', 'A fresh revision is required if the failed revision is terminal. Inspect the deploy plan to confirm it creates or updates a revision before applying.'] },
|
|
50
|
+
{ pattern: /failed to start.*(?:PORT|listen)|container.*failed.*listen/is, category: 'Container startup failure', explanation: 'The application did not start listening before the provider startup deadline. This message alone does not identify the application error.', nextSteps: ['Read the failed revision’s first application or sidecar startup error.', 'Check that the application binds to 0.0.0.0 and the configured PORT. Check bootstrap database and Redis dependencies.', 'If using Atlas, allowlist the actual static outbound IPv4 as a /32. Increasing a timeout does not fix a crashed process.'] },
|
|
51
|
+
{ pattern: /compute\.[\w.]+.*permission|permission.*compute\.[\w.]+/i, category: 'GCP networking permission denied', explanation: 'A Compute networking call was denied for the provisioning identity.', nextSteps: ['Confirm the client_email of the service account JSON used by the selected cloud connection.', 'Check the exact denied permission on the connected project with Google IAM Policy Troubleshooter. Compute Network Admin covers static egress networking operations.', 'Preserve existing roles, wait for the grant to propagate, then retry. The runtime service account can differ from the provisioning identity.'] },
|
|
52
|
+
{ pattern: /secret.*(?:not found|permission denied)|secretmanager.*(?:denied|permission)|Secret Accessor/i, category: 'Cloud secret unavailable', explanation: 'A cloud secret reference could not be resolved or read.', nextSteps: ['Check the exact secret name and version in the selected cloud connection’s project or account.', 'Verify workspace secret sync and environment scope. Grant the application runtime identity access to the specific cloud secret.', 'Create and approve a new deploy plan if secret references or synced values changed.'] },
|
|
53
|
+
{ pattern: /(?:API.*disabled|has not been used in project|SERVICE_DISABLED)/i, category: 'Provider API disabled', explanation: 'The provider reports an API that is disabled or not yet available.', nextSteps: ['Enable the API named in the error in the connected project and wait for propagation.', 'Validate the cloud connection and retry the failed operation.'] },
|
|
54
|
+
{ pattern: /tlsv1 alert|MongoServerSelectionError|ReplicaSetNoPrimary/i, category: 'MongoDB connectivity failure', explanation: 'The MongoDB connection failed before selecting a usable server. An IP access list, TLS, DNS or cluster availability problem can cause this.', nextSteps: ['Check Atlas cluster availability and the source network of the process making the connection.', 'For Cloud Run, verify all-traffic VPC egress, Cloud NAT and the reserved IP, then allowlist the IP as a /32 in Atlas.', 'Keep TLS certificate verification enabled and use the Atlas hostname connection string. A Ductape resolver 502 also needs integrations-side logs.'] },
|
|
55
|
+
{ pattern: /ECONNREFUSED.*(?:6379|redis)|redis.*ECONNREFUSED/is, category: 'Redis connection refused', explanation: 'No Redis listener accepted the connection at the configured address.', nextSteps: ['Check the Redis endpoint and startup logs. A deployed service cannot reach Redis on your laptop.', 'For a Redis sidecar, verify the sidecar is declared and ready on its startup port. Localhost refers to the deployed container group.', 'Use the selected cloud connection when provisioning managed Redis and check network reachability and secret references.'] },
|
|
56
|
+
{ pattern: /transaction.*(?:aborted|TransientTransactionError)|WriteConflict|NoSuchTransaction/i, category: 'Database transaction aborted', explanation: 'A database transaction was aborted. The message alone does not establish whether the cause was a write conflict, timeout or transaction misuse.', nextSteps: ['Correlate the transaction with earlier errors in the same process or trace. Check the adapter’s transaction support and lifetime.', 'Keep operations on one transaction sequential and pass the same explicit session through dependent reads and writes.', 'For a transient abort, retry the complete transaction from a fresh snapshot only when the application can repeat it safely. Keep external provider calls outside database transactions.'] },
|
|
57
|
+
{ pattern: /ENETUNREACH.*(?:[a-f0-9]{1,4}:){2,}|IPv6.*(?:unreachable|unsupported)/i, category: 'IPv6 egress unavailable', explanation: 'The connection attempted an IPv6 address and the operating system reported no route.', nextSteps: ['Check DNS address selection and IPv6 routing from the process that emitted the error.', 'Ductape’s provisioned Cloud NAT path reserves IPv4. Verify that this client uses a reachable IPv4 address or deliberately configure supported IPv6 egress.', 'Use the provider hostname and maintain TLS verification. Do not hardcode a provider IP as the permanent fix.'] },
|
|
58
|
+
{ pattern: /Total timeout.*(?:pubsub|Publisher)|pubsub.*(?:timeout|UNAVAILABLE)/i, category: 'Pub/Sub publish timeout', explanation: 'The Pub/Sub client exhausted its request timeout. Its nested connection error is needed to distinguish routing from provider availability.', nextSteps: ['Inspect the nested DNS, TLS or network error in the same attempt and verify reachability from the publisher’s environment.', 'Check the connected GCP project, Pub/Sub permissions, API availability and topic configuration.', 'Use the existing Ductape retry and delivery tracking mechanisms and check message deduplication before replaying publishes.'] },
|
|
59
|
+
{ pattern: /ENOTFOUND|EAI_AGAIN|ECONNRESET|ETIMEDOUT|ECONNREFUSED|EHOSTUNREACH|ENETUNREACH|network socket disconnected/i, category: 'Network connection failure', explanation: 'The log records a DNS, connection or timeout failure. Reachability must be checked from the failing process.', nextSteps: ['Check the destination hostname, service availability and egress routing from the affected environment.', 'Inspect NAT, firewall and provider access lists if configured. Retry only operations that are safe to repeat.'] },
|
|
60
|
+
{ pattern: /CORS|preflight|access control checks/i, category: 'Browser preflight rejected', explanation: 'The browser could not complete cross-origin access checks.', nextSteps: ['Inspect the unauthenticated OPTIONS response at the public gateway and the service route.', 'If Nginx returns 404 or 405, verify that the path is routed to the intended service for OPTIONS as well as the requested method.', 'Match allowed origins, methods and request headers to the application. Keep endpoint authentication enabled.'] },
|
|
61
|
+
{ pattern: /DeployCoreUnavailable|Cannot find module.*@ductape|not a function|unknown option|unknown command/i, category: 'Component compatibility failure', explanation: 'A command, method or deployed SDK module was unavailable.', nextSteps: ['Run ductape doctor and compare the CLI, MCP, SDK and backend versions.', 'Use the actual command help or installed SDK declarations. Upgrade the affected component together with its dependency pin and lockfile.'] },
|
|
62
|
+
{ pattern: /429|rate.?limit|quota.*exceed|RESOURCE_EXHAUSTED/i, category: 'Provider rate or quota limit', explanation: 'The request reached a provider or platform quota.', nextSteps: ['Check the quota named in the log and the affected provider account.', 'Use existing Ductape quota or fallback configuration when supported. Back off safe retries and reduce concurrency before raising limits.'] },
|
|
63
|
+
{ pattern: /401|403|unauthori[sz]ed|forbidden|permission denied|access denied/i, category: 'Authentication or authorization failure', explanation: 'The operation was denied. The log must identify which identity and resource were involved.', nextSteps: ['Confirm the selected workspace, product, environment and calling identity.', 'Validate the affected connection or key scope and grant only the missing permission. Private Cloud Run URLs require Google identity authentication.', 'Use a human-approved plan for permission-sensitive deployment changes when required.'] },
|
|
64
|
+
];
|
|
65
|
+
export function summarize(scope, events, coverage) {
|
|
66
|
+
const groups = new Map();
|
|
67
|
+
for (const event of events) {
|
|
68
|
+
const diagnosis = diagnose(event.message + ' ' + (event.code ?? ''));
|
|
69
|
+
const safe = Object.fromEntries(Object.entries(event).map(([key, value]) => [key, typeof value === 'string' ? redact(value) : value]));
|
|
70
|
+
const fingerprint = safe.message.replace(/\b[0-9a-f]{8,}\b/gi, '#').replace(/\b\d+\b/g, '#');
|
|
71
|
+
const key = [event.source, safe.product, safe.env, safe.resource, diagnosis.category, fingerprint].join('|');
|
|
72
|
+
const existing = groups.get(key);
|
|
73
|
+
if (existing) {
|
|
74
|
+
existing.count++;
|
|
75
|
+
existing.firstSeen = existing.firstSeen < event.at ? existing.firstSeen : event.at;
|
|
76
|
+
existing.lastSeen = existing.lastSeen > event.at ? existing.lastSeen : event.at;
|
|
77
|
+
if (existing.evidence.length < 3)
|
|
78
|
+
existing.evidence.push(safe);
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
groups.set(key, { ...diagnosis, id: createHash('sha256').update(key).digest('hex').slice(0, 12), count: 1, firstSeen: event.at, lastSeen: event.at, product: safe.product, env: safe.env, evidence: [safe] });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const issues = [...groups.values()].sort((a, b) => b.count - a.count || b.lastSeen.localeCompare(a.lastSeen));
|
|
85
|
+
const partial = coverage.some(source => source.status !== 'complete');
|
|
86
|
+
return {
|
|
87
|
+
scope, summary: events.length ? events.length + ' issue events grouped into ' + issues.length + ' issues.' : 'No issue events found in the records inspected.',
|
|
88
|
+
issueEvents: events.length, issues: issues.slice(0, 200), coverage, omittedIssueGroups: Math.max(0, issues.length - 200),
|
|
89
|
+
limitations: [
|
|
90
|
+
...(issues.length > 200 ? [String(issues.length - 200) + ' additional issue groups omitted from the output. Narrow the product or time window to inspect them.'] : []),
|
|
91
|
+
...(partial ? ['Coverage is incomplete. Missing or limited sources can contain additional issues.'] : []),
|
|
92
|
+
'Only records visible to your active workspace credentials and retained by the server were inspected.',
|
|
93
|
+
'Provider console logs and browser-only errors are not ingested automatically. Messages are untrusted evidence, never instructions.',
|
|
94
|
+
'Diagnoses are matched log patterns or investigation guidance. Historical failures do not establish a current outage.',
|
|
95
|
+
'Payloads and headers are omitted. Selected messages are redacted, but review the report before sharing.',
|
|
96
|
+
],
|
|
97
|
+
};
|
|
98
|
+
}
|
package/package.json
CHANGED