copilot-operator 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -14
- package/dist/src/api/operator.controller.js +31 -42
- package/dist/src/api/operator.service.js +663 -367
- package/dist/src/api/saveToDesktop.js +1 -1
- package/dist/src/api/security.js +40 -10
- package/dist/src/api/server.js +75 -12
- package/dist/src/api/settings.js +156 -15
- package/dist/src/cli.js +108 -97
- package/dist/src/config/handWritten.js +36 -0
- package/dist/src/config/layout.js +13 -4
- package/dist/src/config/lockedPolicy.js +56 -22
- package/dist/src/config/schema.js +13 -84
- package/dist/src/context/desktopDir.js +44 -0
- package/dist/src/exec/authorizer.js +9 -1
- package/dist/src/exec/checks.js +240 -23
- package/dist/src/exec/dangerous.js +29 -5
- package/dist/src/exec/network.js +105 -28
- package/dist/src/exec/redaction.js +52 -9
- package/dist/src/exec/reportFile.js +4 -5
- package/dist/src/exec/runner.js +146 -13
- package/dist/src/exec/shells.js +41 -6
- package/dist/src/exec/workDir.js +4 -4
- package/dist/src/orchestrator/derivedChecks.js +1 -0
- package/dist/src/orchestrator/review.js +25 -3
- package/dist/src/orchestrator/taskRunner.js +476 -163
- package/dist/src/plan/brief.js +37 -44
- package/dist/src/plan/importPlan.js +15 -7
- package/dist/src/plan/schema.js +64 -29
- package/dist/src/plan/script.js +0 -15
- package/dist/src/plan/systemGuide.js +2 -177
- package/dist/src/protocol/parser.js +5 -1
- package/dist/src/session/debugExport.js +1 -1
- package/dist/src/session/exportRecord.js +1 -1
- package/dist/src/session/exports.js +49 -9
- package/dist/src/session/store.js +215 -80
- package/dist/src/vcs/scope.js +29 -6
- package/dist/src/vcs/taskVcs.js +78 -10
- package/dist/web/404.html +2 -2
- package/dist/web/_next/static/chunks/36-94660f1cde2da529.js +1 -0
- package/dist/web/_next/static/chunks/549-9548d9a132c8c7d4.js +1 -0
- package/dist/web/_next/static/chunks/867-8b95ae984befc0e2.js +1 -0
- package/dist/web/_next/static/chunks/app/defaults/page-71a43ed69134b48d.js +1 -0
- package/dist/web/_next/static/chunks/app/sessions/view/page-886f5fc99e146408.js +1 -0
- package/dist/web/_next/static/css/{a0642fdb8adf9edb.css → e2a2c9eb3c226313.css} +1 -1
- package/dist/web/appearance.html +2 -2
- package/dist/web/appearance.txt +12 -12
- package/dist/web/defaults.html +2 -2
- package/dist/web/defaults.txt +12 -12
- package/dist/web/history.html +2 -2
- package/dist/web/history.txt +12 -12
- package/dist/web/import.html +2 -2
- package/dist/web/import.txt +12 -12
- package/dist/web/index.html +2 -2
- package/dist/web/index.txt +12 -12
- package/dist/web/level1.html +2 -2
- package/dist/web/level1.txt +12 -12
- package/dist/web/presets.html +2 -2
- package/dist/web/presets.txt +12 -12
- package/dist/web/sessions/view.html +2 -2
- package/dist/web/sessions/view.txt +12 -12
- package/dist/web/system.html +2 -2
- package/dist/web/system.txt +12 -12
- package/package.json +18 -10
- package/prompts/level1.md +2 -1
- package/prompts/organisation.bg.md +0 -1
- package/prompts/organisation.en.md +0 -1
- package/dist/src/context/contextFiles.js +0 -295
- package/dist/src/context/desktopMirror.js +0 -173
- package/dist/src/context/projectMirror.js +0 -432
- package/dist/web/_next/static/chunks/295-04151ded0741d18d.js +0 -1
- package/dist/web/_next/static/chunks/36-c3dd06c6a459c114.js +0 -1
- package/dist/web/_next/static/chunks/549-a5a59a94f92bb3eb.js +0 -1
- package/dist/web/_next/static/chunks/867-84c19f047972ffc3.js +0 -1
- package/dist/web/_next/static/chunks/app/defaults/page-5f180a88398be4c9.js +0 -1
- package/dist/web/_next/static/chunks/app/sessions/view/page-34d7988849f9aa76.js +0 -1
- /package/dist/web/_next/static/{NosxirjkOgV-Boyq8dGPl → IwNZiBrEMAcXEUFj2miiU}/_buildManifest.js +0 -0
- /package/dist/web/_next/static/{NosxirjkOgV-Boyq8dGPl → IwNZiBrEMAcXEUFj2miiU}/_ssgManifest.js +0 -0
package/dist/src/exec/network.js
CHANGED
|
@@ -33,9 +33,11 @@
|
|
|
33
33
|
* proves the server it started actually answers, and nothing comes back that the project did not
|
|
34
34
|
* itself produce. Holding every health check for approval would turn an unattended run into one
|
|
35
35
|
* that waits all night on its first test. The exception is narrow on purpose: every `http(s)://`
|
|
36
|
-
* address in the command must be loopback,
|
|
37
|
-
*
|
|
38
|
-
*
|
|
36
|
+
* address in the command must be loopback, read the way the program will read it (the host after
|
|
37
|
+
* any `user@`, not the first word after `://`), there must be at least one, no option may send the
|
|
38
|
+
* request somewhere else, and nothing else in the command may look like a host — a bare
|
|
39
|
+
* `example.com` or `10.0.0.5` beside a localhost URL still holds the step, because `iwr` accepts an
|
|
40
|
+
* address without a scheme. Where the gate cannot read an address for certain, it is not local.
|
|
39
41
|
*
|
|
40
42
|
* The same text is screened whole, not command by command. That is deliberate and it is what
|
|
41
43
|
* closes the obvious way round: writing `Invoke-WebRequest …` into a `.ps1` with `Set-Content` and
|
|
@@ -139,39 +141,114 @@ const FILE_EXTENSIONS = new Set([
|
|
|
139
141
|
'png', 'jpg', 'jpeg', 'gif', 'svg', 'ico', 'webp', 'pdf', 'out', 'tmp', 'bak', 'lock',
|
|
140
142
|
'exe', 'cmd', 'bat', 'dll', 'tar', 'gz', 'tgz',
|
|
141
143
|
]);
|
|
144
|
+
/**
|
|
145
|
+
* The host a URL names, read from the text after its `://`, or null when it cannot be read for
|
|
146
|
+
* certain.
|
|
147
|
+
*
|
|
148
|
+
* Everything before the last `@` of an authority is userinfo, not the host: curl and
|
|
149
|
+
* Invoke-WebRequest both fetch `http://localhost:1@evil.example.com/tool.exe` from
|
|
150
|
+
* evil.example.com. Reading the host up to the first `:` called that "localhost", and the one
|
|
151
|
+
* download the hold exists for — an executable from somewhere else — went through unheld, in a
|
|
152
|
+
* check's command with nobody asked (found on 2026-10-01). So the authority is read whole, up to the
|
|
153
|
+
* first `/`, `?`, `#` or white space, and it is only this machine when it is nothing but a host and
|
|
154
|
+
* a port. An `@` anywhere in it, a `\`, a percent-encoded name, or a `$`, a backtick or a caret the
|
|
155
|
+
* shell would expand or escape at run time: the parsers disagree about each of those, or the gate
|
|
156
|
+
* cannot see what the shell will make of it, so none is read as loopback.
|
|
157
|
+
*
|
|
158
|
+
* Quotes are taken out before reading, because to the shell they are not the end of a word:
|
|
159
|
+
* `"http://localhost"@evil.example.com/` is one argument, and it is the URL with userinfo. After the
|
|
160
|
+
* host and port only an end of the argument may follow — `)`, `}`, `]`, `;`, `,`, `|`, `&`, `<`,
|
|
161
|
+
* `>` or nothing — and never another character of a name, so `localhost.evil.example.com`, whose
|
|
162
|
+
* full-width dot a URL parser turns into a plain one, is not "localhost" either.
|
|
163
|
+
*/
|
|
164
|
+
function urlHost(afterScheme) {
|
|
165
|
+
const authority = (/^[^\/?#\s]*/.exec(afterScheme)?.[0] ?? '').replace(/["']/g, '');
|
|
166
|
+
if (/[@\\%$`^]/.test(authority))
|
|
167
|
+
return null;
|
|
168
|
+
return /^(\[[0-9a-f:.]+\]|[a-z0-9._~-]+)(?::\d*)?(?=$|[)}\];,|&<>])/i.exec(authority)?.[1] ?? null;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Whether one piece of a token reads as a host or an IP address that is not this machine. A name
|
|
172
|
+
* with a known file extension (`health.json`), or a .NET namespace (`System.Net`), is not a host.
|
|
173
|
+
*/
|
|
174
|
+
function namesAnotherHost(bare) {
|
|
175
|
+
if (!bare)
|
|
176
|
+
return false;
|
|
177
|
+
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(bare))
|
|
178
|
+
return !isLoopback(bare);
|
|
179
|
+
const labels = bare.split('.');
|
|
180
|
+
if (labels.length < 2 || labels.some((l) => l === ''))
|
|
181
|
+
return false;
|
|
182
|
+
const last = labels[labels.length - 1].toLowerCase();
|
|
183
|
+
if (!/^[a-z]{2,}$/.test(last) || FILE_EXTENSIONS.has(last))
|
|
184
|
+
return false;
|
|
185
|
+
if (/^(system|microsoft|net)$/i.test(labels[0]))
|
|
186
|
+
return false;
|
|
187
|
+
return !isLoopback(bare);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Options that send a request somewhere its URL does not say, or take more of them from a file the
|
|
191
|
+
* line does not show: curl's `--resolve` and `--connect-to` (the URL still says `localhost`, the
|
|
192
|
+
* connection goes to the address the option names), curl's `-K`/`--config`, and wget's
|
|
193
|
+
* `-i`/`--input-file` and `-e`/`--execute`. With any of them the URLs on the line are not the whole
|
|
194
|
+
* story, so the line is not provably local. The options are matched in their own case, as the
|
|
195
|
+
* programs read them: curl's `-k` (skip the certificate check) is an ordinary flag of a local health
|
|
196
|
+
* check and `-K` is not, and PowerShell's `-ErrorAction` after a `wget` alias is not wget's `-e`.
|
|
197
|
+
*
|
|
198
|
+
* And each only after its own program in the same statement, since only that program reads it.
|
|
199
|
+
* `--config` is an ordinary flag of node servers, vite, jest, playwright and webpack, and matched
|
|
200
|
+
* anywhere on the line, `node server.js --config dev.json; Invoke-WebRequest http://localhost:3000/health`
|
|
201
|
+
* was held as if curl had been told to read a file, and refused outright as a check's command. A
|
|
202
|
+
* quote may come before the option, as in `-ArgumentList '--config','x'`, where the program still
|
|
203
|
+
* reads it.
|
|
204
|
+
*/
|
|
205
|
+
const REDIRECTING_OPTIONS = [
|
|
206
|
+
/\bcurl(?:\.exe)?\b[^|;\n]*[\s'"](?:--(?:resolve|connect-to|config)(?=[\s='",]|$)|-K(?![a-zA-Z]))/,
|
|
207
|
+
/\bwget2?(?:\.exe)?\b[^|;\n]*[\s'"](?:-[ie]|--(?:input-file|execute)(?=[\s='",]|$))/,
|
|
208
|
+
];
|
|
142
209
|
/**
|
|
143
210
|
* Whether every place this command could reach is this machine. True only when it names at least
|
|
144
|
-
* one `
|
|
211
|
+
* one `scheme://` address, every one of them a loopback host read for certain (`urlHost`), no option
|
|
212
|
+
* sends the request elsewhere, and no other token reads as a host or an IP.
|
|
145
213
|
*/
|
|
146
214
|
export function onlyLoopbackTargets(command) {
|
|
147
|
-
const
|
|
148
|
-
if (
|
|
215
|
+
const schemes = [...command.matchAll(/\b[a-z][a-z0-9+.-]*:\/\//gi)];
|
|
216
|
+
if (schemes.length === 0)
|
|
149
217
|
return false;
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
for (const token of rest.split(/[\s'"`=,;|(){}]+/)) {
|
|
154
|
-
const bare = token.replace(/^-+/, '').replace(/[:\/].*$/, '');
|
|
155
|
-
if (!bare)
|
|
156
|
-
continue;
|
|
157
|
-
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(bare) && !isLoopback(bare))
|
|
218
|
+
for (const m of schemes) {
|
|
219
|
+
const host = urlHost(command.slice(m.index + m[0].length));
|
|
220
|
+
if (host === null || !isLoopback(host))
|
|
158
221
|
return false;
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
222
|
+
}
|
|
223
|
+
if (REDIRECTING_OPTIONS.some((option) => option.test(command)))
|
|
224
|
+
return false;
|
|
225
|
+
/*
|
|
226
|
+
* The rest of the line, with each URL taken out up to where the shell ends it — white space, a
|
|
227
|
+
* quote, `;`, `|`, `&`, a bracket or a comma — so that a second command glued on after a path
|
|
228
|
+
* (`…/x;iwr('evil.example')`) is still read.
|
|
229
|
+
*/
|
|
230
|
+
const rest = command.replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s'"`;|&(){}<>,]*/gi, ' ');
|
|
231
|
+
for (const token of rest.split(/[\s'"`=,;|&(){}<>]+/)) {
|
|
232
|
+
// A bracketed IPv6 address anywhere in the token: a proxy (`-x [2001:db8::1]:8080`) or an address given to curl.
|
|
233
|
+
for (const v6 of token.matchAll(/\[([0-9a-f:.]*:[0-9a-f:.]*)\]/gi))
|
|
234
|
+
if (!isLoopback(v6[1]))
|
|
235
|
+
return false;
|
|
236
|
+
// PowerShell member access on a variable or a type (`$r.StatusCode`, `[System.Net.Dns]`) is not a host.
|
|
168
237
|
if (token.startsWith('$') || token.startsWith('['))
|
|
169
238
|
continue;
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
239
|
+
/*
|
|
240
|
+
* Every piece between colons and `@`s, not only the first: `localhost:80:93.184.216.34` names an
|
|
241
|
+
* address after its second colon, `bob@evil.example` a host after its `@`, and either reroutes a
|
|
242
|
+
* request whose URL said localhost. Within a piece, only what comes before a `/` or a `\` is a
|
|
243
|
+
* candidate, as in `example.com/x`; a path's folders and its file are not hosts. Cut at `/` alone,
|
|
244
|
+
* a Windows path was not: `C:\work\Shop.Web` splits at its drive's colon into `C` and
|
|
245
|
+
* `\work\Shop.Web`, whose last label `Web` read as a top-level domain, and a localhost health
|
|
246
|
+
* check that saved its answer under a drive was held, or refused as a check's command.
|
|
247
|
+
*/
|
|
248
|
+
for (const piece of token.replace(/^-+/, '').split(/[:@]+/)) {
|
|
249
|
+
if (namesAnotherHost(piece.replace(/[\/\\].*$/, '')))
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
175
252
|
}
|
|
176
253
|
return true;
|
|
177
254
|
}
|
|
@@ -71,6 +71,26 @@ export function findSecrets(text) {
|
|
|
71
71
|
}
|
|
72
72
|
return hits;
|
|
73
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* One of the operator's own patterns, as a regular expression, or null when it is not one. Such a
|
|
76
|
+
* pattern is applied as a plain string instead, as it always was, so a typo in one cannot switch
|
|
77
|
+
* redaction off.
|
|
78
|
+
*/
|
|
79
|
+
function operatorPattern(p) {
|
|
80
|
+
try {
|
|
81
|
+
return new RegExp(p, 'gi');
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/** The text with every built-in shape replaced, before the operator's own patterns are tried. */
|
|
88
|
+
function withoutShapes(text) {
|
|
89
|
+
let out = text;
|
|
90
|
+
for (const s of SHAPES)
|
|
91
|
+
out = out.replace(s.pattern, s.replace);
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
74
94
|
/**
|
|
75
95
|
* The text with every secret-shaped string replaced, then the operator's own patterns.
|
|
76
96
|
*
|
|
@@ -78,16 +98,39 @@ export function findSecrets(text) {
|
|
|
78
98
|
* plain string, as it always was, so a typo in one cannot switch redaction off.
|
|
79
99
|
*/
|
|
80
100
|
export function redactSecrets(text, extraPatterns = []) {
|
|
81
|
-
let out = text;
|
|
82
|
-
for (const s of SHAPES)
|
|
83
|
-
out = out.replace(s.pattern, s.replace);
|
|
101
|
+
let out = withoutShapes(text);
|
|
84
102
|
for (const p of extraPatterns) {
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
88
|
-
catch {
|
|
89
|
-
out = out.split(p).join('[REDACTED]');
|
|
90
|
-
}
|
|
103
|
+
const re = operatorPattern(p);
|
|
104
|
+
out = re ? out.replace(re, '[REDACTED]') : out.split(p).join('[REDACTED]');
|
|
91
105
|
}
|
|
92
106
|
return out;
|
|
93
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* What `redactSecrets` takes out of a text, by name: the built-in shapes as `findSecrets` names
|
|
110
|
+
* them, and each of the operator's own patterns under the setting it comes from.
|
|
111
|
+
*
|
|
112
|
+
* `findSecrets` alone could not see the operator's patterns, so a report whose only secret was one
|
|
113
|
+
* of theirs — a company's ticket or key format, which no built-in shape knows — was redacted without
|
|
114
|
+
* a word, and the event that tells the operator something was taken out never came. Each pattern is
|
|
115
|
+
* counted on the text as `redactSecrets` reaches it, after the shapes, so what a shape already took
|
|
116
|
+
* is not counted a second time under the operator's name.
|
|
117
|
+
*/
|
|
118
|
+
export function findRedactions(text, extraPatterns = []) {
|
|
119
|
+
const hits = findSecrets(text);
|
|
120
|
+
let out = withoutShapes(text);
|
|
121
|
+
for (const p of extraPatterns) {
|
|
122
|
+
const re = operatorPattern(p);
|
|
123
|
+
const count = re ? [...out.matchAll(re)].length : p ? out.split(p).length - 1 : 0;
|
|
124
|
+
if (count > 0)
|
|
125
|
+
hits.push({ name: `report.redactPatterns "${p}"`, count });
|
|
126
|
+
out = re ? out.replace(re, '[REDACTED]') : out.split(p).join('[REDACTED]');
|
|
127
|
+
}
|
|
128
|
+
return hits;
|
|
129
|
+
}
|
|
130
|
+
/** Several counts as one, summed by name, in the order the names were first met. */
|
|
131
|
+
export function mergeRedactions(lists) {
|
|
132
|
+
const byName = new Map();
|
|
133
|
+
for (const hit of lists.flat())
|
|
134
|
+
byName.set(hit.name, (byName.get(hit.name) ?? 0) + hit.count);
|
|
135
|
+
return [...byName.entries()].map(([name, count]) => ({ name, count }));
|
|
136
|
+
}
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
10
10
|
import { join } from 'node:path';
|
|
11
11
|
import { silentFailure } from '../protocol/reporter.js';
|
|
12
|
-
import {
|
|
12
|
+
import { findRedactions, mergeRedactions, redactSecrets } from './redaction.js';
|
|
13
13
|
/**
|
|
14
14
|
* Removes terminal colour codes.
|
|
15
15
|
*
|
|
@@ -119,10 +119,9 @@ export async function writeReport(results, opts) {
|
|
|
119
119
|
const raw = results.map((r) => sectionFor(r, opts.maxOutputChars));
|
|
120
120
|
// Counted before redaction so the event can say what kind of thing was taken out — the
|
|
121
121
|
// operator on a work machine wants to know a token was printed, not only that it was hidden.
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
const redactions = [...byShape.entries()].map(([name, count]) => ({ name, count }));
|
|
122
|
+
// Their own patterns are counted too: a report whose only secret matched one of them was
|
|
123
|
+
// redacted without the event ever saying so.
|
|
124
|
+
const redactions = mergeRedactions(raw.map((s) => findRedactions(s, opts.redactPatterns)));
|
|
126
125
|
const sections = raw.map((s) => redact(s, opts.redactPatterns));
|
|
127
126
|
const chunks = [[]];
|
|
128
127
|
let size = Buffer.byteLength(header) + Buffer.byteLength(footer);
|
package/dist/src/exec/runner.js
CHANGED
|
@@ -23,14 +23,54 @@ import { spawn } from 'node:child_process';
|
|
|
23
23
|
import { createWriteStream, existsSync } from 'node:fs';
|
|
24
24
|
import { mkdir } from 'node:fs/promises';
|
|
25
25
|
import { dirname } from 'node:path';
|
|
26
|
+
import { StringDecoder } from 'node:string_decoder';
|
|
26
27
|
import { effectiveShell, invocationFor, missingShellProblem, resolveShell } from './shells.js';
|
|
27
28
|
import { stopTree } from './processes.js';
|
|
28
29
|
import { stepEnvironment } from './stepEnv.js';
|
|
29
30
|
const DEFAULT_HARD_TIMEOUT_MS = 4 * 60 * 60 * 1000;
|
|
30
31
|
const DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1000;
|
|
32
|
+
/** How often a running step says how it is doing, unless the caller asks for another pace. */
|
|
31
33
|
const HEARTBEAT_EVERY_MS = 30_000;
|
|
32
|
-
/**
|
|
33
|
-
|
|
34
|
+
/**
|
|
35
|
+
* Cap on what is held in memory and echoed into the report body, per stream.
|
|
36
|
+
*
|
|
37
|
+
* Exported because whatever decides something from the captured text has to know where it stops:
|
|
38
|
+
* past it the text is not the whole output, and an answer read from it may not be the true one.
|
|
39
|
+
*/
|
|
40
|
+
export const MAX_CAPTURE_CHARS = 200_000;
|
|
41
|
+
/**
|
|
42
|
+
* How much of a line that has not ended yet is carried over to the next chunk, so the last line a
|
|
43
|
+
* heartbeat shows is a whole line. Bounded, because a program can print a megabyte with no newline.
|
|
44
|
+
*/
|
|
45
|
+
const LINE_CARRIED_CHARS = 2_000;
|
|
46
|
+
/** The longest delay a Node timer can hold: 2^31-1 ms, about 24.8 days. */
|
|
47
|
+
const LONGEST_TIMER_MS = 2_147_483_647;
|
|
48
|
+
/**
|
|
49
|
+
* A delay a timer will really wait.
|
|
50
|
+
*
|
|
51
|
+
* Node does not refuse a delay it cannot hold. Above 2^31-1 ms, Infinity included, and below 1 ms
|
|
52
|
+
* or not a number at all, it sets the timer to 1 ms and prints a warning nobody reads. For the
|
|
53
|
+
* heartbeat that is a beat every millisecond; for the hard clock it is a step killed as it starts,
|
|
54
|
+
* which is what a step allowed a month would get. So a delay too long for a timer becomes the
|
|
55
|
+
* longest one there is, and one that is not a delay at all becomes the default.
|
|
56
|
+
*/
|
|
57
|
+
function timerDelay(ms, fallback) {
|
|
58
|
+
if (ms === undefined || !(ms >= 1))
|
|
59
|
+
return fallback;
|
|
60
|
+
return Math.min(ms, LONGEST_TIMER_MS);
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* A piece of stderr as the log shows it: every line marked once, at its start.
|
|
64
|
+
*
|
|
65
|
+
* It used to be marked with `^` in multiline mode, which also counts a `\r` as the end of a line.
|
|
66
|
+
* On Windows each line came out as `[stderr] text\r[stderr] \n`, a piece that ended a line put a
|
|
67
|
+
* mark at its very end, ahead of whatever was written next, and a piece that went on with a line
|
|
68
|
+
* already begun put another in the middle of it. A pipe hands over pieces, not lines, so whether a
|
|
69
|
+
* piece begins a line is known only from the one before it, and the caller says.
|
|
70
|
+
*/
|
|
71
|
+
function markStderr(text, atLineStart) {
|
|
72
|
+
return `${atLineStart ? '[stderr] ' : ''}${text.replace(/\n(?=[^])/g, '\n[stderr] ')}`;
|
|
73
|
+
}
|
|
34
74
|
/**
|
|
35
75
|
* The result of a step that never started because the machine has no shell for it.
|
|
36
76
|
*
|
|
@@ -56,6 +96,31 @@ function unrunnable(req, problem) {
|
|
|
56
96
|
lastOutputAgoMs: 0,
|
|
57
97
|
};
|
|
58
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* The result of a step the operator stopped before it began.
|
|
101
|
+
*
|
|
102
|
+
* Shaped like a stop in the middle of a run, because that is what the operator asked for and what
|
|
103
|
+
* whoever reads the result has to act on, but with nothing started and nothing logged: there is no
|
|
104
|
+
* process to account for and no stream to point at. Its duration is the time runStep took to find
|
|
105
|
+
* that out, measured like any other, so that a stop which was slow to be noticed shows as slow.
|
|
106
|
+
*/
|
|
107
|
+
function stoppedBeforeStart(req, resolved, startedAt) {
|
|
108
|
+
return {
|
|
109
|
+
id: req.id,
|
|
110
|
+
shell: resolved.shell,
|
|
111
|
+
requestedShell: resolved.requested,
|
|
112
|
+
shellPath: resolved.path,
|
|
113
|
+
command: req.command,
|
|
114
|
+
exitCode: -3,
|
|
115
|
+
outcome: 'aborted',
|
|
116
|
+
durationMs: Date.now() - startedAt,
|
|
117
|
+
stdout: '',
|
|
118
|
+
stderr: '[runner] the command was not run: the operator stopped the run before it started\n',
|
|
119
|
+
truncated: false,
|
|
120
|
+
logPath: '',
|
|
121
|
+
lastOutputAgoMs: 0,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
59
124
|
/**
|
|
60
125
|
* The longest a step that is being stopped may take before its result is given anyway. The stop
|
|
61
126
|
* itself is `taskkill /T`, a grace period, then `/F` (see `processes.ts`), which with reading the
|
|
@@ -67,10 +132,21 @@ export async function runStep(req,
|
|
|
67
132
|
/**
|
|
68
133
|
* `tracker` records the shell this step starts, which is what later lets the runner tell the
|
|
69
134
|
* processes the bot started from ones the operator started by hand. See `processes.ts`.
|
|
135
|
+
*
|
|
136
|
+
* `heartbeatMs` is how often `onHeartbeat` is called while the step runs. The live log wants a
|
|
137
|
+
* beat every half minute and no more; a caller that needs to see what a beat carries during a
|
|
138
|
+
* step of a few seconds asks for a shorter one, since at the default none would come at all.
|
|
139
|
+
*
|
|
140
|
+
* `onOutput` is given every piece of output as it is decoded, all of it, including what falls
|
|
141
|
+
* past MAX_CAPTURE_CHARS and is kept on disk only. It is how a caller decides something about the
|
|
142
|
+
* whole output (an output check does) without the runner holding the whole output in memory. It
|
|
143
|
+
* is called from the stream's handler, so it has to be quick.
|
|
70
144
|
*/
|
|
71
145
|
opts = {}) {
|
|
72
|
-
|
|
73
|
-
const
|
|
146
|
+
// Every clock goes through `timerDelay`: a delay Node cannot hold would fire after 1 ms.
|
|
147
|
+
const hardTimeoutMs = timerDelay(req.hardTimeoutMs, DEFAULT_HARD_TIMEOUT_MS);
|
|
148
|
+
const idleTimeoutMs = timerDelay(req.idleTimeoutMs, DEFAULT_IDLE_TIMEOUT_MS);
|
|
149
|
+
const heartbeatMs = timerDelay(opts.heartbeatMs, HEARTBEAT_EVERY_MS);
|
|
74
150
|
const startedAt = Date.now();
|
|
75
151
|
// Every path into the shell goes through here, which is what makes a step and a post-task
|
|
76
152
|
// check incapable of disagreeing about which interpreter their commands were read by.
|
|
@@ -79,6 +155,18 @@ opts = {}) {
|
|
|
79
155
|
return unrunnable(req, choice.problem);
|
|
80
156
|
const resolved = choice.resolved;
|
|
81
157
|
await mkdir(dirname(req.logPath), { recursive: true });
|
|
158
|
+
/*
|
|
159
|
+
* A Stop that came before the step began means the step never begins.
|
|
160
|
+
*
|
|
161
|
+
* The signal is listened to only once the process exists, and 'abort' is fired once: a signal
|
|
162
|
+
* already aborted by then — Stop pressed while the previous step was ending, or during the await
|
|
163
|
+
* just above — never fires again, and the step used to run its whole course, hours under the
|
|
164
|
+
* default ceiling, with the operator's Stop on record. It is asked here, after the last await and
|
|
165
|
+
* before anything is written or started, because from this line to the listener below everything
|
|
166
|
+
* runs in one go: no abort can arrive in between and be missed.
|
|
167
|
+
*/
|
|
168
|
+
if (opts.signal?.aborted)
|
|
169
|
+
return stoppedBeforeStart(req, resolved, startedAt);
|
|
82
170
|
const log = createWriteStream(req.logPath, { flags: 'a' });
|
|
83
171
|
/*
|
|
84
172
|
* A step log that cannot be written must never take the process down with it.
|
|
@@ -90,13 +178,19 @@ opts = {}) {
|
|
|
90
178
|
*/
|
|
91
179
|
log.on('error', () => undefined);
|
|
92
180
|
log.write(`# step ${req.id} shell=${resolved.shell} exe=${resolved.path} cwd=${req.cwd}\n# ${req.command}\n`);
|
|
93
|
-
const { file, args } = invocationFor(resolved, req.command, req.scriptArgs);
|
|
181
|
+
const { file, args, windowsVerbatimArguments } = invocationFor(resolved, req.command, req.scriptArgs);
|
|
94
182
|
return await new Promise((resolve) => {
|
|
95
183
|
let stdout = '';
|
|
96
184
|
let stderr = '';
|
|
97
185
|
let bytesOut = 0;
|
|
98
186
|
let truncated = false;
|
|
99
187
|
let lastLine = '';
|
|
188
|
+
/*
|
|
189
|
+
* The end of each stream that has no newline yet. A pipe hands over whatever is there, not
|
|
190
|
+
* whole lines, so a line can arrive in two chunks; read chunk by chunk, the last line would be
|
|
191
|
+
* the second half of it, which is text the step never printed on its own.
|
|
192
|
+
*/
|
|
193
|
+
const unfinished = { out: '', err: '' };
|
|
100
194
|
let lastOutputAt = Date.now();
|
|
101
195
|
let settled = false;
|
|
102
196
|
/** Set when the shell itself would not start, which is not a failure of the command. */
|
|
@@ -109,6 +203,9 @@ opts = {}) {
|
|
|
109
203
|
// Named, not inherited: a step must not see the bot's own token or whatever else the
|
|
110
204
|
// operator's shell holds. See `stepEnv.ts`.
|
|
111
205
|
env: stepEnvironment(req.env ?? process.env, req.passEnv),
|
|
206
|
+
// True for `cmd`, whose command line `invocationFor` has already written in full: quoted again
|
|
207
|
+
// here, every double quote in a step would reach `cmd` as `\"`, which it does not understand.
|
|
208
|
+
windowsVerbatimArguments,
|
|
112
209
|
windowsHide: true,
|
|
113
210
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
114
211
|
});
|
|
@@ -144,16 +241,43 @@ opts = {}) {
|
|
|
144
241
|
lastOutputAgoMs: Date.now() - lastOutputAt,
|
|
145
242
|
});
|
|
146
243
|
};
|
|
244
|
+
/*
|
|
245
|
+
* Each stream has a decoder of its own, which holds back the first bytes of a character that a
|
|
246
|
+
* chunk ended in the middle of until the rest arrives. A pipe cuts where it cuts, not between
|
|
247
|
+
* characters, and a chunk decoded by itself turned the two halves of a '✕' or a Cyrillic letter
|
|
248
|
+
* into two replacement characters: in the captured text, the log, the last line, and the output
|
|
249
|
+
* a check is decided on, where an output-omits would then pass on output that did contain it.
|
|
250
|
+
*/
|
|
251
|
+
const decoders = { out: new StringDecoder('utf8'), err: new StringDecoder('utf8') };
|
|
147
252
|
const capture = (chunk, stream) => {
|
|
148
|
-
const text = chunk.toString('utf8');
|
|
149
253
|
lastOutputAt = Date.now();
|
|
150
254
|
bytesOut += chunk.length;
|
|
255
|
+
take(decoders[stream].write(chunk), stream);
|
|
256
|
+
};
|
|
257
|
+
const take = (text, stream) => {
|
|
258
|
+
if (!text)
|
|
259
|
+
return;
|
|
151
260
|
// Output can still arrive after the child has closed and the step has been settled: the
|
|
152
261
|
// pipes flush independently of the close event. Writing then would be a write to an
|
|
153
262
|
// ended stream, so the tail goes to the captured text and not to the file.
|
|
154
|
-
if (!settled)
|
|
155
|
-
log.write(stream === 'err' ? text
|
|
156
|
-
|
|
263
|
+
if (!settled) {
|
|
264
|
+
log.write(stream === 'err' ? markStderr(text, unfinished.err === '') : text);
|
|
265
|
+
/*
|
|
266
|
+
* The caller's listener runs inside this stream handler, where a throw would be an uncaught
|
|
267
|
+
* exception and take the API down in the middle of a run. What it was working out is the
|
|
268
|
+
* caller's to get right; the step itself goes on either way.
|
|
269
|
+
*/
|
|
270
|
+
try {
|
|
271
|
+
opts.onOutput?.(text, stream);
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
/* the listener's fault, not the step's */
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
const joined = unfinished[stream] + text;
|
|
278
|
+
const end = joined.lastIndexOf('\n');
|
|
279
|
+
unfinished[stream] = (end >= 0 ? joined.slice(end + 1) : joined).slice(-LINE_CARRIED_CHARS);
|
|
280
|
+
const trimmed = joined.trimEnd();
|
|
157
281
|
const nl = trimmed.lastIndexOf('\n');
|
|
158
282
|
if (trimmed)
|
|
159
283
|
lastLine = nl >= 0 ? trimmed.slice(nl + 1) : trimmed;
|
|
@@ -195,9 +319,18 @@ opts = {}) {
|
|
|
195
319
|
shellProblem = missingShellProblem(resolved);
|
|
196
320
|
finish('spawn-error', -2);
|
|
197
321
|
});
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
322
|
+
child.on('close', (code) => {
|
|
323
|
+
// Both streams have ended, so whatever a decoder still holds is the last of its stream: the
|
|
324
|
+
// start of a character the process never finished, given as the replacement character.
|
|
325
|
+
take(decoders.out.end(), 'out');
|
|
326
|
+
take(decoders.err.end(), 'err');
|
|
327
|
+
// While the runner is stopping the step the process exiting is the stop working, not the step
|
|
328
|
+
// completing, so the reason recorded is the runner's.
|
|
329
|
+
if (stopping)
|
|
330
|
+
finish(stopping.outcome, stopping.exitCode);
|
|
331
|
+
else
|
|
332
|
+
finish('completed', code ?? -1);
|
|
333
|
+
});
|
|
201
334
|
/*
|
|
202
335
|
* Stops the step's whole tree — `taskkill /T`, then `/F` — and gives the result only
|
|
203
336
|
* once it is down, so the next step does not start while this one still holds a port or a
|
|
@@ -236,7 +369,7 @@ opts = {}) {
|
|
|
236
369
|
bytesOut,
|
|
237
370
|
lastLine,
|
|
238
371
|
});
|
|
239
|
-
},
|
|
372
|
+
}, heartbeatMs);
|
|
240
373
|
const onAbort = () => {
|
|
241
374
|
stop('aborted', -3, 'aborted by the user');
|
|
242
375
|
};
|
package/dist/src/exec/shells.js
CHANGED
|
@@ -262,6 +262,19 @@ export function shellNote(inventory = detectShells(), fallback) {
|
|
|
262
262
|
* a security review can take at face value. The cost is on a machine set to `Restricted` or
|
|
263
263
|
* `AllSigned`: the `.ps1` shims that npm and its kin install are then refused, and the chat is told
|
|
264
264
|
* (in the contract) to call the `.cmd` form of the tool instead.
|
|
265
|
+
*
|
|
266
|
+
* `cmd` is given its command line exactly as written, which is the other thing here that is not
|
|
267
|
+
* incidental. Left to itself, Node quotes every argument by the rules the C runtime splits a command
|
|
268
|
+
* line with, and those rules escape a double quote with a backslash. `cmd` does not split its command
|
|
269
|
+
* line by them and has no such escape, so every quote in a step reached it as `\"`: `echo "hello
|
|
270
|
+
* world"` printed the backslashes, `node -e "console.log(1+1)"` handed node a string literal that
|
|
271
|
+
* printed nothing, and a quoted path such as `"C:\Program Files\nodejs\node.exe"` was looked up with
|
|
272
|
+
* the backslashes in its name and not found. So, as Node does for itself under `shell: true`, the
|
|
273
|
+
* line is passed verbatim and wrapped in one more pair of quotes: with `/s`, `cmd` removes exactly the
|
|
274
|
+
* first and the last quote on the line, which are those two, and reads everything between them as it
|
|
275
|
+
* was written. A script is run the same way, its path quoted and each argument quoted for `cmd` by
|
|
276
|
+
* `cmdArgument`, because nothing else quotes them any more. PowerShell is not touched: it reads its
|
|
277
|
+
* arguments by the C runtime's rules, and Node's quoting is the right quoting for it.
|
|
265
278
|
*/
|
|
266
279
|
export function invocationFor(resolved, command, scriptArgs) {
|
|
267
280
|
const isScript = Array.isArray(scriptArgs);
|
|
@@ -270,15 +283,37 @@ export function invocationFor(resolved, command, scriptArgs) {
|
|
|
270
283
|
case 'powershell': {
|
|
271
284
|
const base = ['-NoProfile', '-NonInteractive'];
|
|
272
285
|
return isScript
|
|
273
|
-
? { file: resolved.path, args: [...base, '-File', command, ...(scriptArgs ?? [])] }
|
|
274
|
-
: { file: resolved.path, args: [...base, '-Command', command] };
|
|
286
|
+
? { file: resolved.path, args: [...base, '-File', command, ...(scriptArgs ?? [])], windowsVerbatimArguments: false }
|
|
287
|
+
: { file: resolved.path, args: [...base, '-Command', command], windowsVerbatimArguments: false };
|
|
288
|
+
}
|
|
289
|
+
case 'cmd': {
|
|
290
|
+
const line = isScript ? [`"${command}"`, ...(scriptArgs ?? []).map(cmdArgument)].join(' ') : command;
|
|
291
|
+
return { file: resolved.path, args: ['/d', '/s', '/c', `"${line}"`], windowsVerbatimArguments: true };
|
|
275
292
|
}
|
|
276
|
-
case 'cmd':
|
|
277
|
-
return isScript
|
|
278
|
-
? { file: resolved.path, args: ['/d', '/c', command, ...(scriptArgs ?? [])] }
|
|
279
|
-
: { file: resolved.path, args: ['/d', '/s', '/c', command] };
|
|
280
293
|
}
|
|
281
294
|
}
|
|
295
|
+
/**
|
|
296
|
+
* One argument to a script run in `cmd`, written so that it arrives as the one argument it is.
|
|
297
|
+
*
|
|
298
|
+
* The line is passed verbatim, so nothing quotes this on the way and it has to be done here, for the
|
|
299
|
+
* reader that is actually there. A space would make one argument two. `&`, `|`, `<` and `>` would
|
|
300
|
+
* make the rest of it a second command or a redirection, `^` would be taken as an escape and the
|
|
301
|
+
* parentheses as a group, and a quote is the only thing that makes `cmd` read any of them as text.
|
|
302
|
+
* `,`, `;` and `=` divide a batch file's arguments the way a space does. `%` quotes an argument too: `%NAME%` is expanded inside quotes as well as outside,
|
|
303
|
+
* as it is anywhere on a `cmd` line, but what it expands to then stays inside this argument.
|
|
304
|
+
*
|
|
305
|
+
* Inside the quotes, a program built on the C runtime gets back exactly what was given. The
|
|
306
|
+
* backslashes in front of a quote are doubled, which its rules ask for, and a quote is written `""`,
|
|
307
|
+
* which it reads as one quote. `\"` would not do: `cmd` counts every quote, with or without a
|
|
308
|
+
* backslash, so it would take that one as the end of the quoted text and run whatever came after it.
|
|
309
|
+
* A batch file sees an inner quote and a closing backslash doubled, which is as near as `cmd` lets
|
|
310
|
+
* anything come.
|
|
311
|
+
*/
|
|
312
|
+
function cmdArgument(arg) {
|
|
313
|
+
if (arg !== '' && !/[\s"&|<>^(),;=%]/.test(arg))
|
|
314
|
+
return arg;
|
|
315
|
+
return `"${arg.replace(/(\\*)"/g, '$1$1""').replace(/(\\+)$/, '$1$1')}"`;
|
|
316
|
+
}
|
|
282
317
|
/**
|
|
283
318
|
* A shell that was found at detection time and would not start when it was wanted.
|
|
284
319
|
*
|
package/dist/src/exec/workDir.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* nothing at all.
|
|
13
13
|
*
|
|
14
14
|
* A session already knows its project — `vcs.repoDir` says where the branches go and
|
|
15
|
-
* `
|
|
15
|
+
* `projectDir` where the work is — so that is where its commands run. The configured
|
|
16
16
|
* `execution.cwd` is the fallback for a session with no project, and this project's own
|
|
17
17
|
* checkout is never a fallback: a session that would land here by default does not run, and
|
|
18
18
|
* says why. Landing here on purpose — the operator pointed the session at the bot itself — is
|
|
@@ -63,11 +63,11 @@ export function isOwnCheckout(dir, own = botRootDir()) {
|
|
|
63
63
|
*/
|
|
64
64
|
export function workingDirFor(session, configCwd, own = botRootDir()) {
|
|
65
65
|
const repo = (session.vcs?.repoDir ?? '').trim();
|
|
66
|
-
const
|
|
66
|
+
const project = (session.projectDir ?? '').trim();
|
|
67
67
|
if (repo)
|
|
68
68
|
return { cwd: resolve(repo), source: 'repository', ownCheckout: isInside(repo, own) };
|
|
69
|
-
if (
|
|
70
|
-
return { cwd: resolve(
|
|
69
|
+
if (project)
|
|
70
|
+
return { cwd: resolve(project), source: 'project', ownCheckout: isInside(project, own) };
|
|
71
71
|
if (isInside(configCwd, own)) {
|
|
72
72
|
return {
|
|
73
73
|
cwd: resolve(configCwd),
|
|
@@ -44,6 +44,7 @@ export async function validateDerivedChecks(findings, opts) {
|
|
|
44
44
|
defaultShell: opts.defaultShell,
|
|
45
45
|
tracker: opts.tracker,
|
|
46
46
|
passEnv: opts.passEnv,
|
|
47
|
+
redactPatterns: opts.redactPatterns,
|
|
47
48
|
});
|
|
48
49
|
if (outcome.refusedBeforeRunning)
|
|
49
50
|
out.blocked.push({ finding, check, outcome });
|