copilot-operator 0.1.12 → 0.1.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +9 -14
  2. package/dist/src/api/operator.controller.js +31 -42
  3. package/dist/src/api/operator.service.js +692 -364
  4. package/dist/src/api/saveToDesktop.js +1 -1
  5. package/dist/src/api/security.js +40 -10
  6. package/dist/src/api/server.js +75 -12
  7. package/dist/src/api/settings.js +156 -15
  8. package/dist/src/cli.js +108 -97
  9. package/dist/src/config/handWritten.js +36 -0
  10. package/dist/src/config/layout.js +13 -4
  11. package/dist/src/config/lockedPolicy.js +56 -22
  12. package/dist/src/config/schema.js +13 -84
  13. package/dist/src/context/desktopDir.js +44 -0
  14. package/dist/src/exec/authorizer.js +9 -1
  15. package/dist/src/exec/checks.js +240 -23
  16. package/dist/src/exec/dangerous.js +29 -5
  17. package/dist/src/exec/network.js +105 -28
  18. package/dist/src/exec/redaction.js +52 -9
  19. package/dist/src/exec/reportFile.js +4 -5
  20. package/dist/src/exec/runner.js +146 -13
  21. package/dist/src/exec/shells.js +41 -6
  22. package/dist/src/exec/workDir.js +4 -4
  23. package/dist/src/orchestrator/derivedChecks.js +1 -0
  24. package/dist/src/orchestrator/review.js +25 -3
  25. package/dist/src/orchestrator/taskRunner.js +482 -150
  26. package/dist/src/plan/brief.js +56 -44
  27. package/dist/src/plan/importPlan.js +18 -7
  28. package/dist/src/plan/schema.js +90 -31
  29. package/dist/src/plan/script.js +79 -90
  30. package/dist/src/plan/systemGuide.js +39 -182
  31. package/dist/src/protocol/parser.js +5 -1
  32. package/dist/src/session/debugExport.js +1 -1
  33. package/dist/src/session/exportRecord.js +1 -1
  34. package/dist/src/session/exports.js +52 -9
  35. package/dist/src/session/store.js +215 -80
  36. package/dist/src/vcs/git.js +77 -2
  37. package/dist/src/vcs/scope.js +29 -6
  38. package/dist/src/vcs/taskVcs.js +149 -23
  39. package/dist/web/404.html +2 -2
  40. package/dist/web/_next/static/chunks/36-94660f1cde2da529.js +1 -0
  41. package/dist/web/_next/static/chunks/549-9548d9a132c8c7d4.js +1 -0
  42. package/dist/web/_next/static/chunks/867-8b95ae984befc0e2.js +1 -0
  43. package/dist/web/_next/static/chunks/app/defaults/page-71a43ed69134b48d.js +1 -0
  44. package/dist/web/_next/static/chunks/app/history/page-d7314ac22d47a3ab.js +1 -0
  45. package/dist/web/_next/static/chunks/app/sessions/view/page-886f5fc99e146408.js +1 -0
  46. package/dist/web/_next/static/css/{a0642fdb8adf9edb.css → e2a2c9eb3c226313.css} +1 -1
  47. package/dist/web/appearance.html +2 -2
  48. package/dist/web/appearance.txt +12 -12
  49. package/dist/web/defaults.html +2 -2
  50. package/dist/web/defaults.txt +12 -12
  51. package/dist/web/history.html +2 -2
  52. package/dist/web/history.txt +12 -12
  53. package/dist/web/import.html +2 -2
  54. package/dist/web/import.txt +12 -12
  55. package/dist/web/index.html +2 -2
  56. package/dist/web/index.txt +12 -12
  57. package/dist/web/level1.html +2 -2
  58. package/dist/web/level1.txt +12 -12
  59. package/dist/web/presets.html +2 -2
  60. package/dist/web/presets.txt +12 -12
  61. package/dist/web/sessions/view.html +2 -2
  62. package/dist/web/sessions/view.txt +12 -12
  63. package/dist/web/system.html +2 -2
  64. package/dist/web/system.txt +12 -12
  65. package/package.json +18 -10
  66. package/prompts/level1.md +2 -1
  67. package/prompts/organisation.bg.md +0 -1
  68. package/prompts/organisation.en.md +0 -1
  69. package/dist/src/context/contextFiles.js +0 -295
  70. package/dist/src/context/desktopMirror.js +0 -135
  71. package/dist/src/context/projectMirror.js +0 -432
  72. package/dist/web/_next/static/chunks/295-04151ded0741d18d.js +0 -1
  73. package/dist/web/_next/static/chunks/36-c3dd06c6a459c114.js +0 -1
  74. package/dist/web/_next/static/chunks/549-a5a59a94f92bb3eb.js +0 -1
  75. package/dist/web/_next/static/chunks/867-b64c436f5b25f39e.js +0 -1
  76. package/dist/web/_next/static/chunks/app/defaults/page-5f180a88398be4c9.js +0 -1
  77. package/dist/web/_next/static/chunks/app/history/page-854e3dde672e38a2.js +0 -1
  78. package/dist/web/_next/static/chunks/app/sessions/view/page-cc99401da5173b36.js +0 -1
  79. /package/dist/web/_next/static/{bcBE4tY_aNh6kzLzN1luP → IwNZiBrEMAcXEUFj2miiU}/_buildManifest.js +0 -0
  80. /package/dist/web/_next/static/{bcBE4tY_aNh6kzLzN1luP → IwNZiBrEMAcXEUFj2miiU}/_ssgManifest.js +0 -0
@@ -33,9 +33,11 @@
33
33
  * proves the server it started actually answers, and nothing comes back that the project did not
34
34
  * itself produce. Holding every health check for approval would turn an unattended run into one
35
35
  * that waits all night on its first test. The exception is narrow on purpose: every `http(s)://`
36
- * address in the command must be loopback, there must be at least one, and nothing else in the
37
- * command may look like a host — a bare `example.com` or `10.0.0.5` beside a localhost URL still
38
- * holds the step, because `iwr` accepts an address without a scheme.
36
+ * address in the command must be loopback, read the way the program will read it (the host after
37
+ * any `user@`, not the first word after `://`), there must be at least one, no option may send the
38
+ * request somewhere else, and nothing else in the command may look like a host — a bare
39
+ * `example.com` or `10.0.0.5` beside a localhost URL still holds the step, because `iwr` accepts an
40
+ * address without a scheme. Where the gate cannot read an address for certain, it is not local.
39
41
  *
40
42
  * The same text is screened whole, not command by command. That is deliberate and it is what
41
43
  * closes the obvious way round: writing `Invoke-WebRequest …` into a `.ps1` with `Set-Content` and
@@ -139,39 +141,114 @@ const FILE_EXTENSIONS = new Set([
139
141
  'png', 'jpg', 'jpeg', 'gif', 'svg', 'ico', 'webp', 'pdf', 'out', 'tmp', 'bak', 'lock',
140
142
  'exe', 'cmd', 'bat', 'dll', 'tar', 'gz', 'tgz',
141
143
  ]);
144
+ /**
145
+ * The host a URL names, read from the text after its `://`, or null when it cannot be read for
146
+ * certain.
147
+ *
148
+ * Everything before the last `@` of an authority is userinfo, not the host: curl and
149
+ * Invoke-WebRequest both fetch `http://localhost:1@evil.example.com/tool.exe` from
150
+ * evil.example.com. Reading the host up to the first `:` called that "localhost", and the one
151
+ * download the hold exists for — an executable from somewhere else — went through unheld, in a
152
+ * check's command with nobody asked (found on 2026-10-01). So the authority is read whole, up to the
153
+ * first `/`, `?`, `#` or white space, and it is only this machine when it is nothing but a host and
154
+ * a port. An `@` anywhere in it, a `\`, a percent-encoded name, or a `$`, a backtick or a caret the
155
+ * shell would expand or escape at run time: the parsers disagree about each of those, or the gate
156
+ * cannot see what the shell will make of it, so none is read as loopback.
157
+ *
158
+ * Quotes are taken out before reading, because to the shell they are not the end of a word:
159
+ * `"http://localhost"@evil.example.com/` is one argument, and it is the URL with userinfo. After the
160
+ * host and port only an end of the argument may follow — `)`, `}`, `]`, `;`, `,`, `|`, `&`, `<`,
161
+ * `>` or nothing — and never another character of a name, so `localhost.evil.example.com`, whose
162
+ * full-width dot a URL parser turns into a plain one, is not "localhost" either.
163
+ */
164
+ function urlHost(afterScheme) {
165
+ const authority = (/^[^\/?#\s]*/.exec(afterScheme)?.[0] ?? '').replace(/["']/g, '');
166
+ if (/[@\\%$`^]/.test(authority))
167
+ return null;
168
+ return /^(\[[0-9a-f:.]+\]|[a-z0-9._~-]+)(?::\d*)?(?=$|[)}\];,|&<>])/i.exec(authority)?.[1] ?? null;
169
+ }
170
+ /**
171
+ * Whether one piece of a token reads as a host or an IP address that is not this machine. A name
172
+ * with a known file extension (`health.json`), or a .NET namespace (`System.Net`), is not a host.
173
+ */
174
+ function namesAnotherHost(bare) {
175
+ if (!bare)
176
+ return false;
177
+ if (/^\d{1,3}(\.\d{1,3}){3}$/.test(bare))
178
+ return !isLoopback(bare);
179
+ const labels = bare.split('.');
180
+ if (labels.length < 2 || labels.some((l) => l === ''))
181
+ return false;
182
+ const last = labels[labels.length - 1].toLowerCase();
183
+ if (!/^[a-z]{2,}$/.test(last) || FILE_EXTENSIONS.has(last))
184
+ return false;
185
+ if (/^(system|microsoft|net)$/i.test(labels[0]))
186
+ return false;
187
+ return !isLoopback(bare);
188
+ }
189
+ /**
190
+ * Options that send a request somewhere its URL does not say, or take more of them from a file the
191
+ * line does not show: curl's `--resolve` and `--connect-to` (the URL still says `localhost`, the
192
+ * connection goes to the address the option names), curl's `-K`/`--config`, and wget's
193
+ * `-i`/`--input-file` and `-e`/`--execute`. With any of them the URLs on the line are not the whole
194
+ * story, so the line is not provably local. The options are matched in their own case, as the
195
+ * programs read them: curl's `-k` (skip the certificate check) is an ordinary flag of a local health
196
+ * check and `-K` is not, and PowerShell's `-ErrorAction` after a `wget` alias is not wget's `-e`.
197
+ *
198
+ * And each only after its own program in the same statement, since only that program reads it.
199
+ * `--config` is an ordinary flag of node servers, vite, jest, playwright and webpack, and matched
200
+ * anywhere on the line, `node server.js --config dev.json; Invoke-WebRequest http://localhost:3000/health`
201
+ * was held as if curl had been told to read a file, and refused outright as a check's command. A
202
+ * quote may come before the option, as in `-ArgumentList '--config','x'`, where the program still
203
+ * reads it.
204
+ */
205
+ const REDIRECTING_OPTIONS = [
206
+ /\bcurl(?:\.exe)?\b[^|;\n]*[\s'"](?:--(?:resolve|connect-to|config)(?=[\s='",]|$)|-K(?![a-zA-Z]))/,
207
+ /\bwget2?(?:\.exe)?\b[^|;\n]*[\s'"](?:-[ie]|--(?:input-file|execute)(?=[\s='",]|$))/,
208
+ ];
142
209
  /**
143
210
  * Whether every place this command could reach is this machine. True only when it names at least
144
- * one `http(s)://` address, all of them loopback, and no other token that reads as a host or an IP.
211
+ * one `scheme://` address, every one of them a loopback host read for certain (`urlHost`), no option
212
+ * sends the request elsewhere, and no other token reads as a host or an IP.
145
213
  */
146
214
  export function onlyLoopbackTargets(command) {
147
- const urls = [...command.matchAll(/\b[a-z][a-z0-9+.-]*:\/\/(\[[^\]]+\]|[^\/\s:'"`;|)]+)/gi)];
148
- if (urls.length === 0)
215
+ const schemes = [...command.matchAll(/\b[a-z][a-z0-9+.-]*:\/\//gi)];
216
+ if (schemes.length === 0)
149
217
  return false;
150
- if (!urls.every((m) => isLoopback(m[1])))
151
- return false;
152
- const rest = command.replace(/\b[a-z][a-z0-9+.-]*:\/\/\S+/gi, ' ');
153
- for (const token of rest.split(/[\s'"`=,;|(){}]+/)) {
154
- const bare = token.replace(/^-+/, '').replace(/[:\/].*$/, '');
155
- if (!bare)
156
- continue;
157
- if (/^\d{1,3}(\.\d{1,3}){3}$/.test(bare) && !isLoopback(bare))
218
+ for (const m of schemes) {
219
+ const host = urlHost(command.slice(m.index + m[0].length));
220
+ if (host === null || !isLoopback(host))
158
221
  return false;
159
- const labels = bare.split('.');
160
- if (labels.length < 2 || labels.some((l) => l === ''))
161
- continue;
162
- const last = labels[labels.length - 1].toLowerCase();
163
- if (!/^[a-z]{2,}$/.test(last))
164
- continue;
165
- if (FILE_EXTENSIONS.has(last))
166
- continue;
167
- // PowerShell member access on a variable or a type (`$r.StatusCode`, `System.Net`) is not a host.
222
+ }
223
+ if (REDIRECTING_OPTIONS.some((option) => option.test(command)))
224
+ return false;
225
+ /*
226
+ * The rest of the line, with each URL taken out up to where the shell ends it — white space, a
227
+ * quote, `;`, `|`, `&`, a bracket or a comma — so that a second command glued on after a path
228
+ * (`…/x;iwr('evil.example')`) is still read.
229
+ */
230
+ const rest = command.replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s'"`;|&(){}<>,]*/gi, ' ');
231
+ for (const token of rest.split(/[\s'"`=,;|&(){}<>]+/)) {
232
+ // A bracketed IPv6 address anywhere in the token: a proxy (`-x [2001:db8::1]:8080`) or an address given to curl.
233
+ for (const v6 of token.matchAll(/\[([0-9a-f:.]*:[0-9a-f:.]*)\]/gi))
234
+ if (!isLoopback(v6[1]))
235
+ return false;
236
+ // PowerShell member access on a variable or a type (`$r.StatusCode`, `[System.Net.Dns]`) is not a host.
168
237
  if (token.startsWith('$') || token.startsWith('['))
169
238
  continue;
170
- if (/^(system|microsoft|net)$/i.test(labels[0]))
171
- continue;
172
- if (isLoopback(bare))
173
- continue;
174
- return false;
239
+ /*
240
+ * Every piece between colons and `@`s, not only the first: `localhost:80:93.184.216.34` names an
241
+ * address after its second colon, `bob@evil.example` a host after its `@`, and either reroutes a
242
+ * request whose URL said localhost. Within a piece, only what comes before a `/` or a `\` is a
243
+ * candidate, as in `example.com/x`; a path's folders and its file are not hosts. Cut at `/` alone,
244
+ * a Windows path was not: `C:\work\Shop.Web` splits at its drive's colon into `C` and
245
+ * `\work\Shop.Web`, whose last label `Web` read as a top-level domain, and a localhost health
246
+ * check that saved its answer under a drive was held, or refused as a check's command.
247
+ */
248
+ for (const piece of token.replace(/^-+/, '').split(/[:@]+/)) {
249
+ if (namesAnotherHost(piece.replace(/[\/\\].*$/, '')))
250
+ return false;
251
+ }
175
252
  }
176
253
  return true;
177
254
  }
@@ -71,6 +71,26 @@ export function findSecrets(text) {
71
71
  }
72
72
  return hits;
73
73
  }
74
+ /**
75
+ * One of the operator's own patterns, as a regular expression, or null when it is not one. Such a
76
+ * pattern is applied as a plain string instead, as it always was, so a typo in one cannot switch
77
+ * redaction off.
78
+ */
79
+ function operatorPattern(p) {
80
+ try {
81
+ return new RegExp(p, 'gi');
82
+ }
83
+ catch {
84
+ return null;
85
+ }
86
+ }
87
+ /** The text with every built-in shape replaced, before the operator's own patterns are tried. */
88
+ function withoutShapes(text) {
89
+ let out = text;
90
+ for (const s of SHAPES)
91
+ out = out.replace(s.pattern, s.replace);
92
+ return out;
93
+ }
74
94
  /**
75
95
  * The text with every secret-shaped string replaced, then the operator's own patterns.
76
96
  *
@@ -78,16 +98,39 @@ export function findSecrets(text) {
78
98
  * plain string, as it always was, so a typo in one cannot switch redaction off.
79
99
  */
80
100
  export function redactSecrets(text, extraPatterns = []) {
81
- let out = text;
82
- for (const s of SHAPES)
83
- out = out.replace(s.pattern, s.replace);
101
+ let out = withoutShapes(text);
84
102
  for (const p of extraPatterns) {
85
- try {
86
- out = out.replace(new RegExp(p, 'gi'), '[REDACTED]');
87
- }
88
- catch {
89
- out = out.split(p).join('[REDACTED]');
90
- }
103
+ const re = operatorPattern(p);
104
+ out = re ? out.replace(re, '[REDACTED]') : out.split(p).join('[REDACTED]');
91
105
  }
92
106
  return out;
93
107
  }
108
+ /**
109
+ * What `redactSecrets` takes out of a text, by name: the built-in shapes as `findSecrets` names
110
+ * them, and each of the operator's own patterns under the setting it comes from.
111
+ *
112
+ * `findSecrets` alone could not see the operator's patterns, so a report whose only secret was one
113
+ * of theirs — a company's ticket or key format, which no built-in shape knows — was redacted without
114
+ * a word, and the event that tells the operator something was taken out never came. Each pattern is
115
+ * counted on the text as `redactSecrets` reaches it, after the shapes, so what a shape already took
116
+ * is not counted a second time under the operator's name.
117
+ */
118
+ export function findRedactions(text, extraPatterns = []) {
119
+ const hits = findSecrets(text);
120
+ let out = withoutShapes(text);
121
+ for (const p of extraPatterns) {
122
+ const re = operatorPattern(p);
123
+ const count = re ? [...out.matchAll(re)].length : p ? out.split(p).length - 1 : 0;
124
+ if (count > 0)
125
+ hits.push({ name: `report.redactPatterns "${p}"`, count });
126
+ out = re ? out.replace(re, '[REDACTED]') : out.split(p).join('[REDACTED]');
127
+ }
128
+ return hits;
129
+ }
130
+ /** Several counts as one, summed by name, in the order the names were first met. */
131
+ export function mergeRedactions(lists) {
132
+ const byName = new Map();
133
+ for (const hit of lists.flat())
134
+ byName.set(hit.name, (byName.get(hit.name) ?? 0) + hit.count);
135
+ return [...byName.entries()].map(([name, count]) => ({ name, count }));
136
+ }
@@ -9,7 +9,7 @@
9
9
  import { mkdir, writeFile } from 'node:fs/promises';
10
10
  import { join } from 'node:path';
11
11
  import { silentFailure } from '../protocol/reporter.js';
12
- import { findSecrets, redactSecrets } from './redaction.js';
12
+ import { findRedactions, mergeRedactions, redactSecrets } from './redaction.js';
13
13
  /**
14
14
  * Removes terminal colour codes.
15
15
  *
@@ -119,10 +119,9 @@ export async function writeReport(results, opts) {
119
119
  const raw = results.map((r) => sectionFor(r, opts.maxOutputChars));
120
120
  // Counted before redaction so the event can say what kind of thing was taken out — the
121
121
  // operator on a work machine wants to know a token was printed, not only that it was hidden.
122
- const byShape = new Map();
123
- for (const hit of raw.flatMap(findSecrets))
124
- byShape.set(hit.name, (byShape.get(hit.name) ?? 0) + hit.count);
125
- const redactions = [...byShape.entries()].map(([name, count]) => ({ name, count }));
122
+ // Their own patterns are counted too: a report whose only secret matched one of them was
123
+ // redacted without the event ever saying so.
124
+ const redactions = mergeRedactions(raw.map((s) => findRedactions(s, opts.redactPatterns)));
126
125
  const sections = raw.map((s) => redact(s, opts.redactPatterns));
127
126
  const chunks = [[]];
128
127
  let size = Buffer.byteLength(header) + Buffer.byteLength(footer);
@@ -23,14 +23,54 @@ import { spawn } from 'node:child_process';
23
23
  import { createWriteStream, existsSync } from 'node:fs';
24
24
  import { mkdir } from 'node:fs/promises';
25
25
  import { dirname } from 'node:path';
26
+ import { StringDecoder } from 'node:string_decoder';
26
27
  import { effectiveShell, invocationFor, missingShellProblem, resolveShell } from './shells.js';
27
28
  import { stopTree } from './processes.js';
28
29
  import { stepEnvironment } from './stepEnv.js';
29
30
  const DEFAULT_HARD_TIMEOUT_MS = 4 * 60 * 60 * 1000;
30
31
  const DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1000;
32
+ /** How often a running step says how it is doing, unless the caller asks for another pace. */
31
33
  const HEARTBEAT_EVERY_MS = 30_000;
32
- /** Cap on what is held in memory and echoed into the report body. */
33
- const MAX_CAPTURE_CHARS = 200_000;
34
+ /**
35
+ * Cap on what is held in memory and echoed into the report body, per stream.
36
+ *
37
+ * Exported because whatever decides something from the captured text has to know where it stops:
38
+ * past it the text is not the whole output, and an answer read from it may not be the true one.
39
+ */
40
+ export const MAX_CAPTURE_CHARS = 200_000;
41
+ /**
42
+ * How much of a line that has not ended yet is carried over to the next chunk, so the last line a
43
+ * heartbeat shows is a whole line. Bounded, because a program can print a megabyte with no newline.
44
+ */
45
+ const LINE_CARRIED_CHARS = 2_000;
46
+ /** The longest delay a Node timer can hold: 2^31-1 ms, about 24.8 days. */
47
+ const LONGEST_TIMER_MS = 2_147_483_647;
48
+ /**
49
+ * A delay a timer will really wait.
50
+ *
51
+ * Node does not refuse a delay it cannot hold. Above 2^31-1 ms, Infinity included, and below 1 ms
52
+ * or not a number at all, it sets the timer to 1 ms and prints a warning nobody reads. For the
53
+ * heartbeat that is a beat every millisecond; for the hard clock it is a step killed as it starts,
54
+ * which is what a step allowed a month would get. So a delay too long for a timer becomes the
55
+ * longest one there is, and one that is not a delay at all becomes the default.
56
+ */
57
+ function timerDelay(ms, fallback) {
58
+ if (ms === undefined || !(ms >= 1))
59
+ return fallback;
60
+ return Math.min(ms, LONGEST_TIMER_MS);
61
+ }
62
+ /**
63
+ * A piece of stderr as the log shows it: every line marked once, at its start.
64
+ *
65
+ * It used to be marked with `^` in multiline mode, which also counts a `\r` as the end of a line.
66
+ * On Windows each line came out as `[stderr] text\r[stderr] \n`, a piece that ended a line put a
67
+ * mark at its very end, ahead of whatever was written next, and a piece that went on with a line
68
+ * already begun put another in the middle of it. A pipe hands over pieces, not lines, so whether a
69
+ * piece begins a line is known only from the one before it, and the caller says.
70
+ */
71
+ function markStderr(text, atLineStart) {
72
+ return `${atLineStart ? '[stderr] ' : ''}${text.replace(/\n(?=[^])/g, '\n[stderr] ')}`;
73
+ }
34
74
  /**
35
75
  * The result of a step that never started because the machine has no shell for it.
36
76
  *
@@ -56,6 +96,31 @@ function unrunnable(req, problem) {
56
96
  lastOutputAgoMs: 0,
57
97
  };
58
98
  }
99
+ /**
100
+ * The result of a step the operator stopped before it began.
101
+ *
102
+ * Shaped like a stop in the middle of a run, because that is what the operator asked for and what
103
+ * whoever reads the result has to act on, but with nothing started and nothing logged: there is no
104
+ * process to account for and no stream to point at. Its duration is the time runStep took to find
105
+ * that out, measured like any other, so that a stop which was slow to be noticed shows as slow.
106
+ */
107
+ function stoppedBeforeStart(req, resolved, startedAt) {
108
+ return {
109
+ id: req.id,
110
+ shell: resolved.shell,
111
+ requestedShell: resolved.requested,
112
+ shellPath: resolved.path,
113
+ command: req.command,
114
+ exitCode: -3,
115
+ outcome: 'aborted',
116
+ durationMs: Date.now() - startedAt,
117
+ stdout: '',
118
+ stderr: '[runner] the command was not run: the operator stopped the run before it started\n',
119
+ truncated: false,
120
+ logPath: '',
121
+ lastOutputAgoMs: 0,
122
+ };
123
+ }
59
124
  /**
60
125
  * The longest a step that is being stopped may take before its result is given anyway. The stop
61
126
  * itself is `taskkill /T`, a grace period, then `/F` (see `processes.ts`), which with reading the
@@ -67,10 +132,21 @@ export async function runStep(req,
67
132
  /**
68
133
  * `tracker` records the shell this step starts, which is what later lets the runner tell the
69
134
  * processes the bot started from ones the operator started by hand. See `processes.ts`.
135
+ *
136
+ * `heartbeatMs` is how often `onHeartbeat` is called while the step runs. The live log wants a
137
+ * beat every half minute and no more; a caller that needs to see what a beat carries during a
138
+ * step of a few seconds asks for a shorter one, since at the default none would come at all.
139
+ *
140
+ * `onOutput` is given every piece of output as it is decoded, all of it, including what falls
141
+ * past MAX_CAPTURE_CHARS and is kept on disk only. It is how a caller decides something about the
142
+ * whole output (an output check does) without the runner holding the whole output in memory. It
143
+ * is called from the stream's handler, so it has to be quick.
70
144
  */
71
145
  opts = {}) {
72
- const hardTimeoutMs = req.hardTimeoutMs ?? DEFAULT_HARD_TIMEOUT_MS;
73
- const idleTimeoutMs = req.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
146
+ // Every clock goes through `timerDelay`: a delay Node cannot hold would fire after 1 ms.
147
+ const hardTimeoutMs = timerDelay(req.hardTimeoutMs, DEFAULT_HARD_TIMEOUT_MS);
148
+ const idleTimeoutMs = timerDelay(req.idleTimeoutMs, DEFAULT_IDLE_TIMEOUT_MS);
149
+ const heartbeatMs = timerDelay(opts.heartbeatMs, HEARTBEAT_EVERY_MS);
74
150
  const startedAt = Date.now();
75
151
  // Every path into the shell goes through here, which is what makes a step and a post-task
76
152
  // check incapable of disagreeing about which interpreter their commands were read by.
@@ -79,6 +155,18 @@ opts = {}) {
79
155
  return unrunnable(req, choice.problem);
80
156
  const resolved = choice.resolved;
81
157
  await mkdir(dirname(req.logPath), { recursive: true });
158
+ /*
159
+ * A Stop that came before the step began means the step never begins.
160
+ *
161
+ * The signal is listened to only once the process exists, and 'abort' is fired once: a signal
162
+ * already aborted by then — Stop pressed while the previous step was ending, or during the await
163
+ * just above — never fires again, and the step used to run its whole course, hours under the
164
+ * default ceiling, with the operator's Stop on record. It is asked here, after the last await and
165
+ * before anything is written or started, because from this line to the listener below everything
166
+ * runs in one go: no abort can arrive in between and be missed.
167
+ */
168
+ if (opts.signal?.aborted)
169
+ return stoppedBeforeStart(req, resolved, startedAt);
82
170
  const log = createWriteStream(req.logPath, { flags: 'a' });
83
171
  /*
84
172
  * A step log that cannot be written must never take the process down with it.
@@ -90,13 +178,19 @@ opts = {}) {
90
178
  */
91
179
  log.on('error', () => undefined);
92
180
  log.write(`# step ${req.id} shell=${resolved.shell} exe=${resolved.path} cwd=${req.cwd}\n# ${req.command}\n`);
93
- const { file, args } = invocationFor(resolved, req.command, req.scriptArgs);
181
+ const { file, args, windowsVerbatimArguments } = invocationFor(resolved, req.command, req.scriptArgs);
94
182
  return await new Promise((resolve) => {
95
183
  let stdout = '';
96
184
  let stderr = '';
97
185
  let bytesOut = 0;
98
186
  let truncated = false;
99
187
  let lastLine = '';
188
+ /*
189
+ * The end of each stream that has no newline yet. A pipe hands over whatever is there, not
190
+ * whole lines, so a line can arrive in two chunks; read chunk by chunk, the last line would be
191
+ * the second half of it, which is text the step never printed on its own.
192
+ */
193
+ const unfinished = { out: '', err: '' };
100
194
  let lastOutputAt = Date.now();
101
195
  let settled = false;
102
196
  /** Set when the shell itself would not start, which is not a failure of the command. */
@@ -109,6 +203,9 @@ opts = {}) {
109
203
  // Named, not inherited: a step must not see the bot's own token or whatever else the
110
204
  // operator's shell holds. See `stepEnv.ts`.
111
205
  env: stepEnvironment(req.env ?? process.env, req.passEnv),
206
+ // True for `cmd`, whose command line `invocationFor` has already written in full: quoted again
207
+ // here, every double quote in a step would reach `cmd` as `\"`, which it does not understand.
208
+ windowsVerbatimArguments,
112
209
  windowsHide: true,
113
210
  stdio: ['ignore', 'pipe', 'pipe'],
114
211
  });
@@ -144,16 +241,43 @@ opts = {}) {
144
241
  lastOutputAgoMs: Date.now() - lastOutputAt,
145
242
  });
146
243
  };
244
+ /*
245
+ * Each stream has a decoder of its own, which holds back the first bytes of a character that a
246
+ * chunk ended in the middle of until the rest arrives. A pipe cuts where it cuts, not between
247
+ * characters, and a chunk decoded by itself turned the two halves of a '✕' or a Cyrillic letter
248
+ * into two replacement characters: in the captured text, the log, the last line, and the output
249
+ * a check is decided on, where an output-omits would then pass on output that did contain it.
250
+ */
251
+ const decoders = { out: new StringDecoder('utf8'), err: new StringDecoder('utf8') };
147
252
  const capture = (chunk, stream) => {
148
- const text = chunk.toString('utf8');
149
253
  lastOutputAt = Date.now();
150
254
  bytesOut += chunk.length;
255
+ take(decoders[stream].write(chunk), stream);
256
+ };
257
+ const take = (text, stream) => {
258
+ if (!text)
259
+ return;
151
260
  // Output can still arrive after the child has closed and the step has been settled: the
152
261
  // pipes flush independently of the close event. Writing then would be a write to an
153
262
  // ended stream, so the tail goes to the captured text and not to the file.
154
- if (!settled)
155
- log.write(stream === 'err' ? text.replace(/^/gm, '[stderr] ') : text);
156
- const trimmed = text.trimEnd();
263
+ if (!settled) {
264
+ log.write(stream === 'err' ? markStderr(text, unfinished.err === '') : text);
265
+ /*
266
+ * The caller's listener runs inside this stream handler, where a throw would be an uncaught
267
+ * exception and take the API down in the middle of a run. What it was working out is the
268
+ * caller's to get right; the step itself goes on either way.
269
+ */
270
+ try {
271
+ opts.onOutput?.(text, stream);
272
+ }
273
+ catch {
274
+ /* the listener's fault, not the step's */
275
+ }
276
+ }
277
+ const joined = unfinished[stream] + text;
278
+ const end = joined.lastIndexOf('\n');
279
+ unfinished[stream] = (end >= 0 ? joined.slice(end + 1) : joined).slice(-LINE_CARRIED_CHARS);
280
+ const trimmed = joined.trimEnd();
157
281
  const nl = trimmed.lastIndexOf('\n');
158
282
  if (trimmed)
159
283
  lastLine = nl >= 0 ? trimmed.slice(nl + 1) : trimmed;
@@ -195,9 +319,18 @@ opts = {}) {
195
319
  shellProblem = missingShellProblem(resolved);
196
320
  finish('spawn-error', -2);
197
321
  });
198
- // While the runner is stopping the step the process exiting is the stop working, not the step
199
- // completing, so the reason recorded is the runner's.
200
- child.on('close', (code) => (stopping ? finish(stopping.outcome, stopping.exitCode) : finish('completed', code ?? -1)));
322
+ child.on('close', (code) => {
323
+ // Both streams have ended, so whatever a decoder still holds is the last of its stream: the
324
+ // start of a character the process never finished, given as the replacement character.
325
+ take(decoders.out.end(), 'out');
326
+ take(decoders.err.end(), 'err');
327
+ // While the runner is stopping the step the process exiting is the stop working, not the step
328
+ // completing, so the reason recorded is the runner's.
329
+ if (stopping)
330
+ finish(stopping.outcome, stopping.exitCode);
331
+ else
332
+ finish('completed', code ?? -1);
333
+ });
201
334
  /*
202
335
  * Stops the step's whole tree — `taskkill /T`, then `/F` — and gives the result only
203
336
  * once it is down, so the next step does not start while this one still holds a port or a
@@ -236,7 +369,7 @@ opts = {}) {
236
369
  bytesOut,
237
370
  lastLine,
238
371
  });
239
- }, HEARTBEAT_EVERY_MS);
372
+ }, heartbeatMs);
240
373
  const onAbort = () => {
241
374
  stop('aborted', -3, 'aborted by the user');
242
375
  };
@@ -262,6 +262,19 @@ export function shellNote(inventory = detectShells(), fallback) {
262
262
  * a security review can take at face value. The cost is on a machine set to `Restricted` or
263
263
  * `AllSigned`: the `.ps1` shims that npm and its kin install are then refused, and the chat is told
264
264
  * (in the contract) to call the `.cmd` form of the tool instead.
265
+ *
266
+ * `cmd` is given its command line exactly as written, which is the other thing here that is not
267
+ * incidental. Left to itself, Node quotes every argument by the rules the C runtime splits a command
268
+ * line with, and those rules escape a double quote with a backslash. `cmd` does not split its command
269
+ * line by them and has no such escape, so every quote in a step reached it as `\"`: `echo "hello
270
+ * world"` printed the backslashes, `node -e "console.log(1+1)"` handed node a string literal that
271
+ * printed nothing, and a quoted path such as `"C:\Program Files\nodejs\node.exe"` was looked up with
272
+ * the backslashes in its name and not found. So, as Node does for itself under `shell: true`, the
273
+ * line is passed verbatim and wrapped in one more pair of quotes: with `/s`, `cmd` removes exactly the
274
+ * first and the last quote on the line, which are those two, and reads everything between them as it
275
+ * was written. A script is run the same way, its path quoted and each argument quoted for `cmd` by
276
+ * `cmdArgument`, because nothing else quotes them any more. PowerShell is not touched: it reads its
277
+ * arguments by the C runtime's rules, and Node's quoting is the right quoting for it.
265
278
  */
266
279
  export function invocationFor(resolved, command, scriptArgs) {
267
280
  const isScript = Array.isArray(scriptArgs);
@@ -270,15 +283,37 @@ export function invocationFor(resolved, command, scriptArgs) {
270
283
  case 'powershell': {
271
284
  const base = ['-NoProfile', '-NonInteractive'];
272
285
  return isScript
273
- ? { file: resolved.path, args: [...base, '-File', command, ...(scriptArgs ?? [])] }
274
- : { file: resolved.path, args: [...base, '-Command', command] };
286
+ ? { file: resolved.path, args: [...base, '-File', command, ...(scriptArgs ?? [])], windowsVerbatimArguments: false }
287
+ : { file: resolved.path, args: [...base, '-Command', command], windowsVerbatimArguments: false };
288
+ }
289
+ case 'cmd': {
290
+ const line = isScript ? [`"${command}"`, ...(scriptArgs ?? []).map(cmdArgument)].join(' ') : command;
291
+ return { file: resolved.path, args: ['/d', '/s', '/c', `"${line}"`], windowsVerbatimArguments: true };
275
292
  }
276
- case 'cmd':
277
- return isScript
278
- ? { file: resolved.path, args: ['/d', '/c', command, ...(scriptArgs ?? [])] }
279
- : { file: resolved.path, args: ['/d', '/s', '/c', command] };
280
293
  }
281
294
  }
295
+ /**
296
+ * One argument to a script run in `cmd`, written so that it arrives as the one argument it is.
297
+ *
298
+ * The line is passed verbatim, so nothing quotes this on the way and it has to be done here, for the
299
+ * reader that is actually there. A space would make one argument two. `&`, `|`, `<` and `>` would
300
+ * make the rest of it a second command or a redirection, `^` would be taken as an escape and the
301
+ * parentheses as a group, and a quote is the only thing that makes `cmd` read any of them as text.
302
+ * `,`, `;` and `=` divide a batch file's arguments the way a space does. `%` quotes an argument too: `%NAME%` is expanded inside quotes as well as outside,
303
+ * as it is anywhere on a `cmd` line, but what it expands to then stays inside this argument.
304
+ *
305
+ * Inside the quotes, a program built on the C runtime gets back exactly what was given. The
306
+ * backslashes in front of a quote are doubled, which its rules ask for, and a quote is written `""`,
307
+ * which it reads as one quote. `\"` would not do: `cmd` counts every quote, with or without a
308
+ * backslash, so it would take that one as the end of the quoted text and run whatever came after it.
309
+ * A batch file sees an inner quote and a closing backslash doubled, which is as near as `cmd` lets
310
+ * anything come.
311
+ */
312
+ function cmdArgument(arg) {
313
+ if (arg !== '' && !/[\s"&|<>^(),;=%]/.test(arg))
314
+ return arg;
315
+ return `"${arg.replace(/(\\*)"/g, '$1$1""').replace(/(\\+)$/, '$1$1')}"`;
316
+ }
282
317
  /**
283
318
  * A shell that was found at detection time and would not start when it was wanted.
284
319
  *
@@ -12,7 +12,7 @@
12
12
  * nothing at all.
13
13
  *
14
14
  * A session already knows its project — `vcs.repoDir` says where the branches go and
15
- * `mirror.rootDir` where the files are — so that is where its commands run. The configured
15
+ * `projectDir` where the work is — so that is where its commands run. The configured
16
16
  * `execution.cwd` is the fallback for a session with no project, and this project's own
17
17
  * checkout is never a fallback: a session that would land here by default does not run, and
18
18
  * says why. Landing here on purpose — the operator pointed the session at the bot itself — is
@@ -63,11 +63,11 @@ export function isOwnCheckout(dir, own = botRootDir()) {
63
63
  */
64
64
  export function workingDirFor(session, configCwd, own = botRootDir()) {
65
65
  const repo = (session.vcs?.repoDir ?? '').trim();
66
- const mirror = (session.mirror?.rootDir ?? '').trim();
66
+ const project = (session.projectDir ?? '').trim();
67
67
  if (repo)
68
68
  return { cwd: resolve(repo), source: 'repository', ownCheckout: isInside(repo, own) };
69
- if (mirror)
70
- return { cwd: resolve(mirror), source: 'mirror', ownCheckout: isInside(mirror, own) };
69
+ if (project)
70
+ return { cwd: resolve(project), source: 'project', ownCheckout: isInside(project, own) };
71
71
  if (isInside(configCwd, own)) {
72
72
  return {
73
73
  cwd: resolve(configCwd),
@@ -44,6 +44,7 @@ export async function validateDerivedChecks(findings, opts) {
44
44
  defaultShell: opts.defaultShell,
45
45
  tracker: opts.tracker,
46
46
  passEnv: opts.passEnv,
47
+ redactPatterns: opts.redactPatterns,
47
48
  });
48
49
  if (outcome.refusedBeforeRunning)
49
50
  out.blocked.push({ finding, check, outcome });