@fungi.computer/oyster 0.46.1 → 0.46.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build/job-log.d.ts +4 -2
- package/dist/build/job-log.js +74 -21
- package/dist/embed-metafiles.js +1 -1
- package/dist/embed-scripts.js +1 -1
- package/dist/page-script.js +3 -3
- package/dist/standalone/agents.js +0 -1
- package/package.json +7 -7
package/dist/build/job-log.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as Schema from "effect/Schema";
|
|
2
|
+
import { type PatternRedactor } from "@fungi.computer/veil/redaction";
|
|
2
3
|
/** Maximum retained combined stdout/stderr bytes, split equally into head and tail. */
|
|
3
4
|
export declare const buildJobLogMaxBytes: number;
|
|
4
5
|
/** Base64url preserves raw command bytes, including split UTF-8 and terminal control bytes. */
|
|
@@ -17,9 +18,10 @@ export declare const redactionWithheld = "[output withheld: redaction unavailabl
|
|
|
17
18
|
/**
|
|
18
19
|
* One bounded sink for a build's combined output. Veil's pattern redactor
|
|
19
20
|
* (S3's gitleaks rules) runs before a byte is retained, so the stored log and
|
|
20
|
-
* every `?log` read of it are already scrubbed.
|
|
21
|
+
* every `?log` read of it are already scrubbed. `loadRedactor` is Veil's
|
|
22
|
+
* loader unless a caller wraps it (a test that counts scanned characters).
|
|
21
23
|
*/
|
|
22
|
-
export declare function createBuildJobLog(): Promise<{
|
|
24
|
+
export declare function createBuildJobLog(loadRedactor?: () => Promise<PatternRedactor>): Promise<{
|
|
23
25
|
/** Retained buffer capacity, distinct from the transient encoded result. */
|
|
24
26
|
readonly allocatedBytes: number;
|
|
25
27
|
append(bytes: Uint8Array): void;
|
package/dist/build/job-log.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as Schema from "effect/Schema";
|
|
2
2
|
import { base64url } from "jose";
|
|
3
3
|
import { boundedDiagnostic } from "@fungi.computer/bird-dog/effect";
|
|
4
|
-
import { loadPatternRedactor, } from "@fungi.computer/veil/redaction";
|
|
4
|
+
import { loadPatternRedactor, maskSpans, } from "@fungi.computer/veil/redaction";
|
|
5
5
|
/** Maximum retained combined stdout/stderr bytes, split equally into head and tail. */
|
|
6
6
|
export const buildJobLogMaxBytes = 2 * 1024 * 1024;
|
|
7
7
|
/** Base64url preserves raw command bytes, including split UTF-8 and terminal control bytes. */
|
|
@@ -102,15 +102,16 @@ function textToBytes(text) {
|
|
|
102
102
|
*/
|
|
103
103
|
const scanWindow = 16 * 1024;
|
|
104
104
|
const maxPendingText = 1024 * 1024;
|
|
105
|
+
const secretsOf = (matches) => matches.map(({ secret }) => secret);
|
|
105
106
|
/** What a host writes where it cannot redact: the ADR's form, never raw output. */
|
|
106
107
|
export const redactionWithheld = "[output withheld: redaction unavailable]";
|
|
107
108
|
/**
|
|
108
109
|
* The redactor, or undefined when its rules cannot load: callers then write
|
|
109
110
|
* {@link redactionWithheld} and never the unredacted text.
|
|
110
111
|
*/
|
|
111
|
-
async function loadRedactorOrWithhold() {
|
|
112
|
+
async function loadRedactorOrWithhold(load) {
|
|
112
113
|
try {
|
|
113
|
-
return { redactor: await
|
|
114
|
+
return { redactor: await load() };
|
|
114
115
|
}
|
|
115
116
|
catch (cause) {
|
|
116
117
|
console.error("redaction rules failed to load", cause instanceof Error ? cause.name : "unknown");
|
|
@@ -120,14 +121,16 @@ async function loadRedactorOrWithhold() {
|
|
|
120
121
|
/**
|
|
121
122
|
* One bounded sink for a build's combined output. Veil's pattern redactor
|
|
122
123
|
* (S3's gitleaks rules) runs before a byte is retained, so the stored log and
|
|
123
|
-
* every `?log` read of it are already scrubbed.
|
|
124
|
+
* every `?log` read of it are already scrubbed. `loadRedactor` is Veil's
|
|
125
|
+
* loader unless a caller wraps it (a test that counts scanned characters).
|
|
124
126
|
*/
|
|
125
|
-
export async function createBuildJobLog() {
|
|
127
|
+
export async function createBuildJobLog(loadRedactor = loadPatternRedactor) {
|
|
126
128
|
const ring = createBoundedLog();
|
|
127
129
|
// A rule table that cannot load stores the withheld notice, never the output.
|
|
128
|
-
const { redactor } = await loadRedactorOrWithhold();
|
|
130
|
+
const { redactor } = await loadRedactorOrWithhold(loadRedactor);
|
|
129
131
|
let pending = "";
|
|
130
132
|
let scannedLength = 0;
|
|
133
|
+
let masking;
|
|
131
134
|
const scan = (final) => {
|
|
132
135
|
if (redactor === undefined) {
|
|
133
136
|
pending = "";
|
|
@@ -136,25 +139,65 @@ export async function createBuildJobLog() {
|
|
|
136
139
|
ring.append(textToBytes(redactionWithheld));
|
|
137
140
|
return;
|
|
138
141
|
}
|
|
139
|
-
|
|
142
|
+
// A private key without its end line, or a curl command with lines still
|
|
143
|
+
// to come, is never released raw: it is held from its opener, and masked
|
|
144
|
+
// whole if the output ends first, or through its end if the held text
|
|
145
|
+
// reaches the ceiling first.
|
|
146
|
+
const open = redactor.openFrom(pending);
|
|
147
|
+
const ceiling = pending.length > maxPendingText;
|
|
148
|
+
// Still held from the start: nothing can leave, so skip the full scan.
|
|
149
|
+
if (!final && !ceiling && open?.opener.start === 0) {
|
|
150
|
+
scannedLength = pending.length;
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
const whole = open !== undefined &&
|
|
154
|
+
(final || (ceiling && open.opener.start < pending.length - scanWindow));
|
|
155
|
+
// A block masked whole needs a full scan only of the text before it.
|
|
156
|
+
const matches = redactor.find(whole ? pending.slice(0, open.opener.start) : pending);
|
|
157
|
+
if (matches === undefined) {
|
|
158
|
+
// Over Veil's size ceiling: withhold this text, never pass it raw.
|
|
159
|
+
ring.append(textToBytes(redactionWithheld));
|
|
160
|
+
pending = "";
|
|
161
|
+
scannedLength = 0;
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const masks = secretsOf(matches);
|
|
165
|
+
if (whole) {
|
|
166
|
+
ring.append(textToBytes(maskSpans(pending, [
|
|
167
|
+
...masks,
|
|
168
|
+
{ start: open.opener.start, end: pending.length },
|
|
169
|
+
])));
|
|
170
|
+
// The rest of the block, through its end, is masked as it arrives.
|
|
171
|
+
if (!final)
|
|
172
|
+
masking = open.follow();
|
|
173
|
+
pending = "";
|
|
174
|
+
scannedLength = 0;
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
140
177
|
if (final) {
|
|
141
|
-
ring.append(textToBytes(
|
|
178
|
+
ring.append(textToBytes(maskSpans(pending, masks)));
|
|
142
179
|
pending = "";
|
|
143
180
|
scannedLength = 0;
|
|
144
181
|
return;
|
|
145
182
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
while (unchanged < text.length &&
|
|
150
|
-
unchanged < pending.length &&
|
|
151
|
-
text[text.length - 1 - unchanged] ===
|
|
152
|
-
pending[pending.length - 1 - unchanged])
|
|
153
|
-
unchanged++;
|
|
154
|
-
if (text.length > scanWindow &&
|
|
155
|
-
(unchanged >= scanWindow || text.length > maxPendingText)) {
|
|
183
|
+
if (ceiling) {
|
|
184
|
+
// Ceiling: emit all but the last window, masked on both sides of the cut.
|
|
185
|
+
const text = maskSpans(pending, masks);
|
|
156
186
|
ring.append(textToBytes(text.slice(0, text.length - scanWindow)));
|
|
157
187
|
pending = text.slice(text.length - scanWindow);
|
|
188
|
+
scannedLength = pending.length;
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
// A match reaching into the last window could still grow with later
|
|
192
|
+
// output, so it stays pending raw, with its whole match as rescan context.
|
|
193
|
+
// Walking by descending start also holds any match that overlaps one held.
|
|
194
|
+
let cut = Math.min(pending.length - scanWindow, open?.opener.start ?? Infinity);
|
|
195
|
+
for (const { match } of [...matches].sort((left, right) => right.match.start - left.match.start))
|
|
196
|
+
if (match.start < cut && match.end > cut)
|
|
197
|
+
cut = match.start;
|
|
198
|
+
if (cut > 0) {
|
|
199
|
+
ring.append(textToBytes(maskSpans(pending.slice(0, cut), masks.filter(({ end }) => end <= cut))));
|
|
200
|
+
pending = pending.slice(cut);
|
|
158
201
|
}
|
|
159
202
|
scannedLength = pending.length;
|
|
160
203
|
};
|
|
@@ -164,8 +207,18 @@ export async function createBuildJobLog() {
|
|
|
164
207
|
return ring.allocatedBytes;
|
|
165
208
|
},
|
|
166
209
|
append(bytes) {
|
|
167
|
-
|
|
168
|
-
if (
|
|
210
|
+
let text = bytesToText(bytes);
|
|
211
|
+
if (masking !== undefined) {
|
|
212
|
+
const end = masking.read(text);
|
|
213
|
+
if (end === undefined)
|
|
214
|
+
return;
|
|
215
|
+
masking = undefined;
|
|
216
|
+
text = text.slice(end);
|
|
217
|
+
}
|
|
218
|
+
pending += text;
|
|
219
|
+
// Held text doubles before its next rescan, so holding stays linear.
|
|
220
|
+
if (pending.length >=
|
|
221
|
+
Math.min(maxPendingText + 1, Math.max(2 * scanWindow, scannedLength + Math.max(scanWindow, scannedLength))))
|
|
169
222
|
scan(false);
|
|
170
223
|
},
|
|
171
224
|
finish() {
|
|
@@ -179,7 +232,7 @@ const buildJobLogReference = (jobId) => `/api/builds/${encodeURIComponent(jobId)
|
|
|
179
232
|
/** Preserve BirdDog's redacted 200-character status/mail budget and name the full log. */
|
|
180
233
|
export async function buildJobDiagnostic(jobId, detail) {
|
|
181
234
|
const reference = `Log: ${buildJobLogReference(jobId)}`;
|
|
182
|
-
const { redactor } = await loadRedactorOrWithhold();
|
|
235
|
+
const { redactor } = await loadRedactorOrWithhold(loadPatternRedactor);
|
|
183
236
|
const diagnostic = boundedDiagnostic(redactor === undefined ? redactionWithheld : redactor.redact(detail).text) ?? "Build finished";
|
|
184
237
|
return `${reference}\n${diagnostic.slice(0, 200 - reference.length - 1)}`;
|
|
185
238
|
}
|