@pdpp/local-collector 0.1.0-beta.6 → 0.1.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/local-collector/bin/pdpp-local-collector.js +580 -22
- package/dist/local-collector/src/runner.d.ts +1 -1
- package/dist/local-collector/src/runner.js +15 -1
- package/dist/polyfill-connectors/connectors/claude_code/index.js +85 -48
- package/dist/polyfill-connectors/connectors/codex/index.js +390 -108
- package/dist/polyfill-connectors/connectors/codex/parsers.js +5 -3
- package/dist/polyfill-connectors/src/bounded-file-preview.js +76 -0
- package/dist/polyfill-connectors/src/browser-handoff.js +38 -5
- package/dist/polyfill-connectors/src/collector-build-info.d.ts +8 -0
- package/dist/polyfill-connectors/src/collector-build-info.js +10 -0
- package/dist/polyfill-connectors/src/collector-runner.d.ts +54 -0
- package/dist/polyfill-connectors/src/collector-runner.js +250 -18
- package/dist/polyfill-connectors/src/connector-exit.js +62 -0
- package/dist/polyfill-connectors/src/connector-runtime-protocol.d.ts +41 -21
- package/dist/polyfill-connectors/src/connector-runtime.js +241 -30
- package/dist/polyfill-connectors/src/fingerprint-cursor.js +107 -0
- package/dist/polyfill-connectors/src/local-device-client.d.ts +17 -0
- package/dist/polyfill-connectors/src/local-device-client.js +69 -9
- package/dist/polyfill-connectors/src/local-device-outbox.d.ts +59 -0
- package/dist/polyfill-connectors/src/local-device-outbox.js +394 -5
- package/dist/polyfill-connectors/src/local-source-inventory.js +8 -1
- package/dist/polyfill-connectors/src/runner/index.d.ts +4 -3
- package/dist/polyfill-connectors/src/runner/index.js +4 -3
- package/dist/polyfill-connectors/src/safe-text-preview.js +13 -0
- package/dist/polyfill-connectors/src/static-secret-injection.js +155 -0
- package/package.json +1 -1
|
@@ -2,7 +2,8 @@ import { rmSync, writeFileSync } from "node:fs";
|
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { createInterface } from "node:readline";
|
|
4
4
|
import { resolveAuth } from "./auth.js";
|
|
5
|
-
import { manualAction } from "./browser-handoff.js";
|
|
5
|
+
import { DEADLINE_TIMEOUT, manualAction, prepareBrowserInteractionTarget, withDeadline } from "./browser-handoff.js";
|
|
6
|
+
import { flushAndExitAfterRuntimeAck } from "./connector-exit.js";
|
|
6
7
|
import { createCaptureSession } from "./fixture-capture.js";
|
|
7
8
|
import { emitToStdout } from "./safe-emit.js";
|
|
8
9
|
import { resourceSet } from "./scope-filters.js";
|
|
@@ -38,6 +39,55 @@ function makeShapeCheckSkip(stream, data, issues) {
|
|
|
38
39
|
diagnostics: { id: data.id, issues, record: data },
|
|
39
40
|
};
|
|
40
41
|
}
|
|
42
|
+
export function buildDetailCoverageMessage(params) {
|
|
43
|
+
const { stream, stateStream, requiredKeys, hydratedKeys, gapKeys, optionalSkipKeys, considered, covered } = params;
|
|
44
|
+
return {
|
|
45
|
+
type: "DETAIL_COVERAGE",
|
|
46
|
+
reference_only: true,
|
|
47
|
+
stream,
|
|
48
|
+
state_stream: stateStream,
|
|
49
|
+
required_keys: [...requiredKeys],
|
|
50
|
+
hydrated_keys: [...hydratedKeys],
|
|
51
|
+
...(typeof considered === "number" && Number.isInteger(considered) && considered >= 0 ? { considered } : {}),
|
|
52
|
+
...(typeof covered === "number" && Number.isInteger(covered) && covered >= 0 ? { covered } : {}),
|
|
53
|
+
...(gapKeys?.length ? { gap_keys: [...gapKeys] } : {}),
|
|
54
|
+
...(optionalSkipKeys?.length ? { optional_skip_keys: [...optionalSkipKeys] } : {}),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
export function emitDetailCoverage(ctx, params) {
|
|
58
|
+
return ctx.emit(buildDetailCoverageMessage(params));
|
|
59
|
+
}
|
|
60
|
+
export function buildDetailGap(params) {
|
|
61
|
+
const { stream, recordKey, reason, locator, parentStream, listCursor, error } = params;
|
|
62
|
+
let errorBlocks = {};
|
|
63
|
+
if (error) {
|
|
64
|
+
const sharedBlock = {
|
|
65
|
+
class: error.class,
|
|
66
|
+
...(error.httpStatus == null ? {} : { http_status: error.httpStatus }),
|
|
67
|
+
...(error.networkPressure == null ? {} : { network_pressure: error.networkPressure }),
|
|
68
|
+
};
|
|
69
|
+
errorBlocks = {
|
|
70
|
+
detail: sharedBlock,
|
|
71
|
+
last_error: error.message == null ? sharedBlock : { ...sharedBlock, message: error.message },
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
type: "DETAIL_GAP",
|
|
76
|
+
stream,
|
|
77
|
+
...(parentStream == null ? {} : { parent_stream: parentStream }),
|
|
78
|
+
record_key: recordKey,
|
|
79
|
+
status: "pending",
|
|
80
|
+
reason,
|
|
81
|
+
detail_locator: locator,
|
|
82
|
+
...(listCursor === undefined ? {} : { list_cursor: listCursor }),
|
|
83
|
+
retryable: true,
|
|
84
|
+
reference_only: true,
|
|
85
|
+
...errorBlocks,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
export function emitDetailGap(ctx, params) {
|
|
89
|
+
return ctx.emit(buildDetailGap(params));
|
|
90
|
+
}
|
|
41
91
|
export const nowIso = () => new Date().toISOString();
|
|
42
92
|
export const politeDelay = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
43
93
|
export function runConnector(config) {
|
|
@@ -66,13 +116,7 @@ export function runConnector(config) {
|
|
|
66
116
|
process.stderr.write(`[capture] ${modeLabel}; writing to ${capture.baseDir}\n`);
|
|
67
117
|
}
|
|
68
118
|
const flushAndExit = (code) => {
|
|
69
|
-
|
|
70
|
-
process.stdout.once("drain", () => process.exit(code));
|
|
71
|
-
setTimeout(() => process.exit(code), 3000).unref();
|
|
72
|
-
}
|
|
73
|
-
else {
|
|
74
|
-
process.exit(code);
|
|
75
|
-
}
|
|
119
|
+
flushAndExitAfterRuntimeAck(code);
|
|
76
120
|
};
|
|
77
121
|
let observedCounters = null;
|
|
78
122
|
const emitFailed = (message, retryable = false, records_emitted = observedCounters?.totalEmitted ?? 0) => {
|
|
@@ -86,6 +130,8 @@ export function runConnector(config) {
|
|
|
86
130
|
};
|
|
87
131
|
let interactionCounter = 0;
|
|
88
132
|
const nextInteractionId = () => `int_${Date.now()}_${++interactionCounter}`;
|
|
133
|
+
let detailGapPageCounter = 0;
|
|
134
|
+
const nextDetailGapPageRequestId = () => `dgp_${Date.now()}_${++detailGapPageCounter}`;
|
|
89
135
|
let assistanceCounter = 0;
|
|
90
136
|
const nextAssistanceId = () => `asst_${Date.now()}_${++assistanceCounter}`;
|
|
91
137
|
const sendInteraction = (req) => {
|
|
@@ -125,6 +171,43 @@ export function runConnector(config) {
|
|
|
125
171
|
}
|
|
126
172
|
});
|
|
127
173
|
});
|
|
174
|
+
const requestDetailGapPage = (req = {}) => {
|
|
175
|
+
const request_id = nextDetailGapPageRequestId();
|
|
176
|
+
const streams = Array.isArray(req.streams)
|
|
177
|
+
? req.streams.filter((stream) => typeof stream === "string" && stream.length > 0)
|
|
178
|
+
: undefined;
|
|
179
|
+
const maxBytes = typeof req.maxBytes === "number" && Number.isFinite(req.maxBytes) && req.maxBytes > 0
|
|
180
|
+
? Math.floor(req.maxBytes)
|
|
181
|
+
: undefined;
|
|
182
|
+
emit({
|
|
183
|
+
type: "DETAIL_GAPS_PAGE_REQUEST",
|
|
184
|
+
reference_only: true,
|
|
185
|
+
request_id,
|
|
186
|
+
...(streams && streams.length > 0 ? { streams } : {}),
|
|
187
|
+
...(maxBytes ? { max_bytes: maxBytes } : {}),
|
|
188
|
+
}).catch(() => undefined);
|
|
189
|
+
return new Promise((resolve, reject) => {
|
|
190
|
+
const onLine = (line) => {
|
|
191
|
+
try {
|
|
192
|
+
const parsed = JSON.parse(line);
|
|
193
|
+
if (parsed.type !== "DETAIL_GAPS_PAGE_RESPONSE" || parsed.request_id !== request_id) {
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
rl.off("line", onLine);
|
|
197
|
+
if (parsed.reference_only !== true || !Array.isArray(parsed.detail_gaps)) {
|
|
198
|
+
reject(new Error("Invalid DETAIL_GAPS_PAGE_RESPONSE envelope"));
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
resolve(parsed.detail_gaps);
|
|
202
|
+
}
|
|
203
|
+
catch (err) {
|
|
204
|
+
rl.off("line", onLine);
|
|
205
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
rl.on("line", onLine);
|
|
209
|
+
});
|
|
210
|
+
};
|
|
128
211
|
const progress = (message, extra = {}) => emit({ type: "PROGRESS", message, ...extra });
|
|
129
212
|
const assist = async (req) => {
|
|
130
213
|
const assistance_request_id = req.assistance_request_id ?? nextAssistanceId();
|
|
@@ -171,6 +254,7 @@ export function runConnector(config) {
|
|
|
171
254
|
sendInteraction,
|
|
172
255
|
emittedAt,
|
|
173
256
|
detailGaps: startMsg.detail_gaps ?? [],
|
|
257
|
+
requestDetailGapPage,
|
|
174
258
|
};
|
|
175
259
|
if (browser) {
|
|
176
260
|
await runInBrowser({
|
|
@@ -272,12 +356,6 @@ async function runInBrowser(args) {
|
|
|
272
356
|
const { browser, name, sendInteraction, assist, completeAssistance, progress, ensureSession, probeSession, collect, baseCtx, } = args;
|
|
273
357
|
const { context: ctx, release } = await acquireBrowser(browser, name);
|
|
274
358
|
const visibility = resolveBrowserRuntimeVisibility(browser, name);
|
|
275
|
-
const browserSendInteraction = makeBrowserInteractionKeepalive({
|
|
276
|
-
context: ctx,
|
|
277
|
-
diagnostics: process.env.PDPP_BROWSER_SURFACE_DIAGNOSTICS === "1",
|
|
278
|
-
progress,
|
|
279
|
-
sendInteraction: (req) => sendInteraction(decorateBrowserManualAction(req, visibility)),
|
|
280
|
-
});
|
|
281
359
|
const { withShutdownRelease } = await import("./shutdown-hook.js");
|
|
282
360
|
const tracer = makeTracer(ctx, name, baseCtx.capture);
|
|
283
361
|
let traceFinalized = false;
|
|
@@ -295,18 +373,41 @@ async function runInBrowser(args) {
|
|
|
295
373
|
let page = null;
|
|
296
374
|
try {
|
|
297
375
|
page = await ctx.newPage();
|
|
376
|
+
const browserSendInteraction = makeBrowserInteractionKeepalive({
|
|
377
|
+
context: ctx,
|
|
378
|
+
diagnostics: process.env.PDPP_BROWSER_SURFACE_DIAGNOSTICS === "1",
|
|
379
|
+
progress,
|
|
380
|
+
sendInteraction: async (req) => {
|
|
381
|
+
const decorated = decorateBrowserManualAction(req, visibility);
|
|
382
|
+
if (decorated.kind !== "otp") {
|
|
383
|
+
return sendInteraction(decorated);
|
|
384
|
+
}
|
|
385
|
+
const { interactionId } = await prepareBrowserInteractionTarget({
|
|
386
|
+
page: page,
|
|
387
|
+
reason: "2fa",
|
|
388
|
+
...(decorated.request_id ? { interactionId: decorated.request_id } : {}),
|
|
389
|
+
});
|
|
390
|
+
return sendInteraction({ ...decorated, request_id: interactionId });
|
|
391
|
+
},
|
|
392
|
+
});
|
|
298
393
|
await captureBrowserPage(baseCtx.capture, page, "runtime-new-page");
|
|
299
394
|
await closeBrowserContextPagesExcept(ctx, page);
|
|
300
|
-
|
|
395
|
+
const watchdog = makeSessionEstablishWatchdog({
|
|
396
|
+
capture: baseCtx.capture,
|
|
397
|
+
name,
|
|
398
|
+
page,
|
|
399
|
+
});
|
|
400
|
+
await watchdog.run(() => establishSession({ ensureSession, probeSession }, {
|
|
301
401
|
assist,
|
|
302
402
|
capture: baseCtx.capture,
|
|
403
|
+
checkpoint: watchdog.checkpoint,
|
|
303
404
|
completeAssistance,
|
|
304
405
|
context: ctx,
|
|
305
|
-
page,
|
|
406
|
+
page: page,
|
|
306
407
|
name,
|
|
307
408
|
progress,
|
|
308
|
-
sendInteraction: browserSendInteraction,
|
|
309
|
-
});
|
|
409
|
+
sendInteraction: watchdog.wrapSendInteraction(browserSendInteraction),
|
|
410
|
+
}));
|
|
310
411
|
await captureBrowserPage(baseCtx.capture, page, "runtime-session-established");
|
|
311
412
|
await captureBrowserPage(baseCtx.capture, page, "runtime-collect-start");
|
|
312
413
|
await collect({ ...baseCtx, context: ctx, page, sendInteraction: browserSendInteraction });
|
|
@@ -328,7 +429,9 @@ async function runInBrowser(args) {
|
|
|
328
429
|
baseCtx.capture?.finalize?.();
|
|
329
430
|
}
|
|
330
431
|
}
|
|
331
|
-
|
|
432
|
+
const CAPTURE_DOM_DEADLINE_MS = 10_000;
|
|
433
|
+
const PAGE_CLOSE_DEADLINE_MS = 10_000;
|
|
434
|
+
export async function captureBrowserPage(capture, page, label, deadlineMs = CAPTURE_DOM_DEADLINE_MS) {
|
|
332
435
|
if (!capture) {
|
|
333
436
|
return;
|
|
334
437
|
}
|
|
@@ -336,9 +439,13 @@ export async function captureBrowserPage(capture, page, label) {
|
|
|
336
439
|
process.stderr.write(`[capture] page already closed at ${label}; skipping dom snapshot\n`);
|
|
337
440
|
return;
|
|
338
441
|
}
|
|
339
|
-
|
|
442
|
+
const captureWork = capture.captureDom(page, label);
|
|
443
|
+
captureWork.catch(() => undefined);
|
|
444
|
+
await withDeadline(captureWork, deadlineMs, () => {
|
|
445
|
+
process.stderr.write(`[capture] dom snapshot for ${label} exceeded ${String(deadlineMs)}ms (wedged renderer?); abandoning this capture.\n`);
|
|
446
|
+
});
|
|
340
447
|
}
|
|
341
|
-
export async function closeBrowserContextPagesExcept(context, keepPage) {
|
|
448
|
+
export async function closeBrowserContextPagesExcept(context, keepPage, deadlineMs = PAGE_CLOSE_DEADLINE_MS) {
|
|
342
449
|
let pages;
|
|
343
450
|
try {
|
|
344
451
|
pages = context.pages();
|
|
@@ -351,22 +458,23 @@ export async function closeBrowserContextPagesExcept(context, keepPage) {
|
|
|
351
458
|
if (page === keepPage || page.isClosed()) {
|
|
352
459
|
continue;
|
|
353
460
|
}
|
|
354
|
-
|
|
355
|
-
await page.close();
|
|
461
|
+
if (await closeBrowserPage(page, deadlineMs)) {
|
|
356
462
|
closed++;
|
|
357
463
|
}
|
|
358
|
-
catch {
|
|
359
|
-
}
|
|
360
464
|
}
|
|
361
465
|
return closed;
|
|
362
466
|
}
|
|
363
|
-
export async function closeBrowserPage(page) {
|
|
467
|
+
export async function closeBrowserPage(page, deadlineMs = PAGE_CLOSE_DEADLINE_MS) {
|
|
364
468
|
if (!page || page.isClosed()) {
|
|
365
469
|
return false;
|
|
366
470
|
}
|
|
367
471
|
try {
|
|
368
|
-
|
|
369
|
-
|
|
472
|
+
const closeWork = page.close();
|
|
473
|
+
closeWork.catch(() => undefined);
|
|
474
|
+
const result = await withDeadline(closeWork, deadlineMs, () => {
|
|
475
|
+
process.stderr.write(`[browser-runtime] page.close() exceeded ${String(deadlineMs)}ms (wedged renderer?); abandoning close.\n`);
|
|
476
|
+
});
|
|
477
|
+
return result !== DEADLINE_TIMEOUT;
|
|
370
478
|
}
|
|
371
479
|
catch {
|
|
372
480
|
return false;
|
|
@@ -670,12 +778,113 @@ async function acquireBrowser(browser, name) {
|
|
|
670
778
|
throw new TerminalError(`could not open browser profile: ${message}`, false);
|
|
671
779
|
}
|
|
672
780
|
}
|
|
781
|
+
const DEFAULT_SESSION_ESTABLISH_WATCHDOG_MS = 120_000;
|
|
782
|
+
const SESSION_ESTABLISH_WATCHDOG_ENV = "PDPP_SESSION_ESTABLISH_WATCHDOG_MS";
|
|
783
|
+
export function resolveSessionEstablishWatchdogMs(env = process.env) {
|
|
784
|
+
const raw = env[SESSION_ESTABLISH_WATCHDOG_ENV]?.trim();
|
|
785
|
+
if (!raw) {
|
|
786
|
+
return DEFAULT_SESSION_ESTABLISH_WATCHDOG_MS;
|
|
787
|
+
}
|
|
788
|
+
const parsed = Number.parseInt(raw, 10);
|
|
789
|
+
if (!(Number.isFinite(parsed) && parsed > 0)) {
|
|
790
|
+
return DEFAULT_SESSION_ESTABLISH_WATCHDOG_MS;
|
|
791
|
+
}
|
|
792
|
+
return parsed;
|
|
793
|
+
}
|
|
794
|
+
export function makeSessionEstablishWatchdog(args) {
|
|
795
|
+
const now = args.now ?? Date.now;
|
|
796
|
+
const deadlineMs = args.deadlineMs ?? resolveSessionEstablishWatchdogMs();
|
|
797
|
+
const pollIntervalMs = args.pollIntervalMs ?? Math.max(1, Math.min(1000, Math.floor(deadlineMs / 4)));
|
|
798
|
+
let lastProgressAt = now();
|
|
799
|
+
let lastLabel = null;
|
|
800
|
+
let openInteractions = 0;
|
|
801
|
+
let tripped = false;
|
|
802
|
+
const markProgress = (label) => {
|
|
803
|
+
lastProgressAt = now();
|
|
804
|
+
if (label !== null) {
|
|
805
|
+
lastLabel = label;
|
|
806
|
+
}
|
|
807
|
+
};
|
|
808
|
+
const checkpoint = async (label) => {
|
|
809
|
+
markProgress(label);
|
|
810
|
+
try {
|
|
811
|
+
await captureBrowserPage(args.capture, args.page, `session-establish-${label}`);
|
|
812
|
+
}
|
|
813
|
+
catch (err) {
|
|
814
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
815
|
+
process.stderr.write(`[session-watchdog] checkpoint capture failed for ${label}: ${message}\n`);
|
|
816
|
+
}
|
|
817
|
+
};
|
|
818
|
+
const wrapSendInteraction = (send) => async (req) => {
|
|
819
|
+
openInteractions++;
|
|
820
|
+
markProgress(null);
|
|
821
|
+
try {
|
|
822
|
+
return await send(req);
|
|
823
|
+
}
|
|
824
|
+
finally {
|
|
825
|
+
openInteractions--;
|
|
826
|
+
markProgress(null);
|
|
827
|
+
}
|
|
828
|
+
};
|
|
829
|
+
const run = async (work) => {
|
|
830
|
+
let timer;
|
|
831
|
+
let tripInfo = null;
|
|
832
|
+
const TRIP = Symbol("session-establish-trip");
|
|
833
|
+
const tripPromise = new Promise((resolve) => {
|
|
834
|
+
const onTick = () => {
|
|
835
|
+
if (tripped || openInteractions > 0) {
|
|
836
|
+
return;
|
|
837
|
+
}
|
|
838
|
+
const sinceMs = now() - lastProgressAt;
|
|
839
|
+
if (sinceMs <= deadlineMs) {
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
tripped = true;
|
|
843
|
+
if (timer) {
|
|
844
|
+
clearInterval(timer);
|
|
845
|
+
}
|
|
846
|
+
tripInfo = { lastLabel, sinceMs };
|
|
847
|
+
args.onTrip?.(tripInfo);
|
|
848
|
+
resolve(TRIP);
|
|
849
|
+
};
|
|
850
|
+
timer = setInterval(onTick, pollIntervalMs);
|
|
851
|
+
});
|
|
852
|
+
const workPromise = work();
|
|
853
|
+
workPromise.catch(() => undefined);
|
|
854
|
+
try {
|
|
855
|
+
const outcome = await Promise.race([workPromise, tripPromise]);
|
|
856
|
+
if (outcome === TRIP) {
|
|
857
|
+
const info = tripInfo;
|
|
858
|
+
const sinceMs = info?.sinceMs ?? deadlineMs;
|
|
859
|
+
const lastCheckpoint = info?.lastLabel ?? "<none>";
|
|
860
|
+
throw new TerminalError(`${args.name}_session_establish_timeout: no session-establishment progress for ${String(sinceMs)}ms ` +
|
|
861
|
+
`(last checkpoint: ${lastCheckpoint}); failing run closed`, true);
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
finally {
|
|
865
|
+
if (timer) {
|
|
866
|
+
clearInterval(timer);
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
};
|
|
870
|
+
return { checkpoint, wrapSendInteraction, run };
|
|
871
|
+
}
|
|
673
872
|
async function establishSession(hooks, args) {
|
|
674
873
|
const { ensureSession, probeSession } = hooks;
|
|
675
|
-
const { assist, capture, completeAssistance, context, page, name, sendInteraction, progress } = args;
|
|
874
|
+
const { assist, capture, checkpoint, completeAssistance, context, page, name, sendInteraction, progress } = args;
|
|
875
|
+
await checkpoint("session-establish:begin");
|
|
676
876
|
if (typeof ensureSession === "function") {
|
|
677
877
|
try {
|
|
678
|
-
await ensureSession({
|
|
878
|
+
await ensureSession({
|
|
879
|
+
assist,
|
|
880
|
+
capture,
|
|
881
|
+
checkpoint,
|
|
882
|
+
completeAssistance,
|
|
883
|
+
context,
|
|
884
|
+
page,
|
|
885
|
+
sendInteraction,
|
|
886
|
+
progress,
|
|
887
|
+
});
|
|
679
888
|
return;
|
|
680
889
|
}
|
|
681
890
|
catch (err) {
|
|
@@ -686,6 +895,7 @@ async function establishSession(hooks, args) {
|
|
|
686
895
|
if (typeof probeSession !== "function") {
|
|
687
896
|
return;
|
|
688
897
|
}
|
|
898
|
+
await checkpoint("session-establish:probe");
|
|
689
899
|
if (await probeSession({ context, page })) {
|
|
690
900
|
return;
|
|
691
901
|
}
|
|
@@ -695,6 +905,7 @@ async function establishSession(hooks, args) {
|
|
|
695
905
|
message: `${name} session expired. Open the browser and re-authenticate, then continue.`,
|
|
696
906
|
timeoutSeconds: 1800,
|
|
697
907
|
}, sendInteraction);
|
|
908
|
+
await checkpoint("session-establish:probe-after-manual");
|
|
698
909
|
if (await probeSession({ context, page })) {
|
|
699
910
|
return;
|
|
700
911
|
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export function openCarryForwardCursor(prior) {
|
|
3
|
+
const next = new Map(prior);
|
|
4
|
+
const seen = new Set();
|
|
5
|
+
return {
|
|
6
|
+
prior(id) {
|
|
7
|
+
return prior.get(id);
|
|
8
|
+
},
|
|
9
|
+
note(id, value) {
|
|
10
|
+
next.set(id, value);
|
|
11
|
+
seen.add(id);
|
|
12
|
+
},
|
|
13
|
+
pruneStale() {
|
|
14
|
+
for (const id of next.keys()) {
|
|
15
|
+
if (!seen.has(id)) {
|
|
16
|
+
next.delete(id);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
size() {
|
|
21
|
+
return next.size;
|
|
22
|
+
},
|
|
23
|
+
toState() {
|
|
24
|
+
const out = {};
|
|
25
|
+
for (const [id, value] of next) {
|
|
26
|
+
out[id] = value;
|
|
27
|
+
}
|
|
28
|
+
return out;
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export function recordFingerprint(record, excludeKeys = []) {
|
|
33
|
+
const exclude = new Set(excludeKeys);
|
|
34
|
+
return createHash("sha1").update(stableStringify(record, exclude)).digest("hex");
|
|
35
|
+
}
|
|
36
|
+
export function openFingerprintCursor(priorState, options = {}) {
|
|
37
|
+
const staticExcludeKeys = options.excludeFromFingerprint ?? [];
|
|
38
|
+
const resolveExcludeKeys = options.resolveExcludeFromFingerprint;
|
|
39
|
+
const prior = options.priorFingerprints ?? decodePriorFingerprints(priorState);
|
|
40
|
+
const cursor = openCarryForwardCursor(prior);
|
|
41
|
+
return {
|
|
42
|
+
shouldEmit(data) {
|
|
43
|
+
const rawId = data.id;
|
|
44
|
+
if (rawId == null) {
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
const id = String(rawId);
|
|
48
|
+
if (id.length === 0) {
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
const record = data;
|
|
52
|
+
const excludeKeys = resolveExcludeKeys ? resolveExcludeKeys(record) : staticExcludeKeys;
|
|
53
|
+
const fingerprint = recordFingerprint(record, excludeKeys);
|
|
54
|
+
cursor.note(id, fingerprint);
|
|
55
|
+
return cursor.prior(id) !== fingerprint;
|
|
56
|
+
},
|
|
57
|
+
priorFingerprint(id) {
|
|
58
|
+
return cursor.prior(id);
|
|
59
|
+
},
|
|
60
|
+
pruneStale() {
|
|
61
|
+
cursor.pruneStale();
|
|
62
|
+
},
|
|
63
|
+
toState() {
|
|
64
|
+
return cursor.toState();
|
|
65
|
+
},
|
|
66
|
+
size() {
|
|
67
|
+
return cursor.size();
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
function decodePriorFingerprints(priorState) {
|
|
72
|
+
const out = new Map();
|
|
73
|
+
if (!priorState || typeof priorState !== "object" || Array.isArray(priorState)) {
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
const raw = priorState.fingerprints;
|
|
77
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
for (const [id, value] of Object.entries(raw)) {
|
|
81
|
+
if (typeof value === "string" && value.length > 0) {
|
|
82
|
+
out.set(id, value);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return out;
|
|
86
|
+
}
|
|
87
|
+
function compareKeys(a, b) {
|
|
88
|
+
if (a < b) {
|
|
89
|
+
return -1;
|
|
90
|
+
}
|
|
91
|
+
if (a > b) {
|
|
92
|
+
return 1;
|
|
93
|
+
}
|
|
94
|
+
return 0;
|
|
95
|
+
}
|
|
96
|
+
function stableStringify(value, exclude) {
|
|
97
|
+
if (value === null || typeof value !== "object") {
|
|
98
|
+
return JSON.stringify(value) ?? "null";
|
|
99
|
+
}
|
|
100
|
+
if (Array.isArray(value)) {
|
|
101
|
+
return `[${value.map((v) => stableStringify(v, exclude)).join(",")}]`;
|
|
102
|
+
}
|
|
103
|
+
const entries = Object.entries(value)
|
|
104
|
+
.filter(([k]) => !exclude.has(k))
|
|
105
|
+
.sort(([a], [b]) => compareKeys(a, b));
|
|
106
|
+
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v, exclude)}`).join(",")}}`;
|
|
107
|
+
}
|
|
@@ -12,7 +12,9 @@ export interface LocalDeviceClientOptions {
|
|
|
12
12
|
deviceId?: string;
|
|
13
13
|
deviceToken?: string;
|
|
14
14
|
fetchImpl?: typeof fetch;
|
|
15
|
+
requestTimeoutMs?: number;
|
|
15
16
|
}
|
|
17
|
+
export declare const DEFAULT_LOCAL_DEVICE_REQUEST_TIMEOUT_MS = 120000;
|
|
16
18
|
export interface EnrollmentExchangeRequest {
|
|
17
19
|
device_label?: string;
|
|
18
20
|
enrollment_code: string;
|
|
@@ -35,8 +37,17 @@ export interface HeartbeatOutboxDiagnostics {
|
|
|
35
37
|
succeeded: number;
|
|
36
38
|
total: number;
|
|
37
39
|
}
|
|
40
|
+
export interface HeartbeatLastError {
|
|
41
|
+
kind: "state_read_failed" | "dead_letter_backlog";
|
|
42
|
+
top_dead_letter_classes?: {
|
|
43
|
+
count: number;
|
|
44
|
+
error_class: string;
|
|
45
|
+
}[];
|
|
46
|
+
}
|
|
38
47
|
export interface HeartbeatRequest {
|
|
48
|
+
agent_version?: string;
|
|
39
49
|
connector_id: string;
|
|
50
|
+
last_error?: HeartbeatLastError | null;
|
|
40
51
|
outbox?: HeartbeatOutboxDiagnostics;
|
|
41
52
|
records_pending?: number;
|
|
42
53
|
source_instance_id: string;
|
|
@@ -105,10 +116,16 @@ export interface RecoverLocalCollectorGapRequest {
|
|
|
105
116
|
}
|
|
106
117
|
export declare class LocalDeviceHttpError extends Error {
|
|
107
118
|
readonly body: string;
|
|
119
|
+
readonly envelopeMessage: string | null;
|
|
120
|
+
readonly param: string | null;
|
|
108
121
|
readonly status: number;
|
|
109
122
|
readonly code: string | null;
|
|
110
123
|
constructor(status: number, body: string);
|
|
111
124
|
}
|
|
125
|
+
export declare class LocalDeviceRequestTimeoutError extends Error {
|
|
126
|
+
readonly timeoutMs: number;
|
|
127
|
+
constructor(timeoutMs: number);
|
|
128
|
+
}
|
|
112
129
|
export declare class LocalDeviceClient {
|
|
113
130
|
#private;
|
|
114
131
|
constructor(options: LocalDeviceClientOptions);
|
|
@@ -7,18 +7,23 @@ export const LOCAL_DEVICE_ENDPOINTS = {
|
|
|
7
7
|
localCollectorGapRecovered: (deviceId, sourceInstanceId) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/source-instances/${encodeURIComponent(sourceInstanceId)}/local-collector-gaps/recovered`,
|
|
8
8
|
sourceInstanceState: (deviceId, sourceInstanceId) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/source-instances/${encodeURIComponent(sourceInstanceId)}/state`,
|
|
9
9
|
};
|
|
10
|
+
export const DEFAULT_LOCAL_DEVICE_REQUEST_TIMEOUT_MS = 120_000;
|
|
10
11
|
export class LocalDeviceHttpError extends Error {
|
|
11
12
|
body;
|
|
13
|
+
envelopeMessage;
|
|
14
|
+
param;
|
|
12
15
|
status;
|
|
13
16
|
code;
|
|
14
17
|
constructor(status, body) {
|
|
15
18
|
const parsed = parseLocalDeviceErrorEnvelope(body);
|
|
16
|
-
const detail = parsed
|
|
19
|
+
const detail = formatLocalDeviceErrorDetail(parsed);
|
|
17
20
|
super(`local device request failed: ${status}${detail}`);
|
|
18
21
|
this.name = "LocalDeviceHttpError";
|
|
19
22
|
this.status = status;
|
|
20
23
|
this.body = body;
|
|
21
24
|
this.code = parsed?.code ?? null;
|
|
25
|
+
this.param = parsed?.param ?? null;
|
|
26
|
+
this.envelopeMessage = parsed?.message ?? null;
|
|
22
27
|
}
|
|
23
28
|
}
|
|
24
29
|
function parseLocalDeviceErrorEnvelope(body) {
|
|
@@ -28,23 +33,55 @@ function parseLocalDeviceErrorEnvelope(body) {
|
|
|
28
33
|
try {
|
|
29
34
|
const parsed = JSON.parse(body);
|
|
30
35
|
if (parsed && typeof parsed === "object" && parsed.error && typeof parsed.error.code === "string") {
|
|
31
|
-
return {
|
|
36
|
+
return {
|
|
37
|
+
code: parsed.error.code,
|
|
38
|
+
message: typeof parsed.error.message === "string" ? sanitizeErrorDetail(parsed.error.message) : null,
|
|
39
|
+
param: typeof parsed.error.param === "string" ? sanitizeErrorDetail(parsed.error.param) : null,
|
|
40
|
+
};
|
|
32
41
|
}
|
|
33
42
|
}
|
|
34
43
|
catch {
|
|
35
44
|
}
|
|
36
45
|
return null;
|
|
37
46
|
}
|
|
47
|
+
function formatLocalDeviceErrorDetail(parsed) {
|
|
48
|
+
if (!parsed) {
|
|
49
|
+
return "";
|
|
50
|
+
}
|
|
51
|
+
const parts = [parsed.code];
|
|
52
|
+
if (parsed.param) {
|
|
53
|
+
parts.push(`param=${parsed.param}`);
|
|
54
|
+
}
|
|
55
|
+
if (parsed.message) {
|
|
56
|
+
parts.push(`message=${parsed.message}`);
|
|
57
|
+
}
|
|
58
|
+
return ` ${parts.join(" ")}`;
|
|
59
|
+
}
|
|
60
|
+
const ERROR_DETAIL_SECRET_RE = /\b(authorization|bearer|token|password|passwd|cookie|secret|otp|api[_-]?key)\b\s*[:=]\s*["']?[^"',\s}]+/gi;
|
|
61
|
+
function sanitizeErrorDetail(value) {
|
|
62
|
+
const compact = value.replace(ERROR_DETAIL_SECRET_RE, "$1=[REDACTED]").replace(/\s+/g, " ").trim();
|
|
63
|
+
return compact.length > 160 ? `${compact.slice(0, 159)}…` : compact;
|
|
64
|
+
}
|
|
65
|
+
export class LocalDeviceRequestTimeoutError extends Error {
|
|
66
|
+
timeoutMs;
|
|
67
|
+
constructor(timeoutMs) {
|
|
68
|
+
super(`local device request timed out after ${timeoutMs}ms`);
|
|
69
|
+
this.name = "LocalDeviceRequestTimeoutError";
|
|
70
|
+
this.timeoutMs = timeoutMs;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
38
73
|
export class LocalDeviceClient {
|
|
39
74
|
#baseUrl;
|
|
40
75
|
#deviceId;
|
|
41
76
|
#deviceToken;
|
|
42
77
|
#fetch;
|
|
78
|
+
#requestTimeoutMs;
|
|
43
79
|
constructor(options) {
|
|
44
80
|
this.#baseUrl = new URL(options.baseUrl);
|
|
45
81
|
this.#deviceId = options.deviceId;
|
|
46
82
|
this.#deviceToken = options.deviceToken;
|
|
47
83
|
this.#fetch = options.fetchImpl ?? fetch;
|
|
84
|
+
this.#requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_LOCAL_DEVICE_REQUEST_TIMEOUT_MS;
|
|
48
85
|
}
|
|
49
86
|
exchangeEnrollment(request) {
|
|
50
87
|
return this.#request(LOCAL_DEVICE_ENDPOINTS.exchangeEnrollment, {
|
|
@@ -119,14 +156,37 @@ export class LocalDeviceClient {
|
|
|
119
156
|
if (options.body !== undefined) {
|
|
120
157
|
init.body = JSON.stringify(options.body);
|
|
121
158
|
}
|
|
122
|
-
const
|
|
123
|
-
const
|
|
124
|
-
|
|
125
|
-
|
|
159
|
+
const controller = this.#requestTimeoutMs > 0 ? new AbortController() : null;
|
|
160
|
+
const timer = controller
|
|
161
|
+
? setTimeout(() => controller.abort(new LocalDeviceRequestTimeoutError(this.#requestTimeoutMs)), this.#requestTimeoutMs)
|
|
162
|
+
: null;
|
|
163
|
+
if (controller) {
|
|
164
|
+
init.signal = controller.signal;
|
|
126
165
|
}
|
|
127
|
-
|
|
128
|
-
|
|
166
|
+
try {
|
|
167
|
+
const response = await this.#fetch(new URL(path, this.#baseUrl), init);
|
|
168
|
+
const text = await response.text();
|
|
169
|
+
if (!response.ok) {
|
|
170
|
+
throw new LocalDeviceHttpError(response.status, text);
|
|
171
|
+
}
|
|
172
|
+
if (!text) {
|
|
173
|
+
return { ok: true };
|
|
174
|
+
}
|
|
175
|
+
return JSON.parse(text);
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
if (controller?.signal.aborted) {
|
|
179
|
+
const reason = controller.signal.reason;
|
|
180
|
+
throw reason instanceof LocalDeviceRequestTimeoutError
|
|
181
|
+
? reason
|
|
182
|
+
: new LocalDeviceRequestTimeoutError(this.#requestTimeoutMs);
|
|
183
|
+
}
|
|
184
|
+
throw error;
|
|
185
|
+
}
|
|
186
|
+
finally {
|
|
187
|
+
if (timer) {
|
|
188
|
+
clearTimeout(timer);
|
|
189
|
+
}
|
|
129
190
|
}
|
|
130
|
-
return JSON.parse(text);
|
|
131
191
|
}
|
|
132
192
|
}
|