@pdpp/local-collector 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/local-collector/bin/pdpp-local-collector.js +251 -16
- package/dist/polyfill-connectors/src/collector-build-info.js +3 -3
- package/dist/polyfill-connectors/src/connector-runtime-protocol.d.ts +24 -0
- package/dist/polyfill-connectors/src/connector-runtime.js +1 -0
- package/dist/polyfill-connectors/src/static-secret-injection.js +8 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -32,6 +32,13 @@ PDPP_CONNECTION_ID=<source_instance_id> \
|
|
|
32
32
|
npx -y @pdpp/local-collector run \
|
|
33
33
|
--base-url https://<reference-host> \
|
|
34
34
|
--connector claude_code
|
|
35
|
+
|
|
36
|
+
# Preview host-local recovery for a stalled collector lane. This loads the
|
|
37
|
+
# enrolled local profile for the source instance and changes nothing.
|
|
38
|
+
npx -y @pdpp/local-collector recover --source-instance-id <source_instance_id>
|
|
39
|
+
|
|
40
|
+
# Apply recovery: requeue failed uploads when present, then run the collector once.
|
|
41
|
+
npx -y @pdpp/local-collector recover --source-instance-id <source_instance_id> --apply
|
|
35
42
|
```
|
|
36
43
|
|
|
37
44
|
The collector sends `X-PDPP-Collector-Protocol` on enrollment and every
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { existsSync, readFileSync, realpathSync } from "node:fs";
|
|
2
|
+
import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
|
|
3
|
+
import { homedir } from "node:os";
|
|
3
4
|
import { basename, dirname, extname, join, sep } from "node:path";
|
|
4
5
|
import { fileURLToPath } from "node:url";
|
|
5
6
|
import { ALLOW_CUSTOM_COMMAND_ENV, CollectorCustomCommandRefusedError, CollectorUsageError, } from "../src/errors.js";
|
|
@@ -8,6 +9,7 @@ const COVERAGE_DIAGNOSTICS_STREAM = "coverage_diagnostics";
|
|
|
8
9
|
const DEFAULT_QUEUE_PATH = join(dirname(fileURLToPath(import.meta.url)), "..", ".pdpp-data", "collector-runner-queue.json");
|
|
9
10
|
const LOCAL_COLLECTOR_PACKAGE_NAME = "@pdpp/local-collector";
|
|
10
11
|
const LOCAL_COLLECTOR_PACKAGE_VERSION_FALLBACK = "0.0.0";
|
|
12
|
+
const LOCAL_COLLECTOR_PROFILE_DIR_ENV = "PDPP_LOCAL_COLLECTOR_PROFILE_DIR";
|
|
11
13
|
const LOCAL_COLLECTOR_PLACEHOLDER_VERSION = "0.0.0";
|
|
12
14
|
const REPO_ONLY_PACKAGE_SIBLINGS = ["src", "bin", "test", "scripts", "tsconfig.build.json"];
|
|
13
15
|
function resolveLocalCollectorManifest(startUrl) {
|
|
@@ -96,18 +98,27 @@ Subcommands:
|
|
|
96
98
|
status Print local durable outbox health as JSON.
|
|
97
99
|
[--queue <path>]
|
|
98
100
|
[--connection-id <id>]
|
|
101
|
+
[--source-instance-id <id>]
|
|
99
102
|
doctor Print local durable outbox operator diagnostics as JSON.
|
|
100
103
|
[--queue <path>]
|
|
101
104
|
[--connection-id <id>]
|
|
105
|
+
[--source-instance-id <id>]
|
|
102
106
|
retry-dead-letters Requeue local dead-letter outbox rows.
|
|
103
107
|
[--queue <path>]
|
|
104
108
|
[--connection-id <id>]
|
|
109
|
+
[--source-instance-id <id>]
|
|
105
110
|
[--kind record_batch|checkpoint|gap|blob_upload]
|
|
106
111
|
[--limit <n>]
|
|
107
112
|
[--apply] Dry-run by default; --apply mutates after a DB backup.
|
|
113
|
+
recover Resolve the enrolled local profile, recover stalled outbox work,
|
|
114
|
+
and run the collector once.
|
|
115
|
+
--source-instance-id <id>
|
|
116
|
+
[--profile <name>] Optional profile name under the collector profile dir.
|
|
117
|
+
[--apply] Dry-run by default; --apply requeues and runs.
|
|
108
118
|
prune-sent Delete sent (succeeded) outbox rows to reclaim disk space.
|
|
109
119
|
[--queue <path>]
|
|
110
120
|
[--connection-id <id>]
|
|
121
|
+
[--source-instance-id <id>]
|
|
111
122
|
[--older-than-days <n>] Delete sent rows older than N days (default: 30).
|
|
112
123
|
[--keep-count <n>] Keep at most N most-recent sent rows per connection.
|
|
113
124
|
[--apply] Dry-run by default; --apply mutates after a DB backup.
|
|
@@ -126,6 +137,7 @@ Subcommands:
|
|
|
126
137
|
--device-id <id>
|
|
127
138
|
--device-token <token>
|
|
128
139
|
--connection-id <id>
|
|
140
|
+
[--source-instance-id <id>]
|
|
129
141
|
[--streams a,b,c]
|
|
130
142
|
[--backfill-streams attachments]
|
|
131
143
|
[--run-id <id>]
|
|
@@ -164,6 +176,11 @@ async function main() {
|
|
|
164
176
|
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
165
177
|
return;
|
|
166
178
|
}
|
|
179
|
+
if (options.command === "recover") {
|
|
180
|
+
const result = await recoverLocalCollector(options);
|
|
181
|
+
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
167
184
|
if (options.command === "prune-sent") {
|
|
168
185
|
const result = pruneSentOutboxRows(options);
|
|
169
186
|
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
@@ -189,14 +206,18 @@ async function main() {
|
|
|
189
206
|
process.stdout.write(`${JSON.stringify(response, null, 2)}\n`);
|
|
190
207
|
return;
|
|
191
208
|
}
|
|
209
|
+
const result = await runCollectorOnce(options);
|
|
210
|
+
process.stdout.write(`${JSON.stringify(summarizeRunResultForCli(result), null, 2)}\n`);
|
|
211
|
+
}
|
|
212
|
+
async function runCollectorOnce(options) {
|
|
192
213
|
if (!(options.deviceId && options.deviceToken && options.sourceInstanceId)) {
|
|
193
|
-
throw new CollectorUsageError("run requires --device-id <id>, --device-token <token>, and --connection-id <id>");
|
|
214
|
+
throw new CollectorUsageError("run requires --device-id <id>, --device-token <token>, and --connection-id/--source-instance-id <id>");
|
|
194
215
|
}
|
|
195
216
|
if (!options.connector) {
|
|
196
217
|
throw new CollectorUsageError("run requires --connector <connector-id>");
|
|
197
218
|
}
|
|
198
219
|
const spec = buildConnectorSpec(options);
|
|
199
|
-
|
|
220
|
+
return runCollectorConnector({
|
|
200
221
|
baseUrl: options.baseUrl,
|
|
201
222
|
connector: spec,
|
|
202
223
|
deviceId: options.deviceId,
|
|
@@ -205,7 +226,6 @@ async function main() {
|
|
|
205
226
|
...(options.runId ? { runId: options.runId } : {}),
|
|
206
227
|
sourceInstanceId: options.sourceInstanceId,
|
|
207
228
|
});
|
|
208
|
-
process.stdout.write(`${JSON.stringify(summarizeRunResultForCli(result), null, 2)}\n`);
|
|
209
229
|
}
|
|
210
230
|
export function summarizeRunResultForCli(result) {
|
|
211
231
|
const summary = result.outboxSummary;
|
|
@@ -251,7 +271,7 @@ function runDrainNote(result, summary, drained) {
|
|
|
251
271
|
parts.push(`${summary.leased} leased (in flight)`);
|
|
252
272
|
}
|
|
253
273
|
if (summary.deadLetter > 0) {
|
|
254
|
-
parts.push(`${summary.deadLetter} dead-letter (run \`
|
|
274
|
+
parts.push(`${summary.deadLetter} dead-letter (run \`recover --source-instance-id <id> --apply\`)`);
|
|
255
275
|
}
|
|
256
276
|
const scanNote = result.scanBudgetExceeded
|
|
257
277
|
? " The connector was stopped by the per-run enqueue budget, so more source work likely remains; re-run to continue."
|
|
@@ -362,9 +382,9 @@ export function buildLocalOutboxDoctor(status, errorSummary) {
|
|
|
362
382
|
? ` Most common cause: ${topClass.error_class} (${topClass.count} row(s)).`
|
|
363
383
|
: "";
|
|
364
384
|
remediation.push(`${status.outbox.counts.dead_letter} dead-letter row(s) need recovery.${causeHint} ` +
|
|
365
|
-
"Preview with `pdpp-local-collector
|
|
366
|
-
"`pdpp-local-collector
|
|
367
|
-
"
|
|
385
|
+
"Preview with `pdpp-local-collector recover --source-instance-id <id>`, then apply with " +
|
|
386
|
+
"`pdpp-local-collector recover --source-instance-id <id> --apply`. The apply step backs up the DB, " +
|
|
387
|
+
"prepares failed uploads for retry when present, and runs the collector once.");
|
|
368
388
|
}
|
|
369
389
|
if (checks.expired_leases === "warn") {
|
|
370
390
|
remediation.push(`${status.outbox.expired_leases} lease(s) are past expiry — a previous run likely crashed mid-drain. ` +
|
|
@@ -440,16 +460,17 @@ export function readLocalOutboxDeadLetterErrorSummary(options) {
|
|
|
440
460
|
}
|
|
441
461
|
const RETRY_DEAD_LETTERS_NO_MATCH_NOTE = "No dead-letter rows matched. If the dashboard shows this connection as " +
|
|
442
462
|
"blocked/stalled, that is a state-read block, not a dead-letter backlog — " +
|
|
443
|
-
"there is nothing to requeue.
|
|
444
|
-
"
|
|
463
|
+
"there is nothing to requeue. Use `pdpp-local-collector recover --source-instance-id <id> --apply` " +
|
|
464
|
+
"to run the collector through the enrolled local profile and clear the block.";
|
|
445
465
|
function retryDeadLettersMatchNote(matched, dryRun) {
|
|
446
466
|
if (matched === 0) {
|
|
447
467
|
return RETRY_DEAD_LETTERS_NO_MATCH_NOTE;
|
|
448
468
|
}
|
|
449
469
|
const requeued = dryRun
|
|
450
|
-
? `${matched} dead-letter row(s) would be requeued (dry run).
|
|
470
|
+
? `${matched} dead-letter row(s) would be requeued (dry run). `
|
|
451
471
|
: `${matched} dead-letter row(s) matched and were requeued to pending. `;
|
|
452
|
-
return `${requeued}
|
|
472
|
+
return (`${requeued}Use \`pdpp-local-collector recover --source-instance-id <id> --apply\` for the dashboard recovery path. ` +
|
|
473
|
+
"This low-level command only moves rows to pending; it does not ingest.");
|
|
453
474
|
}
|
|
454
475
|
export function retryLocalOutboxDeadLetters(options) {
|
|
455
476
|
const dbPath = resolveOutboxPath(options);
|
|
@@ -505,6 +526,205 @@ export function retryLocalOutboxDeadLetters(options) {
|
|
|
505
526
|
outbox.close();
|
|
506
527
|
}
|
|
507
528
|
}
|
|
529
|
+
function defaultCollectorProfileDir() {
|
|
530
|
+
const configHome = process.env.XDG_CONFIG_HOME?.trim() || join(homedir(), ".config");
|
|
531
|
+
return join(configHome, "pdpp", "collectors");
|
|
532
|
+
}
|
|
533
|
+
export function parseCollectorProfileEnv(contents) {
|
|
534
|
+
const env = {};
|
|
535
|
+
for (const rawLine of contents.split(/\r?\n/)) {
|
|
536
|
+
const line = rawLine.trim();
|
|
537
|
+
if (!line || line.startsWith("#")) {
|
|
538
|
+
continue;
|
|
539
|
+
}
|
|
540
|
+
const assignment = line.startsWith("export ") ? line.slice("export ".length).trim() : line;
|
|
541
|
+
const eq = assignment.indexOf("=");
|
|
542
|
+
if (eq <= 0) {
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
const key = assignment.slice(0, eq).trim();
|
|
546
|
+
const rawValue = assignment.slice(eq + 1).trim();
|
|
547
|
+
if (!/^[A-Z0-9_]+$/.test(key)) {
|
|
548
|
+
continue;
|
|
549
|
+
}
|
|
550
|
+
env[key] = unquoteProfileEnvValue(rawValue);
|
|
551
|
+
}
|
|
552
|
+
return env;
|
|
553
|
+
}
|
|
554
|
+
function unquoteProfileEnvValue(rawValue) {
|
|
555
|
+
if (rawValue.length >= 2) {
|
|
556
|
+
const quote = rawValue[0];
|
|
557
|
+
if ((quote === '"' || quote === "'") && rawValue.endsWith(quote)) {
|
|
558
|
+
const inner = rawValue.slice(1, -1);
|
|
559
|
+
return quote === '"' ? inner.replace(/\\"/g, '"').replace(/\\\\/g, "\\") : inner;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
return rawValue;
|
|
563
|
+
}
|
|
564
|
+
function profileSourceInstanceId(env) {
|
|
565
|
+
return env.PDPP_SOURCE_INSTANCE_ID?.trim() || env.PDPP_CONNECTION_ID?.trim() || null;
|
|
566
|
+
}
|
|
567
|
+
function safeProfileFileName(name) {
|
|
568
|
+
const trimmed = name.trim();
|
|
569
|
+
if (!/^[A-Za-z0-9._-]+$/.test(trimmed)) {
|
|
570
|
+
throw new CollectorUsageError("--profile must be a simple profile file name");
|
|
571
|
+
}
|
|
572
|
+
return trimmed.endsWith(".env") ? trimmed : `${trimmed}.env`;
|
|
573
|
+
}
|
|
574
|
+
export function findLocalCollectorProfiles(input) {
|
|
575
|
+
const profileDir = input.profileDir?.trim() || process.env[LOCAL_COLLECTOR_PROFILE_DIR_ENV]?.trim() || defaultCollectorProfileDir();
|
|
576
|
+
const sourceInstanceId = input.sourceInstanceId?.trim() || null;
|
|
577
|
+
const files = input.profileName
|
|
578
|
+
? [safeProfileFileName(input.profileName)]
|
|
579
|
+
: (() => {
|
|
580
|
+
try {
|
|
581
|
+
return readdirSync(profileDir).filter((name) => name.endsWith(".env")).sort();
|
|
582
|
+
}
|
|
583
|
+
catch {
|
|
584
|
+
return [];
|
|
585
|
+
}
|
|
586
|
+
})();
|
|
587
|
+
const matches = [];
|
|
588
|
+
for (const file of files) {
|
|
589
|
+
const path = join(profileDir, file);
|
|
590
|
+
let env;
|
|
591
|
+
try {
|
|
592
|
+
env = parseCollectorProfileEnv(readFileSync(path, "utf8"));
|
|
593
|
+
}
|
|
594
|
+
catch {
|
|
595
|
+
continue;
|
|
596
|
+
}
|
|
597
|
+
const profileSource = profileSourceInstanceId(env);
|
|
598
|
+
if (sourceInstanceId && profileSource !== sourceInstanceId) {
|
|
599
|
+
continue;
|
|
600
|
+
}
|
|
601
|
+
matches.push({
|
|
602
|
+
env,
|
|
603
|
+
name: file.replace(/\.env$/, ""),
|
|
604
|
+
path,
|
|
605
|
+
source_instance_id: profileSource,
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
return { matches, profile_dir: profileDir };
|
|
609
|
+
}
|
|
610
|
+
function applyProfileEnv(options, profile) {
|
|
611
|
+
const env = profile.env;
|
|
612
|
+
const explicit = options.explicitOptions;
|
|
613
|
+
const keep = (flag) => explicit?.has(flag) === true;
|
|
614
|
+
const next = {
|
|
615
|
+
...options,
|
|
616
|
+
baseUrl: keep("--base-url") ? options.baseUrl : env.PDPP_REFERENCE_BASE_URL?.trim() || options.baseUrl,
|
|
617
|
+
queuePath: keep("--queue") ? options.queuePath : env.PDPP_COLLECTOR_QUEUE?.trim() || options.queuePath,
|
|
618
|
+
};
|
|
619
|
+
const sourceInstanceId = profile.source_instance_id ?? options.sourceInstanceId;
|
|
620
|
+
const connector = keep("--connector") ? options.connector : env.PDPP_COLLECTOR_CONNECTOR?.trim() || options.connector;
|
|
621
|
+
const deviceId = keep("--device-id") ? options.deviceId : env.PDPP_LOCAL_DEVICE_ID?.trim() || options.deviceId;
|
|
622
|
+
const deviceToken = keep("--device-token") ? options.deviceToken : env.PDPP_LOCAL_DEVICE_TOKEN?.trim() || options.deviceToken;
|
|
623
|
+
if (sourceInstanceId) {
|
|
624
|
+
next.sourceInstanceId = sourceInstanceId;
|
|
625
|
+
}
|
|
626
|
+
if (connector) {
|
|
627
|
+
next.connector = connector;
|
|
628
|
+
}
|
|
629
|
+
if (deviceId) {
|
|
630
|
+
next.deviceId = deviceId;
|
|
631
|
+
}
|
|
632
|
+
if (deviceToken) {
|
|
633
|
+
next.deviceToken = deviceToken;
|
|
634
|
+
}
|
|
635
|
+
return next;
|
|
636
|
+
}
|
|
637
|
+
function resolveRecoveryOptions(options) {
|
|
638
|
+
const sourceInstanceId = options.sourceInstanceId?.trim();
|
|
639
|
+
if (!sourceInstanceId) {
|
|
640
|
+
throw new CollectorUsageError("recover requires --source-instance-id <id>");
|
|
641
|
+
}
|
|
642
|
+
const lookup = findLocalCollectorProfiles({
|
|
643
|
+
profileName: options.profile ?? null,
|
|
644
|
+
sourceInstanceId,
|
|
645
|
+
});
|
|
646
|
+
if (lookup.matches.length > 1) {
|
|
647
|
+
throw new CollectorUsageError(`recover found ${lookup.matches.length} local collector profiles for source_instance_id '${sourceInstanceId}'. ` +
|
|
648
|
+
"Pass --profile <name> to disambiguate.");
|
|
649
|
+
}
|
|
650
|
+
if (lookup.matches.length === 1) {
|
|
651
|
+
const profile = lookup.matches[0];
|
|
652
|
+
return {
|
|
653
|
+
options: applyProfileEnv(options, profile),
|
|
654
|
+
profileName: profile.name,
|
|
655
|
+
profileSource: "local_profile",
|
|
656
|
+
};
|
|
657
|
+
}
|
|
658
|
+
const configuredQueue = options.queuePath !== DEFAULT_QUEUE_PATH || Boolean(process.env.PDPP_COLLECTOR_QUEUE?.trim());
|
|
659
|
+
if (!configuredQueue) {
|
|
660
|
+
throw new CollectorUsageError(`recover could not find a local collector profile for source_instance_id '${sourceInstanceId}'. ` +
|
|
661
|
+
"Run this on the collector host after enrollment, pass --profile <name>, or set PDPP_COLLECTOR_QUEUE/--queue explicitly. " +
|
|
662
|
+
"Refusing to inspect the package default queue because it is often unrelated to the enrolled collector.");
|
|
663
|
+
}
|
|
664
|
+
return {
|
|
665
|
+
options,
|
|
666
|
+
profileName: null,
|
|
667
|
+
profileSource: "configured_queue",
|
|
668
|
+
};
|
|
669
|
+
}
|
|
670
|
+
function hasDeadLetters(status) {
|
|
671
|
+
return status.outbox.counts.dead_letter > 0;
|
|
672
|
+
}
|
|
673
|
+
function recoverDryRunNote(status) {
|
|
674
|
+
if (hasDeadLetters(status)) {
|
|
675
|
+
return (`${status.outbox.counts.dead_letter} failed upload row(s) would be prepared for retry, then the collector would run once. ` +
|
|
676
|
+
"Dry run only; re-run with --apply to mutate the local outbox and upload.");
|
|
677
|
+
}
|
|
678
|
+
return ("No failed upload rows are present for this source. The recovery apply step would run the collector once on this host " +
|
|
679
|
+
"to refresh state and drain queued work.");
|
|
680
|
+
}
|
|
681
|
+
function recoverAppliedNote(statusBefore, retry) {
|
|
682
|
+
const retried = retry ? `${retry.requeued} failed upload row(s) were prepared for retry. ` : "";
|
|
683
|
+
if (hasDeadLetters(statusBefore)) {
|
|
684
|
+
return `${retried}The collector ran once to upload queued work. Run status again if the dashboard has not refreshed yet.`;
|
|
685
|
+
}
|
|
686
|
+
return "The collector ran once to refresh local state and drain queued work.";
|
|
687
|
+
}
|
|
688
|
+
export async function recoverLocalCollector(options) {
|
|
689
|
+
const resolved = resolveRecoveryOptions(options);
|
|
690
|
+
const sourceInstanceId = resolved.options.sourceInstanceId;
|
|
691
|
+
if (!sourceInstanceId) {
|
|
692
|
+
throw new CollectorUsageError("recover requires --source-instance-id <id>");
|
|
693
|
+
}
|
|
694
|
+
const statusBefore = inspectLocalOutboxStatus(resolved.options);
|
|
695
|
+
const retryPreview = hasDeadLetters(statusBefore) ? retryLocalOutboxDeadLetters({ ...resolved.options, apply: false }) : null;
|
|
696
|
+
if (!options.apply) {
|
|
697
|
+
return {
|
|
698
|
+
applied: false,
|
|
699
|
+
db: statusBefore.db.path ? { exists: statusBefore.db.exists, path: statusBefore.db.path } : { exists: false, path: "" },
|
|
700
|
+
dry_run: true,
|
|
701
|
+
note: recoverDryRunNote(statusBefore),
|
|
702
|
+
object: "local_collector_recovery",
|
|
703
|
+
profile: { name: resolved.profileName, source: resolved.profileSource },
|
|
704
|
+
retry_dead_letters: retryPreview,
|
|
705
|
+
run: null,
|
|
706
|
+
source_instance_id: sourceInstanceId,
|
|
707
|
+
status_after: null,
|
|
708
|
+
status_before: statusBefore,
|
|
709
|
+
};
|
|
710
|
+
}
|
|
711
|
+
const retryApply = hasDeadLetters(statusBefore) ? retryLocalOutboxDeadLetters({ ...resolved.options, apply: true }) : null;
|
|
712
|
+
const run = summarizeRunResultForCli(await runCollectorOnce(resolved.options));
|
|
713
|
+
const statusAfter = inspectLocalOutboxStatus(resolved.options);
|
|
714
|
+
return {
|
|
715
|
+
applied: true,
|
|
716
|
+
db: statusAfter.db.path ? { exists: statusAfter.db.exists, path: statusAfter.db.path } : { exists: false, path: "" },
|
|
717
|
+
dry_run: false,
|
|
718
|
+
note: recoverAppliedNote(statusBefore, retryApply),
|
|
719
|
+
object: "local_collector_recovery",
|
|
720
|
+
profile: { name: resolved.profileName, source: resolved.profileSource },
|
|
721
|
+
retry_dead_letters: retryApply,
|
|
722
|
+
run,
|
|
723
|
+
source_instance_id: sourceInstanceId,
|
|
724
|
+
status_after: statusAfter,
|
|
725
|
+
status_before: statusBefore,
|
|
726
|
+
};
|
|
727
|
+
}
|
|
508
728
|
function summaryCounts(summary) {
|
|
509
729
|
return {
|
|
510
730
|
dead_letter: summary.deadLetter,
|
|
@@ -644,7 +864,7 @@ export function compactOutbox(options) {
|
|
|
644
864
|
db: { exists: true, path: dbPath },
|
|
645
865
|
dry_run: false,
|
|
646
866
|
note: `Refusing to compact: ${nonSucceeded} non-succeeded (ready/leased/dead-letter) row(s) are still in the outbox. ` +
|
|
647
|
-
"Drain the lane first (`pdpp-local-collector
|
|
867
|
+
"Drain the lane first (`pdpp-local-collector recover --source-instance-id <id> --apply` for stalled work), " +
|
|
648
868
|
"or pass --force to compact anyway. VACUUM is lossless — unsent rows are copied, never dropped — but compacting a " +
|
|
649
869
|
"live lane is refused by default so the reclaim runs on a quiet outbox.",
|
|
650
870
|
non_succeeded_rows: nonSucceeded,
|
|
@@ -739,16 +959,19 @@ export function parseArgs(args) {
|
|
|
739
959
|
command !== "advertise" &&
|
|
740
960
|
command !== "status" &&
|
|
741
961
|
command !== "doctor" &&
|
|
962
|
+
command !== "recover" &&
|
|
742
963
|
command !== "retry-dead-letters" &&
|
|
743
964
|
command !== "prune-sent" &&
|
|
744
965
|
command !== "compact") {
|
|
745
|
-
throw new CollectorUsageError(`usage: pdpp-local-collector <enroll|run|advertise|status|doctor|retry-dead-letters|prune-sent|compact> --base-url <url> [options]`);
|
|
966
|
+
throw new CollectorUsageError(`usage: pdpp-local-collector <enroll|run|advertise|status|doctor|recover|retry-dead-letters|prune-sent|compact> --base-url <url> [options]`);
|
|
746
967
|
}
|
|
747
968
|
const options = {
|
|
748
969
|
baseUrl: process.env.PDPP_REFERENCE_BASE_URL ?? "http://127.0.0.1:7662",
|
|
749
970
|
command,
|
|
750
971
|
queuePath: process.env.PDPP_COLLECTOR_QUEUE ?? DEFAULT_QUEUE_PATH,
|
|
751
972
|
};
|
|
973
|
+
const explicitOptions = new Set();
|
|
974
|
+
options.explicitOptions = explicitOptions;
|
|
752
975
|
if (process.env.PDPP_LOCAL_DEVICE_ID) {
|
|
753
976
|
options.deviceId = process.env.PDPP_LOCAL_DEVICE_ID;
|
|
754
977
|
}
|
|
@@ -773,10 +996,12 @@ export function parseArgs(args) {
|
|
|
773
996
|
throw new CollectorUsageError("missing option");
|
|
774
997
|
}
|
|
775
998
|
if (applyFlagOption(options, arg)) {
|
|
999
|
+
explicitOptions.add(arg);
|
|
776
1000
|
continue;
|
|
777
1001
|
}
|
|
778
1002
|
const value = rest[index + 1];
|
|
779
1003
|
applyOption(options, arg, value);
|
|
1004
|
+
explicitOptions.add(arg);
|
|
780
1005
|
index++;
|
|
781
1006
|
}
|
|
782
1007
|
return options;
|
|
@@ -827,14 +1052,17 @@ function applyOption(options, arg, value) {
|
|
|
827
1052
|
"--queue": (next) => {
|
|
828
1053
|
options.queuePath = next;
|
|
829
1054
|
},
|
|
1055
|
+
"--profile": (next) => {
|
|
1056
|
+
options.profile = next;
|
|
1057
|
+
},
|
|
830
1058
|
"--run-id": (next) => {
|
|
831
1059
|
options.runId = next;
|
|
832
1060
|
},
|
|
833
1061
|
"--connection-id": (next) => {
|
|
834
|
-
options
|
|
1062
|
+
setExplicitSourceInstanceId(options, arg, next);
|
|
835
1063
|
},
|
|
836
1064
|
"--source-instance-id": (next) => {
|
|
837
|
-
options
|
|
1065
|
+
setExplicitSourceInstanceId(options, arg, next);
|
|
838
1066
|
},
|
|
839
1067
|
"--streams": (next) => {
|
|
840
1068
|
options.streams = parseCsv(next);
|
|
@@ -858,6 +1086,13 @@ function applyOption(options, arg, value) {
|
|
|
858
1086
|
}
|
|
859
1087
|
set(value);
|
|
860
1088
|
}
|
|
1089
|
+
function setExplicitSourceInstanceId(options, arg, value) {
|
|
1090
|
+
const hadExplicitSource = options.explicitOptions?.has("--connection-id") || options.explicitOptions?.has("--source-instance-id");
|
|
1091
|
+
if (hadExplicitSource && options.sourceInstanceId && options.sourceInstanceId !== value) {
|
|
1092
|
+
throw new CollectorUsageError(`${arg} disagrees with the already supplied source identity '${options.sourceInstanceId}'`);
|
|
1093
|
+
}
|
|
1094
|
+
options.sourceInstanceId = value;
|
|
1095
|
+
}
|
|
861
1096
|
function parseOutboxKind(value) {
|
|
862
1097
|
if (value === "record_batch" || value === "checkpoint" || value === "gap" || value === "blob_upload") {
|
|
863
1098
|
return value;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
const COLLECTOR_BUILD_SOURCE_SENTINEL = "source";
|
|
2
2
|
const COLLECTOR_BUILD_INFO = {
|
|
3
|
-
builtAt: "2026-06-
|
|
4
|
-
revision: "
|
|
5
|
-
version: "0.
|
|
3
|
+
builtAt: "2026-06-17T01:36:27.814Z",
|
|
4
|
+
revision: "1d3825462d7b",
|
|
5
|
+
version: "0.6.0",
|
|
6
6
|
};
|
|
7
7
|
function buildAgentVersion(info = COLLECTOR_BUILD_INFO) {
|
|
8
8
|
return `${info.version}+${info.revision}`;
|
|
@@ -13,6 +13,7 @@ export interface StreamScope {
|
|
|
13
13
|
}
|
|
14
14
|
export interface StartMessage {
|
|
15
15
|
detail_gaps?: readonly DetailGapStartEntry[];
|
|
16
|
+
recovery_only?: boolean;
|
|
16
17
|
scope: {
|
|
17
18
|
streams: readonly StreamScope[];
|
|
18
19
|
};
|
|
@@ -150,6 +151,22 @@ export interface ProviderBudgetProgress {
|
|
|
150
151
|
request_count: number;
|
|
151
152
|
retry_tokens_remaining?: number | "unbounded";
|
|
152
153
|
}
|
|
154
|
+
export interface CollectionRateProgress {
|
|
155
|
+
ceiling_interval_ms: number;
|
|
156
|
+
ceiling_rate_per_min: number;
|
|
157
|
+
current_interval_ms: number;
|
|
158
|
+
effective_rate_per_min: number;
|
|
159
|
+
last_backoff: {
|
|
160
|
+
at_interval_ms: number;
|
|
161
|
+
reason: "retry_after" | "throttle";
|
|
162
|
+
} | null;
|
|
163
|
+
object: "collection_rate";
|
|
164
|
+
}
|
|
165
|
+
export interface ProgressExtra {
|
|
166
|
+
count?: number;
|
|
167
|
+
stream?: string;
|
|
168
|
+
total?: number;
|
|
169
|
+
}
|
|
153
170
|
export type EmittedMessage = {
|
|
154
171
|
type: "RECORD";
|
|
155
172
|
stream: string;
|
|
@@ -164,8 +181,11 @@ export type EmittedMessage = {
|
|
|
164
181
|
} | {
|
|
165
182
|
type: "PROGRESS";
|
|
166
183
|
message: string;
|
|
184
|
+
count?: number;
|
|
167
185
|
stream?: string;
|
|
186
|
+
total?: number;
|
|
168
187
|
provider_budget?: ProviderBudgetProgress;
|
|
188
|
+
collection_rate?: CollectionRateProgress;
|
|
169
189
|
} | ({
|
|
170
190
|
type: "ASSISTANCE";
|
|
171
191
|
} & AssistanceRequest) | ({
|
|
@@ -176,6 +196,10 @@ export type EmittedMessage = {
|
|
|
176
196
|
reason: string;
|
|
177
197
|
message: string;
|
|
178
198
|
diagnostics?: unknown;
|
|
199
|
+
recovery_hint?: string | {
|
|
200
|
+
action?: string;
|
|
201
|
+
retryable?: boolean;
|
|
202
|
+
};
|
|
179
203
|
} | DetailGapMessage | DetailCoverageMessage | DetailGapRecoveredMessage | DetailGapsPageRequestMessage | {
|
|
180
204
|
type: "DONE";
|
|
181
205
|
status: "succeeded" | "failed";
|
|
@@ -40,6 +40,14 @@ export const STATIC_SECRET_CONNECTOR_REGISTRY = Object.freeze({
|
|
|
40
40
|
slack_cookie: ["SLACK_COOKIE"],
|
|
41
41
|
},
|
|
42
42
|
}),
|
|
43
|
+
oura: freezeStaticSecretDescriptor({
|
|
44
|
+
credentialKind: "personal_access_token",
|
|
45
|
+
secretEnvVars: ["OURA_PERSONAL_ACCESS_TOKEN"],
|
|
46
|
+
}),
|
|
47
|
+
notion: freezeStaticSecretDescriptor({
|
|
48
|
+
credentialKind: "personal_access_token",
|
|
49
|
+
secretEnvVars: ["NOTION_API_TOKEN"],
|
|
50
|
+
}),
|
|
43
51
|
reddit: freezeStaticSecretDescriptor({
|
|
44
52
|
credentialKind: "secret_bundle",
|
|
45
53
|
secretFieldEnvVars: {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pdpp/local-collector",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Publishable local collector runtime for PDPP: filesystem-class connectors (Claude Code, Codex) plus the device-exporter ingest client.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"private": false,
|