@pdpp/local-collector 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -32,6 +32,13 @@ PDPP_CONNECTION_ID=<source_instance_id> \
32
32
  npx -y @pdpp/local-collector run \
33
33
  --base-url https://<reference-host> \
34
34
  --connector claude_code
35
+
36
+ # Preview host-local recovery for a stalled collector lane. This loads the
37
+ # enrolled local profile for the source instance and changes nothing.
38
+ npx -y @pdpp/local-collector recover --source-instance-id <source_instance_id>
39
+
40
+ # Apply recovery: requeue failed uploads when present, then run the collector once.
41
+ npx -y @pdpp/local-collector recover --source-instance-id <source_instance_id> --apply
35
42
  ```
36
43
 
37
44
  The collector sends `X-PDPP-Collector-Protocol` on enrollment and every
@@ -1,5 +1,6 @@
1
1
  #!/usr/bin/env node
2
- import { existsSync, readFileSync, realpathSync } from "node:fs";
2
+ import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
3
+ import { homedir } from "node:os";
3
4
  import { basename, dirname, extname, join, sep } from "node:path";
4
5
  import { fileURLToPath } from "node:url";
5
6
  import { ALLOW_CUSTOM_COMMAND_ENV, CollectorCustomCommandRefusedError, CollectorUsageError, } from "../src/errors.js";
@@ -8,6 +9,7 @@ const COVERAGE_DIAGNOSTICS_STREAM = "coverage_diagnostics";
8
9
  const DEFAULT_QUEUE_PATH = join(dirname(fileURLToPath(import.meta.url)), "..", ".pdpp-data", "collector-runner-queue.json");
9
10
  const LOCAL_COLLECTOR_PACKAGE_NAME = "@pdpp/local-collector";
10
11
  const LOCAL_COLLECTOR_PACKAGE_VERSION_FALLBACK = "0.0.0";
12
+ const LOCAL_COLLECTOR_PROFILE_DIR_ENV = "PDPP_LOCAL_COLLECTOR_PROFILE_DIR";
11
13
  const LOCAL_COLLECTOR_PLACEHOLDER_VERSION = "0.0.0";
12
14
  const REPO_ONLY_PACKAGE_SIBLINGS = ["src", "bin", "test", "scripts", "tsconfig.build.json"];
13
15
  function resolveLocalCollectorManifest(startUrl) {
@@ -96,18 +98,27 @@ Subcommands:
96
98
  status Print local durable outbox health as JSON.
97
99
  [--queue <path>]
98
100
  [--connection-id <id>]
101
+ [--source-instance-id <id>]
99
102
  doctor Print local durable outbox operator diagnostics as JSON.
100
103
  [--queue <path>]
101
104
  [--connection-id <id>]
105
+ [--source-instance-id <id>]
102
106
  retry-dead-letters Requeue local dead-letter outbox rows.
103
107
  [--queue <path>]
104
108
  [--connection-id <id>]
109
+ [--source-instance-id <id>]
105
110
  [--kind record_batch|checkpoint|gap|blob_upload]
106
111
  [--limit <n>]
107
112
  [--apply] Dry-run by default; --apply mutates after a DB backup.
113
+ recover Resolve the enrolled local profile, recover stalled outbox work,
114
+ and run the collector once.
115
+ --source-instance-id <id>
116
+ [--profile <name>] Optional profile name under the collector profile dir.
117
+ [--apply] Dry-run by default; --apply requeues and runs.
108
118
  prune-sent Delete sent (succeeded) outbox rows to reclaim disk space.
109
119
  [--queue <path>]
110
120
  [--connection-id <id>]
121
+ [--source-instance-id <id>]
111
122
  [--older-than-days <n>] Delete sent rows older than N days (default: 30).
112
123
  [--keep-count <n>] Keep at most N most-recent sent rows per connection.
113
124
  [--apply] Dry-run by default; --apply mutates after a DB backup.
@@ -126,6 +137,7 @@ Subcommands:
126
137
  --device-id <id>
127
138
  --device-token <token>
128
139
  --connection-id <id>
140
+ [--source-instance-id <id>]
129
141
  [--streams a,b,c]
130
142
  [--backfill-streams attachments]
131
143
  [--run-id <id>]
@@ -164,6 +176,11 @@ async function main() {
164
176
  process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
165
177
  return;
166
178
  }
179
+ if (options.command === "recover") {
180
+ const result = await recoverLocalCollector(options);
181
+ process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
182
+ return;
183
+ }
167
184
  if (options.command === "prune-sent") {
168
185
  const result = pruneSentOutboxRows(options);
169
186
  process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
@@ -189,14 +206,18 @@ async function main() {
189
206
  process.stdout.write(`${JSON.stringify(response, null, 2)}\n`);
190
207
  return;
191
208
  }
209
+ const result = await runCollectorOnce(options);
210
+ process.stdout.write(`${JSON.stringify(summarizeRunResultForCli(result), null, 2)}\n`);
211
+ }
212
+ async function runCollectorOnce(options) {
192
213
  if (!(options.deviceId && options.deviceToken && options.sourceInstanceId)) {
193
- throw new CollectorUsageError("run requires --device-id <id>, --device-token <token>, and --connection-id <id>");
214
+ throw new CollectorUsageError("run requires --device-id <id>, --device-token <token>, and --connection-id/--source-instance-id <id>");
194
215
  }
195
216
  if (!options.connector) {
196
217
  throw new CollectorUsageError("run requires --connector <connector-id>");
197
218
  }
198
219
  const spec = buildConnectorSpec(options);
199
- const result = await runCollectorConnector({
220
+ return runCollectorConnector({
200
221
  baseUrl: options.baseUrl,
201
222
  connector: spec,
202
223
  deviceId: options.deviceId,
@@ -205,7 +226,6 @@ async function main() {
205
226
  ...(options.runId ? { runId: options.runId } : {}),
206
227
  sourceInstanceId: options.sourceInstanceId,
207
228
  });
208
- process.stdout.write(`${JSON.stringify(summarizeRunResultForCli(result), null, 2)}\n`);
209
229
  }
210
230
  export function summarizeRunResultForCli(result) {
211
231
  const summary = result.outboxSummary;
@@ -251,7 +271,7 @@ function runDrainNote(result, summary, drained) {
251
271
  parts.push(`${summary.leased} leased (in flight)`);
252
272
  }
253
273
  if (summary.deadLetter > 0) {
254
- parts.push(`${summary.deadLetter} dead-letter (run \`retry-dead-letters\` then re-run)`);
274
+ parts.push(`${summary.deadLetter} dead-letter (run \`recover --source-instance-id <id> --apply\`)`);
255
275
  }
256
276
  const scanNote = result.scanBudgetExceeded
257
277
  ? " The connector was stopped by the per-run enqueue budget, so more source work likely remains; re-run to continue."
@@ -362,9 +382,9 @@ export function buildLocalOutboxDoctor(status, errorSummary) {
362
382
  ? ` Most common cause: ${topClass.error_class} (${topClass.count} row(s)).`
363
383
  : "";
364
384
  remediation.push(`${status.outbox.counts.dead_letter} dead-letter row(s) need recovery.${causeHint} ` +
365
- "Preview with `pdpp-local-collector retry-dead-letters`, then requeue with " +
366
- "`pdpp-local-collector retry-dead-letters --apply` (backs up the DB first), " +
367
- "then re-run the collector to drain the requeued rows.");
385
+ "Preview with `pdpp-local-collector recover --source-instance-id <id>`, then apply with " +
386
+ "`pdpp-local-collector recover --source-instance-id <id> --apply`. The apply step backs up the DB, " +
387
+ "prepares failed uploads for retry when present, and runs the collector once.");
368
388
  }
369
389
  if (checks.expired_leases === "warn") {
370
390
  remediation.push(`${status.outbox.expired_leases} lease(s) are past expiry — a previous run likely crashed mid-drain. ` +
@@ -440,16 +460,17 @@ export function readLocalOutboxDeadLetterErrorSummary(options) {
440
460
  }
441
461
  const RETRY_DEAD_LETTERS_NO_MATCH_NOTE = "No dead-letter rows matched. If the dashboard shows this connection as " +
442
462
  "blocked/stalled, that is a state-read block, not a dead-letter backlog — " +
443
- "there is nothing to requeue. Recovery is to re-run the collector " +
444
- "(`pdpp-local-collector run …`), which re-reads prior state and clears the block.";
463
+ "there is nothing to requeue. Use `pdpp-local-collector recover --source-instance-id <id> --apply` " +
464
+ "to run the collector through the enrolled local profile and clear the block.";
445
465
  function retryDeadLettersMatchNote(matched, dryRun) {
446
466
  if (matched === 0) {
447
467
  return RETRY_DEAD_LETTERS_NO_MATCH_NOTE;
448
468
  }
449
469
  const requeued = dryRun
450
- ? `${matched} dead-letter row(s) would be requeued (dry run). Re-run with --apply to requeue (backs up the DB first), `
470
+ ? `${matched} dead-letter row(s) would be requeued (dry run). `
451
471
  : `${matched} dead-letter row(s) matched and were requeued to pending. `;
452
- return `${requeued}then re-run the collector (\`pdpp-local-collector run …\`) to drain them requeue moves rows to pending, it does not ingest.`;
472
+ return (`${requeued}Use \`pdpp-local-collector recover --source-instance-id <id> --apply\` for the dashboard recovery path. ` +
473
+ "This low-level command only moves rows to pending; it does not ingest.");
453
474
  }
454
475
  export function retryLocalOutboxDeadLetters(options) {
455
476
  const dbPath = resolveOutboxPath(options);
@@ -505,6 +526,205 @@ export function retryLocalOutboxDeadLetters(options) {
505
526
  outbox.close();
506
527
  }
507
528
  }
529
+ function defaultCollectorProfileDir() {
530
+ const configHome = process.env.XDG_CONFIG_HOME?.trim() || join(homedir(), ".config");
531
+ return join(configHome, "pdpp", "collectors");
532
+ }
533
+ export function parseCollectorProfileEnv(contents) {
534
+ const env = {};
535
+ for (const rawLine of contents.split(/\r?\n/)) {
536
+ const line = rawLine.trim();
537
+ if (!line || line.startsWith("#")) {
538
+ continue;
539
+ }
540
+ const assignment = line.startsWith("export ") ? line.slice("export ".length).trim() : line;
541
+ const eq = assignment.indexOf("=");
542
+ if (eq <= 0) {
543
+ continue;
544
+ }
545
+ const key = assignment.slice(0, eq).trim();
546
+ const rawValue = assignment.slice(eq + 1).trim();
547
+ if (!/^[A-Z0-9_]+$/.test(key)) {
548
+ continue;
549
+ }
550
+ env[key] = unquoteProfileEnvValue(rawValue);
551
+ }
552
+ return env;
553
+ }
554
+ function unquoteProfileEnvValue(rawValue) {
555
+ if (rawValue.length >= 2) {
556
+ const quote = rawValue[0];
557
+ if ((quote === '"' || quote === "'") && rawValue.endsWith(quote)) {
558
+ const inner = rawValue.slice(1, -1);
559
+ return quote === '"' ? inner.replace(/\\"/g, '"').replace(/\\\\/g, "\\") : inner;
560
+ }
561
+ }
562
+ return rawValue;
563
+ }
564
+ function profileSourceInstanceId(env) {
565
+ return env.PDPP_SOURCE_INSTANCE_ID?.trim() || env.PDPP_CONNECTION_ID?.trim() || null;
566
+ }
567
+ function safeProfileFileName(name) {
568
+ const trimmed = name.trim();
569
+ if (!/^[A-Za-z0-9._-]+$/.test(trimmed)) {
570
+ throw new CollectorUsageError("--profile must be a simple profile file name");
571
+ }
572
+ return trimmed.endsWith(".env") ? trimmed : `${trimmed}.env`;
573
+ }
574
+ export function findLocalCollectorProfiles(input) {
575
+ const profileDir = input.profileDir?.trim() || process.env[LOCAL_COLLECTOR_PROFILE_DIR_ENV]?.trim() || defaultCollectorProfileDir();
576
+ const sourceInstanceId = input.sourceInstanceId?.trim() || null;
577
+ const files = input.profileName
578
+ ? [safeProfileFileName(input.profileName)]
579
+ : (() => {
580
+ try {
581
+ return readdirSync(profileDir).filter((name) => name.endsWith(".env")).sort();
582
+ }
583
+ catch {
584
+ return [];
585
+ }
586
+ })();
587
+ const matches = [];
588
+ for (const file of files) {
589
+ const path = join(profileDir, file);
590
+ let env;
591
+ try {
592
+ env = parseCollectorProfileEnv(readFileSync(path, "utf8"));
593
+ }
594
+ catch {
595
+ continue;
596
+ }
597
+ const profileSource = profileSourceInstanceId(env);
598
+ if (sourceInstanceId && profileSource !== sourceInstanceId) {
599
+ continue;
600
+ }
601
+ matches.push({
602
+ env,
603
+ name: file.replace(/\.env$/, ""),
604
+ path,
605
+ source_instance_id: profileSource,
606
+ });
607
+ }
608
+ return { matches, profile_dir: profileDir };
609
+ }
610
+ function applyProfileEnv(options, profile) {
611
+ const env = profile.env;
612
+ const explicit = options.explicitOptions;
613
+ const keep = (flag) => explicit?.has(flag) === true;
614
+ const next = {
615
+ ...options,
616
+ baseUrl: keep("--base-url") ? options.baseUrl : env.PDPP_REFERENCE_BASE_URL?.trim() || options.baseUrl,
617
+ queuePath: keep("--queue") ? options.queuePath : env.PDPP_COLLECTOR_QUEUE?.trim() || options.queuePath,
618
+ };
619
+ const sourceInstanceId = profile.source_instance_id ?? options.sourceInstanceId;
620
+ const connector = keep("--connector") ? options.connector : env.PDPP_COLLECTOR_CONNECTOR?.trim() || options.connector;
621
+ const deviceId = keep("--device-id") ? options.deviceId : env.PDPP_LOCAL_DEVICE_ID?.trim() || options.deviceId;
622
+ const deviceToken = keep("--device-token") ? options.deviceToken : env.PDPP_LOCAL_DEVICE_TOKEN?.trim() || options.deviceToken;
623
+ if (sourceInstanceId) {
624
+ next.sourceInstanceId = sourceInstanceId;
625
+ }
626
+ if (connector) {
627
+ next.connector = connector;
628
+ }
629
+ if (deviceId) {
630
+ next.deviceId = deviceId;
631
+ }
632
+ if (deviceToken) {
633
+ next.deviceToken = deviceToken;
634
+ }
635
+ return next;
636
+ }
637
+ function resolveRecoveryOptions(options) {
638
+ const sourceInstanceId = options.sourceInstanceId?.trim();
639
+ if (!sourceInstanceId) {
640
+ throw new CollectorUsageError("recover requires --source-instance-id <id>");
641
+ }
642
+ const lookup = findLocalCollectorProfiles({
643
+ profileName: options.profile ?? null,
644
+ sourceInstanceId,
645
+ });
646
+ if (lookup.matches.length > 1) {
647
+ throw new CollectorUsageError(`recover found ${lookup.matches.length} local collector profiles for source_instance_id '${sourceInstanceId}'. ` +
648
+ "Pass --profile <name> to disambiguate.");
649
+ }
650
+ if (lookup.matches.length === 1) {
651
+ const profile = lookup.matches[0];
652
+ return {
653
+ options: applyProfileEnv(options, profile),
654
+ profileName: profile.name,
655
+ profileSource: "local_profile",
656
+ };
657
+ }
658
+ const configuredQueue = options.queuePath !== DEFAULT_QUEUE_PATH || Boolean(process.env.PDPP_COLLECTOR_QUEUE?.trim());
659
+ if (!configuredQueue) {
660
+ throw new CollectorUsageError(`recover could not find a local collector profile for source_instance_id '${sourceInstanceId}'. ` +
661
+ "Run this on the collector host after enrollment, pass --profile <name>, or set PDPP_COLLECTOR_QUEUE/--queue explicitly. " +
662
+ "Refusing to inspect the package default queue because it is often unrelated to the enrolled collector.");
663
+ }
664
+ return {
665
+ options,
666
+ profileName: null,
667
+ profileSource: "configured_queue",
668
+ };
669
+ }
670
+ function hasDeadLetters(status) {
671
+ return status.outbox.counts.dead_letter > 0;
672
+ }
673
+ function recoverDryRunNote(status) {
674
+ if (hasDeadLetters(status)) {
675
+ return (`${status.outbox.counts.dead_letter} failed upload row(s) would be prepared for retry, then the collector would run once. ` +
676
+ "Dry run only; re-run with --apply to mutate the local outbox and upload.");
677
+ }
678
+ return ("No failed upload rows are present for this source. The recovery apply step would run the collector once on this host " +
679
+ "to refresh state and drain queued work.");
680
+ }
681
+ function recoverAppliedNote(statusBefore, retry) {
682
+ const retried = retry ? `${retry.requeued} failed upload row(s) were prepared for retry. ` : "";
683
+ if (hasDeadLetters(statusBefore)) {
684
+ return `${retried}The collector ran once to upload queued work. Run status again if the dashboard has not refreshed yet.`;
685
+ }
686
+ return "The collector ran once to refresh local state and drain queued work.";
687
+ }
688
+ export async function recoverLocalCollector(options) {
689
+ const resolved = resolveRecoveryOptions(options);
690
+ const sourceInstanceId = resolved.options.sourceInstanceId;
691
+ if (!sourceInstanceId) {
692
+ throw new CollectorUsageError("recover requires --source-instance-id <id>");
693
+ }
694
+ const statusBefore = inspectLocalOutboxStatus(resolved.options);
695
+ const retryPreview = hasDeadLetters(statusBefore) ? retryLocalOutboxDeadLetters({ ...resolved.options, apply: false }) : null;
696
+ if (!options.apply) {
697
+ return {
698
+ applied: false,
699
+ db: statusBefore.db.path ? { exists: statusBefore.db.exists, path: statusBefore.db.path } : { exists: false, path: "" },
700
+ dry_run: true,
701
+ note: recoverDryRunNote(statusBefore),
702
+ object: "local_collector_recovery",
703
+ profile: { name: resolved.profileName, source: resolved.profileSource },
704
+ retry_dead_letters: retryPreview,
705
+ run: null,
706
+ source_instance_id: sourceInstanceId,
707
+ status_after: null,
708
+ status_before: statusBefore,
709
+ };
710
+ }
711
+ const retryApply = hasDeadLetters(statusBefore) ? retryLocalOutboxDeadLetters({ ...resolved.options, apply: true }) : null;
712
+ const run = summarizeRunResultForCli(await runCollectorOnce(resolved.options));
713
+ const statusAfter = inspectLocalOutboxStatus(resolved.options);
714
+ return {
715
+ applied: true,
716
+ db: statusAfter.db.path ? { exists: statusAfter.db.exists, path: statusAfter.db.path } : { exists: false, path: "" },
717
+ dry_run: false,
718
+ note: recoverAppliedNote(statusBefore, retryApply),
719
+ object: "local_collector_recovery",
720
+ profile: { name: resolved.profileName, source: resolved.profileSource },
721
+ retry_dead_letters: retryApply,
722
+ run,
723
+ source_instance_id: sourceInstanceId,
724
+ status_after: statusAfter,
725
+ status_before: statusBefore,
726
+ };
727
+ }
508
728
  function summaryCounts(summary) {
509
729
  return {
510
730
  dead_letter: summary.deadLetter,
@@ -644,7 +864,7 @@ export function compactOutbox(options) {
644
864
  db: { exists: true, path: dbPath },
645
865
  dry_run: false,
646
866
  note: `Refusing to compact: ${nonSucceeded} non-succeeded (ready/leased/dead-letter) row(s) are still in the outbox. ` +
647
- "Drain the lane first (`pdpp-local-collector run …`, then `retry-dead-letters --apply` for any dead-letter rows), " +
867
+ "Drain the lane first (`pdpp-local-collector recover --source-instance-id <id> --apply` for stalled work), " +
648
868
  "or pass --force to compact anyway. VACUUM is lossless — unsent rows are copied, never dropped — but compacting a " +
649
869
  "live lane is refused by default so the reclaim runs on a quiet outbox.",
650
870
  non_succeeded_rows: nonSucceeded,
@@ -739,16 +959,19 @@ export function parseArgs(args) {
739
959
  command !== "advertise" &&
740
960
  command !== "status" &&
741
961
  command !== "doctor" &&
962
+ command !== "recover" &&
742
963
  command !== "retry-dead-letters" &&
743
964
  command !== "prune-sent" &&
744
965
  command !== "compact") {
745
- throw new CollectorUsageError(`usage: pdpp-local-collector <enroll|run|advertise|status|doctor|retry-dead-letters|prune-sent|compact> --base-url <url> [options]`);
966
+ throw new CollectorUsageError(`usage: pdpp-local-collector <enroll|run|advertise|status|doctor|recover|retry-dead-letters|prune-sent|compact> --base-url <url> [options]`);
746
967
  }
747
968
  const options = {
748
969
  baseUrl: process.env.PDPP_REFERENCE_BASE_URL ?? "http://127.0.0.1:7662",
749
970
  command,
750
971
  queuePath: process.env.PDPP_COLLECTOR_QUEUE ?? DEFAULT_QUEUE_PATH,
751
972
  };
973
+ const explicitOptions = new Set();
974
+ options.explicitOptions = explicitOptions;
752
975
  if (process.env.PDPP_LOCAL_DEVICE_ID) {
753
976
  options.deviceId = process.env.PDPP_LOCAL_DEVICE_ID;
754
977
  }
@@ -773,10 +996,12 @@ export function parseArgs(args) {
773
996
  throw new CollectorUsageError("missing option");
774
997
  }
775
998
  if (applyFlagOption(options, arg)) {
999
+ explicitOptions.add(arg);
776
1000
  continue;
777
1001
  }
778
1002
  const value = rest[index + 1];
779
1003
  applyOption(options, arg, value);
1004
+ explicitOptions.add(arg);
780
1005
  index++;
781
1006
  }
782
1007
  return options;
@@ -827,14 +1052,17 @@ function applyOption(options, arg, value) {
827
1052
  "--queue": (next) => {
828
1053
  options.queuePath = next;
829
1054
  },
1055
+ "--profile": (next) => {
1056
+ options.profile = next;
1057
+ },
830
1058
  "--run-id": (next) => {
831
1059
  options.runId = next;
832
1060
  },
833
1061
  "--connection-id": (next) => {
834
- options.sourceInstanceId = next;
1062
+ setExplicitSourceInstanceId(options, arg, next);
835
1063
  },
836
1064
  "--source-instance-id": (next) => {
837
- options.sourceInstanceId = next;
1065
+ setExplicitSourceInstanceId(options, arg, next);
838
1066
  },
839
1067
  "--streams": (next) => {
840
1068
  options.streams = parseCsv(next);
@@ -858,6 +1086,13 @@ function applyOption(options, arg, value) {
858
1086
  }
859
1087
  set(value);
860
1088
  }
1089
+ function setExplicitSourceInstanceId(options, arg, value) {
1090
+ const hadExplicitSource = options.explicitOptions?.has("--connection-id") || options.explicitOptions?.has("--source-instance-id");
1091
+ if (hadExplicitSource && options.sourceInstanceId && options.sourceInstanceId !== value) {
1092
+ throw new CollectorUsageError(`${arg} disagrees with the already supplied source identity '${options.sourceInstanceId}'`);
1093
+ }
1094
+ options.sourceInstanceId = value;
1095
+ }
861
1096
  function parseOutboxKind(value) {
862
1097
  if (value === "record_batch" || value === "checkpoint" || value === "gap" || value === "blob_upload") {
863
1098
  return value;
@@ -1,8 +1,8 @@
1
1
  const COLLECTOR_BUILD_SOURCE_SENTINEL = "source";
2
2
  const COLLECTOR_BUILD_INFO = {
3
- builtAt: "2026-06-11T00:01:48.219Z",
4
- revision: "5ac4f6d1a3a9",
5
- version: "0.4.0",
3
+ builtAt: "2026-06-17T01:36:27.814Z",
4
+ revision: "1d3825462d7b",
5
+ version: "0.6.0",
6
6
  };
7
7
  function buildAgentVersion(info = COLLECTOR_BUILD_INFO) {
8
8
  return `${info.version}+${info.revision}`;
@@ -13,6 +13,7 @@ export interface StreamScope {
13
13
  }
14
14
  export interface StartMessage {
15
15
  detail_gaps?: readonly DetailGapStartEntry[];
16
+ recovery_only?: boolean;
16
17
  scope: {
17
18
  streams: readonly StreamScope[];
18
19
  };
@@ -150,6 +151,22 @@ export interface ProviderBudgetProgress {
150
151
  request_count: number;
151
152
  retry_tokens_remaining?: number | "unbounded";
152
153
  }
154
+ export interface CollectionRateProgress {
155
+ ceiling_interval_ms: number;
156
+ ceiling_rate_per_min: number;
157
+ current_interval_ms: number;
158
+ effective_rate_per_min: number;
159
+ last_backoff: {
160
+ at_interval_ms: number;
161
+ reason: "retry_after" | "throttle";
162
+ } | null;
163
+ object: "collection_rate";
164
+ }
165
+ export interface ProgressExtra {
166
+ count?: number;
167
+ stream?: string;
168
+ total?: number;
169
+ }
153
170
  export type EmittedMessage = {
154
171
  type: "RECORD";
155
172
  stream: string;
@@ -164,8 +181,11 @@ export type EmittedMessage = {
164
181
  } | {
165
182
  type: "PROGRESS";
166
183
  message: string;
184
+ count?: number;
167
185
  stream?: string;
186
+ total?: number;
168
187
  provider_budget?: ProviderBudgetProgress;
188
+ collection_rate?: CollectionRateProgress;
169
189
  } | ({
170
190
  type: "ASSISTANCE";
171
191
  } & AssistanceRequest) | ({
@@ -176,6 +196,10 @@ export type EmittedMessage = {
176
196
  reason: string;
177
197
  message: string;
178
198
  diagnostics?: unknown;
199
+ recovery_hint?: string | {
200
+ action?: string;
201
+ retryable?: boolean;
202
+ };
179
203
  } | DetailGapMessage | DetailCoverageMessage | DetailGapRecoveredMessage | DetailGapsPageRequestMessage | {
180
204
  type: "DONE";
181
205
  status: "succeeded" | "failed";
@@ -255,6 +255,7 @@ export function runConnector(config) {
255
255
  emittedAt,
256
256
  detailGaps: startMsg.detail_gaps ?? [],
257
257
  requestDetailGapPage,
258
+ recoveryOnly: startMsg.recovery_only === true,
258
259
  };
259
260
  if (browser) {
260
261
  await runInBrowser({
@@ -40,6 +40,14 @@ export const STATIC_SECRET_CONNECTOR_REGISTRY = Object.freeze({
40
40
  slack_cookie: ["SLACK_COOKIE"],
41
41
  },
42
42
  }),
43
+ oura: freezeStaticSecretDescriptor({
44
+ credentialKind: "personal_access_token",
45
+ secretEnvVars: ["OURA_PERSONAL_ACCESS_TOKEN"],
46
+ }),
47
+ notion: freezeStaticSecretDescriptor({
48
+ credentialKind: "personal_access_token",
49
+ secretEnvVars: ["NOTION_API_TOKEN"],
50
+ }),
43
51
  reddit: freezeStaticSecretDescriptor({
44
52
  credentialKind: "secret_bundle",
45
53
  secretFieldEnvVars: {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pdpp/local-collector",
3
- "version": "0.4.0",
3
+ "version": "0.6.0",
4
4
  "description": "Publishable local collector runtime for PDPP: filesystem-class connectors (Claude Code, Codex) plus the device-exporter ingest client.",
5
5
  "type": "module",
6
6
  "private": false,