@indigoai-us/hq-cli 5.345.191 → 5.345.194

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/CHANGELOG.md +431 -0
  2. package/assets/bot-workers/setup/worker.yaml +91 -34
  3. package/dist/command-catalog.generated.d.ts +145 -19
  4. package/dist/command-catalog.generated.js +184 -21
  5. package/dist/commands/bot-jobs.d.ts +21 -18
  6. package/dist/commands/bot-jobs.js +127 -39
  7. package/dist/commands/bot.d.ts +4 -3
  8. package/dist/commands/bot.js +265 -140
  9. package/dist/commands/core-jobs-schedule.js +10 -0
  10. package/dist/commands/daemon.js +37 -0
  11. package/dist/commands/files.d.ts +41 -6
  12. package/dist/commands/files.js +108 -27
  13. package/dist/commands/pack-start-bot.d.ts +4 -6
  14. package/dist/commands/pack-start-bot.js +4 -8
  15. package/dist/commands/signals-run.d.ts +46 -0
  16. package/dist/commands/signals-run.js +180 -0
  17. package/dist/commands/signals.d.ts +1 -0
  18. package/dist/commands/signals.js +3 -0
  19. package/dist/commands/sources.d.ts +58 -0
  20. package/dist/commands/sources.js +294 -26
  21. package/dist/commands/staff/reset-test-account.d.ts +10 -0
  22. package/dist/commands/staff/reset-test-account.js +18 -2
  23. package/dist/commands/whoami.d.ts +7 -0
  24. package/dist/commands/whoami.js +35 -2
  25. package/dist/hq-cloud-sync-runner/bin/sync-runner.js +31171 -52512
  26. package/dist/hq-cloud-sync-runner/package.json +1 -1
  27. package/dist/lib/bot/api.d.ts +3 -2
  28. package/dist/lib/bot/config.d.ts +2 -0
  29. package/dist/lib/bot/fleet-cron.d.ts +121 -0
  30. package/dist/lib/bot/fleet-cron.js +319 -0
  31. package/dist/lib/bot/fleet-local.d.ts +93 -1
  32. package/dist/lib/bot/fleet-local.js +546 -34
  33. package/dist/lib/bot/fleet-owner-companies.d.ts +12 -0
  34. package/dist/lib/bot/fleet-owner-companies.js +30 -0
  35. package/dist/lib/bot/fleet-watch.d.ts +25 -0
  36. package/dist/lib/bot/fleet-watch.js +51 -0
  37. package/dist/lib/bot/index.d.ts +1 -1
  38. package/dist/lib/bot/index.js +1 -1
  39. package/dist/lib/bot/job-run.d.ts +5 -4
  40. package/dist/lib/bot/job-run.js +5 -4
  41. package/dist/lib/bot/jobs/hot-signal-sync.d.ts +57 -3
  42. package/dist/lib/bot/jobs/hot-signal-sync.js +163 -4
  43. package/dist/lib/bot/local-jobs.d.ts +13 -2
  44. package/dist/lib/bot/local-jobs.js +17 -4
  45. package/dist/lib/bot/progress.d.ts +14 -0
  46. package/dist/lib/bot/progress.js +47 -1
  47. package/dist/lib/bot/run.d.ts +55 -2
  48. package/dist/lib/bot/run.js +192 -10
  49. package/dist/lib/bot/setup-tool-offer.d.ts +1 -0
  50. package/dist/lib/bot/setup-tool-offer.js +16 -0
  51. package/dist/lib/bot/signals-engine-flag.d.ts +18 -0
  52. package/dist/lib/bot/signals-engine-flag.js +30 -0
  53. package/dist/lib/daemon/catalog.d.ts +4 -0
  54. package/dist/lib/daemon/catalog.js +29 -9
  55. package/dist/lib/daemon/legacy.d.ts +6 -0
  56. package/dist/lib/daemon/legacy.js +16 -10
  57. package/dist/lib/daemon/run.d.ts +1 -0
  58. package/dist/lib/daemon/run.js +10 -0
  59. package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
  60. package/dist/lib/flag-registry-worker-source.generated.js +1 -1
  61. package/dist/lib/flag-registry.d.ts +13 -1
  62. package/dist/lib/flag-registry.js +46 -0
  63. package/dist/lib/hot-channels/adapters/attio.d.ts +75 -2
  64. package/dist/lib/hot-channels/adapters/attio.js +367 -29
  65. package/dist/lib/hot-channels/adapters/gmail.d.ts +14 -0
  66. package/dist/lib/hot-channels/adapters/gmail.js +59 -2
  67. package/dist/lib/hot-channels/adapters/linear.d.ts +24 -0
  68. package/dist/lib/hot-channels/adapters/linear.js +71 -1
  69. package/dist/lib/hot-channels/adapters/meetings.js +4 -1
  70. package/dist/lib/hot-channels/adapters/slack.js +7 -3
  71. package/dist/lib/hot-channels/alert.js +1 -1
  72. package/dist/lib/hot-channels/contract.d.ts +11 -0
  73. package/dist/lib/hot-channels/contract.js +1 -0
  74. package/dist/lib/hot-channels/cursors.js +1 -1
  75. package/dist/lib/hot-channels/landing.d.ts +6 -2
  76. package/dist/lib/hot-channels/landing.js +13 -6
  77. package/dist/lib/hot-channels/ledger.js +2 -0
  78. package/dist/lib/hot-channels/loop.d.ts +6 -0
  79. package/dist/lib/hot-channels/loop.js +19 -4
  80. package/dist/lib/hot-channels/registry.js +9 -1
  81. package/dist/lib/hot-channels/run.d.ts +18 -0
  82. package/dist/lib/hot-channels/run.js +15 -4
  83. package/dist/lib/hot-channels/signals-pass.d.ts +34 -2
  84. package/dist/lib/hot-channels/signals-pass.js +149 -38
  85. package/dist/lib/hot-channels/signals-redo.d.ts +60 -0
  86. package/dist/lib/hot-channels/signals-redo.js +207 -0
  87. package/dist/lib/hot-channels/sync-lock.js +22 -6
  88. package/dist/lib/mesh/live/daemon/install.d.ts +22 -0
  89. package/dist/lib/mesh/live/daemon/install.js +117 -30
  90. package/dist/lib/signals/adapters-catalog.d.ts +77 -0
  91. package/dist/lib/signals/adapters-catalog.js +194 -0
  92. package/dist/lib/signals/engine.d.ts +202 -0
  93. package/dist/lib/signals/engine.js +504 -0
  94. package/dist/lib/signals/failure-alert.d.ts +89 -0
  95. package/dist/lib/signals/failure-alert.js +175 -0
  96. package/dist/lib/signals/local-date.d.ts +3 -0
  97. package/dist/lib/signals/local-date.js +6 -0
  98. package/dist/lib/signals/projection.d.ts +42 -0
  99. package/dist/lib/signals/projection.js +67 -0
  100. package/dist/lib/signals/reply.d.ts +112 -0
  101. package/dist/lib/signals/reply.js +368 -0
  102. package/dist/lib/signals/run-lock.d.ts +36 -0
  103. package/dist/lib/signals/run-lock.js +148 -0
  104. package/dist/lib/signals/run-record.d.ts +24 -0
  105. package/dist/lib/signals/run-record.js +82 -0
  106. package/dist/lib/signals/sanitize.d.ts +105 -0
  107. package/dist/lib/signals/sanitize.js +439 -0
  108. package/dist/lib/signals/settings.d.ts +10 -0
  109. package/dist/lib/signals/settings.js +48 -0
  110. package/dist/lib/signals/steps/adapters.d.ts +14 -0
  111. package/dist/lib/signals/steps/adapters.js +156 -0
  112. package/dist/lib/signals/steps/entity-similarity.d.ts +10 -0
  113. package/dist/lib/signals/steps/entity-similarity.js +47 -0
  114. package/dist/lib/signals/steps/extract.d.ts +118 -0
  115. package/dist/lib/signals/steps/extract.js +793 -0
  116. package/dist/lib/signals/steps/garden.d.ts +24 -0
  117. package/dist/lib/signals/steps/garden.js +250 -0
  118. package/dist/lib/signals/steps/types.d.ts +117 -0
  119. package/dist/lib/signals/steps/types.js +4 -0
  120. package/dist/lib/signals/steps/upload.d.ts +43 -0
  121. package/dist/lib/signals/steps/upload.js +324 -0
  122. package/dist/lib/signals/write-guard.d.ts +63 -0
  123. package/dist/lib/signals/write-guard.js +287 -0
  124. package/dist/utils/resident-package-preload.js +16 -0
  125. package/dist/utils/vault-api.d.ts +1 -0
  126. package/dist/utils/vault-api.js +1 -1
  127. package/package.json +9 -9
  128. package/dist/lib/bot/fleet-default-flag.d.ts +0 -31
  129. package/dist/lib/bot/fleet-default-flag.js +0 -127
@@ -1,9 +1,22 @@
1
1
  /**
2
- * The local signals pass (US-011). After a hot-channel run lands sources, the
3
- * pass takes the run's landed documents through the ontology worklist, asks
2
+ * The local signals pass (US-011). After each hot-channel adapter run, the
3
+ * pass takes the adapter's channel through the ontology worklist, asks
4
4
  * `claude -p` for signals one document at a time, and writes those signals to
5
5
  * `companies/{co}/signals/`. It returns the count for the run ledger.
6
6
  *
7
+ * Work comes from the worklist, not only from this run's landings: when the
8
+ * worklist script exits 0, the pass takes the channel's unprocessed items in
9
+ * the worklist's order (newest first) up to the run's document cap. Items the
10
+ * cap leaves over stay unmarked, and a later run picks them up, including a
11
+ * run that landed nothing (a bounded backlog drain: at most
12
+ * DEFAULT_MAX_SIGNAL_DOCS claude calls per run, and no claude call at all when
13
+ * the backlog is empty). A channel with no source.yaml (worklist exit 2, the
14
+ * usual case for a hot channel such as `meeting-events`) has no worklist, so
15
+ * the pass keeps its own backlog: landed documents it has not processed yet
16
+ * are listed in `sources/_signals-pending/{channel}.json` and drained, newest
17
+ * first by `when` (else `created_at`), on later runs under the same cap. A
18
+ * cloud-run source (exit 4) is skipped.
19
+ *
7
20
  * A landed document with both `payload_ref` and `raw_ref` is a pointer (the
8
21
  * meetings, Slack and Attio adapters land these). For it, claude gets the
9
22
  * pointer followed by the document `payload_ref` names, capped at
@@ -25,6 +38,7 @@ import * as path from "node:path";
25
38
  import { fileURLToPath } from "node:url";
26
39
  import * as yaml from "js-yaml";
27
40
  import { audienceKey, COMPANY_AUDIENCE_KEY } from "./audience.js";
41
+ import { assertVaultPath } from "../signals/write-guard.js";
28
42
  import { assertPayloadRef, assertSafeSegment, writeFileAtomic } from "./landing.js";
29
43
  /**
30
44
  * A pass failure the run ledger records as `status: failed` with `error` set
@@ -79,8 +93,6 @@ export const WORKLIST_TIMEOUT_MS = 60_000;
79
93
  export const DEFAULT_MAX_SIGNAL_DOCS = 20;
80
94
  /** Payload bytes sent to claude per document (200 KB). A longer payload is cut with a marker. */
81
95
  export const MAX_PAYLOAD_BYTES = 200 * 1024;
82
- /** Items the worklist lists per channel; landed documents must fall inside it. */
83
- const WORKLIST_LIMIT = 500;
84
96
  /** Output kept per child stream. A longer reply is bad output. */
85
97
  const MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
86
98
  /** Signals kept per document. */
@@ -171,8 +183,8 @@ export function runBounded(command, args, options) {
171
183
  export function defaultPromptFile() {
172
184
  return path.join(path.dirname(fileURLToPath(import.meta.url)), "prompts", "signals-pass.md");
173
185
  }
174
- const LOGGED_OUT = /not logged in|log ?in|\/login|invalid api key|authenticat|oauth token/i;
175
- function channelOf(key) {
186
+ export const CLAUDE_LOGGED_OUT_RE = /not logged in|log ?in|\/login|invalid api key|authenticat|oauth token/i;
187
+ export function channelOf(key) {
176
188
  const parts = key.split("/");
177
189
  if (parts.length !== 3 || parts[0] !== "sources" || !parts[2].endsWith(".md"))
178
190
  return null;
@@ -185,7 +197,69 @@ function channelOf(key) {
185
197
  }
186
198
  return parts[1];
187
199
  }
188
- function splitFrontmatter(text) {
200
+ /**
201
+ * Vault-relative file that lists a channel's landed documents the pass has not
202
+ * processed yet. Only channels with no worklist (no source.yaml) use it.
203
+ */
204
+ export function pendingSignalsKey(channel) {
205
+ assertSafeSegment(channel, "channel");
206
+ return `sources/_signals-pending/${channel}.json`;
207
+ }
208
+ /** The channel's pending landed keys; a missing or unreadable file is an empty backlog. */
209
+ export async function readPendingSignals(vaultRoot, channel) {
210
+ let parsed;
211
+ try {
212
+ parsed = JSON.parse(await fs.readFile(path.join(vaultRoot, pendingSignalsKey(channel)), "utf8"));
213
+ }
214
+ catch {
215
+ return [];
216
+ }
217
+ if (!Array.isArray(parsed))
218
+ return [];
219
+ return parsed.filter((key) => typeof key === "string" && channelOf(key) === channel);
220
+ }
221
+ /** Replace the channel's pending list; an empty list removes the file. */
222
+ export async function writePendingSignals(vaultRoot, channel, keys, beforeWrite = async () => undefined) {
223
+ const file = path.join(vaultRoot, pendingSignalsKey(channel));
224
+ assertVaultPath(vaultRoot, file);
225
+ await beforeWrite();
226
+ if (keys.length === 0) {
227
+ await fs.rm(file, { force: true });
228
+ return;
229
+ }
230
+ await writeFileAtomic(vaultRoot, file, `${JSON.stringify([...new Set(keys)], null, 2)}\n`);
231
+ }
232
+ /** Add landed keys to the channel's pending list (kept once each). */
233
+ export async function queuePendingSignals(vaultRoot, channel, keys, beforeWrite) {
234
+ for (const key of keys) {
235
+ if (channelOf(key) !== channel)
236
+ throw new Error(`${key} is not a document in sources/${channel}/`);
237
+ }
238
+ await writePendingSignals(vaultRoot, channel, [...(await readPendingSignals(vaultRoot, channel)), ...keys], beforeWrite);
239
+ }
240
+ /** When a landed document happened: `when`, else `created_at`; unknown sorts last. */
241
+ export function landedDocTime(frontmatter) {
242
+ for (const field of ["when", "created_at"]) {
243
+ const value = frontmatter[field];
244
+ const text = value instanceof Date ? value.toISOString() : typeof value === "string" ? value : undefined;
245
+ if (text === undefined)
246
+ continue;
247
+ const ms = Date.parse(text);
248
+ if (!Number.isNaN(ms))
249
+ return ms;
250
+ }
251
+ return Number.NEGATIVE_INFINITY;
252
+ }
253
+ function isSafeChannel(channel) {
254
+ try {
255
+ assertSafeSegment(channel, "channel");
256
+ return true;
257
+ }
258
+ catch {
259
+ return false;
260
+ }
261
+ }
262
+ export function splitFrontmatter(text) {
189
263
  const match = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/.exec(text);
190
264
  if (!match)
191
265
  return { frontmatter: {}, body: text };
@@ -284,7 +358,7 @@ export function parseClaudeSignals(stdout) {
284
358
  const env = envelope;
285
359
  if (env.is_error === true) {
286
360
  const text = typeof env.result === "string" ? env.result : "";
287
- if (LOGGED_OUT.test(text))
361
+ if (CLAUDE_LOGGED_OUT_RE.test(text))
288
362
  throw new SignalsPassError("claude-logged-out", "claude reported it is not logged in");
289
363
  throw new SignalsPassError("claude-bad-output", "claude returned an error result");
290
364
  }
@@ -353,6 +427,7 @@ export async function ensureAudienceFile(vaultRoot, item) {
353
427
  return;
354
428
  assertSafeSegment(item.audienceKey, "audience key");
355
429
  const file = path.join(vaultRoot, "signals", `@${item.audienceKey}`, "_audience.yaml");
430
+ assertVaultPath(vaultRoot, file);
356
431
  if (await exists(file))
357
432
  return;
358
433
  const principals = [...new Set(item.principals.map((p) => p.trim()).filter((p) => p.length > 0))].sort();
@@ -384,7 +459,7 @@ async function appendSignalIndex(vaultRoot, day, rows) {
384
459
  const added = rows.filter((row) => !seen.has(row.signal_id));
385
460
  if (added.length === 0)
386
461
  return;
387
- await writeFileAtomic(file, `${JSON.stringify([...existing, ...added], null, 2)}\n`);
462
+ await writeFileAtomic(vaultRoot, file, `${JSON.stringify([...existing, ...added], null, 2)}\n`);
388
463
  }
389
464
  /**
390
465
  * Build the claude signals pass for one company. The pass keeps state for one
@@ -431,15 +506,19 @@ export function createClaudeSignalsPass(options) {
431
506
  cwd: options.hqRoot,
432
507
  env: { ...env, HQ_ROOT: options.hqRoot },
433
508
  });
434
- /** Work items for the landed documents of one channel. */
435
- const itemsFor = async (channel, files) => {
436
- const wanted = new Set(files);
437
- const run = await runWorklist(["--company", options.company, "--channel", channel, "--limit", String(WORKLIST_LIMIT)]);
509
+ /**
510
+ * Work items for one channel. With a worklist: its unprocessed items in its
511
+ * order (at most `limit`). Without one: this run's landed `files` plus the
512
+ * channel's pending backlog, newest first, all of them (`pending` is set and
513
+ * the caller applies the cap).
514
+ */
515
+ const itemsFor = async (channel, files, limit) => {
516
+ const run = await runWorklist(["--company", options.company, "--channel", channel, "--limit", String(limit)]);
438
517
  if (run.timedOut)
439
518
  throw new Error(`ontology worklist timed out for channel ${channel}`);
440
519
  // Exit 4: the source is assigned to the cloud agent. Its documents are not this pass's to process.
441
520
  if (run.code === 4)
442
- return [];
521
+ return { items: [], pending: false };
443
522
  if (run.code === 0) {
444
523
  const items = [];
445
524
  for (const line of run.stdout.split("\n")) {
@@ -452,7 +531,7 @@ export function createClaudeSignalsPass(options) {
452
531
  catch {
453
532
  continue;
454
533
  }
455
- if (typeof row.file !== "string" || !wanted.has(row.file) || row.skip !== undefined)
534
+ if (typeof row.file !== "string" || channelOf(row.file) !== channel || row.skip !== undefined)
456
535
  continue;
457
536
  if (typeof row.audience_key !== "string" || !Array.isArray(row.audience))
458
537
  continue;
@@ -465,30 +544,46 @@ export function createClaudeSignalsPass(options) {
465
544
  fromWorklist: true,
466
545
  });
467
546
  }
468
- return items;
547
+ return { items, pending: false };
469
548
  }
470
549
  // Any other exit (usually 2: the channel has no source.yaml yet, which is
471
550
  // normal for a hot channel) falls back to the landed document's own
472
- // audience frontmatter, which landing always writes.
473
- const items = [];
474
- for (const file of files) {
475
- const { frontmatter } = splitFrontmatter(await fs.readFile(path.join(vaultRoot, file), "utf8"));
551
+ // audience frontmatter, which landing always writes. The pass's own
552
+ // pending backlog stands in for the worklist.
553
+ const candidates = [...new Set([...files, ...(await readPendingSignals(vaultRoot, channel))])];
554
+ const dated = [];
555
+ for (const file of candidates) {
556
+ let text;
557
+ try {
558
+ text = await fs.readFile(path.join(vaultRoot, file), "utf8");
559
+ }
560
+ catch (err) {
561
+ // A pending document that is gone has nothing left to process.
562
+ if (isMissing(err))
563
+ continue;
564
+ throw err;
565
+ }
566
+ const { frontmatter } = splitFrontmatter(text);
476
567
  const audience = Array.isArray(frontmatter.audience)
477
568
  ? frontmatter.audience.filter((a) => typeof a === "string" && a.trim().length > 0)
478
569
  : [];
479
570
  // No audience: skipped, never widened to company.
480
571
  if (audience.length === 0)
481
572
  continue;
482
- items.push({
483
- file,
484
- channel,
485
- principals: audience,
486
- audienceKey: audienceKey(audience),
487
- audienceSource: typeof frontmatter.audience_source === "string" ? frontmatter.audience_source : "frontmatter",
488
- fromWorklist: false,
573
+ dated.push({
574
+ item: {
575
+ file,
576
+ channel,
577
+ principals: audience,
578
+ audienceKey: audienceKey(audience),
579
+ audienceSource: typeof frontmatter.audience_source === "string" ? frontmatter.audience_source : "frontmatter",
580
+ fromWorklist: false,
581
+ },
582
+ at: landedDocTime(frontmatter),
489
583
  });
490
584
  }
491
- return items;
585
+ dated.sort((a, b) => b.at - a.at || (a.item.file < b.item.file ? -1 : a.item.file > b.item.file ? 1 : 0));
586
+ return { items: dated.map((d) => d.item), pending: true };
492
587
  };
493
588
  const extract = async (item, input) => {
494
589
  prompt ??= fs.readFile(promptFile, "utf8");
@@ -502,7 +597,7 @@ export function createClaudeSignalsPass(options) {
502
597
  throw new SignalsPassError("claude-timeout", `claude timed out after ${timeoutMs}ms on ${item.file}`);
503
598
  if (run.spawnError !== undefined)
504
599
  throw new SignalsPassError("claude-logged-out", `claude could not start: ${run.spawnError}`);
505
- if (run.code !== 0 && LOGGED_OUT.test(`${run.stderr}\n${run.stdout}`) && !run.stdout.trim().startsWith("{")) {
600
+ if (run.code !== 0 && CLAUDE_LOGGED_OUT_RE.test(`${run.stderr}\n${run.stdout}`) && !run.stdout.trim().startsWith("{")) {
506
601
  throw new SignalsPassError("claude-logged-out", "claude reported it is not logged in");
507
602
  }
508
603
  return parseClaudeSignals(run.stdout);
@@ -537,7 +632,7 @@ export function createClaudeSignalsPass(options) {
537
632
  frontmatter.payload_fallback = input.fallback;
538
633
  const text = `---\n${yaml.dump(frontmatter, { lineWidth: -1, noRefs: true }).trimEnd()}\n---\n## ${heading(signal.type)}\n\n${signal.body || signal.title}\n`;
539
634
  await beforeWrite();
540
- await writeFileAtomic(file, text);
635
+ await writeFileAtomic(vaultRoot, file, text);
541
636
  written += 1;
542
637
  }
543
638
  await beforeWrite();
@@ -547,23 +642,35 @@ export function createClaudeSignalsPass(options) {
547
642
  return async (result, guard) => {
548
643
  const beforeWrite = guard?.beforeWrite ?? (async () => undefined);
549
644
  const byChannel = new Map();
645
+ // The adapter's own channel is listed even when this run landed nothing
646
+ // there, so items an earlier run left over the cap still get processed.
647
+ if (result.channel !== undefined && isSafeChannel(result.channel))
648
+ byChannel.set(result.channel, []);
550
649
  for (const key of result.landedKeys ?? []) {
551
650
  const channel = channelOf(key);
552
651
  if (!channel)
553
652
  continue;
554
653
  byChannel.set(channel, [...(byChannel.get(channel) ?? []), key]);
555
654
  }
556
- // Nothing landed: there is no pass to run, so claude is not called.
557
- if (byChannel.size === 0)
558
- return 0;
559
- preflight ??= checkLogin();
560
- await preflight;
561
655
  let total = 0;
562
656
  for (const [channel, files] of byChannel) {
563
- for (const item of await itemsFor(channel, files)) {
564
- // Documents over the cap stay unmarked in the worklist for a later pass.
657
+ // With the cap spent and nothing landed here, there is nothing to list:
658
+ // the channel's backlog stays where it is (worklist or pending file).
659
+ if (docsSent >= maxDocs && files.length === 0)
660
+ continue;
661
+ const work = await itemsFor(channel, files, Math.max(0, maxDocs - docsSent));
662
+ // No worklist: every candidate is pending until it is processed, so
663
+ // documents over the cap (or left by a failed call) wait for a later run.
664
+ let pending = work.pending ? work.items.map((item) => item.file) : undefined;
665
+ if (pending !== undefined)
666
+ await writePendingSignals(vaultRoot, channel, pending, beforeWrite);
667
+ for (const item of work.items) {
668
+ // Documents over the cap stay in the worklist or the pending file for a later run.
565
669
  if (docsSent >= maxDocs)
566
- return total;
670
+ break;
671
+ // Log in once per run, and only when there is a document to send.
672
+ preflight ??= checkLogin();
673
+ await preflight;
567
674
  docsSent += 1;
568
675
  const input = await buildSignalsInput(vaultRoot, item.file);
569
676
  if (input.fallback !== undefined) {
@@ -573,6 +680,10 @@ export function createClaudeSignalsPass(options) {
573
680
  if (item.fromWorklist) {
574
681
  await runWorklist(["--company", options.company, "--channel", channel, "--mark-done", item.file]);
575
682
  }
683
+ else if (pending !== undefined) {
684
+ pending = pending.filter((file) => file !== item.file);
685
+ await writePendingSignals(vaultRoot, channel, pending, beforeWrite);
686
+ }
576
687
  }
577
688
  }
578
689
  return total;
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Queue landed hot-channel documents for the local signals pass again, and
3
+ * run the pass now (`hq sources signals`). Used when a run's documents never
4
+ * got signals: for example, the 20-document cap of an older build, or a pass
5
+ * that failed. A sync cursor never offers those documents again, so this is
6
+ * the way to reach them.
7
+ *
8
+ * Where a document is queued depends on its channel:
9
+ * - no source.yaml (the usual hot channel, such as `meeting-events`): the
10
+ * pass's own pending file, `sources/_signals-pending/{channel}.json`;
11
+ * - a source.yaml: the ontology worklist's `--redo`, which drops the item
12
+ * from the worklist's ledgers so the next listing includes it.
13
+ */
14
+ import type { HotChannelRunResult } from "./loop.js";
15
+ /** Bad input (key, channel or date): the command exits 2. */
16
+ export declare class SignalsRedoUsageError extends Error {
17
+ constructor(message: string);
18
+ }
19
+ export interface SignalsRedoSelection {
20
+ /** Channels with no source.yaml: keys for the pass's pending file. */
21
+ pending: Map<string, string[]>;
22
+ /** Channels with a source.yaml: keys for the worklist's `--redo`. */
23
+ worklist: Map<string, string[]>;
24
+ notices: string[];
25
+ }
26
+ export interface SelectSignalsRedoInput {
27
+ vaultRoot: string;
28
+ /** Landed keys (`sources/{channel}/{id}.md`) to process again, signals or not. */
29
+ keys?: readonly string[];
30
+ /** Landed documents whose `when` (else `created_at`) is at or after this, with no signals yet. */
31
+ since?: Date;
32
+ /** Limit `since` to one channel. */
33
+ channel?: string;
34
+ }
35
+ /**
36
+ * Parse `--since`: a bare `YYYY-MM-DD` is local midnight, so a day's meetings
37
+ * are the ones in the machine's own time zone. Anything else must be an ISO
38
+ * date-time.
39
+ */
40
+ export declare function parseSince(value: string): Date;
41
+ /** Every `source_ref` in the company's signal index (`signals/_index/*.json`). */
42
+ export declare function signalledSourceRefs(vaultRoot: string): Promise<Set<string>>;
43
+ /** Which landed documents to process again, grouped by how each is queued. */
44
+ export declare function selectSignalsRedo(input: SelectSignalsRedoInput): Promise<SignalsRedoSelection>;
45
+ /**
46
+ * Queue the selection: pending keys into the pass's pending file, worklist
47
+ * keys through `ontology-source-worklist.sh --redo`. Throws when the worklist
48
+ * refuses (an older script has no `--redo`).
49
+ */
50
+ export declare function queueSignalsRedo(input: {
51
+ hqRoot: string;
52
+ company: string;
53
+ selection: SignalsRedoSelection;
54
+ beforeWrite?: () => Promise<void>;
55
+ worklistScript?: string;
56
+ env?: NodeJS.ProcessEnv;
57
+ }): Promise<void>;
58
+ /** A run result with nothing landed, so the signals pass works through one channel's backlog. */
59
+ export declare function backlogRunResult(channel: string): HotChannelRunResult;
60
+ //# sourceMappingURL=signals-redo.d.ts.map
@@ -0,0 +1,207 @@
1
+ /**
2
+ * Queue landed hot-channel documents for the local signals pass again, and
3
+ * run the pass now (`hq sources signals`). Used when a run's documents never
4
+ * got signals: for example, the 20-document cap of an older build, or a pass
5
+ * that failed. A sync cursor never offers those documents again, so this is
6
+ * the way to reach them.
7
+ *
8
+ * Where a document is queued depends on its channel:
9
+ * - no source.yaml (the usual hot channel, such as `meeting-events`): the
10
+ * pass's own pending file, `sources/_signals-pending/{channel}.json`;
11
+ * - a source.yaml: the ontology worklist's `--redo`, which drops the item
12
+ * from the worklist's ledgers so the next listing includes it.
13
+ */
14
+ import * as fs from "node:fs/promises";
15
+ import * as path from "node:path";
16
+ import { assertSafeSegment } from "./landing.js";
17
+ import { channelOf, landedDocTime, queuePendingSignals, runBounded, splitFrontmatter, WORKLIST_TIMEOUT_MS, } from "./signals-pass.js";
18
+ /** Bad input (key, channel or date): the command exits 2. */
19
+ export class SignalsRedoUsageError extends Error {
20
+ constructor(message) {
21
+ super(message);
22
+ this.name = "SignalsRedoUsageError";
23
+ }
24
+ }
25
+ /**
26
+ * Parse `--since`: a bare `YYYY-MM-DD` is local midnight, so a day's meetings
27
+ * are the ones in the machine's own time zone. Anything else must be an ISO
28
+ * date-time.
29
+ */
30
+ export function parseSince(value) {
31
+ const day = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
32
+ const date = day ? new Date(Number(day[1]), Number(day[2]) - 1, Number(day[3])) : new Date(Date.parse(value));
33
+ if (Number.isNaN(date.getTime()))
34
+ throw new SignalsRedoUsageError(`--since must be YYYY-MM-DD or an ISO date-time: ${value}`);
35
+ return date;
36
+ }
37
+ async function hasSourceYaml(vaultRoot, channel) {
38
+ try {
39
+ return (await fs.stat(path.join(vaultRoot, "sources", channel, "source.yaml"))).isFile();
40
+ }
41
+ catch {
42
+ return false;
43
+ }
44
+ }
45
+ async function isFile(file) {
46
+ try {
47
+ return (await fs.stat(file)).isFile();
48
+ }
49
+ catch {
50
+ return false;
51
+ }
52
+ }
53
+ /** Every `source_ref` in the company's signal index (`signals/_index/*.json`). */
54
+ export async function signalledSourceRefs(vaultRoot) {
55
+ const dir = path.join(vaultRoot, "signals", "_index");
56
+ const refs = new Set();
57
+ let names;
58
+ try {
59
+ names = await fs.readdir(dir);
60
+ }
61
+ catch {
62
+ return refs;
63
+ }
64
+ for (const name of names) {
65
+ if (!name.endsWith(".json"))
66
+ continue;
67
+ try {
68
+ const rows = JSON.parse(await fs.readFile(path.join(dir, name), "utf8"));
69
+ if (!Array.isArray(rows))
70
+ continue;
71
+ for (const row of rows) {
72
+ const ref = row?.source_ref;
73
+ if (typeof ref === "string")
74
+ refs.add(ref);
75
+ }
76
+ }
77
+ catch {
78
+ // An unreadable index day says nothing about which sources have signals.
79
+ }
80
+ }
81
+ return refs;
82
+ }
83
+ function add(map, channel, key) {
84
+ const list = map.get(channel) ?? [];
85
+ if (!list.includes(key))
86
+ list.push(key);
87
+ map.set(channel, list);
88
+ }
89
+ /** Which landed documents to process again, grouped by how each is queued. */
90
+ export async function selectSignalsRedo(input) {
91
+ const { vaultRoot } = input;
92
+ const selection = { pending: new Map(), worklist: new Map(), notices: [] };
93
+ if (input.channel !== undefined) {
94
+ try {
95
+ assertSafeSegment(input.channel, "channel");
96
+ }
97
+ catch {
98
+ throw new SignalsRedoUsageError(`not a channel name: ${input.channel}`);
99
+ }
100
+ }
101
+ const worklistChannel = new Map();
102
+ const usesWorklist = async (channel) => {
103
+ if (!worklistChannel.has(channel))
104
+ worklistChannel.set(channel, await hasSourceYaml(vaultRoot, channel));
105
+ return worklistChannel.get(channel) === true;
106
+ };
107
+ for (const key of input.keys ?? []) {
108
+ const channel = channelOf(key);
109
+ if (!channel || channel.startsWith("_"))
110
+ throw new SignalsRedoUsageError(`--key must be sources/<channel>/<id>.md: ${key}`);
111
+ if (!(await isFile(path.join(vaultRoot, key))))
112
+ throw new SignalsRedoUsageError(`no landed document at ${key}`);
113
+ add((await usesWorklist(channel)) ? selection.worklist : selection.pending, channel, key);
114
+ }
115
+ if (input.since !== undefined) {
116
+ const since = input.since.getTime();
117
+ const signalled = await signalledSourceRefs(vaultRoot);
118
+ let channels;
119
+ if (input.channel !== undefined) {
120
+ channels = [input.channel];
121
+ }
122
+ else {
123
+ try {
124
+ const entries = await fs.readdir(path.join(vaultRoot, "sources"), { withFileTypes: true });
125
+ channels = entries
126
+ .filter((e) => e.isDirectory() && !e.name.startsWith("_") && !e.name.startsWith("."))
127
+ .map((e) => e.name)
128
+ .sort();
129
+ }
130
+ catch {
131
+ channels = [];
132
+ }
133
+ }
134
+ const withWorklist = [];
135
+ for (const channel of channels) {
136
+ if (await usesWorklist(channel)) {
137
+ // The worklist already lists its channel's unprocessed items, and the pass drains them.
138
+ withWorklist.push(channel);
139
+ continue;
140
+ }
141
+ let names;
142
+ try {
143
+ names = (await fs.readdir(path.join(vaultRoot, "sources", channel))).filter((n) => n.endsWith(".md")).sort();
144
+ }
145
+ catch {
146
+ continue;
147
+ }
148
+ for (const name of names) {
149
+ const key = `sources/${channel}/${name}`;
150
+ if (channelOf(key) !== channel || signalled.has(key))
151
+ continue;
152
+ let text;
153
+ try {
154
+ text = await fs.readFile(path.join(vaultRoot, key), "utf8");
155
+ }
156
+ catch {
157
+ continue;
158
+ }
159
+ if (landedDocTime(splitFrontmatter(text).frontmatter) >= since)
160
+ add(selection.pending, channel, key);
161
+ }
162
+ }
163
+ if (withWorklist.length > 0) {
164
+ selection.notices.push(`Skipped for --since (their worklist lists unprocessed items itself; use --key to redo one): ${withWorklist.join(", ")}`);
165
+ }
166
+ }
167
+ return selection;
168
+ }
169
+ /**
170
+ * Queue the selection: pending keys into the pass's pending file, worklist
171
+ * keys through `ontology-source-worklist.sh --redo`. Throws when the worklist
172
+ * refuses (an older script has no `--redo`).
173
+ */
174
+ export async function queueSignalsRedo(input) {
175
+ const vaultRoot = path.join(input.hqRoot, "companies", input.company);
176
+ for (const [channel, keys] of input.selection.pending) {
177
+ await queuePendingSignals(vaultRoot, channel, keys, input.beforeWrite);
178
+ }
179
+ const worklist = input.worklistScript ?? path.join(input.hqRoot, "core", "scripts", "ontology-source-worklist.sh");
180
+ for (const [channel, keys] of input.selection.worklist) {
181
+ await input.beforeWrite?.();
182
+ const run = await runBounded("bash", [worklist, "--company", input.company, "--channel", channel, ...keys.flatMap((key) => ["--redo", key])], { timeoutMs: WORKLIST_TIMEOUT_MS, cwd: input.hqRoot, env: { ...(input.env ?? process.env), HQ_ROOT: input.hqRoot } });
183
+ if (run.timedOut || run.code !== 0) {
184
+ const why = run.timedOut ? "timed out" : run.stderr.trim().slice(0, 200) || `exit ${String(run.code)}`;
185
+ throw new Error(`ontology worklist --redo failed for sources/${channel}: ${why}`);
186
+ }
187
+ }
188
+ }
189
+ /** A run result with nothing landed, so the signals pass works through one channel's backlog. */
190
+ export function backlogRunResult(channel) {
191
+ return {
192
+ adapterId: "signals-redo",
193
+ provider: "signals-redo",
194
+ channel,
195
+ connectionId: null,
196
+ transport: null,
197
+ landed: 0,
198
+ duplicates: 0,
199
+ stale: 0,
200
+ rejected: 0,
201
+ pages: 0,
202
+ landedKeys: [],
203
+ cursorBefore: null,
204
+ cursorAfter: null,
205
+ };
206
+ }
207
+ //# sourceMappingURL=signals-redo.js.map