@indigoai-us/hq-cli 5.345.191 → 5.345.194
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +431 -0
- package/assets/bot-workers/setup/worker.yaml +91 -34
- package/dist/command-catalog.generated.d.ts +145 -19
- package/dist/command-catalog.generated.js +184 -21
- package/dist/commands/bot-jobs.d.ts +21 -18
- package/dist/commands/bot-jobs.js +127 -39
- package/dist/commands/bot.d.ts +4 -3
- package/dist/commands/bot.js +265 -140
- package/dist/commands/core-jobs-schedule.js +10 -0
- package/dist/commands/daemon.js +37 -0
- package/dist/commands/files.d.ts +41 -6
- package/dist/commands/files.js +108 -27
- package/dist/commands/pack-start-bot.d.ts +4 -6
- package/dist/commands/pack-start-bot.js +4 -8
- package/dist/commands/signals-run.d.ts +46 -0
- package/dist/commands/signals-run.js +180 -0
- package/dist/commands/signals.d.ts +1 -0
- package/dist/commands/signals.js +3 -0
- package/dist/commands/sources.d.ts +58 -0
- package/dist/commands/sources.js +294 -26
- package/dist/commands/staff/reset-test-account.d.ts +10 -0
- package/dist/commands/staff/reset-test-account.js +18 -2
- package/dist/commands/whoami.d.ts +7 -0
- package/dist/commands/whoami.js +35 -2
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +31171 -52512
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/bot/api.d.ts +3 -2
- package/dist/lib/bot/config.d.ts +2 -0
- package/dist/lib/bot/fleet-cron.d.ts +121 -0
- package/dist/lib/bot/fleet-cron.js +319 -0
- package/dist/lib/bot/fleet-local.d.ts +93 -1
- package/dist/lib/bot/fleet-local.js +546 -34
- package/dist/lib/bot/fleet-owner-companies.d.ts +12 -0
- package/dist/lib/bot/fleet-owner-companies.js +30 -0
- package/dist/lib/bot/fleet-watch.d.ts +25 -0
- package/dist/lib/bot/fleet-watch.js +51 -0
- package/dist/lib/bot/index.d.ts +1 -1
- package/dist/lib/bot/index.js +1 -1
- package/dist/lib/bot/job-run.d.ts +5 -4
- package/dist/lib/bot/job-run.js +5 -4
- package/dist/lib/bot/jobs/hot-signal-sync.d.ts +57 -3
- package/dist/lib/bot/jobs/hot-signal-sync.js +163 -4
- package/dist/lib/bot/local-jobs.d.ts +13 -2
- package/dist/lib/bot/local-jobs.js +17 -4
- package/dist/lib/bot/progress.d.ts +14 -0
- package/dist/lib/bot/progress.js +47 -1
- package/dist/lib/bot/run.d.ts +55 -2
- package/dist/lib/bot/run.js +192 -10
- package/dist/lib/bot/setup-tool-offer.d.ts +1 -0
- package/dist/lib/bot/setup-tool-offer.js +16 -0
- package/dist/lib/bot/signals-engine-flag.d.ts +18 -0
- package/dist/lib/bot/signals-engine-flag.js +30 -0
- package/dist/lib/daemon/catalog.d.ts +4 -0
- package/dist/lib/daemon/catalog.js +29 -9
- package/dist/lib/daemon/legacy.d.ts +6 -0
- package/dist/lib/daemon/legacy.js +16 -10
- package/dist/lib/daemon/run.d.ts +1 -0
- package/dist/lib/daemon/run.js +10 -0
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/flag-registry.d.ts +13 -1
- package/dist/lib/flag-registry.js +46 -0
- package/dist/lib/hot-channels/adapters/attio.d.ts +75 -2
- package/dist/lib/hot-channels/adapters/attio.js +367 -29
- package/dist/lib/hot-channels/adapters/gmail.d.ts +14 -0
- package/dist/lib/hot-channels/adapters/gmail.js +59 -2
- package/dist/lib/hot-channels/adapters/linear.d.ts +24 -0
- package/dist/lib/hot-channels/adapters/linear.js +71 -1
- package/dist/lib/hot-channels/adapters/meetings.js +4 -1
- package/dist/lib/hot-channels/adapters/slack.js +7 -3
- package/dist/lib/hot-channels/alert.js +1 -1
- package/dist/lib/hot-channels/contract.d.ts +11 -0
- package/dist/lib/hot-channels/contract.js +1 -0
- package/dist/lib/hot-channels/cursors.js +1 -1
- package/dist/lib/hot-channels/landing.d.ts +6 -2
- package/dist/lib/hot-channels/landing.js +13 -6
- package/dist/lib/hot-channels/ledger.js +2 -0
- package/dist/lib/hot-channels/loop.d.ts +6 -0
- package/dist/lib/hot-channels/loop.js +19 -4
- package/dist/lib/hot-channels/registry.js +9 -1
- package/dist/lib/hot-channels/run.d.ts +18 -0
- package/dist/lib/hot-channels/run.js +15 -4
- package/dist/lib/hot-channels/signals-pass.d.ts +34 -2
- package/dist/lib/hot-channels/signals-pass.js +149 -38
- package/dist/lib/hot-channels/signals-redo.d.ts +60 -0
- package/dist/lib/hot-channels/signals-redo.js +207 -0
- package/dist/lib/hot-channels/sync-lock.js +22 -6
- package/dist/lib/mesh/live/daemon/install.d.ts +22 -0
- package/dist/lib/mesh/live/daemon/install.js +117 -30
- package/dist/lib/signals/adapters-catalog.d.ts +77 -0
- package/dist/lib/signals/adapters-catalog.js +194 -0
- package/dist/lib/signals/engine.d.ts +202 -0
- package/dist/lib/signals/engine.js +504 -0
- package/dist/lib/signals/failure-alert.d.ts +89 -0
- package/dist/lib/signals/failure-alert.js +175 -0
- package/dist/lib/signals/local-date.d.ts +3 -0
- package/dist/lib/signals/local-date.js +6 -0
- package/dist/lib/signals/projection.d.ts +42 -0
- package/dist/lib/signals/projection.js +67 -0
- package/dist/lib/signals/reply.d.ts +112 -0
- package/dist/lib/signals/reply.js +368 -0
- package/dist/lib/signals/run-lock.d.ts +36 -0
- package/dist/lib/signals/run-lock.js +148 -0
- package/dist/lib/signals/run-record.d.ts +24 -0
- package/dist/lib/signals/run-record.js +82 -0
- package/dist/lib/signals/sanitize.d.ts +105 -0
- package/dist/lib/signals/sanitize.js +439 -0
- package/dist/lib/signals/settings.d.ts +10 -0
- package/dist/lib/signals/settings.js +48 -0
- package/dist/lib/signals/steps/adapters.d.ts +14 -0
- package/dist/lib/signals/steps/adapters.js +156 -0
- package/dist/lib/signals/steps/entity-similarity.d.ts +10 -0
- package/dist/lib/signals/steps/entity-similarity.js +47 -0
- package/dist/lib/signals/steps/extract.d.ts +118 -0
- package/dist/lib/signals/steps/extract.js +793 -0
- package/dist/lib/signals/steps/garden.d.ts +24 -0
- package/dist/lib/signals/steps/garden.js +250 -0
- package/dist/lib/signals/steps/types.d.ts +117 -0
- package/dist/lib/signals/steps/types.js +4 -0
- package/dist/lib/signals/steps/upload.d.ts +43 -0
- package/dist/lib/signals/steps/upload.js +324 -0
- package/dist/lib/signals/write-guard.d.ts +63 -0
- package/dist/lib/signals/write-guard.js +287 -0
- package/dist/utils/resident-package-preload.js +16 -0
- package/dist/utils/vault-api.d.ts +1 -0
- package/dist/utils/vault-api.js +1 -1
- package/package.json +9 -9
- package/dist/lib/bot/fleet-default-flag.d.ts +0 -31
- package/dist/lib/bot/fleet-default-flag.js +0 -127
|
@@ -1,9 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The local signals pass (US-011). After
|
|
3
|
-
* pass takes the
|
|
2
|
+
* The local signals pass (US-011). After each hot-channel adapter run, the
|
|
3
|
+
* pass takes the adapter's channel through the ontology worklist, asks
|
|
4
4
|
* `claude -p` for signals one document at a time, and writes those signals to
|
|
5
5
|
* `companies/{co}/signals/`. It returns the count for the run ledger.
|
|
6
6
|
*
|
|
7
|
+
* Work comes from the worklist, not only from this run's landings: when the
|
|
8
|
+
* worklist script exits 0, the pass takes the channel's unprocessed items in
|
|
9
|
+
* the worklist's order (newest first) up to the run's document cap. Items the
|
|
10
|
+
* cap leaves over stay unmarked, and a later run picks them up, including a
|
|
11
|
+
* run that landed nothing (a bounded backlog drain: at most
|
|
12
|
+
* DEFAULT_MAX_SIGNAL_DOCS claude calls per run, and no claude call at all when
|
|
13
|
+
* the backlog is empty). A channel with no source.yaml (worklist exit 2, the
|
|
14
|
+
* usual case for a hot channel such as `meeting-events`) has no worklist, so
|
|
15
|
+
* the pass keeps its own backlog: landed documents it has not processed yet
|
|
16
|
+
* are listed in `sources/_signals-pending/{channel}.json` and drained, newest
|
|
17
|
+
* first by `when` (else `created_at`), on later runs under the same cap. A
|
|
18
|
+
* cloud-run source (exit 4) is skipped.
|
|
19
|
+
*
|
|
7
20
|
* A landed document with both `payload_ref` and `raw_ref` is a pointer (the
|
|
8
21
|
* meetings, Slack and Attio adapters land these). For it, claude gets the
|
|
9
22
|
* pointer followed by the document `payload_ref` names, capped at
|
|
@@ -25,6 +38,7 @@ import * as path from "node:path";
|
|
|
25
38
|
import { fileURLToPath } from "node:url";
|
|
26
39
|
import * as yaml from "js-yaml";
|
|
27
40
|
import { audienceKey, COMPANY_AUDIENCE_KEY } from "./audience.js";
|
|
41
|
+
import { assertVaultPath } from "../signals/write-guard.js";
|
|
28
42
|
import { assertPayloadRef, assertSafeSegment, writeFileAtomic } from "./landing.js";
|
|
29
43
|
/**
|
|
30
44
|
* A pass failure the run ledger records as `status: failed` with `error` set
|
|
@@ -79,8 +93,6 @@ export const WORKLIST_TIMEOUT_MS = 60_000;
|
|
|
79
93
|
export const DEFAULT_MAX_SIGNAL_DOCS = 20;
|
|
80
94
|
/** Payload bytes sent to claude per document (200 KB). A longer payload is cut with a marker. */
|
|
81
95
|
export const MAX_PAYLOAD_BYTES = 200 * 1024;
|
|
82
|
-
/** Items the worklist lists per channel; landed documents must fall inside it. */
|
|
83
|
-
const WORKLIST_LIMIT = 500;
|
|
84
96
|
/** Output kept per child stream. A longer reply is bad output. */
|
|
85
97
|
const MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
|
|
86
98
|
/** Signals kept per document. */
|
|
@@ -171,8 +183,8 @@ export function runBounded(command, args, options) {
|
|
|
171
183
|
export function defaultPromptFile() {
|
|
172
184
|
return path.join(path.dirname(fileURLToPath(import.meta.url)), "prompts", "signals-pass.md");
|
|
173
185
|
}
|
|
174
|
-
const
|
|
175
|
-
function channelOf(key) {
|
|
186
|
+
export const CLAUDE_LOGGED_OUT_RE = /not logged in|log ?in|\/login|invalid api key|authenticat|oauth token/i;
|
|
187
|
+
export function channelOf(key) {
|
|
176
188
|
const parts = key.split("/");
|
|
177
189
|
if (parts.length !== 3 || parts[0] !== "sources" || !parts[2].endsWith(".md"))
|
|
178
190
|
return null;
|
|
@@ -185,7 +197,69 @@ function channelOf(key) {
|
|
|
185
197
|
}
|
|
186
198
|
return parts[1];
|
|
187
199
|
}
|
|
188
|
-
|
|
200
|
+
/**
|
|
201
|
+
* Vault-relative file that lists a channel's landed documents the pass has not
|
|
202
|
+
* processed yet. Only channels with no worklist (no source.yaml) use it.
|
|
203
|
+
*/
|
|
204
|
+
export function pendingSignalsKey(channel) {
|
|
205
|
+
assertSafeSegment(channel, "channel");
|
|
206
|
+
return `sources/_signals-pending/${channel}.json`;
|
|
207
|
+
}
|
|
208
|
+
/** The channel's pending landed keys; a missing or unreadable file is an empty backlog. */
|
|
209
|
+
export async function readPendingSignals(vaultRoot, channel) {
|
|
210
|
+
let parsed;
|
|
211
|
+
try {
|
|
212
|
+
parsed = JSON.parse(await fs.readFile(path.join(vaultRoot, pendingSignalsKey(channel)), "utf8"));
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
return [];
|
|
216
|
+
}
|
|
217
|
+
if (!Array.isArray(parsed))
|
|
218
|
+
return [];
|
|
219
|
+
return parsed.filter((key) => typeof key === "string" && channelOf(key) === channel);
|
|
220
|
+
}
|
|
221
|
+
/** Replace the channel's pending list; an empty list removes the file. */
|
|
222
|
+
export async function writePendingSignals(vaultRoot, channel, keys, beforeWrite = async () => undefined) {
|
|
223
|
+
const file = path.join(vaultRoot, pendingSignalsKey(channel));
|
|
224
|
+
assertVaultPath(vaultRoot, file);
|
|
225
|
+
await beforeWrite();
|
|
226
|
+
if (keys.length === 0) {
|
|
227
|
+
await fs.rm(file, { force: true });
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
await writeFileAtomic(vaultRoot, file, `${JSON.stringify([...new Set(keys)], null, 2)}\n`);
|
|
231
|
+
}
|
|
232
|
+
/** Add landed keys to the channel's pending list (kept once each). */
|
|
233
|
+
export async function queuePendingSignals(vaultRoot, channel, keys, beforeWrite) {
|
|
234
|
+
for (const key of keys) {
|
|
235
|
+
if (channelOf(key) !== channel)
|
|
236
|
+
throw new Error(`${key} is not a document in sources/${channel}/`);
|
|
237
|
+
}
|
|
238
|
+
await writePendingSignals(vaultRoot, channel, [...(await readPendingSignals(vaultRoot, channel)), ...keys], beforeWrite);
|
|
239
|
+
}
|
|
240
|
+
/** When a landed document happened: `when`, else `created_at`; unknown sorts last. */
|
|
241
|
+
export function landedDocTime(frontmatter) {
|
|
242
|
+
for (const field of ["when", "created_at"]) {
|
|
243
|
+
const value = frontmatter[field];
|
|
244
|
+
const text = value instanceof Date ? value.toISOString() : typeof value === "string" ? value : undefined;
|
|
245
|
+
if (text === undefined)
|
|
246
|
+
continue;
|
|
247
|
+
const ms = Date.parse(text);
|
|
248
|
+
if (!Number.isNaN(ms))
|
|
249
|
+
return ms;
|
|
250
|
+
}
|
|
251
|
+
return Number.NEGATIVE_INFINITY;
|
|
252
|
+
}
|
|
253
|
+
function isSafeChannel(channel) {
|
|
254
|
+
try {
|
|
255
|
+
assertSafeSegment(channel, "channel");
|
|
256
|
+
return true;
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
return false;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
export function splitFrontmatter(text) {
|
|
189
263
|
const match = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/.exec(text);
|
|
190
264
|
if (!match)
|
|
191
265
|
return { frontmatter: {}, body: text };
|
|
@@ -284,7 +358,7 @@ export function parseClaudeSignals(stdout) {
|
|
|
284
358
|
const env = envelope;
|
|
285
359
|
if (env.is_error === true) {
|
|
286
360
|
const text = typeof env.result === "string" ? env.result : "";
|
|
287
|
-
if (
|
|
361
|
+
if (CLAUDE_LOGGED_OUT_RE.test(text))
|
|
288
362
|
throw new SignalsPassError("claude-logged-out", "claude reported it is not logged in");
|
|
289
363
|
throw new SignalsPassError("claude-bad-output", "claude returned an error result");
|
|
290
364
|
}
|
|
@@ -353,6 +427,7 @@ export async function ensureAudienceFile(vaultRoot, item) {
|
|
|
353
427
|
return;
|
|
354
428
|
assertSafeSegment(item.audienceKey, "audience key");
|
|
355
429
|
const file = path.join(vaultRoot, "signals", `@${item.audienceKey}`, "_audience.yaml");
|
|
430
|
+
assertVaultPath(vaultRoot, file);
|
|
356
431
|
if (await exists(file))
|
|
357
432
|
return;
|
|
358
433
|
const principals = [...new Set(item.principals.map((p) => p.trim()).filter((p) => p.length > 0))].sort();
|
|
@@ -384,7 +459,7 @@ async function appendSignalIndex(vaultRoot, day, rows) {
|
|
|
384
459
|
const added = rows.filter((row) => !seen.has(row.signal_id));
|
|
385
460
|
if (added.length === 0)
|
|
386
461
|
return;
|
|
387
|
-
await writeFileAtomic(file, `${JSON.stringify([...existing, ...added], null, 2)}\n`);
|
|
462
|
+
await writeFileAtomic(vaultRoot, file, `${JSON.stringify([...existing, ...added], null, 2)}\n`);
|
|
388
463
|
}
|
|
389
464
|
/**
|
|
390
465
|
* Build the claude signals pass for one company. The pass keeps state for one
|
|
@@ -431,15 +506,19 @@ export function createClaudeSignalsPass(options) {
|
|
|
431
506
|
cwd: options.hqRoot,
|
|
432
507
|
env: { ...env, HQ_ROOT: options.hqRoot },
|
|
433
508
|
});
|
|
434
|
-
/**
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
509
|
+
/**
|
|
510
|
+
* Work items for one channel. With a worklist: its unprocessed items in its
|
|
511
|
+
* order (at most `limit`). Without one: this run's landed `files` plus the
|
|
512
|
+
* channel's pending backlog, newest first, all of them (`pending` is set and
|
|
513
|
+
* the caller applies the cap).
|
|
514
|
+
*/
|
|
515
|
+
const itemsFor = async (channel, files, limit) => {
|
|
516
|
+
const run = await runWorklist(["--company", options.company, "--channel", channel, "--limit", String(limit)]);
|
|
438
517
|
if (run.timedOut)
|
|
439
518
|
throw new Error(`ontology worklist timed out for channel ${channel}`);
|
|
440
519
|
// Exit 4: the source is assigned to the cloud agent. Its documents are not this pass's to process.
|
|
441
520
|
if (run.code === 4)
|
|
442
|
-
return [];
|
|
521
|
+
return { items: [], pending: false };
|
|
443
522
|
if (run.code === 0) {
|
|
444
523
|
const items = [];
|
|
445
524
|
for (const line of run.stdout.split("\n")) {
|
|
@@ -452,7 +531,7 @@ export function createClaudeSignalsPass(options) {
|
|
|
452
531
|
catch {
|
|
453
532
|
continue;
|
|
454
533
|
}
|
|
455
|
-
if (typeof row.file !== "string" ||
|
|
534
|
+
if (typeof row.file !== "string" || channelOf(row.file) !== channel || row.skip !== undefined)
|
|
456
535
|
continue;
|
|
457
536
|
if (typeof row.audience_key !== "string" || !Array.isArray(row.audience))
|
|
458
537
|
continue;
|
|
@@ -465,30 +544,46 @@ export function createClaudeSignalsPass(options) {
|
|
|
465
544
|
fromWorklist: true,
|
|
466
545
|
});
|
|
467
546
|
}
|
|
468
|
-
return items;
|
|
547
|
+
return { items, pending: false };
|
|
469
548
|
}
|
|
470
549
|
// Any other exit (usually 2: the channel has no source.yaml yet, which is
|
|
471
550
|
// normal for a hot channel) falls back to the landed document's own
|
|
472
|
-
// audience frontmatter, which landing always writes.
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
551
|
+
// audience frontmatter, which landing always writes. The pass's own
|
|
552
|
+
// pending backlog stands in for the worklist.
|
|
553
|
+
const candidates = [...new Set([...files, ...(await readPendingSignals(vaultRoot, channel))])];
|
|
554
|
+
const dated = [];
|
|
555
|
+
for (const file of candidates) {
|
|
556
|
+
let text;
|
|
557
|
+
try {
|
|
558
|
+
text = await fs.readFile(path.join(vaultRoot, file), "utf8");
|
|
559
|
+
}
|
|
560
|
+
catch (err) {
|
|
561
|
+
// A pending document that is gone has nothing left to process.
|
|
562
|
+
if (isMissing(err))
|
|
563
|
+
continue;
|
|
564
|
+
throw err;
|
|
565
|
+
}
|
|
566
|
+
const { frontmatter } = splitFrontmatter(text);
|
|
476
567
|
const audience = Array.isArray(frontmatter.audience)
|
|
477
568
|
? frontmatter.audience.filter((a) => typeof a === "string" && a.trim().length > 0)
|
|
478
569
|
: [];
|
|
479
570
|
// No audience: skipped, never widened to company.
|
|
480
571
|
if (audience.length === 0)
|
|
481
572
|
continue;
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
573
|
+
dated.push({
|
|
574
|
+
item: {
|
|
575
|
+
file,
|
|
576
|
+
channel,
|
|
577
|
+
principals: audience,
|
|
578
|
+
audienceKey: audienceKey(audience),
|
|
579
|
+
audienceSource: typeof frontmatter.audience_source === "string" ? frontmatter.audience_source : "frontmatter",
|
|
580
|
+
fromWorklist: false,
|
|
581
|
+
},
|
|
582
|
+
at: landedDocTime(frontmatter),
|
|
489
583
|
});
|
|
490
584
|
}
|
|
491
|
-
|
|
585
|
+
dated.sort((a, b) => b.at - a.at || (a.item.file < b.item.file ? -1 : a.item.file > b.item.file ? 1 : 0));
|
|
586
|
+
return { items: dated.map((d) => d.item), pending: true };
|
|
492
587
|
};
|
|
493
588
|
const extract = async (item, input) => {
|
|
494
589
|
prompt ??= fs.readFile(promptFile, "utf8");
|
|
@@ -502,7 +597,7 @@ export function createClaudeSignalsPass(options) {
|
|
|
502
597
|
throw new SignalsPassError("claude-timeout", `claude timed out after ${timeoutMs}ms on ${item.file}`);
|
|
503
598
|
if (run.spawnError !== undefined)
|
|
504
599
|
throw new SignalsPassError("claude-logged-out", `claude could not start: ${run.spawnError}`);
|
|
505
|
-
if (run.code !== 0 &&
|
|
600
|
+
if (run.code !== 0 && CLAUDE_LOGGED_OUT_RE.test(`${run.stderr}\n${run.stdout}`) && !run.stdout.trim().startsWith("{")) {
|
|
506
601
|
throw new SignalsPassError("claude-logged-out", "claude reported it is not logged in");
|
|
507
602
|
}
|
|
508
603
|
return parseClaudeSignals(run.stdout);
|
|
@@ -537,7 +632,7 @@ export function createClaudeSignalsPass(options) {
|
|
|
537
632
|
frontmatter.payload_fallback = input.fallback;
|
|
538
633
|
const text = `---\n${yaml.dump(frontmatter, { lineWidth: -1, noRefs: true }).trimEnd()}\n---\n## ${heading(signal.type)}\n\n${signal.body || signal.title}\n`;
|
|
539
634
|
await beforeWrite();
|
|
540
|
-
await writeFileAtomic(file, text);
|
|
635
|
+
await writeFileAtomic(vaultRoot, file, text);
|
|
541
636
|
written += 1;
|
|
542
637
|
}
|
|
543
638
|
await beforeWrite();
|
|
@@ -547,23 +642,35 @@ export function createClaudeSignalsPass(options) {
|
|
|
547
642
|
return async (result, guard) => {
|
|
548
643
|
const beforeWrite = guard?.beforeWrite ?? (async () => undefined);
|
|
549
644
|
const byChannel = new Map();
|
|
645
|
+
// The adapter's own channel is listed even when this run landed nothing
|
|
646
|
+
// there, so items an earlier run left over the cap still get processed.
|
|
647
|
+
if (result.channel !== undefined && isSafeChannel(result.channel))
|
|
648
|
+
byChannel.set(result.channel, []);
|
|
550
649
|
for (const key of result.landedKeys ?? []) {
|
|
551
650
|
const channel = channelOf(key);
|
|
552
651
|
if (!channel)
|
|
553
652
|
continue;
|
|
554
653
|
byChannel.set(channel, [...(byChannel.get(channel) ?? []), key]);
|
|
555
654
|
}
|
|
556
|
-
// Nothing landed: there is no pass to run, so claude is not called.
|
|
557
|
-
if (byChannel.size === 0)
|
|
558
|
-
return 0;
|
|
559
|
-
preflight ??= checkLogin();
|
|
560
|
-
await preflight;
|
|
561
655
|
let total = 0;
|
|
562
656
|
for (const [channel, files] of byChannel) {
|
|
563
|
-
|
|
564
|
-
|
|
657
|
+
// With the cap spent and nothing landed here, there is nothing to list:
|
|
658
|
+
// the channel's backlog stays where it is (worklist or pending file).
|
|
659
|
+
if (docsSent >= maxDocs && files.length === 0)
|
|
660
|
+
continue;
|
|
661
|
+
const work = await itemsFor(channel, files, Math.max(0, maxDocs - docsSent));
|
|
662
|
+
// No worklist: every candidate is pending until it is processed, so
|
|
663
|
+
// documents over the cap (or left by a failed call) wait for a later run.
|
|
664
|
+
let pending = work.pending ? work.items.map((item) => item.file) : undefined;
|
|
665
|
+
if (pending !== undefined)
|
|
666
|
+
await writePendingSignals(vaultRoot, channel, pending, beforeWrite);
|
|
667
|
+
for (const item of work.items) {
|
|
668
|
+
// Documents over the cap stay in the worklist or the pending file for a later run.
|
|
565
669
|
if (docsSent >= maxDocs)
|
|
566
|
-
|
|
670
|
+
break;
|
|
671
|
+
// Log in once per run, and only when there is a document to send.
|
|
672
|
+
preflight ??= checkLogin();
|
|
673
|
+
await preflight;
|
|
567
674
|
docsSent += 1;
|
|
568
675
|
const input = await buildSignalsInput(vaultRoot, item.file);
|
|
569
676
|
if (input.fallback !== undefined) {
|
|
@@ -573,6 +680,10 @@ export function createClaudeSignalsPass(options) {
|
|
|
573
680
|
if (item.fromWorklist) {
|
|
574
681
|
await runWorklist(["--company", options.company, "--channel", channel, "--mark-done", item.file]);
|
|
575
682
|
}
|
|
683
|
+
else if (pending !== undefined) {
|
|
684
|
+
pending = pending.filter((file) => file !== item.file);
|
|
685
|
+
await writePendingSignals(vaultRoot, channel, pending, beforeWrite);
|
|
686
|
+
}
|
|
576
687
|
}
|
|
577
688
|
}
|
|
578
689
|
return total;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queue landed hot-channel documents for the local signals pass again, and
|
|
3
|
+
* run the pass now (`hq sources signals`). Used when a run's documents never
|
|
4
|
+
* got signals: for example, the 20-document cap of an older build, or a pass
|
|
5
|
+
* that failed. A sync cursor never offers those documents again, so this is
|
|
6
|
+
* the way to reach them.
|
|
7
|
+
*
|
|
8
|
+
* Where a document is queued depends on its channel:
|
|
9
|
+
* - no source.yaml (the usual hot channel, such as `meeting-events`): the
|
|
10
|
+
* pass's own pending file, `sources/_signals-pending/{channel}.json`;
|
|
11
|
+
* - a source.yaml: the ontology worklist's `--redo`, which drops the item
|
|
12
|
+
* from the worklist's ledgers so the next listing includes it.
|
|
13
|
+
*/
|
|
14
|
+
import type { HotChannelRunResult } from "./loop.js";
|
|
15
|
+
/** Bad input (key, channel or date): the command exits 2. */
|
|
16
|
+
export declare class SignalsRedoUsageError extends Error {
|
|
17
|
+
constructor(message: string);
|
|
18
|
+
}
|
|
19
|
+
export interface SignalsRedoSelection {
|
|
20
|
+
/** Channels with no source.yaml: keys for the pass's pending file. */
|
|
21
|
+
pending: Map<string, string[]>;
|
|
22
|
+
/** Channels with a source.yaml: keys for the worklist's `--redo`. */
|
|
23
|
+
worklist: Map<string, string[]>;
|
|
24
|
+
notices: string[];
|
|
25
|
+
}
|
|
26
|
+
export interface SelectSignalsRedoInput {
|
|
27
|
+
vaultRoot: string;
|
|
28
|
+
/** Landed keys (`sources/{channel}/{id}.md`) to process again, signals or not. */
|
|
29
|
+
keys?: readonly string[];
|
|
30
|
+
/** Landed documents whose `when` (else `created_at`) is at or after this, with no signals yet. */
|
|
31
|
+
since?: Date;
|
|
32
|
+
/** Limit `since` to one channel. */
|
|
33
|
+
channel?: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Parse `--since`: a bare `YYYY-MM-DD` is local midnight, so a day's meetings
|
|
37
|
+
* are the ones in the machine's own time zone. Anything else must be an ISO
|
|
38
|
+
* date-time.
|
|
39
|
+
*/
|
|
40
|
+
export declare function parseSince(value: string): Date;
|
|
41
|
+
/** Every `source_ref` in the company's signal index (`signals/_index/*.json`). */
|
|
42
|
+
export declare function signalledSourceRefs(vaultRoot: string): Promise<Set<string>>;
|
|
43
|
+
/** Which landed documents to process again, grouped by how each is queued. */
|
|
44
|
+
export declare function selectSignalsRedo(input: SelectSignalsRedoInput): Promise<SignalsRedoSelection>;
|
|
45
|
+
/**
|
|
46
|
+
* Queue the selection: pending keys into the pass's pending file, worklist
|
|
47
|
+
* keys through `ontology-source-worklist.sh --redo`. Throws when the worklist
|
|
48
|
+
* refuses (an older script has no `--redo`).
|
|
49
|
+
*/
|
|
50
|
+
export declare function queueSignalsRedo(input: {
|
|
51
|
+
hqRoot: string;
|
|
52
|
+
company: string;
|
|
53
|
+
selection: SignalsRedoSelection;
|
|
54
|
+
beforeWrite?: () => Promise<void>;
|
|
55
|
+
worklistScript?: string;
|
|
56
|
+
env?: NodeJS.ProcessEnv;
|
|
57
|
+
}): Promise<void>;
|
|
58
|
+
/** A run result with nothing landed, so the signals pass works through one channel's backlog. */
|
|
59
|
+
export declare function backlogRunResult(channel: string): HotChannelRunResult;
|
|
60
|
+
//# sourceMappingURL=signals-redo.d.ts.map
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queue landed hot-channel documents for the local signals pass again, and
|
|
3
|
+
* run the pass now (`hq sources signals`). Used when a run's documents never
|
|
4
|
+
* got signals: for example, the 20-document cap of an older build, or a pass
|
|
5
|
+
* that failed. A sync cursor never offers those documents again, so this is
|
|
6
|
+
* the way to reach them.
|
|
7
|
+
*
|
|
8
|
+
* Where a document is queued depends on its channel:
|
|
9
|
+
* - no source.yaml (the usual hot channel, such as `meeting-events`): the
|
|
10
|
+
* pass's own pending file, `sources/_signals-pending/{channel}.json`;
|
|
11
|
+
* - a source.yaml: the ontology worklist's `--redo`, which drops the item
|
|
12
|
+
* from the worklist's ledgers so the next listing includes it.
|
|
13
|
+
*/
|
|
14
|
+
import * as fs from "node:fs/promises";
|
|
15
|
+
import * as path from "node:path";
|
|
16
|
+
import { assertSafeSegment } from "./landing.js";
|
|
17
|
+
import { channelOf, landedDocTime, queuePendingSignals, runBounded, splitFrontmatter, WORKLIST_TIMEOUT_MS, } from "./signals-pass.js";
|
|
18
|
+
/** Bad input (key, channel or date): the command exits 2. */
|
|
19
|
+
export class SignalsRedoUsageError extends Error {
|
|
20
|
+
constructor(message) {
|
|
21
|
+
super(message);
|
|
22
|
+
this.name = "SignalsRedoUsageError";
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Parse `--since`: a bare `YYYY-MM-DD` is local midnight, so a day's meetings
|
|
27
|
+
* are the ones in the machine's own time zone. Anything else must be an ISO
|
|
28
|
+
* date-time.
|
|
29
|
+
*/
|
|
30
|
+
export function parseSince(value) {
|
|
31
|
+
const day = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
|
|
32
|
+
const date = day ? new Date(Number(day[1]), Number(day[2]) - 1, Number(day[3])) : new Date(Date.parse(value));
|
|
33
|
+
if (Number.isNaN(date.getTime()))
|
|
34
|
+
throw new SignalsRedoUsageError(`--since must be YYYY-MM-DD or an ISO date-time: ${value}`);
|
|
35
|
+
return date;
|
|
36
|
+
}
|
|
37
|
+
async function hasSourceYaml(vaultRoot, channel) {
|
|
38
|
+
try {
|
|
39
|
+
return (await fs.stat(path.join(vaultRoot, "sources", channel, "source.yaml"))).isFile();
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
async function isFile(file) {
|
|
46
|
+
try {
|
|
47
|
+
return (await fs.stat(file)).isFile();
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Every `source_ref` in the company's signal index (`signals/_index/*.json`). */
|
|
54
|
+
export async function signalledSourceRefs(vaultRoot) {
|
|
55
|
+
const dir = path.join(vaultRoot, "signals", "_index");
|
|
56
|
+
const refs = new Set();
|
|
57
|
+
let names;
|
|
58
|
+
try {
|
|
59
|
+
names = await fs.readdir(dir);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return refs;
|
|
63
|
+
}
|
|
64
|
+
for (const name of names) {
|
|
65
|
+
if (!name.endsWith(".json"))
|
|
66
|
+
continue;
|
|
67
|
+
try {
|
|
68
|
+
const rows = JSON.parse(await fs.readFile(path.join(dir, name), "utf8"));
|
|
69
|
+
if (!Array.isArray(rows))
|
|
70
|
+
continue;
|
|
71
|
+
for (const row of rows) {
|
|
72
|
+
const ref = row?.source_ref;
|
|
73
|
+
if (typeof ref === "string")
|
|
74
|
+
refs.add(ref);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
// An unreadable index day says nothing about which sources have signals.
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return refs;
|
|
82
|
+
}
|
|
83
|
+
function add(map, channel, key) {
|
|
84
|
+
const list = map.get(channel) ?? [];
|
|
85
|
+
if (!list.includes(key))
|
|
86
|
+
list.push(key);
|
|
87
|
+
map.set(channel, list);
|
|
88
|
+
}
|
|
89
|
+
/** Which landed documents to process again, grouped by how each is queued. */
|
|
90
|
+
export async function selectSignalsRedo(input) {
|
|
91
|
+
const { vaultRoot } = input;
|
|
92
|
+
const selection = { pending: new Map(), worklist: new Map(), notices: [] };
|
|
93
|
+
if (input.channel !== undefined) {
|
|
94
|
+
try {
|
|
95
|
+
assertSafeSegment(input.channel, "channel");
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
throw new SignalsRedoUsageError(`not a channel name: ${input.channel}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const worklistChannel = new Map();
|
|
102
|
+
const usesWorklist = async (channel) => {
|
|
103
|
+
if (!worklistChannel.has(channel))
|
|
104
|
+
worklistChannel.set(channel, await hasSourceYaml(vaultRoot, channel));
|
|
105
|
+
return worklistChannel.get(channel) === true;
|
|
106
|
+
};
|
|
107
|
+
for (const key of input.keys ?? []) {
|
|
108
|
+
const channel = channelOf(key);
|
|
109
|
+
if (!channel || channel.startsWith("_"))
|
|
110
|
+
throw new SignalsRedoUsageError(`--key must be sources/<channel>/<id>.md: ${key}`);
|
|
111
|
+
if (!(await isFile(path.join(vaultRoot, key))))
|
|
112
|
+
throw new SignalsRedoUsageError(`no landed document at ${key}`);
|
|
113
|
+
add((await usesWorklist(channel)) ? selection.worklist : selection.pending, channel, key);
|
|
114
|
+
}
|
|
115
|
+
if (input.since !== undefined) {
|
|
116
|
+
const since = input.since.getTime();
|
|
117
|
+
const signalled = await signalledSourceRefs(vaultRoot);
|
|
118
|
+
let channels;
|
|
119
|
+
if (input.channel !== undefined) {
|
|
120
|
+
channels = [input.channel];
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
try {
|
|
124
|
+
const entries = await fs.readdir(path.join(vaultRoot, "sources"), { withFileTypes: true });
|
|
125
|
+
channels = entries
|
|
126
|
+
.filter((e) => e.isDirectory() && !e.name.startsWith("_") && !e.name.startsWith("."))
|
|
127
|
+
.map((e) => e.name)
|
|
128
|
+
.sort();
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
channels = [];
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
const withWorklist = [];
|
|
135
|
+
for (const channel of channels) {
|
|
136
|
+
if (await usesWorklist(channel)) {
|
|
137
|
+
// The worklist already lists its channel's unprocessed items, and the pass drains them.
|
|
138
|
+
withWorklist.push(channel);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
let names;
|
|
142
|
+
try {
|
|
143
|
+
names = (await fs.readdir(path.join(vaultRoot, "sources", channel))).filter((n) => n.endsWith(".md")).sort();
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
for (const name of names) {
|
|
149
|
+
const key = `sources/${channel}/${name}`;
|
|
150
|
+
if (channelOf(key) !== channel || signalled.has(key))
|
|
151
|
+
continue;
|
|
152
|
+
let text;
|
|
153
|
+
try {
|
|
154
|
+
text = await fs.readFile(path.join(vaultRoot, key), "utf8");
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
if (landedDocTime(splitFrontmatter(text).frontmatter) >= since)
|
|
160
|
+
add(selection.pending, channel, key);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
if (withWorklist.length > 0) {
|
|
164
|
+
selection.notices.push(`Skipped for --since (their worklist lists unprocessed items itself; use --key to redo one): ${withWorklist.join(", ")}`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return selection;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Queue the selection: pending keys into the pass's pending file, worklist
|
|
171
|
+
* keys through `ontology-source-worklist.sh --redo`. Throws when the worklist
|
|
172
|
+
* refuses (an older script has no `--redo`).
|
|
173
|
+
*/
|
|
174
|
+
export async function queueSignalsRedo(input) {
|
|
175
|
+
const vaultRoot = path.join(input.hqRoot, "companies", input.company);
|
|
176
|
+
for (const [channel, keys] of input.selection.pending) {
|
|
177
|
+
await queuePendingSignals(vaultRoot, channel, keys, input.beforeWrite);
|
|
178
|
+
}
|
|
179
|
+
const worklist = input.worklistScript ?? path.join(input.hqRoot, "core", "scripts", "ontology-source-worklist.sh");
|
|
180
|
+
for (const [channel, keys] of input.selection.worklist) {
|
|
181
|
+
await input.beforeWrite?.();
|
|
182
|
+
const run = await runBounded("bash", [worklist, "--company", input.company, "--channel", channel, ...keys.flatMap((key) => ["--redo", key])], { timeoutMs: WORKLIST_TIMEOUT_MS, cwd: input.hqRoot, env: { ...(input.env ?? process.env), HQ_ROOT: input.hqRoot } });
|
|
183
|
+
if (run.timedOut || run.code !== 0) {
|
|
184
|
+
const why = run.timedOut ? "timed out" : run.stderr.trim().slice(0, 200) || `exit ${String(run.code)}`;
|
|
185
|
+
throw new Error(`ontology worklist --redo failed for sources/${channel}: ${why}`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
/** A run result with nothing landed, so the signals pass works through one channel's backlog. */
|
|
190
|
+
export function backlogRunResult(channel) {
|
|
191
|
+
return {
|
|
192
|
+
adapterId: "signals-redo",
|
|
193
|
+
provider: "signals-redo",
|
|
194
|
+
channel,
|
|
195
|
+
connectionId: null,
|
|
196
|
+
transport: null,
|
|
197
|
+
landed: 0,
|
|
198
|
+
duplicates: 0,
|
|
199
|
+
stale: 0,
|
|
200
|
+
rejected: 0,
|
|
201
|
+
pages: 0,
|
|
202
|
+
landedKeys: [],
|
|
203
|
+
cursorBefore: null,
|
|
204
|
+
cursorAfter: null,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
//# sourceMappingURL=signals-redo.js.map
|