@indigoai-us/hq-cli 5.345.191 → 5.345.193
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +398 -0
- package/assets/bot-workers/setup/worker.yaml +91 -34
- package/dist/command-catalog.generated.d.ts +96 -19
- package/dist/command-catalog.generated.js +124 -21
- package/dist/commands/bot-jobs.d.ts +21 -18
- package/dist/commands/bot-jobs.js +127 -39
- package/dist/commands/bot.d.ts +4 -3
- package/dist/commands/bot.js +265 -140
- package/dist/commands/core-jobs-schedule.js +10 -0
- package/dist/commands/daemon.js +37 -0
- package/dist/commands/files.d.ts +41 -6
- package/dist/commands/files.js +108 -27
- package/dist/commands/pack-start-bot.d.ts +4 -6
- package/dist/commands/pack-start-bot.js +4 -8
- package/dist/commands/signals-run.d.ts +45 -0
- package/dist/commands/signals-run.js +175 -0
- package/dist/commands/signals.d.ts +1 -0
- package/dist/commands/signals.js +3 -0
- package/dist/commands/sources.d.ts +11 -0
- package/dist/commands/sources.js +137 -25
- package/dist/commands/staff/reset-test-account.d.ts +10 -0
- package/dist/commands/staff/reset-test-account.js +18 -2
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +31171 -52512
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/bot/api.d.ts +3 -2
- package/dist/lib/bot/config.d.ts +2 -0
- package/dist/lib/bot/fleet-cron.d.ts +121 -0
- package/dist/lib/bot/fleet-cron.js +319 -0
- package/dist/lib/bot/fleet-local.d.ts +93 -1
- package/dist/lib/bot/fleet-local.js +546 -34
- package/dist/lib/bot/fleet-owner-companies.d.ts +12 -0
- package/dist/lib/bot/fleet-owner-companies.js +30 -0
- package/dist/lib/bot/fleet-watch.d.ts +25 -0
- package/dist/lib/bot/fleet-watch.js +51 -0
- package/dist/lib/bot/index.d.ts +1 -1
- package/dist/lib/bot/index.js +1 -1
- package/dist/lib/bot/job-run.d.ts +5 -4
- package/dist/lib/bot/job-run.js +5 -4
- package/dist/lib/bot/jobs/hot-signal-sync.d.ts +57 -3
- package/dist/lib/bot/jobs/hot-signal-sync.js +163 -4
- package/dist/lib/bot/local-jobs.d.ts +13 -2
- package/dist/lib/bot/local-jobs.js +17 -4
- package/dist/lib/bot/progress.d.ts +14 -0
- package/dist/lib/bot/progress.js +47 -1
- package/dist/lib/bot/run.d.ts +55 -2
- package/dist/lib/bot/run.js +192 -10
- package/dist/lib/bot/setup-tool-offer.d.ts +1 -0
- package/dist/lib/bot/setup-tool-offer.js +16 -0
- package/dist/lib/bot/signals-engine-flag.d.ts +18 -0
- package/dist/lib/bot/signals-engine-flag.js +30 -0
- package/dist/lib/daemon/catalog.d.ts +4 -0
- package/dist/lib/daemon/catalog.js +29 -9
- package/dist/lib/daemon/legacy.d.ts +6 -0
- package/dist/lib/daemon/legacy.js +16 -10
- package/dist/lib/daemon/run.d.ts +1 -0
- package/dist/lib/daemon/run.js +10 -0
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/flag-registry.d.ts +13 -1
- package/dist/lib/flag-registry.js +46 -0
- package/dist/lib/hot-channels/adapters/attio.d.ts +75 -2
- package/dist/lib/hot-channels/adapters/attio.js +367 -29
- package/dist/lib/hot-channels/adapters/gmail.d.ts +14 -0
- package/dist/lib/hot-channels/adapters/gmail.js +59 -2
- package/dist/lib/hot-channels/adapters/linear.d.ts +24 -0
- package/dist/lib/hot-channels/adapters/linear.js +71 -1
- package/dist/lib/hot-channels/adapters/meetings.js +4 -1
- package/dist/lib/hot-channels/adapters/slack.js +7 -3
- package/dist/lib/hot-channels/alert.js +1 -1
- package/dist/lib/hot-channels/contract.d.ts +11 -0
- package/dist/lib/hot-channels/contract.js +1 -0
- package/dist/lib/hot-channels/cursors.js +1 -1
- package/dist/lib/hot-channels/landing.d.ts +6 -2
- package/dist/lib/hot-channels/landing.js +13 -6
- package/dist/lib/hot-channels/ledger.js +2 -0
- package/dist/lib/hot-channels/loop.js +18 -4
- package/dist/lib/hot-channels/registry.js +9 -1
- package/dist/lib/hot-channels/run.d.ts +18 -0
- package/dist/lib/hot-channels/run.js +15 -4
- package/dist/lib/hot-channels/signals-pass.d.ts +1 -0
- package/dist/lib/hot-channels/signals-pass.js +7 -5
- package/dist/lib/hot-channels/sync-lock.js +22 -6
- package/dist/lib/mesh/live/daemon/install.d.ts +22 -0
- package/dist/lib/mesh/live/daemon/install.js +117 -30
- package/dist/lib/signals/adapters-catalog.d.ts +77 -0
- package/dist/lib/signals/adapters-catalog.js +194 -0
- package/dist/lib/signals/engine.d.ts +196 -0
- package/dist/lib/signals/engine.js +495 -0
- package/dist/lib/signals/failure-alert.d.ts +89 -0
- package/dist/lib/signals/failure-alert.js +175 -0
- package/dist/lib/signals/local-date.d.ts +3 -0
- package/dist/lib/signals/local-date.js +6 -0
- package/dist/lib/signals/projection.d.ts +42 -0
- package/dist/lib/signals/projection.js +67 -0
- package/dist/lib/signals/reply.d.ts +112 -0
- package/dist/lib/signals/reply.js +368 -0
- package/dist/lib/signals/run-lock.d.ts +36 -0
- package/dist/lib/signals/run-lock.js +148 -0
- package/dist/lib/signals/run-record.d.ts +24 -0
- package/dist/lib/signals/run-record.js +82 -0
- package/dist/lib/signals/sanitize.d.ts +105 -0
- package/dist/lib/signals/sanitize.js +439 -0
- package/dist/lib/signals/settings.d.ts +10 -0
- package/dist/lib/signals/settings.js +48 -0
- package/dist/lib/signals/steps/adapters.d.ts +14 -0
- package/dist/lib/signals/steps/adapters.js +156 -0
- package/dist/lib/signals/steps/entity-similarity.d.ts +10 -0
- package/dist/lib/signals/steps/entity-similarity.js +47 -0
- package/dist/lib/signals/steps/extract.d.ts +113 -0
- package/dist/lib/signals/steps/extract.js +765 -0
- package/dist/lib/signals/steps/garden.d.ts +24 -0
- package/dist/lib/signals/steps/garden.js +250 -0
- package/dist/lib/signals/steps/types.d.ts +112 -0
- package/dist/lib/signals/steps/types.js +4 -0
- package/dist/lib/signals/steps/upload.d.ts +43 -0
- package/dist/lib/signals/steps/upload.js +324 -0
- package/dist/lib/signals/write-guard.d.ts +63 -0
- package/dist/lib/signals/write-guard.js +287 -0
- package/dist/utils/resident-package-preload.js +16 -0
- package/package.json +9 -9
- package/dist/lib/bot/fleet-default-flag.d.ts +0 -31
- package/dist/lib/bot/fleet-default-flag.js +0 -127
|
@@ -34,7 +34,19 @@
|
|
|
34
34
|
* A revoked or expired token (`invalid_grant`) surfaces from the gateway call;
|
|
35
35
|
* run.ts turns it into a reconnect prompt in the run ledger and the cursor
|
|
36
36
|
* does not move.
|
|
37
|
+
*
|
|
38
|
+
* A rate-limited call (HTTP 429, or a rate-limit error from the gateway) is
|
|
39
|
+
* retried up to `LINEAR_MAX_RETRIES` times, waiting for `Retry-After` when the
|
|
40
|
+
* error carries one (capped) or an exponential backoff otherwise. When the
|
|
41
|
+
* retries run out the fetch throws `LinearRateLimitedError`, so the run fails,
|
|
42
|
+
* nothing lands for that page and the cursor does not move.
|
|
43
|
+
*
|
|
44
|
+
* Request budget: a run makes at most `DEFAULT_MAX_PAGES_PER_ADAPTER` (20)
|
|
45
|
+
* pages, each at most `1 + LINEAR_MAX_RETRIES` requests, so
|
|
46
|
+
* `LINEAR_MAX_REQUESTS_PER_RUN` is 60. 41 worst-case runs in one hour still
|
|
47
|
+
* stay under `LINEAR_REQUESTS_PER_HOUR` (2,500).
|
|
37
48
|
*/
|
|
49
|
+
import { DEFAULT_MAX_PAGES_PER_ADAPTER } from "../loop.js";
|
|
38
50
|
export const LINEAR_ADAPTER_ID = "linear";
|
|
39
51
|
export const LINEAR_CHANNEL = "linear";
|
|
40
52
|
export const LINEAR_PROVIDER = "linear";
|
|
@@ -44,6 +56,12 @@ export const LINEAR_LIST_ISSUES_TOOL = "list_issues";
|
|
|
44
56
|
export const LINEAR_REQUESTS_PER_HOUR = 2500;
|
|
45
57
|
export const LINEAR_CATCH_UP_WINDOW_MS = 14 * 24 * 60 * 60 * 1000;
|
|
46
58
|
const SUGGESTION_WINDOW_MS = 14 * 24 * 60 * 60 * 1000;
|
|
59
|
+
/** Retries for one rate-limited `list_issues` call before the run gives up. */
|
|
60
|
+
export const LINEAR_MAX_RETRIES = 2;
|
|
61
|
+
/** Most requests one run can make: every page, every retry. */
|
|
62
|
+
export const LINEAR_MAX_REQUESTS_PER_RUN = DEFAULT_MAX_PAGES_PER_ADAPTER * (1 + LINEAR_MAX_RETRIES);
|
|
63
|
+
const BASE_BACKOFF_MS = 1000;
|
|
64
|
+
const MAX_BACKOFF_MS = 30_000;
|
|
47
65
|
const DEFAULT_PAGE_SIZE = 100;
|
|
48
66
|
const MAX_PAGE_SIZE = 250;
|
|
49
67
|
export const LINEAR_ISSUE_FIELDS = [
|
|
@@ -70,6 +88,39 @@ const PRIORITY_LABELS = {
|
|
|
70
88
|
3: "Medium",
|
|
71
89
|
4: "Low",
|
|
72
90
|
};
|
|
91
|
+
/** Linear kept answering 429 after every retry; the cursor stays put. */
|
|
92
|
+
export class LinearRateLimitedError extends Error {
|
|
93
|
+
retryAfterMs;
|
|
94
|
+
code = "rate_limited";
|
|
95
|
+
status = 429;
|
|
96
|
+
constructor(retryAfterMs) {
|
|
97
|
+
super("Linear rate limit (429): the run stopped and will resume from the same cursor next time");
|
|
98
|
+
this.retryAfterMs = retryAfterMs;
|
|
99
|
+
this.name = "LinearRateLimitedError";
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const RATE_LIMIT_PATTERN = /\b429\b|rate[ _-]?limit|too many requests|RATELIMITED/i;
|
|
103
|
+
function isRateLimited(err) {
|
|
104
|
+
if (!err || typeof err !== "object")
|
|
105
|
+
return false;
|
|
106
|
+
const e = err;
|
|
107
|
+
if (e.status === 429 || e.statusCode === 429)
|
|
108
|
+
return true;
|
|
109
|
+
if (typeof e.code === "string" && RATE_LIMIT_PATTERN.test(e.code))
|
|
110
|
+
return true;
|
|
111
|
+
return typeof e.message === "string" && RATE_LIMIT_PATTERN.test(e.message);
|
|
112
|
+
}
|
|
113
|
+
function retryAfterMs(err) {
|
|
114
|
+
const e = err;
|
|
115
|
+
let raw = e?.retryAfter;
|
|
116
|
+
if (raw === undefined && e?.headers && typeof e.headers === "object") {
|
|
117
|
+
const h = e.headers;
|
|
118
|
+
raw = typeof h.get === "function" ? h.get("retry-after") : (h["retry-after"] ?? h["Retry-After"]);
|
|
119
|
+
}
|
|
120
|
+
const secs = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : NaN;
|
|
121
|
+
return Number.isFinite(secs) && secs >= 0 ? secs * 1000 : null;
|
|
122
|
+
}
|
|
123
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
73
124
|
function nonEmptyString(value) {
|
|
74
125
|
return typeof value === "string" && value.trim() !== "" ? value : undefined;
|
|
75
126
|
}
|
|
@@ -267,6 +318,22 @@ function toEvent(issue, personUid) {
|
|
|
267
318
|
}
|
|
268
319
|
export function createLinearAdapter(options = {}) {
|
|
269
320
|
const pageSize = Math.min(MAX_PAGE_SIZE, Math.max(1, Math.floor(options.pageSize ?? DEFAULT_PAGE_SIZE)));
|
|
321
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
322
|
+
async function listIssues(ctx, args) {
|
|
323
|
+
for (let attempt = 0;; attempt += 1) {
|
|
324
|
+
try {
|
|
325
|
+
return await ctx.gateway.call(LINEAR_LIST_ISSUES_TOOL, args);
|
|
326
|
+
}
|
|
327
|
+
catch (err) {
|
|
328
|
+
if (!isRateLimited(err))
|
|
329
|
+
throw err;
|
|
330
|
+
const wait = retryAfterMs(err);
|
|
331
|
+
if (attempt >= LINEAR_MAX_RETRIES)
|
|
332
|
+
throw new LinearRateLimitedError(wait);
|
|
333
|
+
await sleep(Math.min(MAX_BACKOFF_MS, wait ?? BASE_BACKOFF_MS * 2 ** attempt));
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
270
337
|
async function fetch(cursor, ctx) {
|
|
271
338
|
if (!ctx.gateway)
|
|
272
339
|
throw new Error("Linear hot channel needs a gateway connection");
|
|
@@ -280,7 +347,7 @@ export function createLinearAdapter(options = {}) {
|
|
|
280
347
|
};
|
|
281
348
|
if (state.after)
|
|
282
349
|
args.cursor = state.after;
|
|
283
|
-
const page = parseIssuePage(await ctx
|
|
350
|
+
const page = parseIssuePage(await listIssues(ctx, args));
|
|
284
351
|
const watermarkMs = Date.parse(state.watermark);
|
|
285
352
|
let high = state.high;
|
|
286
353
|
const events = [];
|
|
@@ -311,6 +378,9 @@ export function createLinearAdapter(options = {}) {
|
|
|
311
378
|
}
|
|
312
379
|
return {
|
|
313
380
|
id: LINEAR_ADAPTER_ID,
|
|
381
|
+
displayName: "Linear",
|
|
382
|
+
provider: LINEAR_PROVIDER,
|
|
383
|
+
watermarkKind: "cursor",
|
|
314
384
|
channel: LINEAR_CHANNEL,
|
|
315
385
|
transport: "poll",
|
|
316
386
|
hotEventTypes: [LINEAR_ISSUE_UPDATED_EVENT],
|
|
@@ -372,7 +372,7 @@ export function createMeetingsAdapter(options) {
|
|
|
372
372
|
...(m.calendarEventId ? { calendar_event_id: m.calendarEventId } : {}),
|
|
373
373
|
origin: HOT_SIGNAL_ORIGIN,
|
|
374
374
|
};
|
|
375
|
-
await writeFileAtomic(file, `---\n${yaml.dump(frontmatter, { lineWidth: -1, noRefs: true }).trimEnd()}\n---\n${text}`);
|
|
375
|
+
await writeFileAtomic(options.vaultRoot, file, `---\n${yaml.dump(frontmatter, { lineWidth: -1, noRefs: true }).trimEnd()}\n---\n${text}`);
|
|
376
376
|
return ref;
|
|
377
377
|
}
|
|
378
378
|
function providerEvent(provider, m, ref, text, me) {
|
|
@@ -515,6 +515,9 @@ export function createMeetingsAdapter(options) {
|
|
|
515
515
|
}
|
|
516
516
|
return {
|
|
517
517
|
id: MEETINGS_ADAPTER_ID,
|
|
518
|
+
displayName: "Meetings",
|
|
519
|
+
provider: MEETINGS_PROVIDER,
|
|
520
|
+
watermarkKind: "timestamp",
|
|
518
521
|
channel: MEETINGS_CHANNEL,
|
|
519
522
|
transport: "poll",
|
|
520
523
|
hotEventTypes: [MEETING_TRANSCRIPT_EVENT],
|
|
@@ -149,9 +149,10 @@ function toEvent(t) {
|
|
|
149
149
|
if (fm[field] !== undefined)
|
|
150
150
|
raw[field] = fm[field];
|
|
151
151
|
return {
|
|
152
|
-
|
|
153
|
-
//
|
|
154
|
-
|
|
152
|
+
// The event key is `slack:{channel}:{ts}`: one item per thread. A thread
|
|
153
|
+
// that later gains replies keeps its key, so it does not land again.
|
|
154
|
+
providerId: t.channelId,
|
|
155
|
+
version: t.threadTs,
|
|
155
156
|
who: owner,
|
|
156
157
|
what: SLACK_THREAD_EVENT,
|
|
157
158
|
where,
|
|
@@ -265,6 +266,9 @@ export function createSlackAdapter(options) {
|
|
|
265
266
|
}
|
|
266
267
|
return {
|
|
267
268
|
id: SLACK_ADAPTER_ID,
|
|
269
|
+
displayName: "Slack",
|
|
270
|
+
provider: SLACK_PROVIDER,
|
|
271
|
+
watermarkKind: "timestamp",
|
|
268
272
|
channel: SLACK_CHANNEL,
|
|
269
273
|
transport: "poll",
|
|
270
274
|
hotEventTypes: [SLACK_THREAD_EVENT],
|
|
@@ -119,7 +119,7 @@ export async function raiseSignalsFailureAlerts(input) {
|
|
|
119
119
|
}
|
|
120
120
|
if (changed) {
|
|
121
121
|
try {
|
|
122
|
-
await writeFileAtomic(file, `${JSON.stringify(state, null, 2)}\n`);
|
|
122
|
+
await writeFileAtomic(input.vaultRoot, file, `${JSON.stringify(state, null, 2)}\n`);
|
|
123
123
|
}
|
|
124
124
|
catch (err) {
|
|
125
125
|
log(`signals alert state was not saved: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -123,7 +123,18 @@ export interface HotChannelDeclaration {
|
|
|
123
123
|
* run hands every matching connection to `fetch` as `PollContext.gateways`.
|
|
124
124
|
*/
|
|
125
125
|
companionProviders?: readonly string[];
|
|
126
|
+
/** Optional human label, matching the hq-pro catalog entry's `displayName` (US-008). */
|
|
127
|
+
displayName?: string;
|
|
128
|
+
/** Optional catalog provider id; defaults to `tokenScope.provider` when unset (US-008). */
|
|
129
|
+
provider?: string;
|
|
130
|
+
/**
|
|
131
|
+
* Optional description of what the stored cursor is (US-008): an opaque
|
|
132
|
+
* provider `cursor`, an ISO `timestamp`, or a `webhook` buffer position.
|
|
133
|
+
*/
|
|
134
|
+
watermarkKind?: WatermarkKind;
|
|
126
135
|
}
|
|
136
|
+
export type WatermarkKind = "cursor" | "timestamp" | "webhook";
|
|
137
|
+
export declare const WATERMARK_KINDS: readonly WatermarkKind[];
|
|
127
138
|
/** A feature flag the adapter needs on before it runs. */
|
|
128
139
|
export interface HotChannelGate {
|
|
129
140
|
/** Flag key as registered in hq-pro, e.g. `hq.hot-channel-gmail`. */
|
|
@@ -77,7 +77,7 @@ export function createVaultCursorStore(vaultRoot, options = {}) {
|
|
|
77
77
|
const file = cursorFilePath(vaultRoot, personKey, adapterId);
|
|
78
78
|
const kept = lastEventKey === undefined ? ((await read(personKey, adapterId))?.lastEventKey ?? null) : lastEventKey;
|
|
79
79
|
const record = { cursor, updatedAt: now().toISOString(), lastEventKey: kept };
|
|
80
|
-
await writeFileAtomic(file, `${JSON.stringify(record)}\n`);
|
|
80
|
+
await writeFileAtomic(vaultRoot, file, `${JSON.stringify(record)}\n`);
|
|
81
81
|
},
|
|
82
82
|
};
|
|
83
83
|
}
|
|
@@ -71,8 +71,12 @@ export declare function assertHotEvent(event: HotEvent): void;
|
|
|
71
71
|
* checks a landed `payload_ref` with the same rule before reading it.
|
|
72
72
|
*/
|
|
73
73
|
export declare function assertPayloadRef(ref: string): void;
|
|
74
|
-
/**
|
|
75
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Write via a sibling temp file and rename, so readers never see a partial
|
|
76
|
+
* file. Both paths go through the company-scoped write guard first: a link
|
|
77
|
+
* that would carry the write into another company's folder fails closed.
|
|
78
|
+
*/
|
|
79
|
+
export declare function writeFileAtomic(vaultRoot: string, file: string, contents: string): Promise<void>;
|
|
76
80
|
/**
|
|
77
81
|
* Land one normalized event as a source document under
|
|
78
82
|
* `sources/{channel}/{source_id}.md`, with its raw payload alongside and a row
|
|
@@ -2,6 +2,7 @@ import { createHash, randomBytes } from "node:crypto";
|
|
|
2
2
|
import * as fs from "node:fs/promises";
|
|
3
3
|
import * as path from "node:path";
|
|
4
4
|
import * as yaml from "js-yaml";
|
|
5
|
+
import { assertVaultPath } from "../signals/write-guard.js";
|
|
5
6
|
/** `origin` written on every hot-signal landed document. */
|
|
6
7
|
export const HOT_SIGNAL_ORIGIN = "hot-signal-sync";
|
|
7
8
|
const SAFE_SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
@@ -74,10 +75,16 @@ async function exists(file) {
|
|
|
74
75
|
return false;
|
|
75
76
|
}
|
|
76
77
|
}
|
|
77
|
-
/**
|
|
78
|
-
|
|
79
|
-
|
|
78
|
+
/**
|
|
79
|
+
* Write via a sibling temp file and rename, so readers never see a partial
|
|
80
|
+
* file. Both paths go through the company-scoped write guard first: a link
|
|
81
|
+
* that would carry the write into another company's folder fails closed.
|
|
82
|
+
*/
|
|
83
|
+
export async function writeFileAtomic(vaultRoot, file, contents) {
|
|
80
84
|
const tmp = `${file}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
|
|
85
|
+
assertVaultPath(vaultRoot, file);
|
|
86
|
+
assertVaultPath(vaultRoot, tmp);
|
|
87
|
+
await fs.mkdir(path.dirname(file), { recursive: true });
|
|
81
88
|
try {
|
|
82
89
|
await fs.writeFile(tmp, contents, "utf8");
|
|
83
90
|
await fs.rename(tmp, file);
|
|
@@ -102,7 +109,7 @@ async function appendIndexEntry(vaultRoot, day, entry) {
|
|
|
102
109
|
if (rows.some((row) => row.channel === entry.channel && row.source_id === entry.source_id))
|
|
103
110
|
return;
|
|
104
111
|
rows.push(entry);
|
|
105
|
-
await writeFileAtomic(file, `${JSON.stringify(rows, null, 2)}\n`);
|
|
112
|
+
await writeFileAtomic(vaultRoot, file, `${JSON.stringify(rows, null, 2)}\n`);
|
|
106
113
|
}
|
|
107
114
|
/**
|
|
108
115
|
* Land one normalized event as a source document under
|
|
@@ -158,8 +165,8 @@ export async function landHotEvent(input) {
|
|
|
158
165
|
const markdown = `---\n${yaml.dump(frontmatter, { lineWidth: -1, noRefs: true }).trimEnd()}\n---\n${event.body ?? ""}`;
|
|
159
166
|
// Raw first, markdown second: the markdown is the dedupe marker, so it must
|
|
160
167
|
// only exist once its payload_ref target does.
|
|
161
|
-
await writeFileAtomic(path.join(vaultRoot, rawKey), `${JSON.stringify(event.raw ?? event, null, 2)}\n`);
|
|
162
|
-
await writeFileAtomic(markdownFile, markdown);
|
|
168
|
+
await writeFileAtomic(vaultRoot, path.join(vaultRoot, rawKey), `${JSON.stringify(event.raw ?? event, null, 2)}\n`);
|
|
169
|
+
await writeFileAtomic(vaultRoot, markdownFile, markdown);
|
|
163
170
|
await appendIndexEntry(vaultRoot, now.slice(0, 10), {
|
|
164
171
|
source_ref: frontmatter.source_ref,
|
|
165
172
|
channel: adapter.channel,
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { randomUUID } from "node:crypto";
|
|
9
9
|
import * as fs from "node:fs/promises";
|
|
10
10
|
import * as path from "node:path";
|
|
11
|
+
import { assertVaultPath } from "../signals/write-guard.js";
|
|
11
12
|
import { assertNotIntegrationSecretsPath, personDirName } from "./cursors.js";
|
|
12
13
|
import { runHotChannelLoop } from "./loop.js";
|
|
13
14
|
import { SyncLockLostError } from "./sync-lock.js";
|
|
@@ -35,6 +36,7 @@ export function runLedgerPath(vaultRoot, personKey, day) {
|
|
|
35
36
|
*/
|
|
36
37
|
export async function appendRunLedgerEntry(vaultRoot, personKey, entry) {
|
|
37
38
|
const file = runLedgerPath(vaultRoot, personKey, ledgerDay(new Date(entry.startedAt)));
|
|
39
|
+
assertVaultPath(vaultRoot, file);
|
|
38
40
|
await fs.mkdir(path.dirname(file), { recursive: true });
|
|
39
41
|
await fs.appendFile(file, `${JSON.stringify(entry)}\n`, { encoding: "utf8", flag: "a" });
|
|
40
42
|
return file;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import * as fs from "node:fs/promises";
|
|
3
3
|
import * as path from "node:path";
|
|
4
|
+
import { assertVaultPath } from "../signals/write-guard.js";
|
|
4
5
|
import { createVaultCursorStore, cursorKey } from "./cursors.js";
|
|
5
6
|
import { assertHotEvent, assertSafeSegment, eventKey, HotEventValidationError, landHotEvent } from "./landing.js";
|
|
6
7
|
import { SyncLockLostError } from "./sync-lock.js";
|
|
@@ -73,19 +74,27 @@ async function landOne(ctx, adapter, event, result) {
|
|
|
73
74
|
* copy is neither overwritten nor a reason to abort the drain. `link` fails
|
|
74
75
|
* atomically on an existing target.
|
|
75
76
|
*/
|
|
76
|
-
async function moveToRejected(file) {
|
|
77
|
+
async function moveToRejected(vaultRoot, file) {
|
|
78
|
+
assertVaultPath(vaultRoot, file);
|
|
79
|
+
assertVaultPath(vaultRoot, `${file}.rejected`);
|
|
77
80
|
try {
|
|
78
81
|
await fs.link(file, `${file}.rejected`);
|
|
79
82
|
}
|
|
80
83
|
catch (err) {
|
|
81
84
|
if (err.code !== "EEXIST")
|
|
82
85
|
throw err;
|
|
83
|
-
|
|
86
|
+
const unique = `${file}.${randomUUID()}.rejected`;
|
|
87
|
+
assertVaultPath(vaultRoot, unique);
|
|
88
|
+
await fs.link(file, unique);
|
|
84
89
|
}
|
|
90
|
+
assertVaultPath(vaultRoot, file);
|
|
85
91
|
await fs.rm(file, { force: true });
|
|
86
92
|
}
|
|
87
93
|
async function drainPush(ctx, adapter, result) {
|
|
88
94
|
const dir = pushBufferDir(ctx.vaultRoot, adapter.id, ctx.personUid);
|
|
95
|
+
// Check the buffer folder before reading it: a link here would read another
|
|
96
|
+
// company's buffered events into this run and then delete them there.
|
|
97
|
+
assertVaultPath(ctx.vaultRoot, path.join(dir, ".write-guard-probe"));
|
|
89
98
|
let names;
|
|
90
99
|
try {
|
|
91
100
|
names = await fs.readdir(dir);
|
|
@@ -98,6 +107,10 @@ async function drainPush(ctx, adapter, result) {
|
|
|
98
107
|
const files = names.filter((name) => name.endsWith(".json")).sort().slice(0, ctx.maxPushEvents);
|
|
99
108
|
for (const name of files) {
|
|
100
109
|
const file = path.join(dir, name);
|
|
110
|
+
// Check each buffer file before reading it. readFile follows a link, so a
|
|
111
|
+
// buffer file linked into another company would land that company's
|
|
112
|
+
// event here before any later check ran. A refused file fails the drain.
|
|
113
|
+
assertVaultPath(ctx.vaultRoot, file);
|
|
101
114
|
let event;
|
|
102
115
|
try {
|
|
103
116
|
event = JSON.parse(await fs.readFile(file, "utf8"));
|
|
@@ -105,17 +118,18 @@ async function drainPush(ctx, adapter, result) {
|
|
|
105
118
|
assertHotEvent(event);
|
|
106
119
|
}
|
|
107
120
|
catch (err) {
|
|
108
|
-
await moveToRejected(file);
|
|
121
|
+
await moveToRejected(ctx.vaultRoot, file);
|
|
109
122
|
reject(ctx, adapter, result, `${name}: ${reasonOf(err)}`);
|
|
110
123
|
continue;
|
|
111
124
|
}
|
|
112
125
|
const before = result.rejected;
|
|
113
126
|
await landOne(ctx, adapter, event, result);
|
|
114
127
|
if (result.rejected > before) {
|
|
115
|
-
await moveToRejected(file);
|
|
128
|
+
await moveToRejected(ctx.vaultRoot, file);
|
|
116
129
|
continue;
|
|
117
130
|
}
|
|
118
131
|
// Removed only after the event landed, was a duplicate, or was stale.
|
|
132
|
+
assertVaultPath(ctx.vaultRoot, file);
|
|
119
133
|
await fs.rm(file, { force: true });
|
|
120
134
|
}
|
|
121
135
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { HOT_TRANSPORTS, MAX_CATCH_UP_WINDOW_MS, SUGGESTION_RULE_KINDS, } from "./contract.js";
|
|
1
|
+
import { HOT_TRANSPORTS, MAX_CATCH_UP_WINDOW_MS, SUGGESTION_RULE_KINDS, WATERMARK_KINDS, } from "./contract.js";
|
|
2
2
|
/** Adapter ids and channels become path segments, so keep them to a safe set. */
|
|
3
3
|
const SAFE_SEGMENT = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
|
4
4
|
export class InvalidHotChannelAdapterError extends Error {
|
|
@@ -84,6 +84,14 @@ export function validateHotChannelAdapter(adapter) {
|
|
|
84
84
|
rule.minEvents < 1)) {
|
|
85
85
|
fail(a, "recent-activity rules need withinMs > 0 and an integer minEvents >= 1");
|
|
86
86
|
}
|
|
87
|
+
if (a.watermarkKind !== undefined && !WATERMARK_KINDS.includes(a.watermarkKind)) {
|
|
88
|
+
fail(a, `watermarkKind must be one of ${WATERMARK_KINDS.join(", ")}`);
|
|
89
|
+
}
|
|
90
|
+
for (const field of ["displayName", "provider"]) {
|
|
91
|
+
if (a[field] !== undefined && (typeof a[field] !== "string" || a[field].trim() === "")) {
|
|
92
|
+
fail(a, `${field} must be a non-empty string when set`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
87
95
|
if (a.gate !== undefined) {
|
|
88
96
|
const gate = a.gate;
|
|
89
97
|
if (!gate || typeof gate !== "object" || typeof gate.flag !== "string" || gate.flag.trim() === "") {
|
|
@@ -90,6 +90,13 @@ export interface OnDemandSyncInput {
|
|
|
90
90
|
personKey: string;
|
|
91
91
|
/** `--adapter`: a catalog provider or a registered adapter id. */
|
|
92
92
|
adapter?: string;
|
|
93
|
+
/**
|
|
94
|
+
* Registered adapter ids allowed to run (the signals adapters step's
|
|
95
|
+
* locally enabled list). Catalog items and gated adapters outside the list
|
|
96
|
+
* are left out of this run without an outcome. An id the catalog does not
|
|
97
|
+
* list is not an error: it simply has nothing to run. Unset = no filter.
|
|
98
|
+
*/
|
|
99
|
+
onlyAdapters?: readonly string[];
|
|
93
100
|
now?: () => Date;
|
|
94
101
|
runId?: string;
|
|
95
102
|
maxPagesPerAdapter?: number;
|
|
@@ -124,13 +131,24 @@ export interface SyncOutcome {
|
|
|
124
131
|
reason: ReconnectReason;
|
|
125
132
|
prompt: string;
|
|
126
133
|
};
|
|
134
|
+
/**
|
|
135
|
+
* Which token the run used for this connection (US-010): `user` when the
|
|
136
|
+
* signed-in person created the connection, `workspace` when it is a shared
|
|
137
|
+
* company connection. Absent when no connection was used.
|
|
138
|
+
*/
|
|
139
|
+
tokenLevel?: HotChannelTokenLevel;
|
|
127
140
|
}
|
|
141
|
+
export type HotChannelTokenLevel = "user" | "workspace";
|
|
142
|
+
/** The token level of a connection: the person's own (`user`) or a shared one (`workspace`). */
|
|
143
|
+
export declare function tokenLevelOf(app: Pick<HotChannelApp, "ownedByCaller">): HotChannelTokenLevel;
|
|
128
144
|
export interface OnDemandSyncReport {
|
|
129
145
|
runId: string | null;
|
|
130
146
|
/** True when the HQ session was gone before anything ran; nothing was written. */
|
|
131
147
|
signInRequired: boolean;
|
|
132
148
|
/** True when another sync for this person and company was running; nothing ran. */
|
|
133
149
|
skipped?: boolean;
|
|
150
|
+
/** The signed-in person's cursor directory key (`_cursors/@{personKey}`); unset when nobody is signed in. */
|
|
151
|
+
personKey?: string;
|
|
134
152
|
outcomes: SyncOutcome[];
|
|
135
153
|
entries: RunLedgerEntry[];
|
|
136
154
|
/** Lines for the person (no channels, sign-in needed). */
|
|
@@ -72,6 +72,10 @@ export class UnknownSyncAdapterError extends Error {
|
|
|
72
72
|
this.name = "UnknownSyncAdapterError";
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
|
+
/** The token level of a connection: the person's own (`user`) or a shared one (`workspace`). */
|
|
76
|
+
export function tokenLevelOf(app) {
|
|
77
|
+
return app.ownedByCaller === true ? "user" : "workspace";
|
|
78
|
+
}
|
|
75
79
|
const SIGN_IN_PROMPT = "Your HQ session has expired or you're not signed in. Run `hq login`, then `hq sources sync` again.";
|
|
76
80
|
const REVOKED_PATTERN = /revoked|reconnect|invalid_grant|token (has )?expired|OAUTH_REQUIRED/i;
|
|
77
81
|
function statusOf(err) {
|
|
@@ -102,11 +106,13 @@ export function reconnectPrompt(app, reason) {
|
|
|
102
106
|
* adapter with a `gate` that no catalog item resolves to is gated off. These
|
|
103
107
|
* are reported but never planned: no fetch, no cursor and no ledger line.
|
|
104
108
|
*/
|
|
105
|
-
function gatedOutcomes(registry, resolvedIds, wanted) {
|
|
109
|
+
function gatedOutcomes(registry, resolvedIds, wanted, only) {
|
|
106
110
|
const out = [];
|
|
107
111
|
for (const adapter of registry.list()) {
|
|
108
112
|
if (!adapter.gate || resolvedIds.has(adapter.id))
|
|
109
113
|
continue;
|
|
114
|
+
if (only && !only.has(adapter.id))
|
|
115
|
+
continue;
|
|
110
116
|
const provider = adapter.tokenScope.provider;
|
|
111
117
|
if (wanted !== undefined && adapter.id !== wanted && provider.toLowerCase() !== wanted)
|
|
112
118
|
continue;
|
|
@@ -195,6 +201,7 @@ export async function runOnDemandSync(input) {
|
|
|
195
201
|
const report = {
|
|
196
202
|
runId: null,
|
|
197
203
|
signInRequired: false,
|
|
204
|
+
personKey: input.personKey,
|
|
198
205
|
outcomes: [],
|
|
199
206
|
entries: [],
|
|
200
207
|
notices: [],
|
|
@@ -214,8 +221,9 @@ export async function runOnDemandSync(input) {
|
|
|
214
221
|
return report;
|
|
215
222
|
}
|
|
216
223
|
const wanted = input.adapter?.toLowerCase();
|
|
224
|
+
const only = input.onlyAdapters ? new Set(input.onlyAdapters) : undefined;
|
|
217
225
|
if (!items || items.length === 0) {
|
|
218
|
-
const gated = gatedOutcomes(input.registry, new Set(), wanted);
|
|
226
|
+
const gated = gatedOutcomes(input.registry, new Set(), wanted, only);
|
|
219
227
|
if (wanted !== undefined && gated.length === 0)
|
|
220
228
|
throw new UnknownSyncAdapterError(input.adapter, []);
|
|
221
229
|
if (wanted === undefined)
|
|
@@ -224,7 +232,7 @@ export async function runOnDemandSync(input) {
|
|
|
224
232
|
return report;
|
|
225
233
|
}
|
|
226
234
|
const resolved = items.map((item) => ({ item, hit: resolveAdapter(input.registry, item) }));
|
|
227
|
-
const gated = gatedOutcomes(input.registry, new Set(resolved.flatMap(({ hit }) => (hit ? [hit.adapter.id] : []))), wanted);
|
|
235
|
+
const gated = gatedOutcomes(input.registry, new Set(resolved.flatMap(({ hit }) => (hit ? [hit.adapter.id] : []))), wanted, only);
|
|
228
236
|
if (wanted !== undefined && gated.length === 0) {
|
|
229
237
|
const valid = new Set();
|
|
230
238
|
for (const { item, hit } of resolved) {
|
|
@@ -235,7 +243,8 @@ export async function runOnDemandSync(input) {
|
|
|
235
243
|
if (!valid.has(wanted))
|
|
236
244
|
throw new UnknownSyncAdapterError(input.adapter, [...valid].sort());
|
|
237
245
|
}
|
|
238
|
-
const selected = resolved.filter(({ item, hit }) => wanted === undefined || item.provider.toLowerCase() === wanted || hit?.adapter.id === wanted)
|
|
246
|
+
const selected = resolved.filter(({ item, hit }) => (wanted === undefined || item.provider.toLowerCase() === wanted || hit?.adapter.id === wanted) &&
|
|
247
|
+
(only === undefined || !hit || only.has(hit.adapter.id)));
|
|
239
248
|
const planned = [];
|
|
240
249
|
const skipped = [];
|
|
241
250
|
for (const { item, hit } of selected) {
|
|
@@ -328,6 +337,8 @@ export async function runOnDemandSync(input) {
|
|
|
328
337
|
duplicates: result?.duplicates ?? 0,
|
|
329
338
|
message: result?.error?.message ?? null,
|
|
330
339
|
};
|
|
340
|
+
if (p.app)
|
|
341
|
+
outcome.tokenLevel = tokenLevelOf(p.app);
|
|
331
342
|
if (failed && p.app) {
|
|
332
343
|
const reason = reconnects.get(p.app.connectionId) ?? (p.app.status === "needs-attention" ? "revoked" : undefined);
|
|
333
344
|
if (reason)
|
|
@@ -117,6 +117,7 @@ interface ParsedSignal {
|
|
|
117
117
|
body: string;
|
|
118
118
|
}
|
|
119
119
|
export declare function defaultPromptFile(): string;
|
|
120
|
+
export declare const CLAUDE_LOGGED_OUT_RE: RegExp;
|
|
120
121
|
/**
|
|
121
122
|
* The text claude gets for one landed document. A pointer (both `payload_ref`
|
|
122
123
|
* and `raw_ref` set) gets its payload appended after a separator line. Any
|
|
@@ -25,6 +25,7 @@ import * as path from "node:path";
|
|
|
25
25
|
import { fileURLToPath } from "node:url";
|
|
26
26
|
import * as yaml from "js-yaml";
|
|
27
27
|
import { audienceKey, COMPANY_AUDIENCE_KEY } from "./audience.js";
|
|
28
|
+
import { assertVaultPath } from "../signals/write-guard.js";
|
|
28
29
|
import { assertPayloadRef, assertSafeSegment, writeFileAtomic } from "./landing.js";
|
|
29
30
|
/**
|
|
30
31
|
* A pass failure the run ledger records as `status: failed` with `error` set
|
|
@@ -171,7 +172,7 @@ export function runBounded(command, args, options) {
|
|
|
171
172
|
export function defaultPromptFile() {
|
|
172
173
|
return path.join(path.dirname(fileURLToPath(import.meta.url)), "prompts", "signals-pass.md");
|
|
173
174
|
}
|
|
174
|
-
const
|
|
175
|
+
export const CLAUDE_LOGGED_OUT_RE = /not logged in|log ?in|\/login|invalid api key|authenticat|oauth token/i;
|
|
175
176
|
function channelOf(key) {
|
|
176
177
|
const parts = key.split("/");
|
|
177
178
|
if (parts.length !== 3 || parts[0] !== "sources" || !parts[2].endsWith(".md"))
|
|
@@ -284,7 +285,7 @@ export function parseClaudeSignals(stdout) {
|
|
|
284
285
|
const env = envelope;
|
|
285
286
|
if (env.is_error === true) {
|
|
286
287
|
const text = typeof env.result === "string" ? env.result : "";
|
|
287
|
-
if (
|
|
288
|
+
if (CLAUDE_LOGGED_OUT_RE.test(text))
|
|
288
289
|
throw new SignalsPassError("claude-logged-out", "claude reported it is not logged in");
|
|
289
290
|
throw new SignalsPassError("claude-bad-output", "claude returned an error result");
|
|
290
291
|
}
|
|
@@ -353,6 +354,7 @@ export async function ensureAudienceFile(vaultRoot, item) {
|
|
|
353
354
|
return;
|
|
354
355
|
assertSafeSegment(item.audienceKey, "audience key");
|
|
355
356
|
const file = path.join(vaultRoot, "signals", `@${item.audienceKey}`, "_audience.yaml");
|
|
357
|
+
assertVaultPath(vaultRoot, file);
|
|
356
358
|
if (await exists(file))
|
|
357
359
|
return;
|
|
358
360
|
const principals = [...new Set(item.principals.map((p) => p.trim()).filter((p) => p.length > 0))].sort();
|
|
@@ -384,7 +386,7 @@ async function appendSignalIndex(vaultRoot, day, rows) {
|
|
|
384
386
|
const added = rows.filter((row) => !seen.has(row.signal_id));
|
|
385
387
|
if (added.length === 0)
|
|
386
388
|
return;
|
|
387
|
-
await writeFileAtomic(file, `${JSON.stringify([...existing, ...added], null, 2)}\n`);
|
|
389
|
+
await writeFileAtomic(vaultRoot, file, `${JSON.stringify([...existing, ...added], null, 2)}\n`);
|
|
388
390
|
}
|
|
389
391
|
/**
|
|
390
392
|
* Build the claude signals pass for one company. The pass keeps state for one
|
|
@@ -502,7 +504,7 @@ export function createClaudeSignalsPass(options) {
|
|
|
502
504
|
throw new SignalsPassError("claude-timeout", `claude timed out after ${timeoutMs}ms on ${item.file}`);
|
|
503
505
|
if (run.spawnError !== undefined)
|
|
504
506
|
throw new SignalsPassError("claude-logged-out", `claude could not start: ${run.spawnError}`);
|
|
505
|
-
if (run.code !== 0 &&
|
|
507
|
+
if (run.code !== 0 && CLAUDE_LOGGED_OUT_RE.test(`${run.stderr}\n${run.stdout}`) && !run.stdout.trim().startsWith("{")) {
|
|
506
508
|
throw new SignalsPassError("claude-logged-out", "claude reported it is not logged in");
|
|
507
509
|
}
|
|
508
510
|
return parseClaudeSignals(run.stdout);
|
|
@@ -537,7 +539,7 @@ export function createClaudeSignalsPass(options) {
|
|
|
537
539
|
frontmatter.payload_fallback = input.fallback;
|
|
538
540
|
const text = `---\n${yaml.dump(frontmatter, { lineWidth: -1, noRefs: true }).trimEnd()}\n---\n## ${heading(signal.type)}\n\n${signal.body || signal.title}\n`;
|
|
539
541
|
await beforeWrite();
|
|
540
|
-
await writeFileAtomic(file, text);
|
|
542
|
+
await writeFileAtomic(vaultRoot, file, text);
|
|
541
543
|
written += 1;
|
|
542
544
|
}
|
|
543
545
|
await beforeWrite();
|