@indigoai-us/hq-cli 5.217.0 → 5.219.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/command-catalog.generated.d.ts +18 -0
- package/dist/command-catalog.generated.js +23 -0
- package/dist/commands/lanes-checks.d.ts +13 -0
- package/dist/commands/lanes-checks.js +558 -0
- package/dist/commands/lanes.js +160 -44
- package/dist/lib/lanes/config.d.ts +2 -1
- package/dist/lib/lanes/config.js +30 -0
- package/dist/lib/lanes/questions.d.ts +13 -7
- package/dist/lib/lanes/questions.js +78 -18
- package/dist/lib/lanes/reconcile.d.ts +8 -1
- package/dist/lib/lanes/reconcile.js +51 -5
- package/dist/lib/lanes/spawn.js +28 -16
- package/dist/lib/lanes/store.js +15 -1
- package/dist/lib/lanes/tracker-checks.d.ts +77 -0
- package/dist/lib/lanes/tracker-checks.js +134 -0
- package/dist/lib/lanes/types.d.ts +15 -0
- package/dist/lib/lanes/watcher-main.js +17 -1
- package/dist/lib/lanes/watcher.d.ts +4 -1
- package/dist/lib/lanes/watcher.js +30 -7
- package/package.json +1 -1
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
function timestamp(value, field) {
|
|
2
|
+
const parsed = Date.parse(value);
|
|
3
|
+
if (!Number.isFinite(parsed))
|
|
4
|
+
throw new TypeError(`Invalid tracker check timestamp for ${field}`);
|
|
5
|
+
return parsed;
|
|
6
|
+
}
|
|
7
|
+
function check(id, items) {
|
|
8
|
+
return { id, result: items.length === 0 ? "ok" : "failing", items };
|
|
9
|
+
}
|
|
10
|
+
export function evaluateTrackerChecks(input) {
|
|
11
|
+
const now = input.now.getTime();
|
|
12
|
+
const questions = new Map();
|
|
13
|
+
for (const question of input.questions) {
|
|
14
|
+
if (question.status !== "pending")
|
|
15
|
+
continue;
|
|
16
|
+
const askedAt = timestamp(question.asked_at, `question ${question.id}.asked_at`);
|
|
17
|
+
if (now - askedAt > input.thresholds.open_question_age_ms) {
|
|
18
|
+
questions.set(question.id, {
|
|
19
|
+
id: question.id,
|
|
20
|
+
reason: "open_question_overdue",
|
|
21
|
+
age_ms: now - askedAt,
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
const latestRoute = question.routing_history
|
|
25
|
+
.map((route) => ({ ...route, at_ms: timestamp(route.at, `question ${question.id}.routing_history.at`) }))
|
|
26
|
+
.sort((a, b) => a.at_ms - b.at_ms)
|
|
27
|
+
.at(-1);
|
|
28
|
+
if (!latestRoute || latestRoute.audience_kind !== "user")
|
|
29
|
+
continue;
|
|
30
|
+
const surfacedAt = question.surfaced_at === null
|
|
31
|
+
? null
|
|
32
|
+
: timestamp(question.surfaced_at, `question ${question.id}.surfaced_at`);
|
|
33
|
+
if (now - latestRoute.at_ms >= input.thresholds.owner_question_surface_ms &&
|
|
34
|
+
(surfacedAt === null || surfacedAt < latestRoute.at_ms)) {
|
|
35
|
+
const finding = questions.get(question.id);
|
|
36
|
+
questions.set(question.id, {
|
|
37
|
+
...(finding ?? { id: question.id }),
|
|
38
|
+
reason: finding ? `${finding.reason},owner_not_surfaced` : "owner_not_surfaced",
|
|
39
|
+
routed_to_owner_at: new Date(latestRoute.at_ms).toISOString(),
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const leadLanes = input.teams.flatMap((team) => {
|
|
44
|
+
const lead = team.lead_lane;
|
|
45
|
+
if (!lead || (lead.state !== "done" && lead.state !== "failed"))
|
|
46
|
+
return [];
|
|
47
|
+
const hasOpenWork = team.open_story_ids.length > 0 ||
|
|
48
|
+
team.open_pr_refs.length > 0 || team.running_worker_ids.length > 0;
|
|
49
|
+
if (!hasOpenWork)
|
|
50
|
+
return [];
|
|
51
|
+
return [{
|
|
52
|
+
id: team.team,
|
|
53
|
+
reason: "terminal_lead_with_open_work",
|
|
54
|
+
lead_lane_id: lead.id,
|
|
55
|
+
lead_state: lead.state,
|
|
56
|
+
open_story_ids: team.open_story_ids,
|
|
57
|
+
open_pr_refs: team.open_pr_refs,
|
|
58
|
+
running_worker_ids: team.running_worker_ids,
|
|
59
|
+
}];
|
|
60
|
+
});
|
|
61
|
+
const freshness = [];
|
|
62
|
+
for (const team of input.teams) {
|
|
63
|
+
if (team.status_updated_at === null) {
|
|
64
|
+
freshness.push({ id: team.team, reason: "status_missing", path: team.status_path });
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
const age = now - timestamp(team.status_updated_at, `${team.team}.status.updated_at`);
|
|
68
|
+
if (age >= input.thresholds.status_stale_ms) {
|
|
69
|
+
freshness.push({ id: team.team, reason: "status_stale", path: team.status_path, age_ms: age });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (team.task_list_updated_at === null) {
|
|
73
|
+
freshness.push({ id: team.team, reason: "task_list_missing", path: team.task_list_path });
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
const age = now - timestamp(team.task_list_updated_at, `${team.team}.task_list.mtime`);
|
|
77
|
+
if (age >= input.thresholds.task_list_stale_ms) {
|
|
78
|
+
freshness.push({ id: team.team, reason: "task_list_stale", path: team.task_list_path, age_ms: age });
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
const blockers = input.blockers.flatMap((blocker) => {
|
|
83
|
+
if (blocker.resolved_at !== null)
|
|
84
|
+
return [];
|
|
85
|
+
const reasons = [];
|
|
86
|
+
const age = now - timestamp(blocker.opened_at, `blocker ${blocker.id}.opened_at`);
|
|
87
|
+
if (typeof blocker.next_step !== "string" || blocker.next_step.trim() === "")
|
|
88
|
+
reasons.push("missing_next_step");
|
|
89
|
+
if (age > input.thresholds.blocker_max_age_ms)
|
|
90
|
+
reasons.push("open_overdue");
|
|
91
|
+
return reasons.length === 0 ? [] : [{ id: blocker.id, reason: reasons.join(","), age_ms: age }];
|
|
92
|
+
});
|
|
93
|
+
const kpis = input.kpis.flatMap((kpi) => {
|
|
94
|
+
const latestValue = kpi.values
|
|
95
|
+
.map((value) => timestamp(value.at, `KPI ${kpi.id}.values.at`))
|
|
96
|
+
.reduce((latest, at) => latest === null || at > latest ? at : latest, null);
|
|
97
|
+
const baseline = latestValue ?? timestamp(kpi.created_at, `KPI ${kpi.id}.created_at`);
|
|
98
|
+
const age = now - baseline;
|
|
99
|
+
return age >= input.thresholds.kpi_value_max_age_ms
|
|
100
|
+
? [{ id: kpi.id, reason: latestValue === null ? "no_value_overdue" : "value_stale", age_ms: age }]
|
|
101
|
+
: [];
|
|
102
|
+
});
|
|
103
|
+
const directiveTeams = input.directive_teams ?? input.teams.map((team) => ({
|
|
104
|
+
team: team.team,
|
|
105
|
+
...(team.lead_lane ? { lead_lane_id: team.lead_lane.id } : {}),
|
|
106
|
+
}));
|
|
107
|
+
const teamByName = new Map(directiveTeams.map((team) => [team.team, team.lead_lane_id]));
|
|
108
|
+
const directives = input.directives.flatMap((directive) => {
|
|
109
|
+
const age = now - timestamp(directive.created_at, `directive ${directive.id}.created_at`);
|
|
110
|
+
if (age < input.thresholds.directive_ack_timeout_ms)
|
|
111
|
+
return [];
|
|
112
|
+
return directive.teams.flatMap((teamName) => {
|
|
113
|
+
const leadId = teamByName.get(teamName);
|
|
114
|
+
const acknowledged = leadId !== undefined && directive.acknowledgements.some((ack) => ack.team === teamName && ack.lane_id === leadId);
|
|
115
|
+
return acknowledged ? [] : [{
|
|
116
|
+
id: `${directive.id}:${teamName}`,
|
|
117
|
+
reason: leadId === undefined ? "target_team_lead_unavailable" : "lead_ack_missing",
|
|
118
|
+
directive_id: directive.id,
|
|
119
|
+
team: teamName,
|
|
120
|
+
lead_lane_id: leadId ?? null,
|
|
121
|
+
age_ms: age,
|
|
122
|
+
}];
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
return [
|
|
126
|
+
check("questions", [...questions.values()]),
|
|
127
|
+
check("lead_lanes", leadLanes),
|
|
128
|
+
check("freshness", freshness),
|
|
129
|
+
check("blockers", blockers),
|
|
130
|
+
check("kpis", kpis),
|
|
131
|
+
check("directives", directives),
|
|
132
|
+
];
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=tracker-checks.js.map
|
|
@@ -636,6 +636,8 @@ export interface QuestionRecord {
|
|
|
636
636
|
hop: number;
|
|
637
637
|
status: QuestionStatus;
|
|
638
638
|
version: number;
|
|
639
|
+
/** Null until the owner starts a resume carrying this answer; absent on old records. */
|
|
640
|
+
delivered_at?: string | null;
|
|
639
641
|
body: QuestionBodyRef;
|
|
640
642
|
routing_history: RoutingHistoryEntry[];
|
|
641
643
|
answer?: QuestionAnswer;
|
|
@@ -727,11 +729,24 @@ export interface LivenessWindows {
|
|
|
727
729
|
/** No event at all for this window (and no `session_end`) → lost. Default 12 hours. */
|
|
728
730
|
silent_lost_ms: number;
|
|
729
731
|
}
|
|
732
|
+
/** Age limits used by `hq lanes checks`; each value is a positive millisecond count. */
|
|
733
|
+
export interface TrackerChecksConfig {
|
|
734
|
+
owner_question_surface_ms: number;
|
|
735
|
+
open_question_age_ms: number;
|
|
736
|
+
status_stale_ms: number;
|
|
737
|
+
task_list_stale_ms: number;
|
|
738
|
+
blocker_max_age_ms: number;
|
|
739
|
+
kpi_value_max_age_ms: number;
|
|
740
|
+
directive_ack_timeout_ms: number;
|
|
741
|
+
}
|
|
730
742
|
export interface LanesConfig {
|
|
731
743
|
schema_version: typeof CONFIG_SCHEMA_VERSION | string;
|
|
732
744
|
providers: Record<LaneProvider, ProviderConfig>;
|
|
733
745
|
throttle: ThrottleConfig;
|
|
734
746
|
liveness: LivenessWindows;
|
|
747
|
+
tracker: {
|
|
748
|
+
checks: TrackerChecksConfig;
|
|
749
|
+
};
|
|
735
750
|
/** Seconds an unanswered question waits at a session audience before advancing. */
|
|
736
751
|
session_question_timeout_secs: number;
|
|
737
752
|
}
|
|
@@ -6,6 +6,20 @@
|
|
|
6
6
|
* Requires HQ_ROOT in the environment.
|
|
7
7
|
*/
|
|
8
8
|
import { runWatcherLoop } from "./watcher.js";
|
|
9
|
+
import { LanesError } from "./errors.js";
|
|
10
|
+
import { openMeshTransport } from "../mesh/client.js";
|
|
11
|
+
import { CompanySelectionError } from "../../utils/company-selection-error.js";
|
|
12
|
+
async function resolveQuestionActiveCompany() {
|
|
13
|
+
try {
|
|
14
|
+
const { company } = await openMeshTransport({});
|
|
15
|
+
return { slug: company.companySlug, uid: company.companyUid };
|
|
16
|
+
}
|
|
17
|
+
catch (err) {
|
|
18
|
+
if (!(err instanceof CompanySelectionError))
|
|
19
|
+
throw err;
|
|
20
|
+
throw new LanesError("question_company_unavailable", "Cannot raise a lane question because the active HQ company could not be resolved", {});
|
|
21
|
+
}
|
|
22
|
+
}
|
|
9
23
|
function main() {
|
|
10
24
|
const laneId = process.argv[2]?.trim();
|
|
11
25
|
const hqRoot = process.env.HQ_ROOT?.trim();
|
|
@@ -13,7 +27,9 @@ function main() {
|
|
|
13
27
|
process.stderr.write("hq lanes watcher: requires lane id argv and HQ_ROOT in the environment\n");
|
|
14
28
|
process.exit(1);
|
|
15
29
|
}
|
|
16
|
-
void runWatcherLoop(hqRoot, laneId
|
|
30
|
+
void runWatcherLoop(hqRoot, laneId, {
|
|
31
|
+
resolveQuestionActiveCompany,
|
|
32
|
+
}).catch((err) => {
|
|
17
33
|
process.stderr.write(`hq lanes watcher: ${err instanceof Error ? err.message : String(err)}${err instanceof Error && err.stack ? `\n${err.stack}` : ""}\n`);
|
|
18
34
|
process.exit(1);
|
|
19
35
|
});
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
* Queue admission of the next lane when a slot frees is owned by the
|
|
32
32
|
* periodic reconcile pass and is out of scope here.
|
|
33
33
|
*/
|
|
34
|
-
import { ENVELOPE_INVALID_REASON, type LaneEnvelope, type LaneRecord, type LaneState } from "./types.js";
|
|
34
|
+
import { ENVELOPE_INVALID_REASON, type LaneEnvelope, type LaneCompanyRef, type LaneRecord, type LaneState } from "./types.js";
|
|
35
35
|
import { type QuotaSignal } from "./quota.js";
|
|
36
36
|
import { type ProviderRejection } from "./provider-rejection.js";
|
|
37
37
|
import type { AdmissionHost } from "./admission.js";
|
|
@@ -145,6 +145,7 @@ export interface ApplyTerminalOpts {
|
|
|
145
145
|
envelopeRef?: string;
|
|
146
146
|
envelopeValidationErrors?: string[];
|
|
147
147
|
questionIndexes?: readonly number[];
|
|
148
|
+
activeCompany?: LaneCompanyRef | null;
|
|
148
149
|
at?: string;
|
|
149
150
|
quota?: QuotaSignal | null;
|
|
150
151
|
/** Written to `terminal_details` with the model and account the run used. */
|
|
@@ -219,6 +220,8 @@ export interface WatcherLoopDeps {
|
|
|
219
220
|
resolveAdmissionHost?: () => AdmissionHost;
|
|
220
221
|
/** Spawn seams for account retry; host scope is supplied by the resolver above. */
|
|
221
222
|
retrySpawnDeps?: Omit<SpawnLaneDeps, "admissionHost">;
|
|
223
|
+
/** Active-company fallback when an unscoped lane asks from its envelope. */
|
|
224
|
+
resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
|
|
222
225
|
}
|
|
223
226
|
/**
|
|
224
227
|
* Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
import * as fs from "node:fs";
|
|
35
35
|
import * as path from "node:path";
|
|
36
36
|
import { LanesError } from "./errors.js";
|
|
37
|
-
import { raiseEnvelopeQuestions } from "./questions.js";
|
|
37
|
+
import { questionRecordRefusalCode, raiseEnvelopeQuestions } from "./questions.js";
|
|
38
38
|
import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, } from "./types.js";
|
|
39
39
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
40
40
|
import { readLane, retryUpdateLane } from "./store.js";
|
|
@@ -309,12 +309,27 @@ export function applyTerminalToLane(hqRoot, laneId, classification, opts = {}) {
|
|
|
309
309
|
export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, opts = {}) {
|
|
310
310
|
const at = opts.at ?? new Date().toISOString();
|
|
311
311
|
const existing = readLane(hqRoot, laneId);
|
|
312
|
+
let questionRefusalCode = null;
|
|
312
313
|
if (!isTerminalLaneState(existing.state) &&
|
|
313
314
|
classification.state === "awaiting_input" &&
|
|
314
315
|
opts.envelope &&
|
|
315
316
|
(opts.envelope.questions?.length ?? 0) > 0) {
|
|
316
|
-
|
|
317
|
+
try {
|
|
318
|
+
raiseEnvelopeQuestions(hqRoot, existing, opts.envelope, opts.envelopeRef ?? "envelopes/unknown.json", at, opts.questionIndexes, opts.activeCompany);
|
|
319
|
+
}
|
|
320
|
+
catch (err) {
|
|
321
|
+
questionRefusalCode = questionRecordRefusalCode(err);
|
|
322
|
+
if (questionRefusalCode === null)
|
|
323
|
+
throw err;
|
|
324
|
+
process.stderr.write(`hq lanes watcher: terminal ask refused for ${laneId}: ${questionRefusalCode}\n`);
|
|
325
|
+
}
|
|
317
326
|
}
|
|
327
|
+
const terminalClassification = questionRefusalCode === null
|
|
328
|
+
? classification
|
|
329
|
+
: {
|
|
330
|
+
state: "failed",
|
|
331
|
+
reason: `question_record_refused:${questionRefusalCode}`,
|
|
332
|
+
};
|
|
318
333
|
let changed = false;
|
|
319
334
|
const updated = retryUpdateLane(hqRoot, laneId, "watcher", (current) => {
|
|
320
335
|
const terminal = current.state === "done" ||
|
|
@@ -331,7 +346,7 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
|
|
|
331
346
|
const next = {
|
|
332
347
|
...current,
|
|
333
348
|
...runResultFields(hqRoot, current),
|
|
334
|
-
state:
|
|
349
|
+
state: terminalClassification.state,
|
|
335
350
|
queued_reason: undefined,
|
|
336
351
|
stalled: undefined,
|
|
337
352
|
stalled_at: undefined,
|
|
@@ -340,9 +355,9 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
|
|
|
340
355
|
envelope_validation_errors: opts.envelopeValidationErrors,
|
|
341
356
|
};
|
|
342
357
|
changed = true;
|
|
343
|
-
if (
|
|
358
|
+
if (terminalClassification.reason) {
|
|
344
359
|
next.terminal_reason =
|
|
345
|
-
|
|
360
|
+
terminalClassification.reason;
|
|
346
361
|
}
|
|
347
362
|
if (opts.quota?.quota_reset_at) {
|
|
348
363
|
next.quota_reset_at = opts.quota.quota_reset_at;
|
|
@@ -1054,11 +1069,19 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
|
|
|
1054
1069
|
return;
|
|
1055
1070
|
}
|
|
1056
1071
|
}
|
|
1072
|
+
const terminalLane = readLane(hqRoot, laneId);
|
|
1073
|
+
const activeQuestionCompany = envelope?.decision === "ask" &&
|
|
1074
|
+
(envelope.questions?.length ?? 0) > 0 &&
|
|
1075
|
+
terminalLane.company === null &&
|
|
1076
|
+
terminalLane.mesh_company === undefined
|
|
1077
|
+
? await deps.resolveQuestionActiveCompany?.()
|
|
1078
|
+
: undefined;
|
|
1057
1079
|
const terminalWrite = applyTerminalToLaneWithResult(hqRoot, laneId, classification, {
|
|
1058
1080
|
envelope,
|
|
1059
1081
|
envelopeRef: latest?.ref,
|
|
1060
1082
|
envelopeValidationErrors,
|
|
1061
1083
|
questionIndexes,
|
|
1084
|
+
activeCompany: activeQuestionCompany,
|
|
1062
1085
|
at: tick.emit.at,
|
|
1063
1086
|
quota,
|
|
1064
1087
|
providerRejection,
|
|
@@ -1074,7 +1097,7 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
|
|
|
1074
1097
|
// lane_blocked; ask → lane_ask with the queue ids. After the state
|
|
1075
1098
|
// write, so a mesh failure can never leave the lane unclassified.
|
|
1076
1099
|
if (mesh && terminalWrite.changed) {
|
|
1077
|
-
if (envelope && latest) {
|
|
1100
|
+
if (envelope && latest && updated.state === classification.state) {
|
|
1078
1101
|
const questionIds = envelope.decision === "ask"
|
|
1079
1102
|
? envelopeQuestionIds(laneId, latest.ref, envelope, questionIndexes)
|
|
1080
1103
|
: undefined;
|
|
@@ -1086,7 +1109,7 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
|
|
|
1086
1109
|
}
|
|
1087
1110
|
else if (updated.state === "failed" || updated.state === "interrupted" ||
|
|
1088
1111
|
updated.state === "cancelled" || updated.state === "killed") {
|
|
1089
|
-
await emitMeshSoft(`lane_${updated.state}`, logDecision, tick.emit.at, (signal) => emitLaneTerminalStateEvent(mesh, updated, updated.state, classification.reason, signal), meshRequestTimeoutMs);
|
|
1112
|
+
await emitMeshSoft(`lane_${updated.state}`, logDecision, tick.emit.at, (signal) => emitLaneTerminalStateEvent(mesh, updated, updated.state, updated.terminal_reason ?? classification.reason, signal), meshRequestTimeoutMs);
|
|
1090
1113
|
}
|
|
1091
1114
|
}
|
|
1092
1115
|
}
|