ai-foreman 1.7.12 → 1.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/activity.d.ts +5 -1
- package/dist/activity.js +1 -1
- package/dist/adapters/claude.d.ts +7 -0
- package/dist/adapters/claude.js +133 -7
- package/dist/adapters/codex.d.ts +5 -0
- package/dist/adapters/codex.js +121 -11
- package/dist/adapters/recovering.d.ts +2 -0
- package/dist/adapters/recovering.js +11 -1
- package/dist/adapters/types.d.ts +47 -1
- package/dist/agentRun.d.ts +2 -0
- package/dist/agentRun.js +7 -1
- package/dist/branch/resume.js +11 -1
- package/dist/branch/runner.d.ts +11 -3
- package/dist/branch/runner.js +114 -60
- package/dist/buildRuns.d.ts +6 -2
- package/dist/buildRuns.js +76 -32
- package/dist/cli/doctor.d.ts +7 -0
- package/dist/cli/doctor.js +84 -1
- package/dist/cli/manager.d.ts +17 -0
- package/dist/cli/manager.js +135 -0
- package/dist/cli/recovery.d.ts +4 -0
- package/dist/cli/recovery.js +80 -0
- package/dist/cli/start.js +289 -53
- package/dist/cli/status.js +13 -2
- package/dist/cli/tickets.js +26 -8
- package/dist/config.d.ts +2 -0
- package/dist/config.js +23 -0
- package/dist/continuity.js +14 -9
- package/dist/diagnostics.d.ts +25 -0
- package/dist/diagnostics.js +428 -0
- package/dist/foreman.d.ts +16 -6
- package/dist/foreman.js +70 -82
- package/dist/handoffs.d.ts +5 -1
- package/dist/handoffs.js +4 -1
- package/dist/index.js +6 -0
- package/dist/log.d.ts +2 -2
- package/dist/log.js +4 -1
- package/dist/managerPacket.d.ts +25 -0
- package/dist/managerPacket.js +83 -0
- package/dist/markers.d.ts +1 -1
- package/dist/markers.js +4 -1
- package/dist/observability.d.ts +248 -0
- package/dist/observability.js +1018 -0
- package/dist/planChanges.d.ts +4 -0
- package/dist/planChanges.js +16 -0
- package/dist/projectDiagnostics.d.ts +27 -0
- package/dist/projectDiagnostics.js +407 -0
- package/dist/qaRecovery.d.ts +140 -0
- package/dist/qaRecovery.js +595 -0
- package/dist/qaReview.d.ts +128 -14
- package/dist/qaReview.js +974 -57
- package/dist/qaSnapshot.d.ts +31 -5
- package/dist/qaSnapshot.js +209 -134
- package/dist/recoveryPolicy.d.ts +37 -0
- package/dist/recoveryPolicy.js +251 -0
- package/dist/roles.js +1 -0
- package/dist/supervisor.d.ts +40 -0
- package/dist/supervisor.js +102 -0
- package/dist/ticketHistoryReader.d.ts +19 -0
- package/dist/ticketHistoryReader.js +58 -0
- package/dist/tickets/details.d.ts +8 -0
- package/dist/tickets/details.js +59 -11
- package/dist/tickets/output.d.ts +5 -0
- package/dist/tickets/output.js +73 -0
- package/dist/tickets/stateDb.d.ts +1 -0
- package/dist/tickets/stateDb.js +5 -0
- package/dist/workflowDb.d.ts +31 -2
- package/dist/workflowDb.js +192 -23
- package/dist/workflowReader.d.ts +42 -0
- package/dist/workflowReader.js +171 -0
- package/foreman.yaml +9 -0
- package/package.json +51 -3
|
@@ -0,0 +1,1018 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, statfsSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { gzipSync } from "node:zlib";
|
|
5
|
+
import { hostname, loadavg } from "node:os";
|
|
6
|
+
import { dirname, join, resolve, sep } from "node:path";
|
|
7
|
+
import Database from "better-sqlite3";
|
|
8
|
+
export const OBSERVABILITY_DB_FILE = ".rafi/observability.sqlite3";
|
|
9
|
+
export const OBSERVABILITY_SCHEMA_VERSION = 2;
|
|
10
|
+
export const DEFAULT_OBSERVABILITY_CONFIG = {
|
|
11
|
+
enabled: true,
|
|
12
|
+
sample_interval_seconds: 60,
|
|
13
|
+
detail_retention_days: 30,
|
|
14
|
+
log_retention_days: 30,
|
|
15
|
+
detail_soft_limit_mb: 192,
|
|
16
|
+
detail_hard_limit_mb: 256,
|
|
17
|
+
log_limit_mb: 128,
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Bounded, failure-isolated writer for diagnostic data. Recovery never depends on
|
|
21
|
+
* this database. Boundary records are synchronous; high-rate state and samples
|
|
22
|
+
* are coalesced by primary key and minute bucket.
|
|
23
|
+
*/
|
|
24
|
+
export class ObservabilityStore {
|
|
25
|
+
path;
|
|
26
|
+
config;
|
|
27
|
+
db;
|
|
28
|
+
projectDir;
|
|
29
|
+
now;
|
|
30
|
+
monotonicNow;
|
|
31
|
+
openMonotonic = new Map();
|
|
32
|
+
disabled = false;
|
|
33
|
+
warned = false;
|
|
34
|
+
summaryOnly = false;
|
|
35
|
+
priorCpu;
|
|
36
|
+
constructor(projectDir, options = {}) {
|
|
37
|
+
this.projectDir = resolve(projectDir);
|
|
38
|
+
this.path = options.path ?? join(this.projectDir, OBSERVABILITY_DB_FILE);
|
|
39
|
+
this.config = { ...DEFAULT_OBSERVABILITY_CONFIG, ...options.config };
|
|
40
|
+
this.now = options.now ?? (() => new Date());
|
|
41
|
+
this.monotonicNow = options.monotonicNow ?? (() => performance.now());
|
|
42
|
+
mkdirSync(dirname(this.path), { recursive: true });
|
|
43
|
+
if (options.ensureIgnore !== false)
|
|
44
|
+
ensureObservabilityIgnore(resolve(projectDir));
|
|
45
|
+
this.db = new Database(this.path);
|
|
46
|
+
this.db.pragma("journal_mode = WAL");
|
|
47
|
+
this.db.pragma("synchronous = NORMAL");
|
|
48
|
+
this.db.pragma("foreign_keys = ON");
|
|
49
|
+
this.migrate();
|
|
50
|
+
this.backfillBuildSnapshots();
|
|
51
|
+
this.summaryOnly = this.meta("summary_only") === "1";
|
|
52
|
+
}
|
|
53
|
+
close() { if (this.db.open)
|
|
54
|
+
this.db.close(); }
|
|
55
|
+
isDisabled() { return this.disabled; }
|
|
56
|
+
isSummaryOnly() { return this.summaryOnly; }
|
|
57
|
+
recordCapabilities(runId, capabilities, rafiVersion) {
|
|
58
|
+
this.guard(() => this.db.prepare(`INSERT INTO run_capabilities(run_id,rafi_version,capabilities_json,recorded_at)
|
|
59
|
+
VALUES(?,?,?,?) ON CONFLICT(run_id) DO UPDATE SET rafi_version=excluded.rafi_version,capabilities_json=excluded.capabilities_json,recorded_at=excluded.recorded_at`)
|
|
60
|
+
.run(runId, rafiVersion ?? null, safeJson(capabilities), this.now().toISOString()));
|
|
61
|
+
}
|
|
62
|
+
beginExecution(input) {
|
|
63
|
+
const executionId = input.executionId ?? randomUUID();
|
|
64
|
+
this.guard(() => this.db.prepare(`INSERT INTO run_executions(execution_id,run_id,lease_generation,host,pid,process_start,started_at)
|
|
65
|
+
VALUES(?,?,?,?,?,?,?)`).run(executionId, input.runId, input.leaseGeneration ?? null, input.host ?? hostname(), input.pid ?? process.pid, input.processStart ?? processStartIdentity(input.pid ?? process.pid), this.now().toISOString()));
|
|
66
|
+
return executionId;
|
|
67
|
+
}
|
|
68
|
+
endExecution(executionId, outcome) {
|
|
69
|
+
this.guard(() => this.db.prepare("UPDATE run_executions SET ended_at=?,outcome=? WHERE execution_id=? AND ended_at IS NULL")
|
|
70
|
+
.run(this.now().toISOString(), outcome, executionId));
|
|
71
|
+
}
|
|
72
|
+
attachExecutionLease(executionId, leaseGeneration) {
|
|
73
|
+
if (leaseGeneration === undefined)
|
|
74
|
+
return;
|
|
75
|
+
this.guard(() => this.db.prepare("UPDATE run_executions SET lease_generation=? WHERE execution_id=?")
|
|
76
|
+
.run(leaseGeneration, executionId));
|
|
77
|
+
}
|
|
78
|
+
abandonPriorExecutionSpans(runId, executionId) {
|
|
79
|
+
this.guard(() => this.db.prepare(`UPDATE run_spans SET outcome='abandoned',completion_known=0
|
|
80
|
+
WHERE run_id=? AND execution_id<>? AND ended_at IS NULL`).run(runId, executionId));
|
|
81
|
+
}
|
|
82
|
+
startSpan(context, input) {
|
|
83
|
+
const spanId = input.spanId ?? randomUUID();
|
|
84
|
+
const startedAt = this.now().toISOString();
|
|
85
|
+
const record = () => {
|
|
86
|
+
this.db.prepare(`INSERT INTO run_spans(span_id,run_id,execution_id,parent_span_id,provider_turn_id,ticket_id,delivery_unit_id,role,stream,provider_session_id,kind,name,started_at,completion_known,attributes_json)
|
|
87
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,0,?)`).run(spanId, context.runId, input.executionId ?? context.executionId ?? null, input.parentSpanId ?? context.parentSpanId ?? null, input.providerTurnId ?? context.providerTurnId ?? null, input.ticketId ?? context.ticketId ?? null, input.deliveryUnitId ?? context.deliveryUnitId ?? null, input.role ?? context.role ?? null, input.stream ?? context.stream ?? null, input.providerSessionId ?? context.providerSessionId ?? null, input.kind, input.name, startedAt, input.attributes ? safeJson(sanitizeValue(input.attributes)) : null);
|
|
88
|
+
this.openMonotonic.set(spanId, this.monotonicNow());
|
|
89
|
+
};
|
|
90
|
+
this.guard(record);
|
|
91
|
+
return spanId;
|
|
92
|
+
}
|
|
93
|
+
finishSpan(spanId, input = {}) {
|
|
94
|
+
this.guard(() => {
|
|
95
|
+
const row = this.db.prepare("SELECT run_id,kind,role,ticket_id,attributes_json,started_at,ended_at FROM run_spans WHERE span_id=?").get(spanId);
|
|
96
|
+
if (!row)
|
|
97
|
+
return;
|
|
98
|
+
if (row.ended_at)
|
|
99
|
+
return;
|
|
100
|
+
const monotonicStarted = this.openMonotonic.get(spanId);
|
|
101
|
+
const durationMs = monotonicStarted === undefined
|
|
102
|
+
? Math.max(0, this.now().getTime() - new Date(row.started_at).getTime())
|
|
103
|
+
: Math.max(0, this.monotonicNow() - monotonicStarted);
|
|
104
|
+
const attributes = input.attributes
|
|
105
|
+
? { ...(row.attributes_json ? JSON.parse(row.attributes_json) : {}), ...sanitizeValue(input.attributes) }
|
|
106
|
+
: row.attributes_json ? JSON.parse(row.attributes_json) : undefined;
|
|
107
|
+
this.db.prepare(`UPDATE run_spans SET ended_at=?,duration_ms=?,outcome=?,completion_known=?,attributes_json=?
|
|
108
|
+
WHERE span_id=? AND ended_at IS NULL`).run(this.now().toISOString(), Math.round(durationMs), input.outcome ?? null, input.completionKnown === false ? 0 : 1, attributes ? safeJson(attributes) : null, spanId);
|
|
109
|
+
this.db.prepare(`INSERT INTO phase_rollups(run_id,phase,role,ticket_id,duration_ms,count) VALUES(?,?,?,?,?,1)
|
|
110
|
+
ON CONFLICT(run_id,phase,role,ticket_id) DO UPDATE SET duration_ms=duration_ms+excluded.duration_ms,count=count+1`)
|
|
111
|
+
.run(row.run_id, row.kind, row.role ?? "", row.ticket_id ?? "", Math.round(durationMs));
|
|
112
|
+
this.db.prepare(`UPDATE run_current_state SET active_span_id=NULL,active_span_kind=NULL,updated_at=? WHERE active_span_id=?`)
|
|
113
|
+
.run(this.now().toISOString(), spanId);
|
|
114
|
+
if (this.summaryOnly && row.kind === "tool")
|
|
115
|
+
this.db.prepare(`DELETE FROM run_spans WHERE span_id IN (
|
|
116
|
+
SELECT span_id FROM run_spans WHERE kind='tool' AND ended_at IS NOT NULL AND COALESCE(outcome,'') NOT IN ('failed','error','blocked')
|
|
117
|
+
ORDER BY COALESCE(duration_ms,0) DESC,span_id LIMIT -1 OFFSET 100)`).run();
|
|
118
|
+
this.openMonotonic.delete(spanId);
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
recordEvent(context, kind, input = {}) {
|
|
122
|
+
const eventId = input.eventId ?? randomUUID();
|
|
123
|
+
this.guard(() => this.db.prepare(`INSERT OR IGNORE INTO run_events(event_id,run_id,execution_id,span_id,ticket_id,role,stream,kind,severity,attributes_json,observed_at)
|
|
124
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?)`).run(eventId, context.runId, context.executionId ?? null, input.spanId ?? context.parentSpanId ?? null, context.ticketId ?? null, context.role ?? null, context.stream ?? null, kind, input.severity ?? null, input.attributes ? safeJson(sanitizeValue(input.attributes)) : null, this.now().toISOString()));
|
|
125
|
+
return eventId;
|
|
126
|
+
}
|
|
127
|
+
updateCurrentState(state) {
|
|
128
|
+
this.guard(() => this.db.prepare(`INSERT INTO run_current_state(run_id,role,stream,execution_id,ticket_id,delivery_unit_id,provider_session_id,phase,active_span_id,active_span_kind,last_signal_at,last_semantic_progress_at,updated_at)
|
|
129
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(run_id,role,stream) DO UPDATE SET
|
|
130
|
+
execution_id=excluded.execution_id,ticket_id=excluded.ticket_id,delivery_unit_id=excluded.delivery_unit_id,
|
|
131
|
+
provider_session_id=excluded.provider_session_id,phase=excluded.phase,active_span_id=excluded.active_span_id,
|
|
132
|
+
active_span_kind=excluded.active_span_kind,last_signal_at=COALESCE(excluded.last_signal_at,run_current_state.last_signal_at),
|
|
133
|
+
last_semantic_progress_at=COALESCE(excluded.last_semantic_progress_at,run_current_state.last_semantic_progress_at),updated_at=excluded.updated_at`)
|
|
134
|
+
.run(state.runId, state.role, state.stream, state.executionId ?? null, state.ticketId ?? null, state.deliveryUnitId ?? null, state.providerSessionId ?? null, state.phase ?? null, state.activeSpanId ?? null, state.activeSpanKind ?? null, state.lastSignalAt ?? null, state.lastSemanticProgressAt ?? null, state.updatedAt ?? this.now().toISOString()));
|
|
135
|
+
}
|
|
136
|
+
recordMetric(context, metric, value, input = {}) {
|
|
137
|
+
if (!Number.isFinite(value))
|
|
138
|
+
return;
|
|
139
|
+
this.guard(() => {
|
|
140
|
+
const observedAt = this.now();
|
|
141
|
+
const interval = Math.max(1, this.config.sample_interval_seconds) * 1000;
|
|
142
|
+
const bucket = input.boundary ? observedAt.getTime() : Math.floor(observedAt.getTime() / interval) * interval;
|
|
143
|
+
this.db.prepare(`INSERT INTO metric_samples(run_id,role,stream,provider_session_id,metric,bucket_ms,value,unit,boundary,attributes_json,observed_at)
|
|
144
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(run_id,role,stream,provider_session_id,metric,bucket_ms,boundary)
|
|
145
|
+
DO UPDATE SET value=excluded.value,attributes_json=excluded.attributes_json,observed_at=excluded.observed_at`)
|
|
146
|
+
.run(context.runId, context.role ?? "host", context.stream ?? "host", context.providerSessionId ?? "", metric, bucket, value, input.unit ?? null, input.boundary ? 1 : 0, input.attributes ? safeJson(sanitizeValue(input.attributes)) : null, observedAt.toISOString());
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
sampleProcess(context, eventLoopDelayMs, boundary = false) {
|
|
150
|
+
const usage = process.resourceUsage();
|
|
151
|
+
const mem = process.memoryUsage();
|
|
152
|
+
const disk = availableDiskBytes(dirname(this.path));
|
|
153
|
+
this.recordMetric(context, "process_rss_bytes", mem.rss, { unit: "bytes", boundary });
|
|
154
|
+
this.recordMetric(context, "process_cpu_user_delta_us", Math.max(0, usage.userCPUTime - (this.priorCpu?.userCPUTime ?? usage.userCPUTime)), { unit: "microseconds", boundary });
|
|
155
|
+
this.recordMetric(context, "process_cpu_system_delta_us", Math.max(0, usage.systemCPUTime - (this.priorCpu?.systemCPUTime ?? usage.systemCPUTime)), { unit: "microseconds", boundary });
|
|
156
|
+
this.priorCpu = { userCPUTime: usage.userCPUTime, systemCPUTime: usage.systemCPUTime };
|
|
157
|
+
this.recordMetric(context, "system_load_1m", loadavg()[0] ?? 0, { unit: "load", boundary });
|
|
158
|
+
if (eventLoopDelayMs !== undefined)
|
|
159
|
+
this.recordMetric(context, "event_loop_delay_ms", eventLoopDelayMs, { unit: "milliseconds", boundary });
|
|
160
|
+
if (disk !== undefined)
|
|
161
|
+
this.recordMetric(context, "disk_available_bytes", disk, { unit: "bytes", boundary });
|
|
162
|
+
}
|
|
163
|
+
upsertRollup(rollup) {
|
|
164
|
+
this.guard(() => this.db.prepare(`INSERT INTO run_rollups(run_id,project_id,status,branch_mode,qa_enabled,primary_provider,ticket_count,ticket_count_bucket,ticket_size_bucket,created_at,completed_at,calendar_ms,active_execution_ms,explicit_wait_ms,attributed_ms,unattributed_ms,totals_json)
|
|
165
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(run_id) DO UPDATE SET project_id=excluded.project_id,status=excluded.status,branch_mode=excluded.branch_mode,
|
|
166
|
+
qa_enabled=excluded.qa_enabled,primary_provider=excluded.primary_provider,ticket_count=excluded.ticket_count,ticket_count_bucket=excluded.ticket_count_bucket,
|
|
167
|
+
ticket_size_bucket=excluded.ticket_size_bucket,created_at=excluded.created_at,completed_at=excluded.completed_at,
|
|
168
|
+
calendar_ms=excluded.calendar_ms,active_execution_ms=excluded.active_execution_ms,explicit_wait_ms=excluded.explicit_wait_ms,
|
|
169
|
+
attributed_ms=excluded.attributed_ms,unattributed_ms=excluded.unattributed_ms,totals_json=excluded.totals_json`)
|
|
170
|
+
.run(rollup.runId, rollup.projectId ?? null, rollup.status, rollup.branchMode ?? null, rollup.qaEnabled === undefined ? null : rollup.qaEnabled ? 1 : 0, rollup.primaryProvider ?? null, rollup.ticketCount ?? null, rollup.ticketCountBucket ?? null, rollup.ticketSizeBucket ?? null, rollup.createdAt, rollup.completedAt ?? null, rollup.calendarMs, rollup.activeExecutionMs, rollup.explicitWaitMs, rollup.attributedMs, rollup.unattributedMs, safeJson(rollup.totals)));
|
|
171
|
+
}
|
|
172
|
+
upsertRunSummary(summary) {
|
|
173
|
+
this.guard(() => {
|
|
174
|
+
const transaction = this.db.transaction(() => {
|
|
175
|
+
this.db.prepare(`INSERT INTO run_summaries(run_id,status,created_at,updated_at,completed_at,provider,model,branch_mode,qa_enabled,detail_level,summary_json)
|
|
176
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(run_id) DO UPDATE SET status=excluded.status,created_at=excluded.created_at,
|
|
177
|
+
updated_at=excluded.updated_at,completed_at=excluded.completed_at,provider=excluded.provider,model=excluded.model,
|
|
178
|
+
branch_mode=excluded.branch_mode,qa_enabled=excluded.qa_enabled,detail_level=excluded.detail_level,summary_json=excluded.summary_json`)
|
|
179
|
+
.run(summary.runId, summary.status, summary.createdAt, summary.updatedAt, summary.completedAt ?? null, summary.provider ?? null, summary.model ?? null, summary.branchMode ?? null, summary.qaEnabled === undefined ? null : summary.qaEnabled ? 1 : 0, summary.detailLevel, safeJson(sanitizeValue(summary)));
|
|
180
|
+
this.db.prepare("DELETE FROM run_summary_tickets WHERE run_id=?").run(summary.runId);
|
|
181
|
+
const insert = this.db.prepare("INSERT INTO run_summary_tickets(run_id,ticket_id) VALUES(?,?)");
|
|
182
|
+
for (const ticketId of [...new Set(summary.ticketIds)].sort())
|
|
183
|
+
insert.run(summary.runId, ticketId);
|
|
184
|
+
});
|
|
185
|
+
transaction();
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
finalizeRun(runId, input) {
|
|
189
|
+
let rollup;
|
|
190
|
+
this.guard(() => {
|
|
191
|
+
const spans = this.db.prepare("SELECT * FROM run_spans WHERE run_id=?").all(runId).map(spanFromRow);
|
|
192
|
+
const executions = this.db.prepare("SELECT started_at,ended_at FROM run_executions WHERE run_id=?").all(runId);
|
|
193
|
+
const observedEnd = input.completedAt ?? this.now().toISOString();
|
|
194
|
+
const completedAt = input.completedAt ?? (["completed", "failed", "superseded"].includes(input.status) ? observedEnd : undefined);
|
|
195
|
+
const createdAt = input.createdAt ?? executions[0]?.started_at ?? observedEnd;
|
|
196
|
+
const activeExecutionMs = unionMs(executions.map(item => [new Date(item.started_at).getTime(), new Date(item.ended_at ?? observedEnd).getTime()]));
|
|
197
|
+
const attributedMs = unionMs(spans.map(item => [new Date(item.startedAt).getTime(), new Date(item.endedAt ?? observedEnd).getTime()]));
|
|
198
|
+
const explicitWaitMs = unionMs(spans.filter(item => ["user_wait", "dependency_wait", "external_ci_wait"].includes(item.kind)).map(item => [new Date(item.startedAt).getTime(), new Date(item.endedAt ?? observedEnd).getTime()]));
|
|
199
|
+
const totals = {};
|
|
200
|
+
const phases = this.db.prepare("SELECT phase,SUM(duration_ms) AS duration_ms FROM phase_rollups WHERE run_id=? GROUP BY phase").all(runId);
|
|
201
|
+
for (const phase of phases)
|
|
202
|
+
totals[phase.phase] = Number(phase.duration_ms);
|
|
203
|
+
rollup = { version: 1, runId, status: input.status, branchMode: input.branchMode, qaEnabled: input.qaEnabled, primaryProvider: input.primaryProvider,
|
|
204
|
+
ticketCount: input.ticketCount, ticketCountBucket: input.ticketCount === undefined ? undefined : input.ticketCount <= 1 ? "1" : input.ticketCount <= 3 ? "2-3" : input.ticketCount <= 7 ? "4-7" : "8+",
|
|
205
|
+
ticketSizeBucket: input.ticketSizeBucket, createdAt, completedAt, calendarMs: Math.max(0, new Date(observedEnd).getTime() - new Date(createdAt).getTime()), activeExecutionMs,
|
|
206
|
+
explicitWaitMs, attributedMs, unattributedMs: Math.max(0, activeExecutionMs - attributedMs), totals };
|
|
207
|
+
this.upsertRollup(rollup);
|
|
208
|
+
const summary = buildStoredRunSummary(this.db, spans, executions, rollup, input, this.summaryOnly, this.now());
|
|
209
|
+
this.upsertRunSummary(summary);
|
|
210
|
+
});
|
|
211
|
+
return rollup;
|
|
212
|
+
}
|
|
213
|
+
recordManagerSession(input) {
|
|
214
|
+
this.guard(() => this.db.prepare(`INSERT INTO manager_sessions(session_id,run_id,provider,started_at,ended_at,outcome,report_digest,input_tokens,output_tokens,cost_usd,error_code,scope,latest_focus_run_id,project_report_digest,lookup_rounds,lookup_operations)
|
|
215
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id) DO UPDATE SET ended_at=excluded.ended_at,outcome=excluded.outcome,
|
|
216
|
+
provider=excluded.provider,report_digest=excluded.report_digest,input_tokens=excluded.input_tokens,output_tokens=excluded.output_tokens,cost_usd=excluded.cost_usd,error_code=excluded.error_code,
|
|
217
|
+
scope=excluded.scope,latest_focus_run_id=excluded.latest_focus_run_id,project_report_digest=excluded.project_report_digest,lookup_rounds=excluded.lookup_rounds,lookup_operations=excluded.lookup_operations`)
|
|
218
|
+
.run(input.sessionId, input.runId, input.provider ?? null, input.startedAt, input.endedAt ?? null, input.outcome ?? null, input.reportDigest ?? null, input.inputTokens ?? null, input.outputTokens ?? null, input.costUsd ?? null, input.errorCode ?? null, input.scope ?? null, input.latestFocusRunId ?? null, input.projectReportDigest ?? null, input.lookupRounds ?? 0, input.lookupOperations ?? 0));
|
|
219
|
+
}
|
|
220
|
+
indexLogFile(path, runId) {
|
|
221
|
+
this.guard(() => this.db.prepare(`INSERT INTO log_files(path,run_id,status,indexed,resume_required,bytes,updated_at)
|
|
222
|
+
VALUES(?,?,'active',1,1,?,?) ON CONFLICT(path) DO UPDATE SET run_id=excluded.run_id,status='active',indexed=1,resume_required=1,bytes=excluded.bytes,updated_at=excluded.updated_at`)
|
|
223
|
+
.run(resolve(path), runId, fileSize(path), this.now().toISOString()));
|
|
224
|
+
}
|
|
225
|
+
closeLogFile(path, status) {
|
|
226
|
+
const terminal = status === "completed";
|
|
227
|
+
this.guard(() => this.db.prepare(`UPDATE log_files SET status=?,resume_required=?,bytes=?,closed_at=?,updated_at=? WHERE path=?`)
|
|
228
|
+
.run(status, terminal ? 0 : 1, fileSize(path), this.now().toISOString(), this.now().toISOString(), resolve(path)));
|
|
229
|
+
}
|
|
230
|
+
maintainLogs(projectDir) {
|
|
231
|
+
const result = { compressed: 0, deleted: 0, reclaimedBytes: 0 };
|
|
232
|
+
this.guard(() => {
|
|
233
|
+
const root = resolve(projectDir, ".foreman");
|
|
234
|
+
const compressBefore = this.now().getTime() - 86_400_000;
|
|
235
|
+
const deleteBefore = this.now().getTime() - this.config.log_retention_days * 86_400_000;
|
|
236
|
+
const rows = this.db.prepare(`SELECT path,run_id,closed_at FROM log_files WHERE indexed=1 AND resume_required=0 AND status='completed' ORDER BY closed_at,path`).all();
|
|
237
|
+
for (const row of rows) {
|
|
238
|
+
if (!safeOwnedLogPath(root, row.path) || !row.closed_at || !existsSync(row.path))
|
|
239
|
+
continue;
|
|
240
|
+
const closed = new Date(row.closed_at).getTime();
|
|
241
|
+
if (row.path.endsWith(".jsonl") && closed <= compressBefore) {
|
|
242
|
+
const gzPath = `${row.path}.gz`;
|
|
243
|
+
const temporary = `${gzPath}.${process.pid}.tmp`;
|
|
244
|
+
const before = fileSize(row.path);
|
|
245
|
+
writeFileSync(temporary, gzipSync(readFileSync(row.path), { level: 6 }), { mode: 0o600 });
|
|
246
|
+
renameSync(temporary, gzPath);
|
|
247
|
+
unlinkSync(row.path);
|
|
248
|
+
const after = fileSize(gzPath);
|
|
249
|
+
this.db.prepare("DELETE FROM log_files WHERE path=?").run(row.path);
|
|
250
|
+
this.db.prepare(`INSERT OR REPLACE INTO log_files(path,run_id,status,indexed,resume_required,bytes,closed_at,updated_at)
|
|
251
|
+
VALUES(?,?,'completed',1,0,?,?,?)`)
|
|
252
|
+
.run(gzPath, row.run_id, after, row.closed_at, this.now().toISOString());
|
|
253
|
+
result.compressed += 1;
|
|
254
|
+
result.reclaimedBytes += Math.max(0, before - after);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
const candidates = this.db.prepare(`SELECT path,closed_at,bytes FROM log_files WHERE indexed=1 AND resume_required=0 AND status='completed' AND path LIKE '%.jsonl.gz' ORDER BY closed_at,path`).all();
|
|
258
|
+
let total = directoryIndexedLogBytes(this.db);
|
|
259
|
+
const limit = this.config.log_limit_mb * 1024 * 1024;
|
|
260
|
+
for (const row of candidates) {
|
|
261
|
+
const expired = row.closed_at ? new Date(row.closed_at).getTime() <= deleteBefore : false;
|
|
262
|
+
if (!expired && total <= limit)
|
|
263
|
+
continue;
|
|
264
|
+
if (!safeOwnedLogPath(root, row.path) || !existsSync(row.path))
|
|
265
|
+
continue;
|
|
266
|
+
const bytes = fileSize(row.path);
|
|
267
|
+
unlinkSync(row.path);
|
|
268
|
+
this.db.prepare("DELETE FROM log_files WHERE path=?").run(row.path);
|
|
269
|
+
total = Math.max(0, total - bytes);
|
|
270
|
+
result.deleted += 1;
|
|
271
|
+
result.reclaimedBytes += bytes;
|
|
272
|
+
}
|
|
273
|
+
});
|
|
274
|
+
return result;
|
|
275
|
+
}
|
|
276
|
+
enforceLimits(protectedRunIds = []) {
|
|
277
|
+
let prunedRows = 0;
|
|
278
|
+
this.guard(() => {
|
|
279
|
+
const cutoff = new Date(this.now().getTime() - this.config.detail_retention_days * 86_400_000).toISOString();
|
|
280
|
+
const automaticProtected = this.db.prepare(`SELECT DISTINCT run_id FROM run_executions WHERE ended_at IS NULL
|
|
281
|
+
UNION SELECT run_id FROM run_rollups WHERE status IN ('running','recoverable','interrupted','blocked','paused')`).all();
|
|
282
|
+
const protectedIds = [...new Set([...protectedRunIds, ...automaticProtected.map(row => row.run_id)])];
|
|
283
|
+
const protectedSql = protectedIds.length ? ` AND run_id NOT IN (${protectedIds.map(() => "?").join(",")})` : "";
|
|
284
|
+
for (const table of ["metric_samples", "run_events", "run_spans"]) {
|
|
285
|
+
const column = table === "run_spans" ? "COALESCE(ended_at,started_at)" : "observed_at";
|
|
286
|
+
const terminal = table === "run_spans" ? " AND ended_at IS NOT NULL" : "";
|
|
287
|
+
const result = this.db.prepare(`DELETE FROM ${table} WHERE ${column}<?${terminal}${protectedSql} AND run_id IN (SELECT run_id FROM run_summaries)`).run(cutoff, ...protectedIds);
|
|
288
|
+
prunedRows += result.changes;
|
|
289
|
+
}
|
|
290
|
+
const expiredDetail = this.db.prepare(`SELECT s.run_id FROM run_summaries s WHERE s.detail_level='detailed' AND s.status NOT IN ('running','recoverable','interrupted','blocked','paused') AND NOT EXISTS (SELECT 1 FROM run_spans p WHERE p.run_id=s.run_id)`).all();
|
|
291
|
+
demoteRunSummaries(this.db, expiredDetail.map(row => row.run_id), this.now(), "retained detail expired; permanent summary remains available");
|
|
292
|
+
const bytes = databaseBytes(this.path);
|
|
293
|
+
if (bytes >= this.config.detail_hard_limit_mb * 1024 * 1024) {
|
|
294
|
+
this.summaryOnly = true;
|
|
295
|
+
this.setMeta("summary_only", "1");
|
|
296
|
+
this.setMeta("summary_only_reason", "detail_hard_limit");
|
|
297
|
+
const protectedKinds = ["provider_turn", "qa_attempt", "retry", "user_wait", "dependency_wait", "external_ci_wait"];
|
|
298
|
+
const marks = protectedKinds.map(() => "?").join(",");
|
|
299
|
+
const result = this.db.prepare(`DELETE FROM run_spans WHERE ended_at IS NOT NULL AND kind NOT IN (${marks}) AND COALESCE(outcome,'') NOT IN ('failed','error','blocked')${protectedSql} AND run_id IN (SELECT run_id FROM run_summaries)
|
|
300
|
+
AND span_id NOT IN (SELECT span_id FROM run_spans WHERE kind='tool' ORDER BY duration_ms DESC LIMIT 100)`).run(...protectedKinds, ...protectedIds);
|
|
301
|
+
prunedRows += result.changes;
|
|
302
|
+
const partialDetail = this.db.prepare(`SELECT run_id FROM run_summaries WHERE detail_level='detailed' AND status NOT IN ('running','recoverable','interrupted','blocked','paused')`).all();
|
|
303
|
+
demoteRunSummaries(this.db, partialDetail.map(row => row.run_id), this.now(), "detail hard limit pruned low-priority rows; permanent summary remains available");
|
|
304
|
+
}
|
|
305
|
+
this.setMeta("last_maintenance_at", this.now().toISOString());
|
|
306
|
+
});
|
|
307
|
+
return { summaryOnly: this.summaryOnly, prunedRows };
|
|
308
|
+
}
|
|
309
|
+
checkpoint() { this.guard(() => { this.db.pragma("wal_checkpoint(PASSIVE)"); }); }
|
|
310
|
+
compact() { this.guard(() => { this.db.pragma("wal_checkpoint(TRUNCATE)"); this.db.exec("VACUUM"); }); }
|
|
311
|
+
meta(key) {
|
|
312
|
+
return this.db.prepare("SELECT value FROM observability_meta WHERE key=?").get(key)?.value;
|
|
313
|
+
}
|
|
314
|
+
setMeta(key, value) {
|
|
315
|
+
this.db.prepare("INSERT INTO observability_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(key, value);
|
|
316
|
+
}
|
|
317
|
+
guard(action) {
|
|
318
|
+
if (this.disabled || !this.config.enabled)
|
|
319
|
+
return;
|
|
320
|
+
try {
|
|
321
|
+
action();
|
|
322
|
+
}
|
|
323
|
+
catch (error) {
|
|
324
|
+
this.disabled = true;
|
|
325
|
+
if (!this.warned) {
|
|
326
|
+
this.warned = true;
|
|
327
|
+
process.stderr.write(`rafi: observability disabled after a storage error: ${sanitizeText(String(error)).slice(0, 300)}\n`);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
migrate() {
|
|
332
|
+
this.db.exec(`
|
|
333
|
+
CREATE TABLE IF NOT EXISTS observability_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL);
|
|
334
|
+
INSERT OR IGNORE INTO observability_meta(key,value) VALUES('schema_version','2');
|
|
335
|
+
CREATE TABLE IF NOT EXISTS run_capabilities(run_id TEXT PRIMARY KEY,rafi_version TEXT,capabilities_json TEXT NOT NULL,recorded_at TEXT NOT NULL);
|
|
336
|
+
CREATE TABLE IF NOT EXISTS run_executions(execution_id TEXT PRIMARY KEY,run_id TEXT NOT NULL,lease_generation INTEGER,host TEXT NOT NULL,pid INTEGER NOT NULL,process_start TEXT NOT NULL,started_at TEXT NOT NULL,ended_at TEXT,outcome TEXT);
|
|
337
|
+
CREATE TABLE IF NOT EXISTS run_current_state(run_id TEXT NOT NULL,role TEXT NOT NULL,stream TEXT NOT NULL,execution_id TEXT,ticket_id TEXT,delivery_unit_id TEXT,provider_session_id TEXT,phase TEXT,active_span_id TEXT,active_span_kind TEXT,last_signal_at TEXT,last_semantic_progress_at TEXT,updated_at TEXT NOT NULL,PRIMARY KEY(run_id,role,stream));
|
|
338
|
+
CREATE TABLE IF NOT EXISTS run_spans(span_id TEXT PRIMARY KEY,run_id TEXT NOT NULL,execution_id TEXT,parent_span_id TEXT,provider_turn_id TEXT,ticket_id TEXT,delivery_unit_id TEXT,role TEXT,stream TEXT,provider_session_id TEXT,kind TEXT NOT NULL,name TEXT NOT NULL,started_at TEXT NOT NULL,ended_at TEXT,duration_ms INTEGER,outcome TEXT,completion_known INTEGER NOT NULL DEFAULT 0,attributes_json TEXT);
|
|
339
|
+
CREATE TABLE IF NOT EXISTS run_events(event_id TEXT PRIMARY KEY,run_id TEXT NOT NULL,execution_id TEXT,span_id TEXT,ticket_id TEXT,role TEXT,stream TEXT,kind TEXT NOT NULL,severity TEXT,attributes_json TEXT,observed_at TEXT NOT NULL);
|
|
340
|
+
CREATE TABLE IF NOT EXISTS metric_samples(run_id TEXT NOT NULL,role TEXT NOT NULL,stream TEXT NOT NULL,provider_session_id TEXT NOT NULL DEFAULT '',metric TEXT NOT NULL,bucket_ms INTEGER NOT NULL,value REAL NOT NULL,unit TEXT,boundary INTEGER NOT NULL DEFAULT 0,attributes_json TEXT,observed_at TEXT NOT NULL,PRIMARY KEY(run_id,role,stream,provider_session_id,metric,bucket_ms,boundary));
|
|
341
|
+
CREATE TABLE IF NOT EXISTS run_rollups(run_id TEXT PRIMARY KEY,project_id TEXT,status TEXT NOT NULL,branch_mode TEXT,qa_enabled INTEGER,primary_provider TEXT,ticket_count INTEGER,ticket_count_bucket TEXT,ticket_size_bucket TEXT,created_at TEXT NOT NULL,completed_at TEXT,calendar_ms INTEGER NOT NULL,active_execution_ms INTEGER NOT NULL,explicit_wait_ms INTEGER NOT NULL,attributed_ms INTEGER NOT NULL,unattributed_ms INTEGER NOT NULL,totals_json TEXT NOT NULL);
|
|
342
|
+
CREATE TABLE IF NOT EXISTS phase_rollups(run_id TEXT NOT NULL,phase TEXT NOT NULL,role TEXT NOT NULL DEFAULT '',ticket_id TEXT NOT NULL DEFAULT '',duration_ms INTEGER NOT NULL,count INTEGER NOT NULL,PRIMARY KEY(run_id,phase,role,ticket_id));
|
|
343
|
+
CREATE TABLE IF NOT EXISTS log_files(path TEXT PRIMARY KEY,run_id TEXT,status TEXT NOT NULL,indexed INTEGER NOT NULL,resume_required INTEGER NOT NULL,bytes INTEGER NOT NULL,closed_at TEXT,updated_at TEXT NOT NULL);
|
|
344
|
+
CREATE TABLE IF NOT EXISTS manager_sessions(session_id TEXT PRIMARY KEY,run_id TEXT NOT NULL,provider TEXT,started_at TEXT NOT NULL,ended_at TEXT,outcome TEXT,report_digest TEXT,input_tokens INTEGER,output_tokens INTEGER,cost_usd REAL,error_code TEXT);
|
|
345
|
+
CREATE TABLE IF NOT EXISTS run_summaries(run_id TEXT PRIMARY KEY,status TEXT NOT NULL,created_at TEXT NOT NULL,updated_at TEXT NOT NULL,completed_at TEXT,provider TEXT,model TEXT,branch_mode TEXT,qa_enabled INTEGER,detail_level TEXT NOT NULL,summary_json TEXT NOT NULL);
|
|
346
|
+
CREATE TABLE IF NOT EXISTS run_summary_tickets(run_id TEXT NOT NULL,ticket_id TEXT NOT NULL,PRIMARY KEY(run_id,ticket_id));
|
|
347
|
+
CREATE INDEX IF NOT EXISTS run_spans_run_time ON run_spans(run_id,started_at);
|
|
348
|
+
CREATE INDEX IF NOT EXISTS run_spans_kind_duration ON run_spans(run_id,kind,duration_ms);
|
|
349
|
+
CREATE INDEX IF NOT EXISTS run_events_run_time ON run_events(run_id,observed_at);
|
|
350
|
+
CREATE INDEX IF NOT EXISTS run_executions_run ON run_executions(run_id,started_at);
|
|
351
|
+
CREATE INDEX IF NOT EXISTS run_rollups_cohort ON run_rollups(status,completed_at,branch_mode,qa_enabled,primary_provider);
|
|
352
|
+
CREATE INDEX IF NOT EXISTS run_summaries_completed ON run_summaries(completed_at DESC,run_id);
|
|
353
|
+
CREATE INDEX IF NOT EXISTS run_summaries_status ON run_summaries(status,updated_at DESC,run_id);
|
|
354
|
+
CREATE INDEX IF NOT EXISTS run_summaries_provider_model ON run_summaries(provider,model,updated_at DESC);
|
|
355
|
+
CREATE INDEX IF NOT EXISTS run_summaries_branch_qa ON run_summaries(branch_mode,qa_enabled,updated_at DESC);
|
|
356
|
+
CREATE INDEX IF NOT EXISTS run_summary_tickets_ticket ON run_summary_tickets(ticket_id,run_id);
|
|
357
|
+
`);
|
|
358
|
+
addColumn(this.db, "manager_sessions", "scope", "TEXT");
|
|
359
|
+
addColumn(this.db, "manager_sessions", "latest_focus_run_id", "TEXT");
|
|
360
|
+
addColumn(this.db, "manager_sessions", "project_report_digest", "TEXT");
|
|
361
|
+
addColumn(this.db, "manager_sessions", "lookup_rounds", "INTEGER NOT NULL DEFAULT 0");
|
|
362
|
+
addColumn(this.db, "manager_sessions", "lookup_operations", "INTEGER NOT NULL DEFAULT 0");
|
|
363
|
+
this.setMeta("schema_version", String(OBSERVABILITY_SCHEMA_VERSION));
|
|
364
|
+
const legacy = this.db.prepare("SELECT r.* FROM run_rollups r LEFT JOIN run_summaries s ON s.run_id=r.run_id WHERE s.run_id IS NULL").all();
|
|
365
|
+
for (const row of legacy)
|
|
366
|
+
this.upsertRunSummary(legacySummaryFromRollup(rollupFromRow(row), this.now()));
|
|
367
|
+
}
|
|
368
|
+
backfillBuildSnapshots() {
|
|
369
|
+
const directory = join(this.projectDir, ".foreman", "runs");
|
|
370
|
+
if (!existsSync(directory))
|
|
371
|
+
return;
|
|
372
|
+
for (const name of readdirSync(directory).filter(item => item.endsWith(".json")).sort()) {
|
|
373
|
+
let run;
|
|
374
|
+
try {
|
|
375
|
+
run = JSON.parse(readFileSync(join(directory, name), "utf8"));
|
|
376
|
+
}
|
|
377
|
+
catch {
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
if (typeof run.runId !== "string" || typeof run.createdAt !== "string" || typeof run.updatedAt !== "string" || typeof run.status !== "string")
|
|
381
|
+
continue;
|
|
382
|
+
let existing;
|
|
383
|
+
try {
|
|
384
|
+
const row = this.db.prepare("SELECT summary_json FROM run_summaries WHERE run_id=?").get(run.runId);
|
|
385
|
+
if (row?.summary_json)
|
|
386
|
+
existing = JSON.parse(row.summary_json);
|
|
387
|
+
}
|
|
388
|
+
catch {
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
const builder = isRecord(run.builder) && isRecord(run.builder.settings) ? run.builder.settings : undefined;
|
|
392
|
+
const repository = isRecord(run.repository) ? run.repository : undefined;
|
|
393
|
+
const git = repository && isRecord(repository.git) ? repository.git : undefined;
|
|
394
|
+
const tickets = Array.isArray(run.tickets) ? run.tickets.filter((item) => typeof item === "string") : existing?.ticketIds ?? [];
|
|
395
|
+
const completedAt = typeof run.completedAt === "string" ? run.completedAt : existing?.completedAt;
|
|
396
|
+
const calendarMs = Math.max(0, Date.parse(completedAt ?? run.updatedAt) - Date.parse(run.createdAt));
|
|
397
|
+
const withoutDigest = existing ? {
|
|
398
|
+
...existing, status: run.status, createdAt: run.createdAt, updatedAt: run.updatedAt, completedAt,
|
|
399
|
+
...(typeof run.checkpoint === "string" ? { checkpoint: run.checkpoint } : {}), ticketIds: tickets,
|
|
400
|
+
...(typeof run.deliveryUnit === "string" ? { deliveryUnit: run.deliveryUnit } : {}), ...(typeof run.branchMode === "string" ? { branchMode: run.branchMode } : {}),
|
|
401
|
+
qaEnabled: Boolean(run.qa), ...(builder?.make === "claude" || builder?.make === "codex" ? { provider: builder.make } : {}), ...(typeof builder?.model === "string" ? { model: builder.model } : {}),
|
|
402
|
+
...(isRecord(run.failure) && typeof run.failure.category === "string" ? { failureCategory: run.failure.category } : isRecord(run.interruption) && typeof run.interruption.category === "string" ? { failureCategory: run.interruption.category } : {}),
|
|
403
|
+
} : {
|
|
404
|
+
version: 1, runId: run.runId, status: run.status, createdAt: run.createdAt, updatedAt: run.updatedAt, ...(completedAt ? { completedAt } : {}), ...(typeof run.checkpoint === "string" ? { checkpoint: run.checkpoint } : {}), activeState: "inactive", ticketIds: tickets,
|
|
405
|
+
...(typeof run.deliveryUnit === "string" ? { deliveryUnit: run.deliveryUnit } : {}), ...(typeof run.branchMode === "string" ? { branchMode: run.branchMode } : {}), qaEnabled: Boolean(run.qa),
|
|
406
|
+
...(builder?.make === "claude" || builder?.make === "codex" ? { provider: builder.make } : {}), ...(typeof builder?.model === "string" ? { model: builder.model } : {}),
|
|
407
|
+
timing: { calendarMs, inclusiveByKind: {}, exclusiveByKind: {} }, counts: { byKind: {}, byOutcome: {} }, usage: { scope: "unavailable" }, retry: {},
|
|
408
|
+
...(isRecord(run.failure) && typeof run.failure.category === "string" ? { failureCategory: run.failure.category } : isRecord(run.interruption) && typeof run.interruption.category === "string" ? { failureCategory: run.interruption.category } : {}),
|
|
409
|
+
topOperations: [], ...(git || repository ? { git: { ...(typeof git?.branch === "string" ? { branch: git.branch } : typeof repository?.branch === "string" ? { branch: repository.branch } : {}), changedPathCount: Array.isArray(git?.runOwnedPaths) ? git.runOwnedPaths.length : undefined, historical: true } } : {}), detailLevel: "legacy",
|
|
410
|
+
metricCoverage: { calendarMs: "available", timing: "partial", counts: "unavailable", usage: "unavailable", git: git || repository ? "partial" : "unavailable" },
|
|
411
|
+
capabilities: { version: 1, sources: { migration: { source: "build_snapshot", state: "partial", observedAt: this.now().toISOString(), detail: "backfilled only exact snapshot facts" } } }, evidenceIds: [`snapshot:${run.runId}`],
|
|
412
|
+
};
|
|
413
|
+
this.upsertRunSummary({ ...withoutDigest, digest: diagnosticDigest(withoutDigest) });
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
/** Strictly read-only access. Construction never creates directories, journals, migrations, or ignore entries. */
|
|
418
|
+
export class ObservabilityReader {
|
|
419
|
+
path;
|
|
420
|
+
db;
|
|
421
|
+
constructor(projectDir, path = join(resolve(projectDir), OBSERVABILITY_DB_FILE)) {
|
|
422
|
+
this.path = path;
|
|
423
|
+
if (!existsSync(path))
|
|
424
|
+
return;
|
|
425
|
+
this.db = new Database(path, { readonly: true, fileMustExist: true });
|
|
426
|
+
this.db.pragma("query_only = ON");
|
|
427
|
+
}
|
|
428
|
+
close() { this.db?.close(); }
|
|
429
|
+
available() { return Boolean(this.db); }
|
|
430
|
+
schemaVersion() {
|
|
431
|
+
if (!this.db)
|
|
432
|
+
return undefined;
|
|
433
|
+
try {
|
|
434
|
+
return Number(this.db.prepare("SELECT value FROM observability_meta WHERE key='schema_version'").get()?.value);
|
|
435
|
+
}
|
|
436
|
+
catch {
|
|
437
|
+
return undefined;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
currentState(runId) {
|
|
441
|
+
if (!this.db)
|
|
442
|
+
return [];
|
|
443
|
+
try {
|
|
444
|
+
return this.db.prepare("SELECT * FROM run_current_state WHERE run_id=? ORDER BY role,stream").all(runId).map(row => ({
|
|
445
|
+
version: 1, runId: String(row.run_id), role: String(row.role), stream: String(row.stream),
|
|
446
|
+
...optional("executionId", row.execution_id), ...optional("ticketId", row.ticket_id), ...optional("deliveryUnitId", row.delivery_unit_id),
|
|
447
|
+
...optional("providerSessionId", row.provider_session_id), ...optional("phase", row.phase), ...optional("activeSpanId", row.active_span_id),
|
|
448
|
+
...optional("activeSpanKind", row.active_span_kind), ...optional("lastSignalAt", row.last_signal_at),
|
|
449
|
+
...optional("lastSemanticProgressAt", row.last_semantic_progress_at), updatedAt: String(row.updated_at),
|
|
450
|
+
}));
|
|
451
|
+
}
|
|
452
|
+
catch {
|
|
453
|
+
return [];
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
spans(runId) {
|
|
457
|
+
if (!this.db)
|
|
458
|
+
return [];
|
|
459
|
+
try {
|
|
460
|
+
return this.db.prepare("SELECT * FROM run_spans WHERE run_id=? ORDER BY started_at,span_id").all(runId).map(spanFromRow);
|
|
461
|
+
}
|
|
462
|
+
catch {
|
|
463
|
+
return [];
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
executions(runId) {
|
|
467
|
+
if (!this.db)
|
|
468
|
+
return [];
|
|
469
|
+
try {
|
|
470
|
+
return this.db.prepare("SELECT * FROM run_executions WHERE run_id=? ORDER BY started_at").all(runId).map(row => ({
|
|
471
|
+
executionId: String(row.execution_id), startedAt: String(row.started_at), ...optional("endedAt", row.ended_at),
|
|
472
|
+
...optional("outcome", row.outcome), ...(row.lease_generation === null ? {} : { leaseGeneration: Number(row.lease_generation) }),
|
|
473
|
+
}));
|
|
474
|
+
}
|
|
475
|
+
catch {
|
|
476
|
+
return [];
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
rollups() {
|
|
480
|
+
if (!this.db)
|
|
481
|
+
return [];
|
|
482
|
+
try {
|
|
483
|
+
return this.db.prepare("SELECT * FROM run_rollups ORDER BY COALESCE(completed_at,created_at) DESC").all().map(rollupFromRow);
|
|
484
|
+
}
|
|
485
|
+
catch {
|
|
486
|
+
return [];
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
runSummaries(input = {}) {
|
|
490
|
+
if (!this.db)
|
|
491
|
+
return [];
|
|
492
|
+
try {
|
|
493
|
+
const clauses = [];
|
|
494
|
+
const parameters = [];
|
|
495
|
+
const values = (column, items) => {
|
|
496
|
+
if (!items?.length)
|
|
497
|
+
return;
|
|
498
|
+
clauses.push(`${column} IN (${items.map(() => "?").join(",")})`);
|
|
499
|
+
parameters.push(...items);
|
|
500
|
+
};
|
|
501
|
+
values("s.run_id", input.runIds);
|
|
502
|
+
values("s.status", input.statuses);
|
|
503
|
+
values("s.provider", input.providers);
|
|
504
|
+
values("s.model", input.models);
|
|
505
|
+
values("s.branch_mode", input.branchModes);
|
|
506
|
+
values("s.detail_level", input.detailLevels);
|
|
507
|
+
if (input.qaEnabled !== undefined) {
|
|
508
|
+
clauses.push("s.qa_enabled=?");
|
|
509
|
+
parameters.push(input.qaEnabled ? 1 : 0);
|
|
510
|
+
}
|
|
511
|
+
if (input.createdFrom) {
|
|
512
|
+
clauses.push("s.created_at>=?");
|
|
513
|
+
parameters.push(input.createdFrom);
|
|
514
|
+
}
|
|
515
|
+
if (input.createdTo) {
|
|
516
|
+
clauses.push("s.created_at<=?");
|
|
517
|
+
parameters.push(input.createdTo);
|
|
518
|
+
}
|
|
519
|
+
if (input.completedFrom) {
|
|
520
|
+
clauses.push("s.completed_at>=?");
|
|
521
|
+
parameters.push(input.completedFrom);
|
|
522
|
+
}
|
|
523
|
+
if (input.completedTo) {
|
|
524
|
+
clauses.push("s.completed_at<=?");
|
|
525
|
+
parameters.push(input.completedTo);
|
|
526
|
+
}
|
|
527
|
+
if (input.ticketIds?.length) {
|
|
528
|
+
clauses.push(`EXISTS (SELECT 1 FROM run_summary_tickets t WHERE t.run_id=s.run_id AND t.ticket_id IN (${input.ticketIds.map(() => "?").join(",")}))`);
|
|
529
|
+
parameters.push(...input.ticketIds);
|
|
530
|
+
}
|
|
531
|
+
const limit = Math.max(1, Math.min(10_000, Math.trunc(input.limit ?? 10_000)));
|
|
532
|
+
const offset = Math.max(0, Math.trunc(input.offset ?? 0));
|
|
533
|
+
const sql = `SELECT s.summary_json FROM run_summaries s${clauses.length ? ` WHERE ${clauses.join(" AND ")}` : ""} ORDER BY s.updated_at DESC,s.run_id LIMIT ? OFFSET ?`;
|
|
534
|
+
parameters.push(limit, offset);
|
|
535
|
+
return this.db.prepare(sql).all(...parameters).flatMap(row => {
|
|
536
|
+
try {
|
|
537
|
+
return [JSON.parse(row.summary_json)];
|
|
538
|
+
}
|
|
539
|
+
catch {
|
|
540
|
+
return [];
|
|
541
|
+
}
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
return [];
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
phaseRollups(runIds) {
|
|
549
|
+
if (!this.db || !runIds.length)
|
|
550
|
+
return [];
|
|
551
|
+
try {
|
|
552
|
+
const ids = [...new Set(runIds)].slice(0, 100);
|
|
553
|
+
return this.db.prepare(`SELECT * FROM phase_rollups WHERE run_id IN (${ids.map(() => "?").join(",")}) ORDER BY run_id,phase,role,ticket_id`).all(...ids)
|
|
554
|
+
.map(row => ({ runId: String(row.run_id), phase: String(row.phase), role: String(row.role), ticketId: String(row.ticket_id), durationMs: Number(row.duration_ms), count: Number(row.count) }));
|
|
555
|
+
}
|
|
556
|
+
catch {
|
|
557
|
+
return [];
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
boundedSpans(runIds, perRunLimit = 100) {
|
|
561
|
+
const result = {};
|
|
562
|
+
if (!this.db || !runIds.length)
|
|
563
|
+
return result;
|
|
564
|
+
try {
|
|
565
|
+
const ids = [...new Set(runIds)].slice(0, 5);
|
|
566
|
+
const rows = this.db.prepare(`SELECT * FROM (SELECT *,ROW_NUMBER() OVER (PARTITION BY run_id ORDER BY COALESCE(duration_ms,0) DESC,started_at DESC,span_id) AS rn FROM run_spans WHERE run_id IN (${ids.map(() => "?").join(",")})) WHERE rn<=? ORDER BY run_id,rn`).all(...ids, Math.max(1, Math.min(500, perRunLimit)));
|
|
567
|
+
for (const row of rows)
|
|
568
|
+
(result[row.run_id] ??= []).push(spanFromRow(row));
|
|
569
|
+
}
|
|
570
|
+
catch { /* older SQLite or schema: bounded detail is unavailable */ }
|
|
571
|
+
return result;
|
|
572
|
+
}
|
|
573
|
+
boundedEvidence(runIds, perRunLimit = 100) {
|
|
574
|
+
const result = {};
|
|
575
|
+
if (!this.db || !runIds.length)
|
|
576
|
+
return result;
|
|
577
|
+
const ids = [...new Set(runIds)].slice(0, 5);
|
|
578
|
+
const marks = ids.map(() => "?").join(",");
|
|
579
|
+
const limit = Math.max(1, Math.min(500, perRunLimit));
|
|
580
|
+
for (const id of ids)
|
|
581
|
+
result[id] = { spans: [], events: [], metrics: [], currentState: [], executions: [] };
|
|
582
|
+
const spans = this.boundedSpans(ids, limit);
|
|
583
|
+
for (const id of ids)
|
|
584
|
+
result[id].spans = spans[id] ?? [];
|
|
585
|
+
try {
|
|
586
|
+
const events = this.db.prepare(`SELECT * FROM (SELECT *,ROW_NUMBER() OVER(PARTITION BY run_id ORDER BY observed_at DESC,event_id) rn FROM run_events WHERE run_id IN (${marks})) WHERE rn<=? ORDER BY run_id,observed_at,event_id`).all(...ids, limit);
|
|
587
|
+
for (const row of events)
|
|
588
|
+
result[String(row.run_id)]?.events.push({ eventId: String(row.event_id), kind: String(row.kind), ...optional("severity", row.severity), observedAt: String(row.observed_at), ...(row.attributes_json ? { attributes: JSON.parse(String(row.attributes_json)) } : {}) });
|
|
589
|
+
const metrics = this.db.prepare(`SELECT * FROM (SELECT *,ROW_NUMBER() OVER(PARTITION BY run_id ORDER BY observed_at DESC,metric) rn FROM metric_samples WHERE run_id IN (${marks})) WHERE rn<=? ORDER BY run_id,observed_at,metric`).all(...ids, limit);
|
|
590
|
+
for (const row of metrics)
|
|
591
|
+
result[String(row.run_id)]?.metrics.push({ metric: String(row.metric), value: Number(row.value), ...optional("unit", row.unit), observedAt: String(row.observed_at) });
|
|
592
|
+
const states = this.db.prepare(`SELECT * FROM run_current_state WHERE run_id IN (${marks}) ORDER BY run_id,role,stream`).all(...ids);
|
|
593
|
+
for (const row of states)
|
|
594
|
+
result[String(row.run_id)]?.currentState.push({ version: 1, runId: String(row.run_id), role: String(row.role), stream: String(row.stream), ...optional("executionId", row.execution_id), ...optional("ticketId", row.ticket_id), ...optional("deliveryUnitId", row.delivery_unit_id), ...optional("providerSessionId", row.provider_session_id), ...optional("phase", row.phase), ...optional("activeSpanId", row.active_span_id), ...optional("activeSpanKind", row.active_span_kind), ...optional("lastSignalAt", row.last_signal_at), ...optional("lastSemanticProgressAt", row.last_semantic_progress_at), updatedAt: String(row.updated_at) });
|
|
595
|
+
const executions = this.db.prepare(`SELECT * FROM run_executions WHERE run_id IN (${marks}) ORDER BY run_id,started_at`).all(...ids);
|
|
596
|
+
for (const row of executions)
|
|
597
|
+
result[String(row.run_id)]?.executions.push({ executionId: String(row.execution_id), startedAt: String(row.started_at), ...optional("endedAt", row.ended_at), ...optional("outcome", row.outcome), ...(row.lease_generation === null ? {} : { leaseGeneration: Number(row.lease_generation) }) });
|
|
598
|
+
}
|
|
599
|
+
catch { /* bounded evidence remains empty against older schemas */ }
|
|
600
|
+
return result;
|
|
601
|
+
}
|
|
602
|
+
capability(runId) {
|
|
603
|
+
if (!this.db)
|
|
604
|
+
return undefined;
|
|
605
|
+
try {
|
|
606
|
+
const row = this.db.prepare("SELECT * FROM run_capabilities WHERE run_id=?").get(runId);
|
|
607
|
+
return row ? { ...(row.rafi_version ? { rafiVersion: row.rafi_version } : {}), sources: JSON.parse(row.capabilities_json) } : undefined;
|
|
608
|
+
}
|
|
609
|
+
catch {
|
|
610
|
+
return undefined;
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
storage() {
|
|
614
|
+
const databaseBytesValue = existsSync(this.path) ? statSync(this.path).size : 0;
|
|
615
|
+
const wal = `${this.path}-wal`;
|
|
616
|
+
let summaryOnly = false;
|
|
617
|
+
if (this.db)
|
|
618
|
+
try {
|
|
619
|
+
summaryOnly = this.db.prepare("SELECT value FROM observability_meta WHERE key='summary_only'").get()?.value === "1";
|
|
620
|
+
}
|
|
621
|
+
catch { /* legacy */ }
|
|
622
|
+
return { databaseBytes: databaseBytesValue, walBytes: existsSync(wal) ? statSync(wal).size : 0, summaryOnly };
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
/** Records Manager session accounting only when observability already exists. It never creates or migrates schema. */
|
|
626
|
+
export class ManagerSessionRecorder {
|
|
627
|
+
db;
|
|
628
|
+
v2;
|
|
629
|
+
constructor(projectDir, path = join(resolve(projectDir), OBSERVABILITY_DB_FILE)) {
|
|
630
|
+
if (!existsSync(path)) {
|
|
631
|
+
this.v2 = false;
|
|
632
|
+
return;
|
|
633
|
+
}
|
|
634
|
+
let db;
|
|
635
|
+
try {
|
|
636
|
+
db = new Database(path, { fileMustExist: true });
|
|
637
|
+
const columns = db.prepare("PRAGMA table_info(manager_sessions)").all();
|
|
638
|
+
if (!columns.some(item => item.name === "session_id")) {
|
|
639
|
+
db.close();
|
|
640
|
+
this.v2 = false;
|
|
641
|
+
return;
|
|
642
|
+
}
|
|
643
|
+
this.db = db;
|
|
644
|
+
this.v2 = columns.some(item => item.name === "project_report_digest");
|
|
645
|
+
}
|
|
646
|
+
catch {
|
|
647
|
+
db?.close();
|
|
648
|
+
this.v2 = false;
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
close() { try {
|
|
652
|
+
this.db?.close();
|
|
653
|
+
}
|
|
654
|
+
catch { /* diagnostics must not fail on accounting */ } }
|
|
655
|
+
record(input) {
|
|
656
|
+
if (!this.db)
|
|
657
|
+
return;
|
|
658
|
+
try {
|
|
659
|
+
if (this.v2)
|
|
660
|
+
this.db.prepare(`INSERT INTO manager_sessions(session_id,run_id,provider,started_at,ended_at,outcome,report_digest,input_tokens,output_tokens,cost_usd,error_code,scope,latest_focus_run_id,project_report_digest,lookup_rounds,lookup_operations)
|
|
661
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id) DO UPDATE SET provider=excluded.provider,ended_at=excluded.ended_at,outcome=excluded.outcome,report_digest=excluded.report_digest,input_tokens=excluded.input_tokens,output_tokens=excluded.output_tokens,cost_usd=excluded.cost_usd,error_code=excluded.error_code,scope=excluded.scope,latest_focus_run_id=excluded.latest_focus_run_id,project_report_digest=excluded.project_report_digest,lookup_rounds=excluded.lookup_rounds,lookup_operations=excluded.lookup_operations`)
|
|
662
|
+
.run(input.sessionId, input.runId, input.provider ?? null, input.startedAt, input.endedAt ?? null, input.outcome ?? null, input.reportDigest ?? null, input.inputTokens ?? null, input.outputTokens ?? null, input.costUsd ?? null, input.errorCode ?? null, input.scope ?? null, input.latestFocusRunId ?? null, input.projectReportDigest ?? null, input.lookupRounds ?? 0, input.lookupOperations ?? 0);
|
|
663
|
+
else
|
|
664
|
+
this.db.prepare(`INSERT INTO manager_sessions(session_id,run_id,provider,started_at,ended_at,outcome,report_digest,input_tokens,output_tokens,cost_usd,error_code)
|
|
665
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id) DO UPDATE SET provider=excluded.provider,ended_at=excluded.ended_at,outcome=excluded.outcome,report_digest=excluded.report_digest,input_tokens=excluded.input_tokens,output_tokens=excluded.output_tokens,cost_usd=excluded.cost_usd,error_code=excluded.error_code`)
|
|
666
|
+
.run(input.sessionId, input.runId, input.provider ?? null, input.startedAt, input.endedAt ?? null, input.outcome ?? null, input.reportDigest ?? null, input.inputTokens ?? null, input.outputTokens ?? null, input.costUsd ?? null, input.errorCode ?? null);
|
|
667
|
+
}
|
|
668
|
+
catch { /* session accounting is best-effort and never weakens Manager isolation */ }
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
/** One observer is owned by one build process attachment. */
|
|
672
|
+
export class RunObserver {
|
|
673
|
+
store;
|
|
674
|
+
runId;
|
|
675
|
+
executionId;
|
|
676
|
+
storage = new AsyncLocalStorage();
|
|
677
|
+
resourceTimer;
|
|
678
|
+
constructor(store, runId, input = {}) {
|
|
679
|
+
this.store = store;
|
|
680
|
+
this.runId = runId;
|
|
681
|
+
this.executionId = store.beginExecution({ runId, ...input });
|
|
682
|
+
store.abandonPriorExecutionSpans(runId, this.executionId);
|
|
683
|
+
}
|
|
684
|
+
context() { return this.storage.getStore() ?? { runId: this.runId, executionId: this.executionId, role: "host", stream: "host" }; }
|
|
685
|
+
withContext(values, fn) {
|
|
686
|
+
return this.storage.run({ ...this.context(), ...values, runId: this.runId, executionId: this.executionId }, fn);
|
|
687
|
+
}
|
|
688
|
+
async span(kind, name, fn, attributes) {
|
|
689
|
+
const parent = this.context();
|
|
690
|
+
const spanId = this.store.startSpan(parent, { kind, name, attributes });
|
|
691
|
+
this.store.updateCurrentState({ runId: this.runId, role: parent.role ?? "host", stream: parent.stream ?? "host", executionId: this.executionId, ticketId: parent.ticketId, deliveryUnitId: parent.deliveryUnitId, providerSessionId: parent.providerSessionId, phase: name, activeSpanId: spanId, activeSpanKind: kind, lastSemanticProgressAt: new Date().toISOString() });
|
|
692
|
+
try {
|
|
693
|
+
const value = await this.withContext({ parentSpanId: spanId }, fn);
|
|
694
|
+
this.store.finishSpan(spanId, { outcome: "completed" });
|
|
695
|
+
return value;
|
|
696
|
+
}
|
|
697
|
+
catch (error) {
|
|
698
|
+
this.store.finishSpan(spanId, { outcome: "failed", attributes: { error: sanitizeText(String(error)).slice(0, 500) } });
|
|
699
|
+
throw error;
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
signal(semantic = false) {
|
|
703
|
+
const context = this.context();
|
|
704
|
+
const at = new Date().toISOString();
|
|
705
|
+
this.store.updateCurrentState({ runId: this.runId, role: context.role ?? "host", stream: context.stream ?? "host", executionId: this.executionId, ticketId: context.ticketId, deliveryUnitId: context.deliveryUnitId, providerSessionId: context.providerSessionId, lastSignalAt: at, ...(semantic ? { lastSemanticProgressAt: at } : {}) });
|
|
706
|
+
}
|
|
707
|
+
startResourceSampling() {
|
|
708
|
+
if (this.resourceTimer)
|
|
709
|
+
return;
|
|
710
|
+
this.store.sampleProcess(this.context(), 0, true);
|
|
711
|
+
let scheduled = performance.now();
|
|
712
|
+
this.resourceTimer = setInterval(() => {
|
|
713
|
+
const now = performance.now();
|
|
714
|
+
const delay = Math.max(0, now - scheduled - this.store.config.sample_interval_seconds * 1000);
|
|
715
|
+
scheduled = now;
|
|
716
|
+
this.store.sampleProcess(this.context(), delay);
|
|
717
|
+
}, this.store.config.sample_interval_seconds * 1000);
|
|
718
|
+
this.resourceTimer.unref();
|
|
719
|
+
}
|
|
720
|
+
finish(outcome, metadata = {}) {
|
|
721
|
+
if (this.resourceTimer)
|
|
722
|
+
clearInterval(this.resourceTimer);
|
|
723
|
+
this.store.sampleProcess(this.context(), undefined, true);
|
|
724
|
+
this.store.endExecution(this.executionId, outcome);
|
|
725
|
+
this.store.finalizeRun(this.runId, { status: outcome, ...metadata });
|
|
726
|
+
this.store.checkpoint();
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
function spanFromRow(row) {
|
|
730
|
+
return { version: 1, spanId: row.span_id, runId: row.run_id, ...optional("executionId", row.execution_id),
|
|
731
|
+
...optional("parentSpanId", row.parent_span_id), ...optional("providerTurnId", row.provider_turn_id), ...optional("ticketId", row.ticket_id),
|
|
732
|
+
...optional("deliveryUnitId", row.delivery_unit_id), ...(row.role ? { role: row.role } : {}), ...optional("stream", row.stream),
|
|
733
|
+
...optional("providerSessionId", row.provider_session_id), kind: row.kind, name: row.name, startedAt: row.started_at,
|
|
734
|
+
...optional("endedAt", row.ended_at), ...(row.duration_ms === null ? {} : { durationMs: row.duration_ms }), ...optional("outcome", row.outcome),
|
|
735
|
+
completionKnown: Boolean(row.completion_known), ...(row.attributes_json ? { attributes: JSON.parse(row.attributes_json) } : {}) };
|
|
736
|
+
}
|
|
737
|
+
function rollupFromRow(row) {
|
|
738
|
+
return { version: 1, runId: String(row.run_id), ...optional("projectId", row.project_id), status: String(row.status),
|
|
739
|
+
...optional("branchMode", row.branch_mode), ...(row.qa_enabled === null ? {} : { qaEnabled: Boolean(row.qa_enabled) }),
|
|
740
|
+
...(row.primary_provider ? { primaryProvider: String(row.primary_provider) } : {}),
|
|
741
|
+
...(row.ticket_count === null ? {} : { ticketCount: Number(row.ticket_count) }), ...optional("ticketCountBucket", row.ticket_count_bucket),
|
|
742
|
+
...optional("ticketSizeBucket", row.ticket_size_bucket), createdAt: String(row.created_at), ...optional("completedAt", row.completed_at),
|
|
743
|
+
calendarMs: Number(row.calendar_ms), activeExecutionMs: Number(row.active_execution_ms), explicitWaitMs: Number(row.explicit_wait_ms),
|
|
744
|
+
attributedMs: Number(row.attributed_ms), unattributedMs: Number(row.unattributed_ms), totals: JSON.parse(String(row.totals_json)) };
|
|
745
|
+
}
|
|
746
|
+
function addColumn(db, table, column, definition) {
|
|
747
|
+
const columns = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
748
|
+
if (!columns.some(item => item.name === column))
|
|
749
|
+
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
750
|
+
}
|
|
751
|
+
function legacySummaryFromRollup(rollup, now) {
|
|
752
|
+
const withoutDigest = {
|
|
753
|
+
version: 1, runId: rollup.runId, status: rollup.status, createdAt: rollup.createdAt,
|
|
754
|
+
updatedAt: rollup.completedAt ?? rollup.createdAt, ...(rollup.completedAt ? { completedAt: rollup.completedAt } : {}), activeState: "inactive",
|
|
755
|
+
ticketIds: [], ...(rollup.branchMode ? { branchMode: rollup.branchMode } : {}), ...(rollup.qaEnabled === undefined ? {} : { qaEnabled: rollup.qaEnabled }),
|
|
756
|
+
...(rollup.primaryProvider ? { provider: rollup.primaryProvider } : {}),
|
|
757
|
+
timing: { calendarMs: rollup.calendarMs, activeExecutionMs: rollup.activeExecutionMs, pausedOfflineMs: Math.max(0, rollup.calendarMs - rollup.activeExecutionMs), explicitWaitMs: rollup.explicitWaitMs, attributedMs: rollup.attributedMs, unattributedMs: rollup.unattributedMs, inclusiveByKind: rollup.totals, exclusiveByKind: rollup.totals },
|
|
758
|
+
counts: { byKind: {}, byOutcome: {} }, usage: { scope: "unavailable" }, retry: {}, topOperations: [], detailLevel: "rollup",
|
|
759
|
+
metricCoverage: { timing: "partial", counts: "unavailable", usage: "unavailable", git: "unavailable" },
|
|
760
|
+
capabilities: { version: 1, sources: { migration: { source: "run_rollup_v1", state: "partial", observedAt: now.toISOString(), detail: "backfilled exactly from V1 rollup; unavailable facts were not reconstructed" } } }, evidenceIds: [`rollup:${rollup.runId}`],
|
|
761
|
+
};
|
|
762
|
+
return { ...withoutDigest, digest: diagnosticDigest(withoutDigest) };
|
|
763
|
+
}
|
|
764
|
+
function demoteRunSummaries(db, runIds, now, detail) {
|
|
765
|
+
const read = db.prepare("SELECT summary_json FROM run_summaries WHERE run_id=?");
|
|
766
|
+
const write = db.prepare("UPDATE run_summaries SET detail_level='rollup',summary_json=? WHERE run_id=?");
|
|
767
|
+
for (const runId of runIds) {
|
|
768
|
+
try {
|
|
769
|
+
const row = read.get(runId);
|
|
770
|
+
if (!row?.summary_json)
|
|
771
|
+
continue;
|
|
772
|
+
const summary = JSON.parse(row.summary_json);
|
|
773
|
+
const withoutDigest = { ...summary, detailLevel: "rollup", capabilities: { ...summary.capabilities, sources: { ...summary.capabilities.sources, retained_detail: { source: "retained_detail", state: "partial", observedAt: now.toISOString(), detail } } } };
|
|
774
|
+
write.run(safeJson({ ...withoutDigest, digest: diagnosticDigest(withoutDigest) }), runId);
|
|
775
|
+
}
|
|
776
|
+
catch { /* one corrupt summary must not disable retention for other runs */ }
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
function buildStoredRunSummary(db, spans, executions, rollup, input, summaryOnly, now) {
|
|
780
|
+
let existing;
|
|
781
|
+
try {
|
|
782
|
+
const row = db.prepare("SELECT summary_json FROM run_summaries WHERE run_id=?").get(rollup.runId);
|
|
783
|
+
if (row?.summary_json)
|
|
784
|
+
existing = JSON.parse(row.summary_json);
|
|
785
|
+
}
|
|
786
|
+
catch { /* first summary */ }
|
|
787
|
+
const completedAt = rollup.completedAt ?? now.toISOString();
|
|
788
|
+
const inclusiveByKind = durationUnionsByKind(spans, completedAt);
|
|
789
|
+
const exclusiveByKind = exclusiveDurationTotals(spans, completedAt);
|
|
790
|
+
const byKind = {};
|
|
791
|
+
const byOutcome = {};
|
|
792
|
+
try {
|
|
793
|
+
const phases = db.prepare("SELECT phase,SUM(count) count FROM phase_rollups WHERE run_id=? GROUP BY phase").all(rollup.runId);
|
|
794
|
+
for (const phase of phases)
|
|
795
|
+
byKind[phase.phase] = Number(phase.count);
|
|
796
|
+
}
|
|
797
|
+
catch { /* optional legacy table */ }
|
|
798
|
+
for (const span of spans) {
|
|
799
|
+
if (byKind[span.kind] === undefined)
|
|
800
|
+
byKind[span.kind] = spans.filter(item => item.kind === span.kind).length;
|
|
801
|
+
const outcome = span.outcome ?? (span.endedAt ? "unknown" : "open");
|
|
802
|
+
byOutcome[outcome] = (byOutcome[outcome] ?? 0) + 1;
|
|
803
|
+
}
|
|
804
|
+
const measuredUsage = storedUsage(spans);
|
|
805
|
+
const usage = measuredUsage.scope === "unavailable" && existing?.usage ? existing.usage : measuredUsage;
|
|
806
|
+
const topOperations = spans.filter(span => span.durationMs !== undefined).sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0) || a.spanId.localeCompare(b.spanId)).slice(0, 10)
|
|
807
|
+
.map(span => ({ kind: span.kind, name: span.name.slice(0, 160), durationMs: span.durationMs ?? 0, ...(span.outcome ? { outcome: span.outcome } : {}), evidenceId: `span:${span.spanId}` }));
|
|
808
|
+
let capabilitySources = {};
|
|
809
|
+
try {
|
|
810
|
+
capabilitySources = JSON.parse(db.prepare("SELECT capabilities_json FROM run_capabilities WHERE run_id=?").get(rollup.runId)?.capabilities_json ?? "{}");
|
|
811
|
+
}
|
|
812
|
+
catch { /* optional */ }
|
|
813
|
+
const timingState = summaryOnly ? "partial" : "available";
|
|
814
|
+
const usageState = usage.scope === "unavailable" ? "unavailable" : summaryOnly ? "partial" : "available";
|
|
815
|
+
const withoutDigest = {
|
|
816
|
+
version: 1, runId: rollup.runId, status: input.status, createdAt: rollup.createdAt, updatedAt: input.updatedAt ?? completedAt,
|
|
817
|
+
...(rollup.completedAt ? { completedAt: rollup.completedAt } : {}), ...(input.checkpoint ? { checkpoint: input.checkpoint } : {}), activeState: "inactive",
|
|
818
|
+
ticketIds: [...new Set(input.ticketIds ?? existing?.ticketIds ?? [])].sort(), ...(input.deliveryUnit ?? existing?.deliveryUnit ? { deliveryUnit: input.deliveryUnit ?? existing?.deliveryUnit } : {}),
|
|
819
|
+
...(input.branchMode ?? existing?.branchMode ? { branchMode: input.branchMode ?? existing?.branchMode } : {}), ...(input.qaEnabled === undefined ? existing?.qaEnabled === undefined ? {} : { qaEnabled: existing.qaEnabled } : { qaEnabled: input.qaEnabled }),
|
|
820
|
+
...(input.primaryProvider ?? existing?.provider ? { provider: input.primaryProvider ?? existing?.provider } : {}), ...(input.model ?? existing?.model ? { model: input.model ?? existing?.model } : {}),
|
|
821
|
+
timing: { calendarMs: rollup.calendarMs, activeExecutionMs: rollup.activeExecutionMs,
|
|
822
|
+
pausedOfflineMs: Math.max(0, rollup.calendarMs - rollup.activeExecutionMs), explicitWaitMs: rollup.explicitWaitMs,
|
|
823
|
+
attributedMs: rollup.attributedMs, unattributedMs: rollup.unattributedMs, inclusiveByKind, exclusiveByKind },
|
|
824
|
+
counts: { byKind, byOutcome, qaAttempts: byKind.qa_attempt ?? 0, qaFailures: spans.filter(span => span.kind === "qa_attempt" && ["failed", "rejected"].includes(span.outcome ?? "")).length,
|
|
825
|
+
fixes: (byKind.qa_fix ?? 0) + (byKind.fix ?? 0), retries: byKind.retry ?? 0, tools: byKind.tool ?? 0,
|
|
826
|
+
providerTurns: byKind.provider_turn ?? 0, waits: [...WAIT_SUMMARY_KINDS].reduce((sum, kind) => sum + (byKind[kind] ?? 0), 0), executions: executions.length },
|
|
827
|
+
usage, retry: { observedMs: inclusiveByKind.retry ?? 0, reportedDelayMs: spans.filter(span => span.kind === "retry").reduce((sum, span) => sum + finiteAttribute(span.attributes?.reportedDelayMs), 0) },
|
|
828
|
+
...(input.failureCategory ?? existing?.failureCategory ? { failureCategory: input.failureCategory ?? existing?.failureCategory } : {}), topOperations: topOperations.length ? topOperations : existing?.topOperations ?? [], ...(input.git ?? existing?.git ? { git: input.git ?? existing?.git } : {}),
|
|
829
|
+
detailLevel: summaryOnly ? "rollup" : "detailed",
|
|
830
|
+
metricCoverage: { timing: timingState, counts: timingState, usage: usageState, git: input.git ? "available" : "unavailable" },
|
|
831
|
+
capabilities: { version: 1, sources: capabilitySources }, evidenceIds: topOperations.map(item => item.evidenceId),
|
|
832
|
+
};
|
|
833
|
+
return { ...withoutDigest, digest: diagnosticDigest(withoutDigest) };
|
|
834
|
+
}
|
|
835
|
+
const WAIT_SUMMARY_KINDS = new Set(["user_wait", "dependency_wait", "external_ci_wait"]);
|
|
836
|
+
function durationUnionsByKind(spans, fallbackEnd) {
|
|
837
|
+
const grouped = new Map();
|
|
838
|
+
for (const span of spans)
|
|
839
|
+
(grouped.get(span.kind) ?? (grouped.set(span.kind, []), grouped.get(span.kind))).push([Date.parse(span.startedAt), Date.parse(span.endedAt ?? fallbackEnd)]);
|
|
840
|
+
return Object.fromEntries([...grouped].map(([kind, intervals]) => [kind, unionMs(intervals)]));
|
|
841
|
+
}
|
|
842
|
+
function exclusiveDurationTotals(spans, fallbackEnd) {
|
|
843
|
+
const children = new Map();
|
|
844
|
+
for (const span of spans)
|
|
845
|
+
if (span.parentSpanId)
|
|
846
|
+
(children.get(span.parentSpanId) ?? (children.set(span.parentSpanId, []), children.get(span.parentSpanId))).push([Date.parse(span.startedAt), Date.parse(span.endedAt ?? fallbackEnd)]);
|
|
847
|
+
const grouped = new Map();
|
|
848
|
+
for (const span of spans) {
|
|
849
|
+
const start = Date.parse(span.startedAt);
|
|
850
|
+
const end = Date.parse(span.endedAt ?? fallbackEnd);
|
|
851
|
+
const cuts = (children.get(span.spanId) ?? []).map(([a, b]) => [Math.max(a, start), Math.min(b, end)]).filter(([a, b]) => b >= a);
|
|
852
|
+
const segments = subtractDurationIntervals([start, end], cuts);
|
|
853
|
+
grouped.set(span.kind, [...(grouped.get(span.kind) ?? []), ...segments]);
|
|
854
|
+
}
|
|
855
|
+
return Object.fromEntries([...grouped].map(([kind, intervals]) => [kind, unionMs(intervals)]));
|
|
856
|
+
}
|
|
857
|
+
function subtractDurationIntervals(base, cuts) {
|
|
858
|
+
const sorted = cuts.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
859
|
+
const merged = [];
|
|
860
|
+
for (const cut of sorted) {
|
|
861
|
+
const last = merged.at(-1);
|
|
862
|
+
if (!last || cut[0] > last[1])
|
|
863
|
+
merged.push([...cut]);
|
|
864
|
+
else
|
|
865
|
+
last[1] = Math.max(last[1], cut[1]);
|
|
866
|
+
}
|
|
867
|
+
const result = [];
|
|
868
|
+
let cursor = base[0];
|
|
869
|
+
for (const cut of merged) {
|
|
870
|
+
if (cut[0] > cursor)
|
|
871
|
+
result.push([cursor, cut[0]]);
|
|
872
|
+
cursor = Math.max(cursor, cut[1]);
|
|
873
|
+
}
|
|
874
|
+
if (cursor < base[1])
|
|
875
|
+
result.push([cursor, base[1]]);
|
|
876
|
+
return result;
|
|
877
|
+
}
|
|
878
|
+
function storedUsage(spans) {
|
|
879
|
+
const delta = { inputTokens: 0, outputTokens: 0, totalTokens: 0, authoritativeCostUsd: 0 };
|
|
880
|
+
const observed = { inputTokens: false, outputTokens: false, totalTokens: false, authoritativeCostUsd: false };
|
|
881
|
+
let deltaObserved = false;
|
|
882
|
+
const cumulative = new Map();
|
|
883
|
+
for (const span of spans.filter(item => item.kind === "provider_turn")) {
|
|
884
|
+
const attributes = span.attributes ?? {};
|
|
885
|
+
const turn = isRecord(attributes.turnDelta) ? attributes.turnDelta : isRecord(attributes.usage) && attributes.usage.scope === "turn-delta" ? attributes.usage : undefined;
|
|
886
|
+
if (turn && [turn.inputTokens, turn.outputTokens, turn.totalTokens, turn.costUsd].some(item => optionalFinite(item) !== undefined)) {
|
|
887
|
+
deltaObserved = true;
|
|
888
|
+
const input = optionalFinite(turn.inputTokens);
|
|
889
|
+
const output = optionalFinite(turn.outputTokens);
|
|
890
|
+
const total = optionalFinite(turn.totalTokens);
|
|
891
|
+
const cost = optionalFinite(turn.costUsd);
|
|
892
|
+
if (input !== undefined) {
|
|
893
|
+
observed.inputTokens = true;
|
|
894
|
+
delta.inputTokens += input;
|
|
895
|
+
}
|
|
896
|
+
if (output !== undefined) {
|
|
897
|
+
observed.outputTokens = true;
|
|
898
|
+
delta.outputTokens += output;
|
|
899
|
+
}
|
|
900
|
+
if (total !== undefined) {
|
|
901
|
+
observed.totalTokens = true;
|
|
902
|
+
delta.totalTokens += total;
|
|
903
|
+
}
|
|
904
|
+
else if (input !== undefined && output !== undefined) {
|
|
905
|
+
observed.totalTokens = true;
|
|
906
|
+
delta.totalTokens += input + output;
|
|
907
|
+
}
|
|
908
|
+
if (cost !== undefined) {
|
|
909
|
+
observed.authoritativeCostUsd = true;
|
|
910
|
+
delta.authoritativeCostUsd += cost;
|
|
911
|
+
}
|
|
912
|
+
continue;
|
|
913
|
+
}
|
|
914
|
+
const session = isRecord(attributes.cumulativeUsage) ? attributes.cumulativeUsage : isRecord(attributes.usage) && attributes.usage.scope === "session-cumulative" ? attributes.usage : undefined;
|
|
915
|
+
if (session && span.providerSessionId)
|
|
916
|
+
cumulative.set(span.providerSessionId, { inputTokens: optionalFinite(session.inputTokens), outputTokens: optionalFinite(session.outputTokens), totalTokens: optionalFinite(session.totalTokens), authoritativeCostUsd: optionalFinite(session.costUsd) });
|
|
917
|
+
}
|
|
918
|
+
if (!deltaObserved && !cumulative.size)
|
|
919
|
+
return { scope: "unavailable" };
|
|
920
|
+
const values = [...cumulative.values()];
|
|
921
|
+
const inputValues = values.map(item => item.inputTokens).filter((item) => item !== undefined);
|
|
922
|
+
const outputValues = values.map(item => item.outputTokens).filter((item) => item !== undefined);
|
|
923
|
+
const totalValues = values.map(item => item.totalTokens).filter((item) => item !== undefined);
|
|
924
|
+
const costValues = values.map(item => item.authoritativeCostUsd).filter((item) => item !== undefined);
|
|
925
|
+
const result = {
|
|
926
|
+
...(observed.inputTokens || inputValues.length ? { inputTokens: delta.inputTokens + inputValues.reduce((sum, item) => sum + item, 0) } : {}),
|
|
927
|
+
...(observed.outputTokens || outputValues.length ? { outputTokens: delta.outputTokens + outputValues.reduce((sum, item) => sum + item, 0) } : {}),
|
|
928
|
+
...(observed.totalTokens || totalValues.length ? { totalTokens: delta.totalTokens + totalValues.reduce((sum, item) => sum + item, 0) } : {}),
|
|
929
|
+
...(observed.authoritativeCostUsd || costValues.length ? { authoritativeCostUsd: delta.authoritativeCostUsd + costValues.reduce((sum, item) => sum + item, 0) } : {}),
|
|
930
|
+
};
|
|
931
|
+
return { ...result, scope: deltaObserved && cumulative.size ? "mixed" : deltaObserved ? "turn-deltas" : "session-cumulative-deduplicated" };
|
|
932
|
+
}
|
|
933
|
+
function isRecord(value) { return Boolean(value) && typeof value === "object" && !Array.isArray(value); }
|
|
934
|
+
function optionalFinite(value) { const number = Number(value); return Number.isFinite(number) && number >= 0 ? number : undefined; }
|
|
935
|
+
function finiteAttribute(value) { return optionalFinite(value) ?? 0; }
|
|
936
|
+
function optional(key, value) { return value === null || value === undefined ? {} : { [key]: String(value) }; }
|
|
937
|
+
function sanitizeText(value) {
|
|
938
|
+
return value
|
|
939
|
+
.replace(/\b(?:sk|ghp|glpat|xox[baprs])[-_][A-Za-z0-9_-]{8,}\b/gi, "[REDACTED]")
|
|
940
|
+
.replace(/\b(?:api[_-]?key|token|password|secret|authorization)\s*[:=]\s*[^\s,;]+/gi, "$1=[REDACTED]")
|
|
941
|
+
.replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@:]+:[^\s/@]+@/gi, "$1[REDACTED]@");
|
|
942
|
+
}
|
|
943
|
+
export function sanitizeDiagnosticValue(value) { return sanitizeValue(value); }
|
|
944
|
+
function sanitizeValue(value, depth = 0) {
|
|
945
|
+
if (depth > 8)
|
|
946
|
+
return "[TRUNCATED]";
|
|
947
|
+
if (typeof value === "string")
|
|
948
|
+
return sanitizeText(value).slice(0, 4096);
|
|
949
|
+
if (Array.isArray(value))
|
|
950
|
+
return value.slice(0, 100).map(item => sanitizeValue(item, depth + 1));
|
|
951
|
+
if (value && typeof value === "object")
|
|
952
|
+
return Object.fromEntries(Object.entries(value)
|
|
953
|
+
.filter(([key]) => !/^(prompt|response|reasoning|transcript|credential|secret|token|password|api[_-]?key)$/i.test(key))
|
|
954
|
+
.slice(0, 100).map(([key, item]) => [key, sanitizeValue(item, depth + 1)]));
|
|
955
|
+
return value;
|
|
956
|
+
}
|
|
957
|
+
function safeJson(value) { return JSON.stringify(value ?? null); }
|
|
958
|
+
export function diagnosticDigest(value) { return createHash("sha256").update(stableJson(value)).digest("hex"); }
|
|
959
|
+
function stableJson(value) {
|
|
960
|
+
if (Array.isArray(value))
|
|
961
|
+
return `[${value.map(stableJson).join(",")}]`;
|
|
962
|
+
if (value && typeof value === "object")
|
|
963
|
+
return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`;
|
|
964
|
+
return JSON.stringify(value ?? null);
|
|
965
|
+
}
|
|
966
|
+
function processStartIdentity(pid) { try {
|
|
967
|
+
return readFileSync(`/proc/${pid}/stat`, "utf8").split(" ")[21] ?? "unavailable";
|
|
968
|
+
}
|
|
969
|
+
catch {
|
|
970
|
+
return "unavailable";
|
|
971
|
+
} }
|
|
972
|
+
function databaseBytes(path) { return [path, `${path}-wal`, `${path}-shm`].reduce((sum, item) => sum + (existsSync(item) ? statSync(item).size : 0), 0); }
|
|
973
|
+
function fileSize(path) { try {
|
|
974
|
+
return statSync(path).size;
|
|
975
|
+
}
|
|
976
|
+
catch {
|
|
977
|
+
return 0;
|
|
978
|
+
} }
|
|
979
|
+
function safeOwnedLogPath(root, path) { const target = resolve(path); return target.startsWith(`${root}${sep}`) && (target.endsWith(".jsonl") || target.endsWith(".jsonl.gz")); }
|
|
980
|
+
function directoryIndexedLogBytes(db) { return Number(db.prepare("SELECT COALESCE(SUM(bytes),0) AS bytes FROM log_files").get().bytes); }
|
|
981
|
+
function availableDiskBytes(path) {
|
|
982
|
+
try {
|
|
983
|
+
const stats = statfsSync(path);
|
|
984
|
+
return Number(stats.bavail) * Number(stats.bsize);
|
|
985
|
+
}
|
|
986
|
+
catch {
|
|
987
|
+
return undefined;
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
function unionMs(intervals) {
|
|
991
|
+
const sorted = intervals.filter(item => item.length === 2 && item.every(Number.isFinite) && item[1] >= item[0]).sort((a, b) => a[0] - b[0]);
|
|
992
|
+
let total = 0;
|
|
993
|
+
let start;
|
|
994
|
+
let end = 0;
|
|
995
|
+
for (const [a, b] of sorted) {
|
|
996
|
+
if (start === undefined) {
|
|
997
|
+
start = a;
|
|
998
|
+
end = b;
|
|
999
|
+
}
|
|
1000
|
+
else if (a <= end)
|
|
1001
|
+
end = Math.max(end, b);
|
|
1002
|
+
else {
|
|
1003
|
+
total += end - start;
|
|
1004
|
+
start = a;
|
|
1005
|
+
end = b;
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
return total + (start === undefined ? 0 : end - start);
|
|
1009
|
+
}
|
|
1010
|
+
function ensureObservabilityIgnore(projectDir) {
|
|
1011
|
+
const localExclude = join(projectDir, ".git", "info", "exclude");
|
|
1012
|
+
const path = existsSync(localExclude) ? localExclude : join(projectDir, ".gitignore");
|
|
1013
|
+
const entries = [OBSERVABILITY_DB_FILE, `${OBSERVABILITY_DB_FILE}-wal`, `${OBSERVABILITY_DB_FILE}-shm`];
|
|
1014
|
+
const existing = existsSync(path) ? readFileSync(path, "utf8") : "";
|
|
1015
|
+
const missing = entries.filter(entry => !existing.split(/\r?\n/).includes(entry));
|
|
1016
|
+
if (missing.length)
|
|
1017
|
+
appendFileSync(path, `${existing && !existing.endsWith("\n") ? "\n" : ""}${missing.join("\n")}\n`, "utf8");
|
|
1018
|
+
}
|