@pasko70/pibo 3.4.4 → 3.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-runtime/routed-session.js +111 -25
- package/dist/agent-runtimes/codex-native/turn.js +27 -3
- package/dist/agent-runtimes/pi/adapter.js +1 -0
- package/dist/agent-runtimes/pi/routed-session.js +2 -2
- package/dist/apps/chat/bounded-event-stream.js +98 -0
- package/dist/apps/chat/chat-settings-routes.js +3 -3
- package/dist/apps/chat/data/chat-data-mappers.js +40 -14
- package/dist/apps/chat/data/event-command-service.js +30 -21
- package/dist/apps/chat/data/history-query-service.js +34 -25
- package/dist/apps/chat/data/read-state-service.js +37 -3
- package/dist/apps/chat/data/session-query-service.js +19 -11
- package/dist/apps/chat/data/timeline-query-service.js +19 -7
- package/dist/apps/chat/message-command-dispatcher.js +149 -0
- package/dist/apps/chat/output-compactor.js +9 -0
- package/dist/apps/chat/output-event-policy.js +9 -1
- package/dist/apps/chat/stream.js +21 -3
- package/dist/apps/chat/telemetry-retention-service.js +113 -8
- package/dist/apps/chat/trace-response-cache.js +46 -0
- package/dist/apps/chat/trace-v2.js +12 -6
- package/dist/apps/chat/trace.js +1 -1
- package/dist/apps/chat/web-app.js +645 -297
- package/dist/apps/chat-ui/assets/{dist-V06sfuZa.js → dist-B-auLrzD.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-CH3SvpYV.js → dist-BA_dsINH.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-23lt_7qm.js → dist-eJZar_0-.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-Bf2KScPo.js → dist-wNNR2Bci.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-DsgL8w-W.js → dist-zcmEsIEp.js} +1 -1
- package/dist/apps/chat-ui/assets/index-DEkbN5Vo.js +229 -0
- package/dist/apps/chat-ui/assets/index-DZK6Tzil.css +1 -0
- package/dist/apps/chat-ui/index.html +2 -2
- package/dist/apps/chat-vscode-web/assets/index-DZgW1fCB.js +44 -0
- package/dist/apps/chat-vscode-web/index.html +1 -1
- package/dist/cli-session/localSessionSource.js +3 -2
- package/dist/compute/pool/seeds.js +25 -3
- package/dist/core/events.js +4 -0
- package/dist/core/output-render-sequence.js +65 -6
- package/dist/core/provider-capacity.js +33 -0
- package/dist/core/provider-telemetry.js +21 -6
- package/dist/core/runtime-capacity.js +174 -0
- package/dist/core/runtime-telemetry.js +40 -12
- package/dist/core/session-router.js +178 -13
- package/dist/data/async-chat-reads.js +38 -0
- package/dist/data/async-chat-storage.js +96 -0
- package/dist/data/async-telemetry-maintenance.js +9 -0
- package/dist/data/bounded-worker-client.js +256 -0
- package/dist/data/chat-read-projections.js +159 -0
- package/dist/data/chat-read-worker.js +73 -0
- package/dist/data/chat-storage-worker.js +167 -0
- package/dist/data/ingest-service.js +152 -18
- package/dist/data/message-command-store.js +290 -0
- package/dist/data/payload-store.js +92 -11
- package/dist/data/pibo-store.js +10 -8
- package/dist/data/schema.js +39 -4
- package/dist/data/session-store.js +3 -1
- package/dist/data/storage-backup.js +278 -0
- package/dist/data/storage-maintenance.js +344 -0
- package/dist/data/storage-verification-worker.js +25 -0
- package/dist/data/telemetry-capture.js +188 -0
- package/dist/data/telemetry-command.js +3 -0
- package/dist/data/telemetry-maintenance-worker.js +40 -0
- package/dist/data/telemetry-maintenance.js +110 -0
- package/dist/data/telemetry-retention.js +16 -7
- package/dist/data/telemetry-worker.js +111 -0
- package/dist/data/telemetry-writer.js +150 -83
- package/dist/data/telemetry.js +5 -0
- package/dist/debug/index.js +207 -1
- package/dist/debug/message-queue.js +108 -0
- package/dist/debug/output-collision-repair.js +140 -0
- package/dist/debug/output-integrity.js +38 -2
- package/dist/debug/output-repair.js +1 -0
- package/dist/debug/storage-backup.js +41 -0
- package/dist/debug/storage-maintenance.js +78 -0
- package/dist/debug/telemetry-capture.js +66 -0
- package/dist/gateway/cli.js +71 -7
- package/dist/gateway/server.js +3 -0
- package/dist/providers/openai-gpt56.js +11 -6
- package/dist/reliability/store.js +119 -23
- package/dist/session-ui/terminalRows.js +54 -13
- package/dist/sessions/pibo-data-store.js +19 -14
- package/dist/shared/debug-features.js +4 -0
- package/dist/shared/model-inference-metrics.js +23 -0
- package/dist/shared/trace-event-projection.js +69 -2
- package/dist/shared/trace-history.js +9 -0
- package/dist/shared/trace-live-reducer.js +1 -0
- package/dist/shared/trace-patch-nodes.js +19 -0
- package/dist/web/channel.js +117 -12
- package/dist/web/http.js +135 -41
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/dist/apps/chat-ui/assets/index-BOceJ0jM.css +0 -1
- package/dist/apps/chat-ui/assets/index-BOemYq-V.js +0 -228
- package/dist/apps/chat-vscode-web/assets/index-CMwTB8o8.js +0 -43
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
export const TELEMETRY_MAINTENANCE_SCHEMA = `CREATE TABLE IF NOT EXISTS telemetry_maintenance_job (
|
|
2
|
+
id INTEGER PRIMARY KEY CHECK(id=1),retention_scope TEXT,cutoff TEXT NOT NULL,status TEXT NOT NULL,table_index INTEGER NOT NULL DEFAULT 0,class_index INTEGER NOT NULL DEFAULT 0,cursor_time TEXT NOT NULL DEFAULT '',cursor_row INTEGER NOT NULL DEFAULT 0,
|
|
3
|
+
scanned INTEGER NOT NULL DEFAULT 0,deleted INTEGER NOT NULL DEFAULT 0,protected INTEGER NOT NULL DEFAULT 0,batches INTEGER NOT NULL DEFAULT 0,updated_at TEXT NOT NULL
|
|
4
|
+
);`;
|
|
5
|
+
const CLASSES = ["live", "diagnostic", "provider_event", "payload_preview", "incident"];
|
|
6
|
+
/** A live owner updates the job on every bounded batch, so a stale stamp proves it stopped. */
|
|
7
|
+
export const TELEMETRY_MAINTENANCE_OWNER_LEASE_MS = 30_000;
|
|
8
|
+
const TABLES = [
|
|
9
|
+
{ name: "telemetry_provider_events", time: "received_at", status: false },
|
|
10
|
+
{ name: "telemetry_tool_calls", time: "updated_at", status: true },
|
|
11
|
+
{ name: "telemetry_provider_requests", time: "updated_at", status: true },
|
|
12
|
+
{ name: "telemetry_phases", time: "updated_at", status: true },
|
|
13
|
+
{ name: "telemetry_turns", time: "updated_at", status: true },
|
|
14
|
+
];
|
|
15
|
+
/** Only optional telemetry rows are eligible. No product, receipt, retry or payload deletion. */
|
|
16
|
+
export class TelemetryMaintenance {
|
|
17
|
+
db;
|
|
18
|
+
constructor(db) {
|
|
19
|
+
this.db = db;
|
|
20
|
+
}
|
|
21
|
+
status() { return this.db.prepare("SELECT * FROM telemetry_maintenance_job WHERE id=1").get(); }
|
|
22
|
+
start(cutoff, retentionScope) {
|
|
23
|
+
if (retentionScope && !CLASSES.includes(retentionScope))
|
|
24
|
+
throw Error("Invalid maintenance retention scope");
|
|
25
|
+
if (!Number.isFinite(Date.parse(cutoff)))
|
|
26
|
+
throw Error("Invalid telemetry cutoff");
|
|
27
|
+
const existing = this.status();
|
|
28
|
+
if (existing && (existing.status === "running" || existing.status === "paused")) {
|
|
29
|
+
if (existing.retention_scope === (retentionScope ?? null))
|
|
30
|
+
return existing;
|
|
31
|
+
if (existing.status === "paused")
|
|
32
|
+
throw Error("Telemetry maintenance is paused; resume or cancel it explicitly before changing scope");
|
|
33
|
+
// A different scope only conflicts while its owner still steps the job. A persisted job whose
|
|
34
|
+
// owner stopped updating it is reclaimable, so one narrow manual prune that outlived its process
|
|
35
|
+
// cannot disable automatic retention permanently.
|
|
36
|
+
if (Date.now() - Date.parse(existing.updated_at) < TELEMETRY_MAINTENANCE_OWNER_LEASE_MS)
|
|
37
|
+
throw Error("Another retention scope is already running");
|
|
38
|
+
}
|
|
39
|
+
this.db.prepare(`INSERT INTO telemetry_maintenance_job(id,cutoff,status,updated_at,retention_scope,class_index) VALUES(1,?,'running',?,?,?)
|
|
40
|
+
ON CONFLICT(id) DO UPDATE SET cutoff=excluded.cutoff,status='running',table_index=0,retention_scope=excluded.retention_scope,class_index=excluded.class_index,cursor_time='',cursor_row=0,scanned=0,deleted=0,protected=0,batches=0,updated_at=excluded.updated_at`).run(cutoff, new Date().toISOString(), retentionScope ?? null, retentionScope ? CLASSES.indexOf(retentionScope) : 0);
|
|
41
|
+
return this.status();
|
|
42
|
+
}
|
|
43
|
+
control(action) {
|
|
44
|
+
const status = action === "pause" ? "paused" : action === "resume" ? "running" : "cancelled";
|
|
45
|
+
this.db.prepare("UPDATE telemetry_maintenance_job SET status=?,updated_at=? WHERE id=1 AND status IN ('running','paused')").run(status, new Date().toISOString());
|
|
46
|
+
return this.status();
|
|
47
|
+
}
|
|
48
|
+
step(options = {}) {
|
|
49
|
+
const limit = options.rows ?? 128, budget = options.milliseconds ?? 4;
|
|
50
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 512 || !Number.isFinite(budget) || budget <= 0 || budget > 20)
|
|
51
|
+
throw Error("Invalid maintenance batch budget");
|
|
52
|
+
if (this.status()?.status !== "running")
|
|
53
|
+
return this.status();
|
|
54
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
55
|
+
try {
|
|
56
|
+
const state = this.status();
|
|
57
|
+
if (state.status !== "running") {
|
|
58
|
+
this.db.exec("COMMIT");
|
|
59
|
+
return state;
|
|
60
|
+
}
|
|
61
|
+
const spec = TABLES[state.table_index];
|
|
62
|
+
const rows = this.db.prepare(`SELECT rowid AS cursor_row,${spec.time} AS cursor_time,pibo_session_id,turn_id ${spec.name !== "telemetry_turns" ? ",provider_request_id" : ""} ${spec.status ? ',status' : ''} FROM ${spec.name}
|
|
63
|
+
WHERE retention_class=? AND (${spec.time},rowid)>(?,?) AND ${spec.time}<? ORDER BY ${spec.time},rowid LIMIT ?`).all(CLASSES[state.class_index], state.cursor_time, state.cursor_row, state.cutoff, limit);
|
|
64
|
+
const activeSession = this.db.prepare("SELECT 1 FROM sessions WHERE id=? AND status='running'");
|
|
65
|
+
const activeTurn = this.db.prepare("SELECT 1 FROM telemetry_turns WHERE turn_id=? AND status IN ('queued','running')");
|
|
66
|
+
const activeProvider = this.db.prepare("SELECT 1 FROM telemetry_provider_requests p WHERE p.provider_request_id=? AND (p.status NOT IN ('completed','error','aborted','timeout') OR EXISTS (SELECT 1 FROM telemetry_turns t WHERE t.turn_id=p.turn_id AND t.status IN ('queued','running'))) ");
|
|
67
|
+
const remove = this.db.prepare(`DELETE FROM ${spec.name} WHERE rowid=? AND ${spec.time}=?`);
|
|
68
|
+
const started = performance.now();
|
|
69
|
+
let scanned = 0, deleted = 0, protectedRows = 0;
|
|
70
|
+
let cursorTime = state.cursor_time, cursorRow = state.cursor_row;
|
|
71
|
+
for (const row of rows) {
|
|
72
|
+
const terminal = !spec.status || ["ok", "completed", "error", "aborted", "timeout"].includes(row.status ?? "");
|
|
73
|
+
if (!terminal || (row.pibo_session_id && activeSession.get(row.pibo_session_id)) || (row.turn_id && activeTurn.get(row.turn_id)) || (row.provider_request_id && activeProvider.get(row.provider_request_id)))
|
|
74
|
+
protectedRows++;
|
|
75
|
+
else
|
|
76
|
+
deleted += Number(remove.run(row.cursor_row, row.cursor_time).changes);
|
|
77
|
+
scanned++;
|
|
78
|
+
cursorTime = row.cursor_time;
|
|
79
|
+
cursorRow = row.cursor_row;
|
|
80
|
+
if (performance.now() - started >= budget)
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
let tableIndex = state.table_index, classIndex = state.class_index;
|
|
84
|
+
let status = state.status;
|
|
85
|
+
if (scanned === rows.length && rows.length < limit) {
|
|
86
|
+
cursorTime = "";
|
|
87
|
+
cursorRow = 0;
|
|
88
|
+
if (state.retention_scope) {
|
|
89
|
+
tableIndex++;
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
classIndex++;
|
|
93
|
+
if (classIndex >= CLASSES.length) {
|
|
94
|
+
classIndex = 0;
|
|
95
|
+
tableIndex++;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if (tableIndex >= TABLES.length)
|
|
99
|
+
status = "completed";
|
|
100
|
+
}
|
|
101
|
+
this.db.prepare(`UPDATE telemetry_maintenance_job SET table_index=?,class_index=?,cursor_time=?,cursor_row=?,status=?,scanned=scanned+?,deleted=deleted+?,protected=protected+?,batches=batches+1,updated_at=? WHERE id=1`).run(tableIndex, classIndex, cursorTime, cursorRow, status, scanned, deleted, protectedRows, new Date().toISOString());
|
|
102
|
+
this.db.exec("COMMIT");
|
|
103
|
+
return this.status();
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
this.db.exec("ROLLBACK");
|
|
107
|
+
throw error;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { TelemetryMaintenance, TELEMETRY_MAINTENANCE_SCHEMA } from "./telemetry-maintenance.js";
|
|
1
2
|
const RETENTION_CLASSES = ["live", "diagnostic", "provider_event", "incident", "payload_preview"];
|
|
2
3
|
const PRUNE_TABLES = [
|
|
3
4
|
{ table: "telemetry_provider_events", cutoffColumn: "received_at", byteExpression: "byte_size" },
|
|
@@ -23,16 +24,24 @@ export function getTelemetryRetentionStats(db) {
|
|
|
23
24
|
};
|
|
24
25
|
}
|
|
25
26
|
export function pruneTelemetryRetention(db, input) {
|
|
26
|
-
const plan = telemetryPrunePlan(db, input.retentionClass, input.before);
|
|
27
27
|
if (!input.apply) {
|
|
28
|
+
const plan = telemetryPrunePlan(db, input.retentionClass, input.before);
|
|
28
29
|
return { retentionClass: input.retentionClass, before: input.before, applied: false, rowsMatched: plan.rows, bytesMatched: plan.bytes, rowsDeleted: 0 };
|
|
29
30
|
}
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
31
|
+
db.exec(TELEMETRY_MAINTENANCE_SCHEMA);
|
|
32
|
+
const maintenance = new TelemetryMaintenance(db);
|
|
33
|
+
let state = maintenance.start(input.before, input.retentionClass);
|
|
34
|
+
if (state.status === "paused")
|
|
35
|
+
state = maintenance.control("resume");
|
|
36
|
+
const previous = state.deleted, started = performance.now();
|
|
37
|
+
let scanned = 0;
|
|
38
|
+
do {
|
|
39
|
+
const before = state.scanned;
|
|
40
|
+
state = maintenance.step({ rows: Math.min(128 - scanned, 128), milliseconds: 4 });
|
|
41
|
+
scanned += state.scanned - before;
|
|
42
|
+
} while (state.status === "running" && scanned < 128 && performance.now() - started < 20);
|
|
43
|
+
const rowsDeleted = state.deleted - previous;
|
|
44
|
+
return { retentionClass: input.retentionClass, before: state.cutoff, applied: true, rowsMatched: scanned, bytesMatched: 0, rowsDeleted, completed: state.status === "completed" };
|
|
36
45
|
}
|
|
37
46
|
function statsForTable(db, table, sqlTable, byteExpression, retentionClass) {
|
|
38
47
|
const row = db.prepare(`SELECT COUNT(*) AS row_count, COALESCE(SUM(${byteExpression}), 0) AS byte_count FROM ${sqlTable} WHERE retention_class = ?`).get(retentionClass);
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { dirname, join } from "node:path";
|
|
2
|
+
import { TelemetryCaptureWriter } from "./telemetry-capture.js";
|
|
3
|
+
import { statSync } from "node:fs";
|
|
4
|
+
import { parentPort, workerData, threadId } from "node:worker_threads";
|
|
5
|
+
import { DatabaseSync } from "node:sqlite";
|
|
6
|
+
import { TelemetryStore } from "./telemetry.js";
|
|
7
|
+
import { PiboRuntimeTelemetryRecorder } from "../core/runtime-telemetry.js";
|
|
8
|
+
import { PiboProviderTelemetryRecorder } from "../core/provider-telemetry.js";
|
|
9
|
+
const db = new DatabaseSync(workerData.path);
|
|
10
|
+
db.exec("PRAGMA busy_timeout=10; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON");
|
|
11
|
+
const store = new TelemetryStore(db);
|
|
12
|
+
const capture = new TelemetryCaptureWriter(join(dirname(workerData.path), "telemetry-captures"));
|
|
13
|
+
const captureTimer = setInterval(() => capture.append([]), 1000);
|
|
14
|
+
captureTimer.unref();
|
|
15
|
+
const measured = Boolean(workerData.measure);
|
|
16
|
+
let currentKind = "startup";
|
|
17
|
+
const measurements = {};
|
|
18
|
+
if (measured) {
|
|
19
|
+
const prepare = db.prepare.bind(db);
|
|
20
|
+
db.prepare = (sql) => {
|
|
21
|
+
const statement = prepare(sql);
|
|
22
|
+
for (const method of ["run", "get", "all"]) {
|
|
23
|
+
const original = statement[method].bind(statement);
|
|
24
|
+
statement[method] = (...args) => {
|
|
25
|
+
const kind = measurements[currentKind];
|
|
26
|
+
const verb = sql.trim().split(/\s/, 1)[0].toLowerCase();
|
|
27
|
+
if (kind) {
|
|
28
|
+
const label = ["insert", "update", "delete", "select"].includes(verb) ? verb : "other";
|
|
29
|
+
kind.sql[label] = (kind.sql[label] ?? 0) + 1;
|
|
30
|
+
}
|
|
31
|
+
return original(...args);
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
return statement;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
let errors = 0;
|
|
38
|
+
let transactions = 0;
|
|
39
|
+
let operations = 0;
|
|
40
|
+
let busy = 0;
|
|
41
|
+
const runtimes = new Map();
|
|
42
|
+
function runtimeFor(input) {
|
|
43
|
+
const key = `${input.providerEventMode}:${input.progressFlushIntervalMs}`;
|
|
44
|
+
let runtime = runtimes.get(key);
|
|
45
|
+
if (!runtime) {
|
|
46
|
+
if (runtimes.size >= 4)
|
|
47
|
+
runtimes.delete(runtimes.keys().next().value);
|
|
48
|
+
runtime = new PiboRuntimeTelemetryRecorder(store, () => { errors++; }, { providerEventMode: input.providerEventMode, progressFlushIntervalMs: input.progressFlushIntervalMs });
|
|
49
|
+
runtimes.set(key, runtime);
|
|
50
|
+
}
|
|
51
|
+
return runtime;
|
|
52
|
+
}
|
|
53
|
+
parentPort.on("message", (request) => {
|
|
54
|
+
const started = performance.now();
|
|
55
|
+
let processed = 0;
|
|
56
|
+
const initialErrors = errors;
|
|
57
|
+
try {
|
|
58
|
+
if (request.command.commands.length > 64)
|
|
59
|
+
throw Error("Telemetry batch count exceeded");
|
|
60
|
+
if (started > request.deadline)
|
|
61
|
+
throw Error("Telemetry batch expired");
|
|
62
|
+
try {
|
|
63
|
+
store.transaction(() => {
|
|
64
|
+
for (const input of request.command.commands) {
|
|
65
|
+
if (processed && performance.now() - started >= 8)
|
|
66
|
+
break;
|
|
67
|
+
currentKind = input.recorder === "runtime" && input.command.kind === "output" ? input.command.event.type : `${input.recorder}.${input.command.kind}`;
|
|
68
|
+
if (measured && !measurements[currentKind]) {
|
|
69
|
+
if (Object.keys(measurements).length >= 32)
|
|
70
|
+
currentKind = "other";
|
|
71
|
+
measurements[currentKind] ??= { operations: 0, sql: {}, executionMs: 0 };
|
|
72
|
+
}
|
|
73
|
+
const operationStart = performance.now();
|
|
74
|
+
if (input.recorder === "runtime")
|
|
75
|
+
runtimeFor(input).executeTelemetryCommand(input.command);
|
|
76
|
+
else
|
|
77
|
+
new PiboProviderTelemetryRecorder({ store, session: input.session, model: input.model, onError: () => { errors++; } }).executeTelemetryCommand(input.command);
|
|
78
|
+
if (measured) {
|
|
79
|
+
measurements[currentKind].operations++;
|
|
80
|
+
measurements[currentKind].executionMs += performance.now() - operationStart;
|
|
81
|
+
}
|
|
82
|
+
processed++;
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
transactions++;
|
|
86
|
+
operations += processed;
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
runtimes.clear();
|
|
90
|
+
if (error instanceof Error && /busy|locked/i.test(error.message)) {
|
|
91
|
+
busy++;
|
|
92
|
+
processed = 0;
|
|
93
|
+
}
|
|
94
|
+
else
|
|
95
|
+
throw error;
|
|
96
|
+
}
|
|
97
|
+
if (processed)
|
|
98
|
+
capture.append(request.command.commands.slice(0, processed));
|
|
99
|
+
const result = { processed, errors: Math.min(processed, errors - initialErrors), ms: performance.now() - started, stats: { pid: process.pid, threadId, capture: capture.status(), transactions, operations, busy, rssBytes: process.memoryUsage.rss(), heapUsedBytes: process.memoryUsage().heapUsed, synchronous: "FULL", ...(measured ? { measurements, walBytes: (() => { try {
|
|
100
|
+
return statSync(workerData.path + "-wal").size;
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return 0;
|
|
104
|
+
} })() } : {}) } };
|
|
105
|
+
parentPort.postMessage({ id: request.id, value: result });
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
parentPort.postMessage({ id: request.id, error: { code: "telemetry_failed", message: "Optional telemetry batch failed" } });
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
parentPort.postMessage({ ready: true, worker: { pid: process.pid, threadId, kind: "telemetry" } });
|
|
@@ -1,114 +1,181 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Gateway-scoped, ordered telemetry writer.
|
|
5
|
-
*
|
|
6
|
-
* Normal writes are deferred briefly so telemetry from multiple routed sessions
|
|
7
|
-
* shares one SQLite transaction. The queue never drops lifecycle events: when
|
|
8
|
-
* the hard bound is reached, it drains immediately in the caller instead.
|
|
9
|
-
*/
|
|
1
|
+
import { BoundedWorkerClient, boundedMessageBytes } from "./bounded-worker-client.js";
|
|
2
|
+
import { isTelemetryProgress } from "./telemetry-command.js";
|
|
3
|
+
/** Optional diagnostic projection. Product lifecycle/final events remain owned by the durable output outbox. */
|
|
10
4
|
export class AsyncTelemetryWriter {
|
|
11
5
|
store;
|
|
12
6
|
options;
|
|
13
|
-
|
|
14
|
-
|
|
7
|
+
client;
|
|
8
|
+
path;
|
|
9
|
+
restartAfter = 0;
|
|
10
|
+
restarts = 0;
|
|
15
11
|
pending = [];
|
|
12
|
+
pendingBytes = 0;
|
|
13
|
+
inFlight = 0;
|
|
16
14
|
flushTimer;
|
|
17
|
-
flushing
|
|
15
|
+
flushing;
|
|
18
16
|
closed = false;
|
|
17
|
+
limit;
|
|
18
|
+
byteLimit;
|
|
19
|
+
ageLimit;
|
|
20
|
+
interval;
|
|
21
|
+
accepted = 0;
|
|
22
|
+
completed = 0;
|
|
23
|
+
rejected = 0;
|
|
24
|
+
failed = 0;
|
|
25
|
+
expired = 0;
|
|
26
|
+
batches = 0;
|
|
27
|
+
maxBatchMs = 0;
|
|
28
|
+
workerStats;
|
|
19
29
|
constructor(store, options = {}) {
|
|
20
30
|
this.store = store;
|
|
21
31
|
this.options = options;
|
|
22
|
-
this.
|
|
23
|
-
this.
|
|
32
|
+
this.limit = options.maxPendingOperations ?? 1024;
|
|
33
|
+
this.byteLimit = options.maxPendingBytes ?? 4 * 1024 * 1024;
|
|
34
|
+
this.ageLimit = options.maxAgeMs ?? 2000;
|
|
35
|
+
this.interval = options.flushIntervalMs ?? 25;
|
|
36
|
+
for (const n of [this.limit, this.byteLimit, this.ageLimit])
|
|
37
|
+
if (!Number.isSafeInteger(n) || n <= 0)
|
|
38
|
+
throw Error("Telemetry budgets must be positive integers");
|
|
39
|
+
if (!Number.isFinite(this.interval) || this.interval < 0)
|
|
40
|
+
throw Error("Invalid telemetry batching interval");
|
|
41
|
+
this.path = store.databasePath;
|
|
42
|
+
if (this.path)
|
|
43
|
+
this.startWorker();
|
|
24
44
|
}
|
|
45
|
+
startWorker() {
|
|
46
|
+
this.restartAfter = Date.now() + 1000;
|
|
47
|
+
this.client = new BoundedWorkerClient(new URL("./telemetry-worker.js", import.meta.url), { maxPending: 1, maxPendingBytes: 512 * 1024, maxMessageBytes: 512 * 1024, maxAgeMs: this.ageLimit, workerOptions: { workerData: { path: this.path, measure: this.options.measure === true } } });
|
|
48
|
+
}
|
|
49
|
+
/** Compatibility for the local in-memory adapter only; closures never execute on a file-backed producer. */
|
|
25
50
|
enqueue(write, onError) {
|
|
26
|
-
if (this.
|
|
27
|
-
this.
|
|
51
|
+
if (this.client) {
|
|
52
|
+
this.reject(Error("File telemetry requires a structured command"), onError);
|
|
28
53
|
return false;
|
|
29
54
|
}
|
|
30
|
-
this.
|
|
31
|
-
|
|
32
|
-
|
|
55
|
+
return this.append({ write, onError, bytes: 64, at: performance.now() }, false);
|
|
56
|
+
}
|
|
57
|
+
record(command, fallback, onError) {
|
|
58
|
+
try {
|
|
59
|
+
const bytes = boundedMessageBytes(command, 64 * 1024);
|
|
60
|
+
return this.append({ ...(this.client ? { command: structuredClone(command) } : { write: fallback }), bytes, onError, at: performance.now() }, isTelemetryProgress(command));
|
|
33
61
|
}
|
|
34
|
-
|
|
35
|
-
this.
|
|
62
|
+
catch (error) {
|
|
63
|
+
this.reject(error, onError);
|
|
64
|
+
return false;
|
|
36
65
|
}
|
|
37
|
-
return true;
|
|
38
66
|
}
|
|
39
|
-
|
|
40
|
-
this.
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
return;
|
|
45
|
-
|
|
46
|
-
|
|
67
|
+
append(item, progress) {
|
|
68
|
+
const reserve = progress ? Math.min(128, Math.floor(this.limit / 4)) : 0;
|
|
69
|
+
const reserveBytes = progress ? Math.min(512 * 1024, Math.floor(this.byteLimit / 4)) : 0;
|
|
70
|
+
if (this.closed || this.pending.length + this.inFlight >= this.limit - reserve || this.pendingBytes + item.bytes > this.byteLimit - reserveBytes) {
|
|
71
|
+
this.reject(Error(this.closed ? "Telemetry writer is closed" : "Optional telemetry capacity exhausted"), item.onError);
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
item.sequence = ++this.accepted;
|
|
75
|
+
this.pending.push(item);
|
|
76
|
+
this.pendingBytes += item.bytes;
|
|
77
|
+
// Pressure only schedules asynchronous work; it never drains SQLite in the producer.
|
|
78
|
+
this.schedule(this.pending.length >= 64 ? 0 : this.interval);
|
|
79
|
+
return true;
|
|
47
80
|
}
|
|
48
|
-
|
|
49
|
-
if (this.flushTimer)
|
|
81
|
+
schedule(delay) {
|
|
82
|
+
if (this.flushTimer || this.flushing || this.closed)
|
|
50
83
|
return;
|
|
51
|
-
this.flushTimer = setTimeout(() => {
|
|
52
|
-
|
|
53
|
-
this.flushNow();
|
|
54
|
-
}, this.flushIntervalMs);
|
|
55
|
-
this.flushTimer.unref();
|
|
84
|
+
this.flushTimer = setTimeout(() => { this.flushTimer = undefined; void this.flush(); }, delay);
|
|
85
|
+
this.flushTimer.unref?.();
|
|
56
86
|
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
return;
|
|
87
|
+
async flush() {
|
|
88
|
+
const target = this.accepted;
|
|
60
89
|
if (this.flushTimer)
|
|
61
90
|
clearTimeout(this.flushTimer);
|
|
62
91
|
this.flushTimer = undefined;
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
92
|
+
for (;;) {
|
|
93
|
+
if (this.flushing) {
|
|
94
|
+
await this.flushing;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if (!this.pending.length || this.pending[0].sequence > target)
|
|
98
|
+
return;
|
|
99
|
+
this.flushing = this.drain(target).finally(() => { this.flushing = undefined; if (this.pending.length && !this.closed)
|
|
100
|
+
this.schedule(0); });
|
|
101
|
+
await this.flushing;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
async drain(target) {
|
|
105
|
+
while (this.pending.length && this.pending[0].sequence <= target) {
|
|
106
|
+
const batch = [];
|
|
107
|
+
let bytes = 0;
|
|
108
|
+
while (this.pending.length && this.pending[0].sequence <= target && batch.length < 64 && bytes + this.pending[0].bytes <= 256 * 1024) {
|
|
109
|
+
const item = this.pending.shift();
|
|
110
|
+
bytes += item.bytes;
|
|
111
|
+
batch.push(item);
|
|
112
|
+
}
|
|
113
|
+
this.inFlight = batch.length;
|
|
114
|
+
const live = batch.filter(item => { if (performance.now() - item.at <= this.ageLimit)
|
|
115
|
+
return true; this.expired++; return false; });
|
|
116
|
+
let processed = live.length;
|
|
117
|
+
try {
|
|
118
|
+
if (live.length && this.client) {
|
|
119
|
+
if (this.client.status().closed && this.client.status().exited && Date.now() >= this.restartAfter) {
|
|
120
|
+
this.restarts++;
|
|
121
|
+
this.startWorker();
|
|
122
|
+
}
|
|
123
|
+
// Startup consumes the same bounded queue age, without loading a second schema owner on the gateway thread.
|
|
124
|
+
while (!this.client.status().ready && !this.client.status().closed && performance.now() - live[0].at < this.ageLimit)
|
|
125
|
+
await new Promise(resolve => setTimeout(resolve, 5));
|
|
126
|
+
const remainingAge = this.ageLimit - (performance.now() - live[0].at);
|
|
127
|
+
if (remainingAge < Math.min(50, this.ageLimit / 4)) {
|
|
128
|
+
this.expired += live.length;
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
const result = await this.client.request({ commands: live.map(item => item.command) }, { priority: "background", timeoutMs: remainingAge });
|
|
132
|
+
processed = result.processed;
|
|
133
|
+
this.failed += result.errors;
|
|
134
|
+
this.completed += processed - result.errors;
|
|
135
|
+
this.maxBatchMs = Math.max(this.maxBatchMs, result.ms);
|
|
136
|
+
this.workerStats = result.stats;
|
|
137
|
+
this.batches++;
|
|
138
|
+
}
|
|
79
139
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
140
|
+
else if (live.length) {
|
|
141
|
+
// Explicit in-memory test adapter uses the same bounded batches and asynchronous entry boundary.
|
|
142
|
+
await new Promise(resolve => setImmediate(resolve));
|
|
143
|
+
this.store.transaction(() => { for (const item of live) {
|
|
144
|
+
try {
|
|
145
|
+
item.write?.();
|
|
146
|
+
this.completed++;
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
this.failed++;
|
|
150
|
+
this.report(error, item.onError);
|
|
151
|
+
}
|
|
152
|
+
} });
|
|
153
|
+
this.batches++;
|
|
83
154
|
}
|
|
84
155
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
156
|
+
catch (error) {
|
|
157
|
+
this.failed += live.length;
|
|
158
|
+
for (const item of live)
|
|
159
|
+
this.report(error, item.onError);
|
|
160
|
+
}
|
|
161
|
+
const remaining = live.slice(processed);
|
|
162
|
+
this.pending.unshift(...remaining);
|
|
163
|
+
this.pendingBytes -= bytes - remaining.reduce((sum, item) => sum + item.bytes, 0);
|
|
164
|
+
this.inFlight = 0;
|
|
165
|
+
if (remaining.length)
|
|
166
|
+
await new Promise(resolve => setTimeout(resolve, 10));
|
|
90
167
|
}
|
|
91
168
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
return;
|
|
169
|
+
status() { return { mode: this.client ? "worker" : "in-memory", closed: this.closed, queued: this.pending.length, inFlight: this.inFlight, pendingBytes: this.pendingBytes, oldestAgeMs: this.pending.length ? performance.now() - this.pending[0].at : 0, accepted: this.accepted, restarts: this.restarts, completed: this.completed, rejected: this.rejected, failed: this.failed, expired: this.expired, batches: this.batches, maxBatchMs: this.maxBatchMs, worker: this.workerStats, transport: this.client ? { ready: this.client.status().ready, closed: this.client.status().closed, exited: this.client.status().exited } : undefined, limits: { count: this.limit, bytes: this.byteLimit, ageMs: this.ageLimit, batchCount: 64, batchBytes: 256 * 1024, batchTimeMs: 8 } }; }
|
|
170
|
+
async dispose() { if (this.closed)
|
|
171
|
+
return; this.closed = true; await this.flush(); await this.client?.close(); }
|
|
172
|
+
reject(error, handler) { this.rejected++; this.report(error, handler); }
|
|
173
|
+
report(error, handler) { try {
|
|
174
|
+
handler?.(error);
|
|
175
|
+
}
|
|
176
|
+
catch { } if (handler !== this.options.onError)
|
|
101
177
|
try {
|
|
102
178
|
this.options.onError?.(error);
|
|
103
179
|
}
|
|
104
|
-
catch {
|
|
105
|
-
// Telemetry error reporting must not affect runtime work.
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
function nonNegativeFinite(value, fallback) {
|
|
110
|
-
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
111
|
-
}
|
|
112
|
-
function positiveInteger(value, fallback) {
|
|
113
|
-
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : fallback;
|
|
180
|
+
catch { } }
|
|
114
181
|
}
|
package/dist/data/telemetry.js
CHANGED
|
@@ -8,6 +8,11 @@ export class TelemetryStore {
|
|
|
8
8
|
constructor(db) {
|
|
9
9
|
this.db = db;
|
|
10
10
|
}
|
|
11
|
+
/** SQLite-owned path; empty for the explicitly local in-memory test adapter. */
|
|
12
|
+
get databasePath() {
|
|
13
|
+
const row = this.db.prepare("PRAGMA database_list").all().find(row => row.name === "main");
|
|
14
|
+
return typeof row?.file === "string" && row.file ? row.file : undefined;
|
|
15
|
+
}
|
|
11
16
|
transaction(action) {
|
|
12
17
|
if (this.db.isTransaction)
|
|
13
18
|
return action();
|