@igorjpimenta/manifold 0.0.0-bootstrap.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.migrations/ops/0000_futuristic_karma.sql +111 -0
- package/.migrations/ops/0001_low_maximus.sql +7 -0
- package/.migrations/ops/meta/0000_snapshot.json +763 -0
- package/.migrations/ops/meta/0001_snapshot.json +173 -0
- package/.migrations/ops/meta/_journal.json +20 -0
- package/.migrations/runtime/0000_real_boomer.sql +180 -0
- package/.migrations/runtime/0001_hot_dakota_north.sql +2 -0
- package/.migrations/runtime/0002_breezy_vertigo.sql +8 -0
- package/.migrations/runtime/0003_dashing_jack_flag.sql +5 -0
- package/.migrations/runtime/0004_panoramic_ego.sql +1 -0
- package/.migrations/runtime/0005_tidy_ben_grimm.sql +10 -0
- package/.migrations/runtime/0006_wet_nightmare.sql +57 -0
- package/.migrations/runtime/0007_ambiguous_chronomancer.sql +8 -0
- package/.migrations/runtime/0008_wealthy_thundra.sql +301 -0
- package/.migrations/runtime/0009_lying_shocker.sql +96 -0
- package/.migrations/runtime/0010_solid_phantom_reporter.sql +160 -0
- package/.migrations/runtime/0011_windy_demogoblin.sql +175 -0
- package/.migrations/runtime/0012_restore_harness_fks.sql +103 -0
- package/.migrations/runtime/meta/0000_snapshot.json +1181 -0
- package/.migrations/runtime/meta/0001_snapshot.json +1209 -0
- package/.migrations/runtime/meta/0002_snapshot.json +1261 -0
- package/.migrations/runtime/meta/0003_snapshot.json +1292 -0
- package/.migrations/runtime/meta/0004_snapshot.json +1240 -0
- package/.migrations/runtime/meta/0006_snapshot.json +1310 -0
- package/.migrations/runtime/meta/0007_snapshot.json +1362 -0
- package/.migrations/runtime/meta/0008_snapshot.json +1742 -0
- package/.migrations/runtime/meta/0009_snapshot.json +2413 -0
- package/.migrations/runtime/meta/0010_snapshot.json +2562 -0
- package/.migrations/runtime/meta/0011_snapshot.json +2812 -0
- package/.migrations/runtime/meta/0012_snapshot.json +2812 -0
- package/.migrations/runtime/meta/_journal.json +97 -0
- package/README.md +1584 -0
- package/RELEASING.md +89 -0
- package/agent/fleet.js +100 -0
- package/bin/cli.js +854 -0
- package/bin/manifold.js +4 -0
- package/boot/flow-revival.js +536 -0
- package/budget/index.js +495 -0
- package/budget/lib.js +59 -0
- package/dashboard/cost.js +528 -0
- package/dashboard/dist/assets/index-CUfp63sj.css +1 -0
- package/dashboard/dist/assets/index-Qy9JMpma.js +371 -0
- package/dashboard/dist/index.html +13 -0
- package/dashboard/event-log.js +267 -0
- package/dashboard/index.html +12 -0
- package/dashboard/package-lock.json +2877 -0
- package/dashboard/package.json +37 -0
- package/dashboard/pipeline-graph.js +131 -0
- package/dashboard/postcss.config.js +6 -0
- package/dashboard/search.js +183 -0
- package/dashboard/server.js +1015 -0
- package/dashboard/src/App.tsx +42 -0
- package/dashboard/src/api/client.ts +118 -0
- package/dashboard/src/api/types.ts +510 -0
- package/dashboard/src/api/ws.tsx +112 -0
- package/dashboard/src/components/CancelButton.tsx +63 -0
- package/dashboard/src/components/ClickToCopy.tsx +68 -0
- package/dashboard/src/components/CopyButton.tsx +61 -0
- package/dashboard/src/components/DailyCostTrend.tsx +85 -0
- package/dashboard/src/components/DailyTimeTrend.tsx +94 -0
- package/dashboard/src/components/Funnel.tsx +149 -0
- package/dashboard/src/components/GateForm.tsx +158 -0
- package/dashboard/src/components/GraphView.tsx +302 -0
- package/dashboard/src/components/GroupPathsSummary.tsx +47 -0
- package/dashboard/src/components/Histogram.tsx +292 -0
- package/dashboard/src/components/Layout.tsx +28 -0
- package/dashboard/src/components/NeedsAttention.tsx +98 -0
- package/dashboard/src/components/NodeContextCard.tsx +132 -0
- package/dashboard/src/components/PathTrail.tsx +35 -0
- package/dashboard/src/components/PipelineCostBreakdown.tsx +277 -0
- package/dashboard/src/components/RerunButton.tsx +46 -0
- package/dashboard/src/components/RunComparison.tsx +53 -0
- package/dashboard/src/components/Sparkline.tsx +94 -0
- package/dashboard/src/components/StatStrip.tsx +147 -0
- package/dashboard/src/components/StateTree.tsx +84 -0
- package/dashboard/src/components/StatusBadge.tsx +18 -0
- package/dashboard/src/components/SubflowPathPills.tsx +38 -0
- package/dashboard/src/components/SubflowSummary.tsx +24 -0
- package/dashboard/src/components/SubflowsCard.tsx +327 -0
- package/dashboard/src/components/TodayStrip.tsx +86 -0
- package/dashboard/src/components/TranscriptPane.tsx +99 -0
- package/dashboard/src/components/TranscriptRow.tsx +69 -0
- package/dashboard/src/components/ui/badge.tsx +38 -0
- package/dashboard/src/components/ui/button.tsx +28 -0
- package/dashboard/src/components/ui/card.tsx +42 -0
- package/dashboard/src/components/ui/skeleton.tsx +9 -0
- package/dashboard/src/components/ui/table.tsx +62 -0
- package/dashboard/src/index.css +26 -0
- package/dashboard/src/lib/anomaly.ts +42 -0
- package/dashboard/src/lib/format.ts +30 -0
- package/dashboard/src/lib/step-history.ts +13 -0
- package/dashboard/src/lib/subflow.ts +154 -0
- package/dashboard/src/lib/use-click-outside.ts +28 -0
- package/dashboard/src/lib/use-ignored-clusters.ts +67 -0
- package/dashboard/src/lib/utils.ts +6 -0
- package/dashboard/src/main.tsx +10 -0
- package/dashboard/src/types/shims.d.ts +4 -0
- package/dashboard/src/views/Home.tsx +478 -0
- package/dashboard/src/views/PipelineDetail.tsx +1185 -0
- package/dashboard/src/views/PipelinesList.tsx +111 -0
- package/dashboard/src/views/RunDetail.tsx +248 -0
- package/dashboard/src/views/Search.tsx +196 -0
- package/dashboard/stats.js +668 -0
- package/dashboard/tailwind.config.ts +43 -0
- package/dashboard/tree-stats.js +12 -0
- package/dashboard/tsconfig.json +25 -0
- package/dashboard/views.js +231 -0
- package/dashboard/vite.config.ts +34 -0
- package/drizzle.config.js +12 -0
- package/drizzle.config.ops.js +12 -0
- package/elevation.js +116 -0
- package/engine/admission-promotion-resume.js +127 -0
- package/engine/agent-dispatch.js +64 -0
- package/engine/child-resume-trigger.js +315 -0
- package/engine/executor-budget.js +87 -0
- package/engine/executor-cancel.js +83 -0
- package/engine/executor-errors.js +122 -0
- package/engine/executor-helpers.js +254 -0
- package/engine/executor-nodes.js +292 -0
- package/engine/executor-persist.js +212 -0
- package/engine/executor-subflow-children.js +275 -0
- package/engine/executor.js +1013 -0
- package/engine/global-concurrency.js +33 -0
- package/engine/loader.js +528 -0
- package/engine/needs-resolver.js +229 -0
- package/engine/pipelines-watcher.js +133 -0
- package/engine/result-replay.js +34 -0
- package/engine/revival-messages.js +11 -0
- package/engine/spawn-child.js +55 -0
- package/engine/wait-for-flow-settlement.js +109 -0
- package/events/catalog.js +26 -0
- package/events/emit.js +44 -0
- package/events/kinds.js +118 -0
- package/events/serve-ws.js +74 -0
- package/events/stream.js +35 -0
- package/events/ws-dialog.js +257 -0
- package/events/ws-frame.js +194 -0
- package/events/ws-run-events.js +70 -0
- package/events.js +32 -0
- package/gates/gate-resolver.js +576 -0
- package/gates/paused-flows.js +55 -0
- package/graphql/columns.js +67 -0
- package/graphql/schema.js +333 -0
- package/index.js +49 -0
- package/mcp/http.js +159 -0
- package/mcp/identity.js +94 -0
- package/mcp/stdio.js +80 -0
- package/mcp/tools.js +245 -0
- package/mcp/wire-ops.js +178 -0
- package/package.json +51 -0
- package/routes/bearer-auth.js +57 -0
- package/routes/discuss-audio.js +87 -0
- package/routes/discuss-component.js +39 -0
- package/routes/discuss-draft.js +159 -0
- package/routes/discuss-helpers.js +20 -0
- package/routes/discuss.js +848 -0
- package/routes/failures.js +104 -0
- package/routes/graphql.js +61 -0
- package/routes/health.js +54 -0
- package/routes/operator-discuss.js +193 -0
- package/routes/operator-resume.js +32 -0
- package/routes/operator-retry.js +301 -0
- package/routes/operator.js +539 -0
- package/routes/pipelines.js +118 -0
- package/routes/runs.js +1129 -0
- package/scripts/ci/import-check.mjs +50 -0
- package/scripts/link-core-self.mjs +31 -0
- package/scripts/with-guard.sh +24 -0
- package/server/app.js +503 -0
- package/server/registry.js +53 -0
- package/server/serve.js +163 -0
- package/state/admission-queue-store.js +325 -0
- package/state/attribution-registries.js +102 -0
- package/state/batch-writer.js +162 -0
- package/state/db-guard.js +63 -0
- package/state/db-health.js +27 -0
- package/state/db.js +122 -0
- package/state/discuss-store.js +299 -0
- package/state/failure-store.js +427 -0
- package/state/migrate.js +104 -0
- package/state/node-cost.js +67 -0
- package/state/node-executions.js +775 -0
- package/state/node-health-store.js +124 -0
- package/state/ops-merge.js +93 -0
- package/state/paths.js +146 -0
- package/state/pipeline-dirs.js +99 -0
- package/state/pipeline-registry.js +449 -0
- package/state/run-store.js +697 -0
- package/state/schema.js +438 -0
- package/state/schema.ops.js +44 -0
- package/state/session-index.js +177 -0
- package/state/session-keys.js +39 -0
- package/state/state-store.js +311 -0
- package/state/subflow-state.js +126 -0
- package/state/tool-handoff.js +169 -0
- package/telemetry/budget-alert.js +353 -0
- package/telemetry/cli.js +331 -0
- package/telemetry/grammar.js +119 -0
- package/telemetry/http-budget.js +132 -0
- package/telemetry/http-cost.js +670 -0
- package/telemetry/http-helpers.js +134 -0
- package/telemetry/http-tree.js +349 -0
- package/telemetry/http.js +44 -0
- package/telemetry/index.js +307 -0
- package/telemetry/ingest.js +134 -0
- package/telemetry/labels.js +115 -0
- package/telemetry/pricer.js +77 -0
- package/telemetry/resolver.js +314 -0
- package/telemetry/store.js +147 -0
- package/utils/run-patch.js +43 -0
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
// Daily-budget guard. Called from the before_message_write hook after each
|
|
2
|
+
// cost insert. When today's total exceeds CAP_USD, writes the
|
|
3
|
+
// budget_alert_events row and emits a manifold.budget.alert event for a
|
|
4
|
+
// consumer to deliver (once per day per threshold — debounced via
|
|
5
|
+
// lastAlertedLocalDay). Delivery is not manifold's job.
|
|
6
|
+
|
|
7
|
+
import fs from "node:fs";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import os from "node:os";
|
|
10
|
+
import { randomUUID } from "node:crypto";
|
|
11
|
+
import { displayRootLabel } from "./labels.js";
|
|
12
|
+
import { execGet } from "./http-helpers.js";
|
|
13
|
+
import { getSharedDb, withSharedWrite } from "../state/db.js";
|
|
14
|
+
import { migrateDatabase, OPS_MIGRATIONS_FOLDER } from "../state/migrate.js";
|
|
15
|
+
|
|
16
|
+
const DEFAULT_CAP_USD = 3.0;
|
|
17
|
+
const DEFAULT_LOCK_THRESHOLD = 0.8; // fraction of cap; operator-configurable via dashboard
|
|
18
|
+
|
|
19
|
+
// Thresholds — fraction of cap. Each fires at most once per local day.
|
|
20
|
+
// 50% lets the operator intervene before damage is done; 80% is the
|
|
21
|
+
// "approaching" warning; 100% means we've crossed the line.
|
|
22
|
+
const THRESHOLDS = [0.1, 0.5, 0.8, 1.0];
|
|
23
|
+
|
|
24
|
+
// Persisted across gateway restarts so we don't re-alert on every reload of
|
|
25
|
+
// the same already-over-budget day. The once-per-day-per-threshold debounce
|
|
26
|
+
// is the only silencing mechanism — no operator-facing snooze (removed:
|
|
27
|
+
// redundant with the debounce, no clear product job).
|
|
28
|
+
const STATE_FILE = path.join(os.homedir(), ".openclaw", "lineage", ".alerted-thresholds.json");
|
|
29
|
+
// Same physical file budget/index.js's gate_state lives in — each module
|
|
30
|
+
// owns its own tables there.
|
|
31
|
+
const DB_PATH = path.join(os.homedir(), ".openclaw", "ops.sqlite");
|
|
32
|
+
|
|
33
|
+
let state = readState(); // { day: "YYYY-MM-DD", fired: [0.5, 0.8, 1.0] }
|
|
34
|
+
let lastObservedTotal = 0;
|
|
35
|
+
|
|
36
|
+
// In-memory cache — read on the hot checkBudget path with zero I/O.
|
|
37
|
+
// Populated by initCapStore() at service start; updated optimistically
|
|
38
|
+
// (before the DB write lands) by the setters below.
|
|
39
|
+
let configuredCapUSD = DEFAULT_CAP_USD;
|
|
40
|
+
let configuredLockThreshold = DEFAULT_LOCK_THRESHOLD;
|
|
41
|
+
|
|
42
|
+
let _dbPath = DB_PATH;
|
|
43
|
+
let _client = null;
|
|
44
|
+
async function _getClient() {
|
|
45
|
+
if (!_client) {
|
|
46
|
+
await migrateDatabase({ dbPath: _dbPath, migrationsFolder: OPS_MIGRATIONS_FOLDER });
|
|
47
|
+
const { client } = await getSharedDb(_dbPath);
|
|
48
|
+
_client = client;
|
|
49
|
+
}
|
|
50
|
+
return _client;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Loads the active cap_versions row into the in-memory cache, seeding one
|
|
55
|
+
* row from the hardcoded defaults on a bare table. Must complete before
|
|
56
|
+
* checkBudget()/the HTTP cap routes see traffic; registered as its own
|
|
57
|
+
* api.registerService start hook.
|
|
58
|
+
*
|
|
59
|
+
* dbPath override is test-only (isolated per-test files) — production
|
|
60
|
+
* always uses the ~/.openclaw default.
|
|
61
|
+
*/
|
|
62
|
+
export async function initCapStore({ dbPath = DB_PATH } = {}) {
|
|
63
|
+
_dbPath = dbPath;
|
|
64
|
+
_client = null; // re-resolve against the (possibly new) path
|
|
65
|
+
const client = await _getClient();
|
|
66
|
+
const { rows } = await client.execute(
|
|
67
|
+
`SELECT cap_usd AS capUSD, lock_threshold AS lockThreshold FROM cap_versions WHERE retired_at IS NULL`
|
|
68
|
+
);
|
|
69
|
+
if (rows.length > 0) {
|
|
70
|
+
configuredCapUSD = rows[0].capUSD;
|
|
71
|
+
configuredLockThreshold = rows[0].lockThreshold;
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
await client.execute({
|
|
75
|
+
sql: `INSERT INTO cap_versions (id, cap_usd, lock_threshold, registered_at, retired_at) VALUES (?, ?, ?, ?, NULL)`,
|
|
76
|
+
args: [randomUUID(), DEFAULT_CAP_USD, DEFAULT_LOCK_THRESHOLD, Date.now()],
|
|
77
|
+
});
|
|
78
|
+
configuredCapUSD = DEFAULT_CAP_USD;
|
|
79
|
+
configuredLockThreshold = DEFAULT_LOCK_THRESHOLD;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* SCD2 write — retires the current active row and inserts a new one in a
|
|
84
|
+
* single atomic batch, same pattern as pipeline_versions. A no-op when the
|
|
85
|
+
* patch doesn't actually change anything, so re-PUTting the same value
|
|
86
|
+
* doesn't churn out a spurious history row.
|
|
87
|
+
*/
|
|
88
|
+
async function _writeCapVersion(patch) {
|
|
89
|
+
const client = await _getClient();
|
|
90
|
+
const { rows } = await client.execute(
|
|
91
|
+
`SELECT id, cap_usd AS capUSD, lock_threshold AS lockThreshold FROM cap_versions WHERE retired_at IS NULL`
|
|
92
|
+
);
|
|
93
|
+
const active = rows[0] ?? null;
|
|
94
|
+
const nextCap = patch.capUSD ?? active?.capUSD ?? configuredCapUSD;
|
|
95
|
+
const nextThreshold = patch.lockThreshold ?? active?.lockThreshold ?? configuredLockThreshold;
|
|
96
|
+
if (active && active.capUSD === nextCap && active.lockThreshold === nextThreshold) return;
|
|
97
|
+
const now = Date.now();
|
|
98
|
+
const queries = [];
|
|
99
|
+
if (active) queries.push({ sql: `UPDATE cap_versions SET retired_at = ? WHERE id = ?`, args: [now, active.id] });
|
|
100
|
+
queries.push({
|
|
101
|
+
sql: `INSERT INTO cap_versions (id, cap_usd, lock_threshold, registered_at, retired_at) VALUES (?, ?, ?, ?, NULL)`,
|
|
102
|
+
args: [randomUUID(), nextCap, nextThreshold, now],
|
|
103
|
+
});
|
|
104
|
+
await withSharedWrite(_dbPath, () => client.batch(queries));
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Set cap, persist as a new cap_versions row. Only ever called from the
|
|
108
|
+
* (already-async) PUT /budget/cap route handler — awaited there so the
|
|
109
|
+
* response only confirms ok:true once the write actually landed, same
|
|
110
|
+
* durable-before-response guarantee the old synchronous fs.writeFileSync
|
|
111
|
+
* gave for free. */
|
|
112
|
+
export async function setCapAndPersist(usd) {
|
|
113
|
+
if (typeof usd !== "number" || usd <= 0 || !Number.isFinite(usd)) return;
|
|
114
|
+
configuredCapUSD = usd;
|
|
115
|
+
await _writeCapVersion({ capUSD: usd });
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export function currentCapUSD() { return configuredCapUSD; }
|
|
119
|
+
|
|
120
|
+
/** Fraction of daily cap at which the gate auto-locks (0 < x ≤ 1). */
|
|
121
|
+
export function currentLockThreshold() { return configuredLockThreshold; }
|
|
122
|
+
|
|
123
|
+
/** Set gate-lock threshold, persist as a new cap_versions row. Same
|
|
124
|
+
* await-before-respond contract as setCapAndPersist above. */
|
|
125
|
+
export async function setLockThresholdAndPersist(fraction) {
|
|
126
|
+
if (typeof fraction !== "number" || fraction <= 0 || fraction > 1 || !Number.isFinite(fraction)) return;
|
|
127
|
+
configuredLockThreshold = fraction;
|
|
128
|
+
await _writeCapVersion({ lockThreshold: fraction });
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Full cap/threshold history, oldest first — e.g. for rendering the actual
|
|
132
|
+
* cap in place at each point on a spend-over-time chart, rather than
|
|
133
|
+
* assuming today's cap applied retroactively. */
|
|
134
|
+
export async function getCapHistory() {
|
|
135
|
+
const client = await _getClient();
|
|
136
|
+
const { rows } = await client.execute(
|
|
137
|
+
`SELECT cap_usd AS capUSD, lock_threshold AS lockThreshold, registered_at AS registeredAt, retired_at AS retiredAt
|
|
138
|
+
FROM cap_versions ORDER BY registered_at ASC`
|
|
139
|
+
);
|
|
140
|
+
return rows;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Records one threshold crossing. Fire-and-forget from checkBudget — a
|
|
144
|
+
* failed write here should never block the alert push itself. */
|
|
145
|
+
async function _recordAlertEvent({ threshold, total, cap }) {
|
|
146
|
+
const client = await _getClient();
|
|
147
|
+
await withSharedWrite(_dbPath, () => client.execute({
|
|
148
|
+
sql: `INSERT INTO budget_alert_events (id, threshold, total_usd, cap_usd, fired_at) VALUES (?, ?, ?, ?, ?)`,
|
|
149
|
+
args: [randomUUID(), threshold, total, cap, Date.now()],
|
|
150
|
+
}));
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Threshold-crossing history, newest first — powers a cap-hit chart's
|
|
154
|
+
* tap-to-detail (which threshold, what time), not just "was that day bad". */
|
|
155
|
+
export async function getAlertHistory({ limit = 60 } = {}) {
|
|
156
|
+
const client = await _getClient();
|
|
157
|
+
const { rows } = await client.execute({
|
|
158
|
+
sql: `SELECT threshold, total_usd AS totalUSD, cap_usd AS capUSD, fired_at AS firedAt
|
|
159
|
+
FROM budget_alert_events ORDER BY fired_at DESC LIMIT ?`,
|
|
160
|
+
args: [limit],
|
|
161
|
+
});
|
|
162
|
+
return rows;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function readState() {
|
|
166
|
+
try {
|
|
167
|
+
const raw = fs.readFileSync(STATE_FILE, "utf8");
|
|
168
|
+
const j = JSON.parse(raw);
|
|
169
|
+
if (j && typeof j === "object" && typeof j.day === "string" && Array.isArray(j.fired)) {
|
|
170
|
+
return { day: j.day, fired: j.fired.filter((x) => typeof x === "number") };
|
|
171
|
+
}
|
|
172
|
+
} catch { /* fall through */ }
|
|
173
|
+
return { day: null, fired: [] };
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function writeState(next) {
|
|
177
|
+
try {
|
|
178
|
+
fs.writeFileSync(STATE_FILE, JSON.stringify(next), "utf8");
|
|
179
|
+
} catch { /* best-effort */ }
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const _GW_PORT = Number(process.env.OPENCLAW_GATEWAY_PORT) || 18789;
|
|
183
|
+
|
|
184
|
+
async function _lockGate(logger) {
|
|
185
|
+
const token = process.env.OPENCLAW_GATEWAY_TOKEN ?? "";
|
|
186
|
+
try {
|
|
187
|
+
await fetch(`http://127.0.0.1:${_GW_PORT}/manifold/budget/gate/lock`, {
|
|
188
|
+
method: "POST",
|
|
189
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
|
|
190
|
+
body: "{}",
|
|
191
|
+
signal: AbortSignal.timeout(3000),
|
|
192
|
+
});
|
|
193
|
+
} catch (err) {
|
|
194
|
+
logger?.warn?.(`lineage[budget-alert] gate lock failed: ${err?.message ?? err}`);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
async function _unlockGate(logger) {
|
|
199
|
+
const token = process.env.OPENCLAW_GATEWAY_TOKEN ?? "";
|
|
200
|
+
try {
|
|
201
|
+
await fetch(`http://127.0.0.1:${_GW_PORT}/manifold/budget/gate/unlock`, {
|
|
202
|
+
method: "POST",
|
|
203
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
|
|
204
|
+
body: "{}",
|
|
205
|
+
signal: AbortSignal.timeout(3000),
|
|
206
|
+
});
|
|
207
|
+
} catch (err) {
|
|
208
|
+
logger?.warn?.(`lineage[budget-alert] gate unlock failed: ${err?.message ?? err}`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function localDayKey(d = new Date()) {
|
|
213
|
+
// Local-time YYYY-MM-DD (the spend rolls over at local midnight, not UTC).
|
|
214
|
+
const y = d.getFullYear();
|
|
215
|
+
const m = String(d.getMonth() + 1).padStart(2, "0");
|
|
216
|
+
const day = String(d.getDate()).padStart(2, "0");
|
|
217
|
+
return `${y}-${m}-${day}`;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Returns today's total spend in USD by summing cost_by_call rows whose
|
|
222
|
+
* ts falls within the local-time day boundaries. Uses sqlite's localtime
|
|
223
|
+
* modifier so the boundary is consistent regardless of timezone.
|
|
224
|
+
*/
|
|
225
|
+
async function todaysTotalUSD(store) {
|
|
226
|
+
const row = await execGet(store._client, `
|
|
227
|
+
SELECT COALESCE(SUM(cost_total), 0) AS total
|
|
228
|
+
FROM cost_by_call
|
|
229
|
+
WHERE date(ts/1000, 'unixepoch', 'localtime') = date('now', 'localtime')
|
|
230
|
+
`);
|
|
231
|
+
return Number(row?.total ?? 0);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Should be called after every successful cost insert. Cheap — one indexed
|
|
236
|
+
* SELECT and an in-memory threshold check. No-op when already alerted for
|
|
237
|
+
* every threshold today.
|
|
238
|
+
*
|
|
239
|
+
* Behavior:
|
|
240
|
+
* - On a NEW local day with prior history, silently mark already-crossed
|
|
241
|
+
* thresholds as fired (so a mid-day gateway restart doesn't retroactively
|
|
242
|
+
* re-fire pushes for thresholds the operator already saw).
|
|
243
|
+
* - For each threshold (50% / 80% / 100%), fire the alert at most once
|
|
244
|
+
* per local day. Crossing two thresholds in one cost insert (rare) fires
|
|
245
|
+
* both notifications.
|
|
246
|
+
*/
|
|
247
|
+
export async function checkBudget({ store, logger, justWroteUsd = 0, notify }) {
|
|
248
|
+
try {
|
|
249
|
+
const today = localDayKey();
|
|
250
|
+
|
|
251
|
+
// New-day rollover: reset fired list. To avoid spamming pushes after a
|
|
252
|
+
// mid-day restart of an already-over-budget day, silently mark any
|
|
253
|
+
// already-crossed thresholds as fired without dispatching.
|
|
254
|
+
if (state.day !== today) {
|
|
255
|
+
state = { day: today, fired: [] };
|
|
256
|
+
const totalNow = await todaysTotalUSD(store);
|
|
257
|
+
const fracNow = configuredCapUSD > 0 ? totalNow / configuredCapUSD : 0;
|
|
258
|
+
for (const t of THRESHOLDS) if (fracNow >= t) state.fired.push(t);
|
|
259
|
+
writeState(state);
|
|
260
|
+
lastObservedTotal = totalNow;
|
|
261
|
+
// New day: unlock the gate so pipelines run freely again
|
|
262
|
+
_unlockGate(logger).catch(() => {});
|
|
263
|
+
// Don't fire on the first call of a new day even if we crossed —
|
|
264
|
+
// alerts only fire on transitions observed during this process.
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
if (state.fired.length === THRESHOLDS.length) return; // all fired today
|
|
269
|
+
|
|
270
|
+
const total = await todaysTotalUSD(store);
|
|
271
|
+
lastObservedTotal = total;
|
|
272
|
+
const frac = configuredCapUSD > 0 ? total / configuredCapUSD : 0;
|
|
273
|
+
|
|
274
|
+
for (const t of THRESHOLDS) {
|
|
275
|
+
if (frac < t) continue;
|
|
276
|
+
if (state.fired.includes(t)) continue;
|
|
277
|
+
state.fired.push(t);
|
|
278
|
+
writeState(state);
|
|
279
|
+
_recordAlertEvent({ threshold: t, total, cap: configuredCapUSD })
|
|
280
|
+
.catch((err) => logger?.warn?.(`lineage[budget-alert] record(${t}) error: ${err?.message ?? err}`));
|
|
281
|
+
// Fire-and-forget so the hook isn't blocked on push round-trip.
|
|
282
|
+
fireAlert({ threshold: t, total, cap: configuredCapUSD, store, logger, notify })
|
|
283
|
+
.catch((err) => logger?.warn?.(`lineage[budget-alert] fire(${t}) error: ${err?.message ?? err}`));
|
|
284
|
+
}
|
|
285
|
+
} catch (err) {
|
|
286
|
+
logger?.warn?.(`lineage[budget-alert]: ${err?.message ?? err}`);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// ─── alert composition + dispatch ──────────────────────────────────────────
|
|
291
|
+
|
|
292
|
+
async function fireAlert({ threshold, total, cap, store, logger, notify }) {
|
|
293
|
+
const text = await composeText({ threshold, total, cap, store });
|
|
294
|
+
const title = composeTitle({ threshold, total, cap });
|
|
295
|
+
if (threshold >= configuredLockThreshold) {
|
|
296
|
+
_lockGate(logger).catch(() => {});
|
|
297
|
+
logger?.info?.(`lineage[budget-alert] approval gate locked (threshold=${threshold} >= lock=${configuredLockThreshold})`);
|
|
298
|
+
}
|
|
299
|
+
// The budget_alert_events row is written unconditionally, so the threshold
|
|
300
|
+
// crossing is never lost even when no consumer is attached. Delivery
|
|
301
|
+
// belongs to whoever subscribes (app bridge, notifier); manifold only
|
|
302
|
+
// states the event.
|
|
303
|
+
if (typeof notify === "function") {
|
|
304
|
+
try {
|
|
305
|
+
notify({ title, body: text.body, threshold, total, cap });
|
|
306
|
+
logger?.info?.(`lineage[budget-alert] emitted manifold.budget.alert threshold=${threshold}`);
|
|
307
|
+
return;
|
|
308
|
+
} catch (err) {
|
|
309
|
+
logger?.warn?.(`lineage[budget-alert] budget.alert emit failed: ${err?.message ?? err}`);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
logger?.warn?.(`lineage[budget-alert] no consumer attached — threshold=${threshold} alert not delivered`);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function composeTitle({ threshold }) {
|
|
316
|
+
if (threshold >= 1.0) return "🚨 LLM budget exceeded";
|
|
317
|
+
if (threshold >= 0.8) return "⚠️ LLM budget at 80%";
|
|
318
|
+
if (threshold >= 0.5) return "📊 LLM budget at 50%";
|
|
319
|
+
return "LLM budget alert";
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
async function composeText({ threshold, total, cap, store }) {
|
|
323
|
+
const pct = Math.round((total / Math.max(cap, 0.0001)) * 100);
|
|
324
|
+
const top = await topDriverToday(store);
|
|
325
|
+
let body = `$${total.toFixed(2)} of $${cap.toFixed(2)} today (${pct}%)`;
|
|
326
|
+
if (top) body += `\nTop: ${top.label} $${top.cost.toFixed(2)}`;
|
|
327
|
+
return { body };
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** Highest-spending root in today's data — used to point the operator at
|
|
331
|
+
* the driver without having to open the app first. Best-effort. */
|
|
332
|
+
async function topDriverToday(store) {
|
|
333
|
+
try {
|
|
334
|
+
const row = await execGet(store._client, `
|
|
335
|
+
SELECT root_label, root_kind, SUM(cost_total) AS cost
|
|
336
|
+
FROM cost_by_call
|
|
337
|
+
WHERE date(ts/1000, 'unixepoch', 'localtime') = date('now', 'localtime')
|
|
338
|
+
AND root_label IS NOT NULL
|
|
339
|
+
GROUP BY root_label, root_kind ORDER BY cost DESC LIMIT 1
|
|
340
|
+
`);
|
|
341
|
+
if (!row || !row.cost) return null;
|
|
342
|
+
return { label: displayRootLabel(row.root_kind, row.root_label), cost: row.cost };
|
|
343
|
+
} catch { return null; }
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** Fire a one-off alert through the normal event path, ignoring the
|
|
347
|
+
* once-per-day debounce. For operator validation only — does not mutate the
|
|
348
|
+
* persistent threshold-fired state. */
|
|
349
|
+
export async function testFireAlert({ store, logger, threshold = 0.5, notify }) {
|
|
350
|
+
const total = await todaysTotalUSD(store);
|
|
351
|
+
await fireAlert({ threshold, total, cap: configuredCapUSD, store, logger, notify });
|
|
352
|
+
}
|
|
353
|
+
|