cartethyia 2.5.3 → 2.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dashboard/assets/{BannedPage-hfCzi-Ux.js → BannedPage-C2orBsI2.js} +1 -1
- package/dist/dashboard/assets/BannedPage-C2orBsI2.js.gz +0 -0
- package/dist/dashboard/assets/CliToolDetailPage-DUcZ6pMB.js +1 -0
- package/dist/dashboard/assets/CliToolDetailPage-DUcZ6pMB.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolsPage-DEDyKBlC.js → CliToolsPage-BV6pxVQo.js} +1 -1
- package/dist/dashboard/assets/CliToolsPage-BV6pxVQo.js.gz +0 -0
- package/dist/dashboard/assets/{CombosPage-CfuYiQ8Z.js → CombosPage-Bsan4679.js} +1 -1
- package/dist/dashboard/assets/CombosPage-Bsan4679.js.gz +0 -0
- package/dist/dashboard/assets/{ConsoleLogPage-bsLuZmYH.js → ConsoleLogPage-CW-wPmJ6.js} +1 -1
- package/dist/dashboard/assets/ConsoleLogPage-CW-wPmJ6.js.gz +0 -0
- package/dist/dashboard/assets/{CustomizationPage-_UWhwziJ.js → CustomizationPage-0p8uX7vB.js} +1 -1
- package/dist/dashboard/assets/CustomizationPage-0p8uX7vB.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-BjqSMfko.js +1 -0
- package/dist/dashboard/assets/HealthEventsModal-BjqSMfko.js.gz +0 -0
- package/dist/dashboard/assets/{LoginPage-DDVIfZGW.js → LoginPage-BvzCx8Kr.js} +1 -1
- package/dist/dashboard/assets/LoginPage-BvzCx8Kr.js.gz +0 -0
- package/dist/dashboard/assets/{ModelPicker-CWkTUtpX.js → ModelPicker-BY1aKEkP.js} +1 -1
- package/dist/dashboard/assets/ModelPicker-BY1aKEkP.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-DOQpy1y0.js +1 -0
- package/dist/dashboard/assets/OverviewPage-DOQpy1y0.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-ecxyUMzt.js +3 -0
- package/dist/dashboard/assets/ProviderDetailPage-ecxyUMzt.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-Js9jUPSb.js +1 -0
- package/dist/dashboard/assets/ProvidersPage-Js9jUPSb.js.gz +0 -0
- package/dist/dashboard/assets/ProxyPage-J3TBi3Kx.js +7 -0
- package/dist/dashboard/assets/ProxyPage-J3TBi3Kx.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-C1gZS6xr.js +1 -0
- package/dist/dashboard/assets/QuotaPage-C1gZS6xr.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-CS86ho5X.js +1 -0
- package/dist/dashboard/assets/SettingsPage-CS86ho5X.js.gz +0 -0
- package/dist/dashboard/assets/{SetupPage-D6pmQ86L.js → SetupPage-DDbMQsoo.js} +1 -1
- package/dist/dashboard/assets/SetupPage-DDbMQsoo.js.gz +0 -0
- package/dist/dashboard/assets/{SortableList-DWvKiuSl.js → SortableList-CB3Z7VTf.js} +1 -1
- package/dist/dashboard/assets/SortableList-CB3Z7VTf.js.gz +0 -0
- package/dist/dashboard/assets/{StudioPage-CmkUrgdj.js → StudioPage-D95UCPHI.js} +3 -3
- package/dist/dashboard/assets/StudioPage-D95UCPHI.js.gz +0 -0
- package/dist/dashboard/assets/{UsagePage-B47uOydr.js → UsagePage-Cnae9SWA.js} +13 -13
- package/dist/dashboard/assets/UsagePage-Cnae9SWA.js.gz +0 -0
- package/dist/dashboard/assets/{api-keys-uNywYlfv.js → api-keys-VY68Mhd1.js} +1 -1
- package/dist/dashboard/assets/api-keys-VY68Mhd1.js.gz +0 -0
- package/dist/dashboard/assets/app-COlK95oi.css +2 -0
- package/dist/dashboard/assets/app-COlK95oi.css.gz +0 -0
- package/dist/dashboard/assets/{assert-NHwYq95t.js → assert-Chi7pCnt.js} +1 -1
- package/dist/dashboard/assets/assert-Chi7pCnt.js.gz +0 -0
- package/dist/dashboard/assets/{badge-6uzDDvqs.js → badge-D4pQapKj.js} +1 -1
- package/dist/dashboard/assets/badge-D4pQapKj.js.gz +0 -0
- package/dist/dashboard/assets/{banner-4_WNg2WZ.js → banner-DH6cg-_B.js} +1 -1
- package/dist/dashboard/assets/banner-DH6cg-_B.js.gz +0 -0
- package/dist/dashboard/assets/{card--AtFedAq.js → card-BKqlUOe4.js} +1 -1
- package/dist/dashboard/assets/card-BKqlUOe4.js.gz +0 -0
- package/dist/dashboard/assets/{cli-tools-B3e7dx3N.js → cli-tools-fTgljTGA.js} +1 -1
- package/dist/dashboard/assets/cli-tools-fTgljTGA.js.gz +0 -0
- package/dist/dashboard/assets/{clipboard-button-NreUx6c8.js → clipboard-button-D5Ys_sM0.js} +1 -1
- package/dist/dashboard/assets/clipboard-button-D5Ys_sM0.js.gz +0 -0
- package/dist/dashboard/assets/database-Dz9N3zGd.js +1 -0
- package/dist/dashboard/assets/database-Dz9N3zGd.js.gz +0 -0
- package/dist/dashboard/assets/{dialog-BtpsJU98.js → dialog-BYj-8WuK.js} +1 -1
- package/dist/dashboard/assets/dialog-BYj-8WuK.js.gz +0 -0
- package/dist/dashboard/assets/{drawer-WwwtzihF.js → drawer-CmXIZ6H1.js} +1 -1
- package/dist/dashboard/assets/drawer-CmXIZ6H1.js.gz +0 -0
- package/dist/dashboard/assets/entry-CBg5zeu8.js +1 -0
- package/dist/dashboard/assets/entry-CBg5zeu8.js.gz +0 -0
- package/dist/dashboard/assets/entry-CQBr6U9c.js +1 -0
- package/dist/dashboard/assets/entry-CQBr6U9c.js.gz +0 -0
- package/dist/dashboard/assets/entry-CgdSAzEV.js +4 -0
- package/dist/dashboard/assets/entry-CgdSAzEV.js.gz +0 -0
- package/dist/dashboard/assets/{github-badge-D_eM3IgG.js → github-badge-BlptL8Uf.js} +1 -1
- package/dist/dashboard/assets/github-badge-BlptL8Uf.js.gz +0 -0
- package/dist/dashboard/assets/{index-BydyKaDz.js → index-js6TJvdr.js} +2 -2
- package/dist/dashboard/assets/index-js6TJvdr.js.gz +0 -0
- package/dist/dashboard/assets/{inline-BcNGF8Gr.js → inline-DUWDSmDV.js} +1 -1
- package/dist/dashboard/assets/inline-DUWDSmDV.js.gz +0 -0
- package/dist/dashboard/assets/{input-BG41et9G.js → input-LVjgZq4f.js} +1 -1
- package/dist/dashboard/assets/input-LVjgZq4f.js.gz +0 -0
- package/dist/dashboard/assets/{layout-C8mer_Ak.js → layout-CPfxpsfy.js} +1 -1
- package/dist/dashboard/assets/layout-CPfxpsfy.js.gz +0 -0
- package/dist/dashboard/assets/{list-row-Gh_W4OR_.js → list-row-DupvLfDx.js} +1 -1
- package/dist/dashboard/assets/list-row-DupvLfDx.js.gz +0 -0
- package/dist/dashboard/assets/{live-BJGYRBnR.js → live-C4pEJUqI.js} +1 -1
- package/dist/dashboard/assets/live-C4pEJUqI.js.gz +0 -0
- package/dist/dashboard/assets/model-bans-BRiuufkB.js +1 -0
- package/dist/dashboard/assets/model-bans-BRiuufkB.js.gz +0 -0
- package/dist/dashboard/assets/{mutation-BkoLM5Ba.js → mutation-B4ROwk8L.js} +1 -1
- package/dist/dashboard/assets/mutation-B4ROwk8L.js.gz +0 -0
- package/dist/dashboard/assets/numbers-YNcHvRY7.js +2 -0
- package/dist/dashboard/assets/numbers-YNcHvRY7.js.gz +0 -0
- package/dist/dashboard/assets/{query-keys-D7wX2vM5.js → query-keys-gmjzC3RS.js} +1 -1
- package/dist/dashboard/assets/{query-keys-D7wX2vM5.js.gz → query-keys-gmjzC3RS.js.gz} +0 -0
- package/dist/dashboard/assets/{routing-CcusqXA7.js → routing-C7tBT5o-.js} +1 -1
- package/dist/dashboard/assets/routing-C7tBT5o-.js.gz +0 -0
- package/dist/dashboard/assets/{select-BO6h8yNv.js → select-WUCXorXw.js} +2 -2
- package/dist/dashboard/assets/select-WUCXorXw.js.gz +0 -0
- package/dist/dashboard/assets/{settings-mpmXTgUa.js → settings-C_zKl1GS.js} +1 -1
- package/dist/dashboard/assets/settings-C_zKl1GS.js.gz +0 -0
- package/dist/dashboard/assets/{stack-DwLmjUFK.js → stack-_3O0F8Xn.js} +1 -1
- package/dist/dashboard/assets/stack-_3O0F8Xn.js.gz +0 -0
- package/dist/dashboard/assets/{state-DYdQmdAl.js → state-BgTxHcJS.js} +1 -1
- package/dist/dashboard/assets/state-BgTxHcJS.js.gz +0 -0
- package/dist/dashboard/assets/switch-DEjTtOWf.js +1 -0
- package/dist/dashboard/assets/switch-DEjTtOWf.js.gz +0 -0
- package/dist/dashboard/assets/terminal-0_IjPWTk.js +1 -0
- package/dist/dashboard/assets/terminal-0_IjPWTk.js.gz +0 -0
- package/dist/dashboard/assets/{useMutation-w4u4CVue.js → useMutation-BWSC0UAS.js} +1 -1
- package/dist/dashboard/assets/useMutation-BWSC0UAS.js.gz +0 -0
- package/dist/dashboard/assets/zap-C4EeVW4q.js +1 -0
- package/dist/dashboard/assets/zap-C4EeVW4q.js.gz +0 -0
- package/dist/dashboard/donation/qris-risun-marke.png +0 -0
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/index.html.gz +0 -0
- package/dist/dashboard/precompressed-assets.json +54 -54
- package/package.json +1 -1
- package/scripts/cli/autostart.ts +237 -0
- package/scripts/cli/command/console-menu.ts +4 -8
- package/scripts/cli/command/update.ts +27 -10
- package/scripts/cli/main.ts +28 -1
- package/src/boot/app.ts +693 -693
- package/src/boot/dependencies.ts +646 -616
- package/src/config/spec.ts +55 -0
- package/src/config/telemetry.ts +46 -0
- package/src/console/api-keys/bucket-spend.ts +15 -4
- package/src/console/backup/contracts.ts +15 -2
- package/src/console/console-router.ts +8 -0
- package/src/console/data-management/contracts.ts +93 -0
- package/src/console/data-management/operations.ts +229 -0
- package/src/console/data-management/routes.ts +104 -0
- package/src/console/domain-registration.ts +39 -0
- package/src/console/observability/contracts.ts +524 -485
- package/src/console/observability/store.ts +1115 -1018
- package/src/console/providers/catalog/contracts.ts +6 -0
- package/src/console/providers/catalog/provider-operations.ts +20 -0
- package/src/console/providers/catalog/routes.ts +7 -0
- package/src/console/providers/catalog/store.ts +22 -1
- package/src/console/quota/account-quota-shared.ts +52 -39
- package/src/console/quota/refresh.ts +441 -429
- package/src/console/routing/pools/import-parse.ts +206 -0
- package/src/console/routing/pools/operations.ts +543 -501
- package/src/console/routing/pools/routes.ts +181 -164
- package/src/console/settings/contracts.ts +37 -0
- package/src/console/settings/store.ts +51 -0
- package/src/console/updates/routes.ts +16 -0
- package/src/main.ts +10 -0
- package/src/observability/event-compaction.ts +315 -0
- package/src/observability/event-day-index.ts +442 -0
- package/src/observability/event-log.ts +152 -3
- package/src/observability/payload-capture.ts +8 -1
- package/src/observability/payload-store.ts +440 -376
- package/src/persistence/health-event-flush.ts +36 -0
- package/src/persistence/schema.ts +1402 -1360
- package/src/persistence/stores/telemetry-store.ts +237 -214
- package/src/protocol/codecs.ts +1 -2
- package/src/protocol/image-clamp.ts +19 -1
- package/src/protocol/registry.ts +8 -0
- package/src/protocol/request/gemini.ts +9 -2
- package/src/protocol/request/messages.ts +14 -2
- package/src/providers/adapters/compatible-adapter.ts +3 -0
- package/src/providers/bundled-discovery-providers.ts +122 -119
- package/src/providers/configured-openai-providers.ts +136 -134
- package/src/providers/default-registry.ts +5 -4
- package/src/providers/discovery/openai-model-discovery.ts +5 -0
- package/src/providers/discovery/openrouter-free-roster.ts +15 -17
- package/src/providers/integrations/cline/cline-quota.ts +132 -125
- package/src/providers/integrations/codex/codex-headers.ts +25 -39
- package/src/providers/integrations/codex/codex.ts +2 -29
- package/src/providers/integrations/cursor/cursor-quota.ts +310 -0
- package/src/providers/integrations/cursor/cursor-request.ts +616 -471
- package/src/providers/integrations/cursor/cursor-session-manager.ts +319 -281
- package/src/providers/integrations/cursor/cursor.ts +1485 -960
- package/src/providers/integrations/cursor/protocol/messages.ts +25 -0
- package/src/providers/integrations/factory/factory.ts +8 -1
- package/src/providers/integrations/gemini.ts +6 -1
- package/src/providers/integrations/grok/grok-quota.ts +279 -177
- package/src/providers/integrations/grok/grok-ratelimit-store.ts +73 -0
- package/src/providers/integrations/grok/grok-ratelimit.ts +218 -0
- package/src/providers/integrations/opencode/provider.ts +12 -13
- package/src/providers/integrations/qoder/qoder-quota.ts +192 -0
- package/src/providers/integrations/trae/trae-quota.ts +384 -0
- package/src/providers/operations/account-health-service.ts +1523 -1480
- package/src/providers/provider-contracts.ts +7 -0
- package/src/providers/provider-metadata.ts +6 -0
- package/src/providers/quota/quota-support.ts +8 -0
- package/src/security/api-key-auth.ts +504 -494
- package/src/transport/dispatch/generation.ts +6 -0
- package/src/transport/dispatch/proxy-request.ts +24 -1
- package/src/transport/dispatch/ratelimit-capture.ts +37 -0
- package/src/transport/dispatch/streaming/stream.ts +14 -0
- package/src/transport/dispatch/upstream.ts +9 -0
- package/src/transport/request/preparer.ts +7 -6
- package/src/transport/request/state.ts +9 -0
- package/src/transport/request/tenant-preferences.ts +60 -0
- package/src/transport/routing/route-catalog.ts +58 -10
- package/dist/dashboard/assets/BannedPage-hfCzi-Ux.js.gz +0 -0
- package/dist/dashboard/assets/CliToolDetailPage-DLGqYEwI.js +0 -1
- package/dist/dashboard/assets/CliToolDetailPage-DLGqYEwI.js.gz +0 -0
- package/dist/dashboard/assets/CliToolsPage-DEDyKBlC.js.gz +0 -0
- package/dist/dashboard/assets/CombosPage-CfuYiQ8Z.js.gz +0 -0
- package/dist/dashboard/assets/ConsoleLogPage-bsLuZmYH.js.gz +0 -0
- package/dist/dashboard/assets/CustomizationPage-_UWhwziJ.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-Cjfc5WFS.js +0 -1
- package/dist/dashboard/assets/HealthEventsModal-Cjfc5WFS.js.gz +0 -0
- package/dist/dashboard/assets/LoginPage-DDVIfZGW.js.gz +0 -0
- package/dist/dashboard/assets/ModelPicker-CWkTUtpX.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-B53YYMng.js +0 -1
- package/dist/dashboard/assets/OverviewPage-B53YYMng.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-B0KaAQf0.js +0 -3
- package/dist/dashboard/assets/ProviderDetailPage-B0KaAQf0.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-Dx31DgIO.js +0 -1
- package/dist/dashboard/assets/ProvidersPage-Dx31DgIO.js.gz +0 -0
- package/dist/dashboard/assets/ProxyPage-BpnR4kXT.js +0 -7
- package/dist/dashboard/assets/ProxyPage-BpnR4kXT.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-BpOQCnun.js +0 -1
- package/dist/dashboard/assets/QuotaPage-BpOQCnun.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-DV1eo2J_.js +0 -1
- package/dist/dashboard/assets/SettingsPage-DV1eo2J_.js.gz +0 -0
- package/dist/dashboard/assets/SetupPage-D6pmQ86L.js.gz +0 -0
- package/dist/dashboard/assets/SortableList-DWvKiuSl.js.gz +0 -0
- package/dist/dashboard/assets/StudioPage-CmkUrgdj.js.gz +0 -0
- package/dist/dashboard/assets/UsagePage-B47uOydr.js.gz +0 -0
- package/dist/dashboard/assets/api-keys-uNywYlfv.js.gz +0 -0
- package/dist/dashboard/assets/app-DxivpfYG.css +0 -2
- package/dist/dashboard/assets/app-DxivpfYG.css.gz +0 -0
- package/dist/dashboard/assets/assert-NHwYq95t.js.gz +0 -0
- package/dist/dashboard/assets/badge-6uzDDvqs.js.gz +0 -0
- package/dist/dashboard/assets/banner-4_WNg2WZ.js.gz +0 -0
- package/dist/dashboard/assets/card--AtFedAq.js.gz +0 -0
- package/dist/dashboard/assets/cli-tools-B3e7dx3N.js.gz +0 -0
- package/dist/dashboard/assets/clipboard-button-NreUx6c8.js.gz +0 -0
- package/dist/dashboard/assets/dialog-BtpsJU98.js.gz +0 -0
- package/dist/dashboard/assets/download-A0SW9OCW.js +0 -1
- package/dist/dashboard/assets/download-A0SW9OCW.js.gz +0 -0
- package/dist/dashboard/assets/drawer-WwwtzihF.js.gz +0 -0
- package/dist/dashboard/assets/entry-CkryrIea.js +0 -4
- package/dist/dashboard/assets/entry-CkryrIea.js.gz +0 -0
- package/dist/dashboard/assets/entry-TJIye_jT.js +0 -1
- package/dist/dashboard/assets/entry-TJIye_jT.js.gz +0 -0
- package/dist/dashboard/assets/entry-jy9_4-Ke.js +0 -1
- package/dist/dashboard/assets/entry-jy9_4-Ke.js.gz +0 -0
- package/dist/dashboard/assets/github-badge-D_eM3IgG.js.gz +0 -0
- package/dist/dashboard/assets/index-BydyKaDz.js.gz +0 -0
- package/dist/dashboard/assets/inline-BcNGF8Gr.js.gz +0 -0
- package/dist/dashboard/assets/input-BG41et9G.js.gz +0 -0
- package/dist/dashboard/assets/layout-C8mer_Ak.js.gz +0 -0
- package/dist/dashboard/assets/list-row-Gh_W4OR_.js.gz +0 -0
- package/dist/dashboard/assets/live-BJGYRBnR.js.gz +0 -0
- package/dist/dashboard/assets/model-bans-DeVLDA7I.js +0 -1
- package/dist/dashboard/assets/model-bans-DeVLDA7I.js.gz +0 -0
- package/dist/dashboard/assets/mutation-BkoLM5Ba.js.gz +0 -0
- package/dist/dashboard/assets/numbers-Bs4LOOuf.js +0 -2
- package/dist/dashboard/assets/numbers-Bs4LOOuf.js.gz +0 -0
- package/dist/dashboard/assets/routing-CcusqXA7.js.gz +0 -0
- package/dist/dashboard/assets/select-BO6h8yNv.js.gz +0 -0
- package/dist/dashboard/assets/settings-mpmXTgUa.js.gz +0 -0
- package/dist/dashboard/assets/sparkles-BRaRUxAU.js +0 -1
- package/dist/dashboard/assets/sparkles-BRaRUxAU.js.gz +0 -0
- package/dist/dashboard/assets/stack-DwLmjUFK.js.gz +0 -0
- package/dist/dashboard/assets/state-DYdQmdAl.js.gz +0 -0
- package/dist/dashboard/assets/switch-CidSfbG7.js +0 -1
- package/dist/dashboard/assets/switch-CidSfbG7.js.gz +0 -0
- package/dist/dashboard/assets/terminal-Bd4ee8TA.js +0 -1
- package/dist/dashboard/assets/terminal-Bd4ee8TA.js.gz +0 -0
- package/dist/dashboard/assets/useMutation-w4u4CVue.js.gz +0 -0
- /package/dist/dashboard/assets/{app-CafWtVXA.js → app-DCQZRmHY.js} +0 -0
- /package/dist/dashboard/assets/{app-CafWtVXA.js.gz → app-DCQZRmHY.js.gz} +0 -0
package/src/config/spec.ts
CHANGED
|
@@ -148,7 +148,62 @@ export const CONFIG_SPEC = {
|
|
|
148
148
|
*/
|
|
149
149
|
CARTETHYIA_DISABLED_TASKS: { kind: "list" },
|
|
150
150
|
|
|
151
|
+
/**
|
|
152
|
+
* How long a day's telemetry is kept, in days.
|
|
153
|
+
*
|
|
154
|
+
* The retention unit is the whole day: a day's event file and its index are
|
|
155
|
+
* deleted together, and the current day is never touched. Past this age an
|
|
156
|
+
* individual request no longer exists anywhere, so its detail is gone —
|
|
157
|
+
* totals, breakdowns and spend are unaffected, because the index keeps
|
|
158
|
+
* running sums rather than the rows.
|
|
159
|
+
*
|
|
160
|
+
* This is the knob that decides how long the request list, per-request
|
|
161
|
+
* filters, and the detail drawer stay complete.
|
|
162
|
+
*/
|
|
151
163
|
CARTETHYIA_TELEMETRY_RETENTION_DAYS: { kind: "int", default: 30, min: 3, max: 365 },
|
|
164
|
+
/**
|
|
165
|
+
* Age at which a request's detail is described as no longer stored.
|
|
166
|
+
*
|
|
167
|
+
* Retention itself is `CARTETHYIA_TELEMETRY_RETENTION_DAYS`; this only words
|
|
168
|
+
* the message shown when a request that old is asked for. Kept because the
|
|
169
|
+
* message is operator-facing and was written against this knob.
|
|
170
|
+
*/
|
|
171
|
+
CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS: { kind: "int", default: 7, min: 1, max: 365 },
|
|
172
|
+
/**
|
|
173
|
+
* How often each day's index is brought up to date, in minutes.
|
|
174
|
+
*
|
|
175
|
+
* Bringing an index up to date means folding the bytes appended since the
|
|
176
|
+
* last pass — not re-reading the day. Reads fold before they answer too, so
|
|
177
|
+
* raising this does not make the numbers stale; it only leaves the on-disk
|
|
178
|
+
* index further behind, and the next read pays for the remainder.
|
|
179
|
+
*/
|
|
180
|
+
CARTETHYIA_TELEMETRY_COMPACT_INTERVAL_MINUTES: {
|
|
181
|
+
kind: "int",
|
|
182
|
+
default: 60,
|
|
183
|
+
min: 5,
|
|
184
|
+
max: 1440,
|
|
185
|
+
},
|
|
186
|
+
/**
|
|
187
|
+
* Inert. Accepted so existing deployments keep booting; nothing reads it.
|
|
188
|
+
*
|
|
189
|
+
* It sized the one-file-per-window scheme — events were written to a file
|
|
190
|
+
* per this many minutes, 96 a day at the default. Events now append to one
|
|
191
|
+
* file per UTC day, so there is no window left to size.
|
|
192
|
+
*/
|
|
193
|
+
CARTETHYIA_TELEMETRY_WINDOW_MINUTES: {
|
|
194
|
+
kind: "int",
|
|
195
|
+
default: 15,
|
|
196
|
+
min: 1,
|
|
197
|
+
max: 1440,
|
|
198
|
+
},
|
|
199
|
+
/**
|
|
200
|
+
* Inert. Accepted so existing deployments keep booting; nothing reads it.
|
|
201
|
+
*
|
|
202
|
+
* It capped how many window files one pass could fold. Folding is driven by
|
|
203
|
+
* a byte offset into the day file now and never deletes the file, so there
|
|
204
|
+
* is no per-pass batch to bound.
|
|
205
|
+
*/
|
|
206
|
+
CARTETHYIA_TELEMETRY_COMPACT_MAX_FILES: { kind: "int", default: 48, min: 1, max: 10_000 },
|
|
152
207
|
CARTETHYIA_TELEMETRY_PAYLOAD_MAX_BYTES: {
|
|
153
208
|
kind: "int",
|
|
154
209
|
default: 16 * 1024 * 1024,
|
package/src/config/telemetry.ts
CHANGED
|
@@ -19,3 +19,49 @@ export function resolveTelemetryPayloadMaxBytes(): number {
|
|
|
19
19
|
CONFIG_SPEC.CARTETHYIA_TELEMETRY_PAYLOAD_MAX_BYTES,
|
|
20
20
|
);
|
|
21
21
|
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Age at which hourly telemetry files are compacted into a per-day summary.
|
|
25
|
+
*
|
|
26
|
+
* Clamped below the retention window: compacting at or past the delete age
|
|
27
|
+
* would fold a file and throw it away in the same pass, spending the read for
|
|
28
|
+
* totals that were about to be discarded.
|
|
29
|
+
*/
|
|
30
|
+
export function resolveTelemetryCompactAfterDays(): number {
|
|
31
|
+
const compacted = readInt(
|
|
32
|
+
"CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS",
|
|
33
|
+
CONFIG_SPEC.CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS,
|
|
34
|
+
);
|
|
35
|
+
return Math.min(compacted, resolveTelemetryRetentionDays());
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Minutes between compaction passes. */
|
|
39
|
+
export function resolveTelemetryCompactIntervalMs(): number {
|
|
40
|
+
const minutes = readInt(
|
|
41
|
+
"CARTETHYIA_TELEMETRY_COMPACT_INTERVAL_MINUTES",
|
|
42
|
+
CONFIG_SPEC.CARTETHYIA_TELEMETRY_COMPACT_INTERVAL_MINUTES,
|
|
43
|
+
);
|
|
44
|
+
return minutes * 60_000;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Minutes of events per source file, before it is folded into the day summary.
|
|
49
|
+
*
|
|
50
|
+
* Defaults to the sweep interval so exactly one window closes per pass: a
|
|
51
|
+
* window longer than the interval would leave files waiting several passes, and
|
|
52
|
+
* a shorter one would let several close between passes.
|
|
53
|
+
*/
|
|
54
|
+
export function resolveTelemetryWindowMinutes(): number {
|
|
55
|
+
return readInt(
|
|
56
|
+
"CARTETHYIA_TELEMETRY_WINDOW_MINUTES",
|
|
57
|
+
CONFIG_SPEC.CARTETHYIA_TELEMETRY_WINDOW_MINUTES,
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Maximum hourly files folded in one compaction pass. */
|
|
62
|
+
export function resolveTelemetryCompactMaxFiles(): number {
|
|
63
|
+
return readInt(
|
|
64
|
+
"CARTETHYIA_TELEMETRY_COMPACT_MAX_FILES",
|
|
65
|
+
CONFIG_SPEC.CARTETHYIA_TELEMETRY_COMPACT_MAX_FILES,
|
|
66
|
+
);
|
|
67
|
+
}
|
|
@@ -2,7 +2,7 @@ import { eq } from "drizzle-orm";
|
|
|
2
2
|
|
|
3
3
|
import type { CartethyiaDatabase } from "../../persistence/db-handle";
|
|
4
4
|
import { apiKeys } from "../../persistence/schema";
|
|
5
|
-
import {
|
|
5
|
+
import { queryIndexedBuckets } from "../../observability/event-day-index";
|
|
6
6
|
import { utcDayStart, utcMonthStart } from "../shared/day";
|
|
7
7
|
|
|
8
8
|
/**
|
|
@@ -33,9 +33,20 @@ export async function familyBucketSpend(
|
|
|
33
33
|
.where(eq(apiKeys.parentKeyId, keyId));
|
|
34
34
|
const keyIds = [keyId, ...children.map((child) => child.id)];
|
|
35
35
|
const spentSince = async (start: Date): Promise<number> => {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
// Folded first, so the index covers every event written so far including the
|
|
37
|
+
// current day's — then read from the index alone.
|
|
38
|
+
//
|
|
39
|
+
// Deliberately not index *plus* events: they hold the same requests, and
|
|
40
|
+
// adding them doubled every bucket. Folding first is what makes the index
|
|
41
|
+
// sufficient on its own; the watermark is the point up to which it is
|
|
42
|
+
// complete.
|
|
43
|
+
const allowed = new Set(keyIds);
|
|
44
|
+
const indexed = await queryIndexedBuckets({ since: start, foldFirst: true }).catch(
|
|
45
|
+
() => [],
|
|
46
|
+
);
|
|
47
|
+
return indexed
|
|
48
|
+
.filter((bucket) => bucket.apiKeyId !== null && allowed.has(bucket.apiKeyId))
|
|
49
|
+
.reduce((sum, bucket) => sum + bucket.inputTokens + bucket.outputTokens, 0);
|
|
39
50
|
};
|
|
40
51
|
const [daily, monthly] = await Promise.all([
|
|
41
52
|
spentSince(utcDayStart(now)),
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
* as sensitive as the database.
|
|
21
21
|
*/
|
|
22
22
|
import { getTableColumns, getTableName, type Column, type Table } from "drizzle-orm";
|
|
23
|
+
import { resolveMaxBodyBytes } from "../../config/runtime";
|
|
23
24
|
import {
|
|
24
25
|
apiKeys,
|
|
25
26
|
cliToolMappings,
|
|
@@ -45,8 +46,20 @@ import {
|
|
|
45
46
|
export const BACKUP_APP = "cartethyia";
|
|
46
47
|
export const BACKUP_VERSION = 1;
|
|
47
48
|
|
|
48
|
-
/**
|
|
49
|
-
|
|
49
|
+
/**
|
|
50
|
+
* Upper bound for restore payloads.
|
|
51
|
+
*
|
|
52
|
+
* Deliberately not larger than the listener's own body cap. It used to claim
|
|
53
|
+
* 8 GiB while the server admitted 4 MiB, so the number described a limit that
|
|
54
|
+
* could never be reached: a file above the listener's cap is cut off before
|
|
55
|
+
* this check runs, and a large upload fails as a reset connection rather than
|
|
56
|
+
* the 413 this constant promises. The bound now states what the install
|
|
57
|
+
* actually admits, so the message an operator sees matches what happened.
|
|
58
|
+
*
|
|
59
|
+
* Raise `CARTETHYIA_SERVER_MAX_BODY_BYTES` to accept larger backups; this
|
|
60
|
+
* constant follows it rather than the two drifting apart again.
|
|
61
|
+
*/
|
|
62
|
+
export const MAX_BACKUP_BYTES = resolveMaxBodyBytes();
|
|
50
63
|
|
|
51
64
|
/** Defense-in-depth row cap per table, after the byte bound. */
|
|
52
65
|
export const MAX_ROWS_PER_TABLE = 500_000;
|
|
@@ -48,6 +48,13 @@ export interface ConsoleApiCompositionDeps {
|
|
|
48
48
|
readonly routeSnapshotService: { invalidate(): Promise<number> };
|
|
49
49
|
readonly poolSelector: NetworkPoolSelector;
|
|
50
50
|
readonly telemetryBuffer: TelemetryBatchBuffer;
|
|
51
|
+
/**
|
|
52
|
+
* Queued writers the console can drain on demand (telemetry, usage, health
|
|
53
|
+
* events). Optional for the same reason the other reduced composition
|
|
54
|
+
* surface is: absent means the flush control is not offered rather than the
|
|
55
|
+
* console failing to build.
|
|
56
|
+
*/
|
|
57
|
+
readonly flushBuffers?: { flush(): Promise<void> };
|
|
51
58
|
/**
|
|
52
59
|
* The request state store owns the live in-flight gauge, so the console reads
|
|
53
60
|
* its snapshot/subscription from here rather than a free-standing module.
|
|
@@ -189,6 +196,7 @@ export function createConsoleRouter(deps: ConsoleApiCompositionDeps): Elysia {
|
|
|
189
196
|
admissionService: deps.admissionService,
|
|
190
197
|
...(deps.modelStrikes ? { modelStrikes: deps.modelStrikes } : {}),
|
|
191
198
|
readRoutingAccountInflight: deps.readRoutingAccountInflight,
|
|
199
|
+
...(deps.flushBuffers ? { flushBuffers: deps.flushBuffers } : {}),
|
|
192
200
|
credentialService,
|
|
193
201
|
// The backup surface re-authenticates the operator, so it needs the current
|
|
194
202
|
// user's hash. Read from the session on the request that asks for it, never
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Disk housekeeping: what the telemetry and payload directories hold, and the
|
|
3
|
+
* reclaim actions an operator can run from the console.
|
|
4
|
+
*
|
|
5
|
+
* Everything here is files on disk, except one deliberate row-level action.
|
|
6
|
+
* Reclaiming a file is safe because the data it held is already past the point
|
|
7
|
+
* where the gateway would read it: payload frames expire 15 minutes after they
|
|
8
|
+
* are written, and telemetry files past the retention window are outside every
|
|
9
|
+
* query's scan bound. Deleting rows is a different decision with a different
|
|
10
|
+
* blast radius, and the only row data the console offers purging is the health
|
|
11
|
+
* & error log — purely an operator's diagnostic trail with no request-path
|
|
12
|
+
* reader, unlike everything else in this panel.
|
|
13
|
+
*
|
|
14
|
+
* Sizes are bytes as `number`. A directory is scanned on each read rather than
|
|
15
|
+
* cached, because the point of the panel is to show what is true now — a
|
|
16
|
+
* stale size would be worse than none.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** One measured category of on-disk data. */
|
|
20
|
+
export interface DiskUsageBucket {
|
|
21
|
+
/** Stable id, so the dashboard can key rows without matching on copy. */
|
|
22
|
+
readonly id: "telemetry_hourly" | "telemetry_compacted" | "payload_frames";
|
|
23
|
+
readonly label: string;
|
|
24
|
+
readonly bytes: number;
|
|
25
|
+
readonly files: number;
|
|
26
|
+
/** Why this data exists, shown next to the size. */
|
|
27
|
+
readonly note: string;
|
|
28
|
+
/** Bytes reclaimable right now, when that is knowable without a dry run. */
|
|
29
|
+
readonly reclaimableBytes: number | null;
|
|
30
|
+
readonly reclaimableFiles: number | null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface DiskUsageResponse {
|
|
34
|
+
readonly buckets: readonly DiskUsageBucket[];
|
|
35
|
+
readonly totalBytes: number;
|
|
36
|
+
/** Retention in days; telemetry older than this is outside every scan. */
|
|
37
|
+
readonly retentionDays: number;
|
|
38
|
+
/** Payload frames live this long before they are reclaimable. */
|
|
39
|
+
readonly payloadRetentionMs: number;
|
|
40
|
+
/**
|
|
41
|
+
* Stored health & error rows, for the purge row's label. Absent when the
|
|
42
|
+
* composition does not wire the health-event store (a reduced console),
|
|
43
|
+
* which is also when the purge control is not offered.
|
|
44
|
+
*/
|
|
45
|
+
readonly healthEventCount?: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export type ReclaimAction = "payload_frames" | "telemetry_retention" | "telemetry_hourly";
|
|
49
|
+
|
|
50
|
+
export interface ReclaimResponse {
|
|
51
|
+
readonly action: ReclaimAction;
|
|
52
|
+
readonly deletedFiles: number;
|
|
53
|
+
readonly reclaimedBytes: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Row-level deletion targets. The health & error log is the only one. */
|
|
57
|
+
export type PurgeTarget = "health_events";
|
|
58
|
+
|
|
59
|
+
export interface PurgeResponse {
|
|
60
|
+
readonly target: PurgeTarget;
|
|
61
|
+
/** Rows actually deleted — the number the operator's decision produced. */
|
|
62
|
+
readonly deleted: number;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface FlushResponse {
|
|
66
|
+
readonly flushed: boolean;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* A reclaim the caller is not allowed to make.
|
|
71
|
+
*
|
|
72
|
+
* Kept as a named class so the route maps it to 403 rather than letting it
|
|
73
|
+
* surface as a 500 — an operator asking for something out of policy is a
|
|
74
|
+
* normal request, not a failure.
|
|
75
|
+
*/
|
|
76
|
+
export class ReclaimNotAllowedError extends Error {
|
|
77
|
+
readonly status = 403;
|
|
78
|
+
|
|
79
|
+
constructor(message: string) {
|
|
80
|
+
super(message);
|
|
81
|
+
this.name = "ReclaimNotAllowedError";
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** A purge whose backing store is not wired in this composition. */
|
|
86
|
+
export class PurgeNotWiredError extends Error {
|
|
87
|
+
readonly status = 501;
|
|
88
|
+
|
|
89
|
+
constructor() {
|
|
90
|
+
super("Health & error log purging is not offered in this deployment's console composition.");
|
|
91
|
+
this.name = "PurgeNotWiredError";
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import { readdir, stat, unlink } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { payloadDirectory, prunePayloadFrames } from "../../observability/payload-store";
|
|
4
|
+
import type {
|
|
5
|
+
DiskUsageBucket,
|
|
6
|
+
DiskUsageResponse,
|
|
7
|
+
PurgeTarget,
|
|
8
|
+
ReclaimAction,
|
|
9
|
+
ReclaimResponse,
|
|
10
|
+
} from "./contracts";
|
|
11
|
+
import { PurgeNotWiredError, ReclaimNotAllowedError } from "./contracts";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Reads and reclaims the on-disk telemetry and payload directories, and
|
|
15
|
+
* purges the health & error log — the console's one row-level deletion.
|
|
16
|
+
*
|
|
17
|
+
* Reclaiming files is safe because the data is already past the point the
|
|
18
|
+
* gateway would read it: payload frames expire minutes after they are written,
|
|
19
|
+
* and telemetry past the retention window sits outside every query's scan
|
|
20
|
+
* bound. The health & error log purge is the deliberate exception: the rows are
|
|
21
|
+
* an operator's diagnostic trail that no request-path code reads, so clearing
|
|
22
|
+
* them costs nothing but the history itself.
|
|
23
|
+
*
|
|
24
|
+
* The telemetry directory is injected rather than read from its module
|
|
25
|
+
* constant so the scanner can be pointed at a fixture. The payload directory
|
|
26
|
+
* is not: reclaim goes through `prunePayloadFrames`, which resolves the
|
|
27
|
+
* directory itself, and a second path would let the two drift so the scanner
|
|
28
|
+
* measured one place while the prune emptied another. The health-event store
|
|
29
|
+
* hook is injected for the same reason: this module should know its numbers,
|
|
30
|
+
* not how storage reaches them.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
export interface DataManagementConfig {
|
|
34
|
+
readonly telemetryDir: string;
|
|
35
|
+
readonly retentionDays: number;
|
|
36
|
+
readonly payloadRetentionMs: number;
|
|
37
|
+
/**
|
|
38
|
+
* Whether hourly files may be dropped from the console. Off unless a caller
|
|
39
|
+
* opts in: an hourly file still inside the compaction window is the only
|
|
40
|
+
* copy of its requests, so removing one loses real history.
|
|
41
|
+
*/
|
|
42
|
+
readonly allowHourlyReclaim: boolean;
|
|
43
|
+
/**
|
|
44
|
+
* The health & error log store, backing the console's only row-level purge.
|
|
45
|
+
* Absent (a reduced console composition) means the purge is not offered
|
|
46
|
+
* rather than failing to boot.
|
|
47
|
+
*/
|
|
48
|
+
readonly healthEvents?:
|
|
49
|
+
| {
|
|
50
|
+
count(): Promise<number>;
|
|
51
|
+
purgeAll(): Promise<number>;
|
|
52
|
+
}
|
|
53
|
+
| undefined;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
interface Scan {
|
|
57
|
+
readonly bytes: number;
|
|
58
|
+
readonly files: number;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function scan(
|
|
62
|
+
directory: string,
|
|
63
|
+
match: (name: string) => boolean,
|
|
64
|
+
modifiedBefore?: number,
|
|
65
|
+
): Promise<Scan> {
|
|
66
|
+
const names = await readdir(directory).catch(() => [] as string[]);
|
|
67
|
+
let bytes = 0;
|
|
68
|
+
let files = 0;
|
|
69
|
+
for (const name of names) {
|
|
70
|
+
if (!match(name)) continue;
|
|
71
|
+
const info = await stat(join(directory, name)).catch(() => undefined);
|
|
72
|
+
if (!info?.isFile()) continue;
|
|
73
|
+
if (modifiedBefore !== undefined && info.mtimeMs >= modifiedBefore) continue;
|
|
74
|
+
bytes += info.size;
|
|
75
|
+
files += 1;
|
|
76
|
+
}
|
|
77
|
+
return { bytes, files };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const isIndex = (name: string): boolean => name.endsWith(".index.json");
|
|
81
|
+
const isHourly = (name: string): boolean => name.endsWith(".events.jsonl");
|
|
82
|
+
// Both extensions: old `.jsonb` files are still on disk and still counted.
|
|
83
|
+
const isFrame = (name: string): boolean =>
|
|
84
|
+
name.endsWith(".jsonl") || name.endsWith(".jsonb");
|
|
85
|
+
|
|
86
|
+
/** The UTC day a telemetry file name starts with, compared to a cutoff. */
|
|
87
|
+
function dayBefore(name: string, cutoff: number): boolean {
|
|
88
|
+
const at = Date.parse(`${name.slice(0, 10)}T00:00:00Z`);
|
|
89
|
+
return Number.isFinite(at) && at < cutoff;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function createDataManagementOperations(config: DataManagementConfig) {
|
|
93
|
+
const payloadDir = (): string => payloadDirectory();
|
|
94
|
+
|
|
95
|
+
async function usage(): Promise<DiskUsageResponse> {
|
|
96
|
+
const retentionCutoff = Date.now() - config.retentionDays * 24 * 60 * 60_000;
|
|
97
|
+
const payloadCutoff = Date.now() - config.payloadRetentionMs;
|
|
98
|
+
|
|
99
|
+
const [hourly, compacted, staleCompacted, frames, staleFrames, healthEventCount] = await Promise.all([
|
|
100
|
+
scan(config.telemetryDir, isHourly),
|
|
101
|
+
scan(config.telemetryDir, isIndex),
|
|
102
|
+
scan(config.telemetryDir, (name) => isIndex(name) && dayBefore(name, retentionCutoff)),
|
|
103
|
+
scan(payloadDir(), isFrame),
|
|
104
|
+
scan(payloadDir(), isFrame, payloadCutoff),
|
|
105
|
+
config.healthEvents?.count(),
|
|
106
|
+
]);
|
|
107
|
+
|
|
108
|
+
const buckets: DiskUsageBucket[] = [
|
|
109
|
+
{
|
|
110
|
+
id: "telemetry_hourly",
|
|
111
|
+
label: "Daily request files",
|
|
112
|
+
bytes: hourly.bytes,
|
|
113
|
+
files: hourly.files,
|
|
114
|
+
note: "Feeds the request list and detail drawer.",
|
|
115
|
+
reclaimableBytes: null,
|
|
116
|
+
reclaimableFiles: null,
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
id: "telemetry_compacted",
|
|
120
|
+
label: "Day indexes",
|
|
121
|
+
bytes: compacted.bytes,
|
|
122
|
+
files: compacted.files,
|
|
123
|
+
note: "Feeds usage totals, breakdowns and the chart.",
|
|
124
|
+
reclaimableBytes: staleCompacted.bytes,
|
|
125
|
+
reclaimableFiles: staleCompacted.files,
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
id: "payload_frames",
|
|
129
|
+
label: "Payload frames",
|
|
130
|
+
bytes: frames.bytes,
|
|
131
|
+
files: frames.files,
|
|
132
|
+
note: `Captured bodies, expiring ${Math.round(
|
|
133
|
+
config.payloadRetentionMs / 60_000,
|
|
134
|
+
)} min after write.`,
|
|
135
|
+
reclaimableBytes: staleFrames.bytes,
|
|
136
|
+
reclaimableFiles: staleFrames.files,
|
|
137
|
+
},
|
|
138
|
+
];
|
|
139
|
+
|
|
140
|
+
return {
|
|
141
|
+
buckets,
|
|
142
|
+
totalBytes: buckets.reduce((sum, bucket) => sum + bucket.bytes, 0),
|
|
143
|
+
retentionDays: config.retentionDays,
|
|
144
|
+
payloadRetentionMs: config.payloadRetentionMs,
|
|
145
|
+
...(healthEventCount === undefined ? {} : { healthEventCount }),
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function reclaim(action: ReclaimAction): Promise<ReclaimResponse> {
|
|
150
|
+
if (action === "payload_frames") {
|
|
151
|
+
// Delegated rather than reimplemented: frame expiry lives inside each
|
|
152
|
+
// file, so a file's mtime alone cannot say whether it still holds a live
|
|
153
|
+
// frame. Measuring around the call is how the bytes are reported without
|
|
154
|
+
// duplicating that rule.
|
|
155
|
+
const before = await scan(payloadDir(), isFrame);
|
|
156
|
+
// `before` is the expiry bound: a frame still live at this instant is
|
|
157
|
+
// kept. The second argument covers files that cannot be parsed — one
|
|
158
|
+
// written after that bound might still hold a live frame in the part
|
|
159
|
+
// that failed to parse, so it is left for a later pass.
|
|
160
|
+
const now = new Date();
|
|
161
|
+
const deletedFiles = await prunePayloadFrames(
|
|
162
|
+
now,
|
|
163
|
+
new Date(now.getTime() - config.payloadRetentionMs),
|
|
164
|
+
);
|
|
165
|
+
const after = await scan(payloadDir(), isFrame);
|
|
166
|
+
return {
|
|
167
|
+
action,
|
|
168
|
+
deletedFiles,
|
|
169
|
+
reclaimedBytes: Math.max(0, before.bytes - after.bytes),
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
if (action === "telemetry_retention") {
|
|
173
|
+
// Only day indexes past retention, and only together with their events.
|
|
174
|
+
// Deleting one without the other is the failure this avoids: an index
|
|
175
|
+
// with no events reports totals for requests that are gone.
|
|
176
|
+
const cutoff = Date.now() - config.retentionDays * 24 * 60 * 60_000;
|
|
177
|
+
return unlinkWhere(action, config.telemetryDir, (name) =>
|
|
178
|
+
(isIndex(name) || isHourly(name)) && dayBefore(name, cutoff),
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
if (!config.allowHourlyReclaim) {
|
|
182
|
+
throw new ReclaimNotAllowedError(
|
|
183
|
+
"Hourly files are the only copy of their requests until compaction folds them up, so the console will not delete them.",
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
return unlinkWhere(action, config.telemetryDir, isHourly);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** The one row-level deletion the console offers. */
|
|
190
|
+
async function purge(target: PurgeTarget): Promise<number> {
|
|
191
|
+
switch (target) {
|
|
192
|
+
case "health_events": {
|
|
193
|
+
const store = config.healthEvents;
|
|
194
|
+
if (store === undefined) throw new PurgeNotWiredError();
|
|
195
|
+
return await store.purgeAll();
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
return { usage, reclaim, purge };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
async function unlinkWhere(
|
|
204
|
+
action: ReclaimAction,
|
|
205
|
+
directory: string,
|
|
206
|
+
match: (name: string) => boolean,
|
|
207
|
+
): Promise<ReclaimResponse> {
|
|
208
|
+
const names = await readdir(directory).catch(() => [] as string[]);
|
|
209
|
+
let deletedFiles = 0;
|
|
210
|
+
let reclaimedBytes = 0;
|
|
211
|
+
for (const name of names) {
|
|
212
|
+
if (!match(name)) continue;
|
|
213
|
+
const path = join(directory, name);
|
|
214
|
+
// Measured before the unlink, since the size is gone afterwards.
|
|
215
|
+
const size = await stat(path)
|
|
216
|
+
.then((value) => value.size)
|
|
217
|
+
.catch(() => 0);
|
|
218
|
+
const removed = await unlink(path).then(
|
|
219
|
+
() => true,
|
|
220
|
+
() => false,
|
|
221
|
+
);
|
|
222
|
+
if (!removed) continue;
|
|
223
|
+
deletedFiles += 1;
|
|
224
|
+
reclaimedBytes += size;
|
|
225
|
+
}
|
|
226
|
+
// The response names the action, so a caller that fired two reclaims can
|
|
227
|
+
// tell which result is which rather than both reading the same thing.
|
|
228
|
+
return { action, deletedFiles, reclaimedBytes };
|
|
229
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { Elysia } from "elysia";
|
|
2
|
+
import { t } from "elysia";
|
|
3
|
+
import { errorResponse, requireAnyScope } from "../shared/errors";
|
|
4
|
+
import type { AccessDecision } from "../../security/access-control";
|
|
5
|
+
import { createDataManagementOperations, type DataManagementConfig } from "./operations";
|
|
6
|
+
import type { PurgeTarget, ReclaimAction, ReclaimResponse } from "./contracts";
|
|
7
|
+
|
|
8
|
+
const WRITE_SCOPES = ["dashboard:write"] as const;
|
|
9
|
+
|
|
10
|
+
const reclaimBody = t.Object({
|
|
11
|
+
action: t.Union([
|
|
12
|
+
t.Literal("payload_frames"),
|
|
13
|
+
t.Literal("telemetry_retention"),
|
|
14
|
+
t.Literal("telemetry_hourly"),
|
|
15
|
+
]),
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
const purgeBody = t.Object({
|
|
19
|
+
target: t.Literal("health_events"),
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Console routes for disk housekeeping.
|
|
24
|
+
*
|
|
25
|
+
* Reclaim and flush both need `dashboard:write`. Reading is deliberately the
|
|
26
|
+
* same scope as the rest of the console rather than something narrower: the
|
|
27
|
+
* sizes are not sensitive, but splitting the two would mean a caller who can
|
|
28
|
+
* already write could not see what they are about to affect.
|
|
29
|
+
*/
|
|
30
|
+
export function createDataManagementRoutes(
|
|
31
|
+
config: DataManagementConfig & {
|
|
32
|
+
readonly accessResolver: (request: Request) => AccessDecision | undefined;
|
|
33
|
+
readonly auditSink?: {
|
|
34
|
+
record(entry: {
|
|
35
|
+
readonly access: AccessDecision;
|
|
36
|
+
readonly action: string;
|
|
37
|
+
readonly target: string | null;
|
|
38
|
+
readonly detail?: Record<string, unknown>;
|
|
39
|
+
}): Promise<void> | void;
|
|
40
|
+
} | undefined;
|
|
41
|
+
readonly flusher?: { flush(): Promise<void> } | undefined;
|
|
42
|
+
},
|
|
43
|
+
): Elysia {
|
|
44
|
+
const operations = createDataManagementOperations(config);
|
|
45
|
+
|
|
46
|
+
return new Elysia({ prefix: "/data-management" })
|
|
47
|
+
.get("/usage", async ({ request, set }) => {
|
|
48
|
+
try {
|
|
49
|
+
const access = requireAnyScope(config.accessResolver(request), WRITE_SCOPES);
|
|
50
|
+
void access;
|
|
51
|
+
return await operations.usage();
|
|
52
|
+
} catch (error) {
|
|
53
|
+
return errorResponse(error, set, "Could not read disk usage");
|
|
54
|
+
}
|
|
55
|
+
})
|
|
56
|
+
.post("/reclaim", { body: reclaimBody }, async ({ request, body, set }) => {
|
|
57
|
+
try {
|
|
58
|
+
const access = requireAnyScope(config.accessResolver(request), WRITE_SCOPES);
|
|
59
|
+
const action = body.action as ReclaimAction;
|
|
60
|
+
const result: ReclaimResponse = await operations.reclaim(action);
|
|
61
|
+
await config.auditSink?.record({
|
|
62
|
+
access,
|
|
63
|
+
action: `data_management.reclaim.${action}`,
|
|
64
|
+
target: access.tenantId ?? null,
|
|
65
|
+
detail: { deletedFiles: result.deletedFiles, reclaimedBytes: result.reclaimedBytes },
|
|
66
|
+
});
|
|
67
|
+
set.status = 200;
|
|
68
|
+
return result;
|
|
69
|
+
} catch (error) {
|
|
70
|
+
return errorResponse(error, set, "Reclaim failed");
|
|
71
|
+
}
|
|
72
|
+
})
|
|
73
|
+
.post("/flush", async ({ request, set }) => {
|
|
74
|
+
try {
|
|
75
|
+
const access = requireAnyScope(config.accessResolver(request), WRITE_SCOPES);
|
|
76
|
+
await config.flusher?.flush();
|
|
77
|
+
await config.auditSink?.record({
|
|
78
|
+
access,
|
|
79
|
+
action: "data_management.flush",
|
|
80
|
+
target: access.tenantId ?? null,
|
|
81
|
+
});
|
|
82
|
+
set.status = 200;
|
|
83
|
+
return { flushed: true };
|
|
84
|
+
} catch (error) {
|
|
85
|
+
return errorResponse(error, set, "Flush failed");
|
|
86
|
+
}
|
|
87
|
+
})
|
|
88
|
+
.post("/purge", { body: purgeBody }, async ({ request, body, set }) => {
|
|
89
|
+
try {
|
|
90
|
+
const access = requireAnyScope(config.accessResolver(request), WRITE_SCOPES);
|
|
91
|
+
const deleted: number = await operations.purge(body.target as PurgeTarget);
|
|
92
|
+
await config.auditSink?.record({
|
|
93
|
+
access,
|
|
94
|
+
action: `data_management.purge.${body.target}`,
|
|
95
|
+
target: access.tenantId ?? null,
|
|
96
|
+
detail: { deleted },
|
|
97
|
+
});
|
|
98
|
+
set.status = 200;
|
|
99
|
+
return { target: body.target, deleted } satisfies { target: PurgeTarget; deleted: number };
|
|
100
|
+
} catch (error) {
|
|
101
|
+
return errorResponse(error, set, "Purge failed");
|
|
102
|
+
}
|
|
103
|
+
}) as unknown as Elysia;
|
|
104
|
+
}
|