@bitkyc08/opencodex 2.99.0-preview.20261108 → 2.100.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS_INSTALL.md +19 -1
- package/README.md +37 -33
- package/SPONSORS.md +5 -1
- package/bin/ocx.mjs +1 -1
- package/gui/dist/assets/{App-C1hGwaN1.css → App-COo8Yopd.css} +1 -1
- package/gui/dist/assets/App-CitvEefy.js +59 -0
- package/gui/dist/assets/Tray-CeKVWQyB.js +1 -0
- package/gui/dist/assets/{brand-mark-DpiBmFGI.js → brand-mark-M92UnxX2.js} +1 -1
- package/gui/dist/assets/index-BP47do6-.js +94 -0
- package/gui/dist/assets/index-DmDooSAG.css +2 -0
- package/gui/dist/assets/{segment-thumb-features-JcszQ18T.js → segment-thumb-features-CcxZBnxv.js} +1 -1
- package/gui/dist/classic/assets/App-aYgIQMxI.js +58 -0
- package/gui/dist/classic/assets/{Tray-5v86r9Az.js → Tray-D8jfUXpQ.js} +1 -1
- package/gui/dist/classic/assets/index-CfGqHooN.js +94 -0
- package/gui/dist/classic/assets/quota-summary-R-ih7OlS.js +1 -0
- package/gui/dist/classic/index.html +1 -1
- package/gui/dist/index.html +2 -2
- package/native/ocx-child/Cargo.lock +215 -0
- package/native/ocx-child/Cargo.toml +31 -0
- package/native/ocx-child/src/cli.rs +1901 -0
- package/native/ocx-child/src/codex_home.rs +429 -0
- package/native/ocx-child/src/codex_settings.rs +231 -0
- package/native/ocx-child/src/daemon.rs +363 -0
- package/native/ocx-child/src/http_wire.rs +619 -0
- package/native/ocx-child/src/journal.rs +977 -0
- package/native/ocx-child/src/lib.rs +22 -0
- package/native/ocx-child/src/main.rs +9 -0
- package/native/ocx-child/src/parity.rs +102 -0
- package/native/ocx-child/src/platform.rs +1487 -0
- package/native/ocx-child/src/relay.rs +1288 -0
- package/native/ocx-child/src/relay_auth.rs +166 -0
- package/native/ocx-child/src/routes.rs +236 -0
- package/native/ocx-child/src/state.rs +190 -0
- package/native/ocx-child/src/test_hooks.rs +58 -0
- package/native/ocx-child/src/toml_surgeon.rs +1156 -0
- package/package.json +8 -2
- package/src/adapters/anthropic.ts +28 -2
- package/src/adapters/claude-cli/adapter.ts +5 -25
- package/src/adapters/claude-cli/instruction-images.ts +31 -0
- package/src/adapters/claude-cli/stable-replay.ts +42 -14
- package/src/adapters/claude-cli/usage-admission.ts +53 -64
- package/src/adapters/coding-agent/protocol.ts +1 -1
- package/src/adapters/devin/live-models.ts +3 -0
- package/src/adapters/google-http.ts +3 -0
- package/src/adapters/google-vertex-key-project.ts +10 -2
- package/src/adapters/google.ts +40 -8
- package/src/adapters/kiro/usage.ts +3 -13
- package/src/adapters/openai-responses/passthrough.ts +5 -2
- package/src/adapters/openai-responses/request-strips.ts +32 -0
- package/src/adapters/openai-responses/tool-schema.ts +9 -4
- package/src/chat/inbound.ts +1 -1
- package/src/claude/auth-detect.ts +26 -8
- package/src/claude/inbound-content-options.ts +18 -3
- package/src/claude/inbound.ts +5 -2
- package/src/cli/bundled-bun-diagnostic.ts +91 -0
- package/src/cli/capabilities-agents-routing.ts +2 -2
- package/src/cli/capabilities-observe-system.ts +60 -0
- package/src/cli/combo-input.ts +6 -0
- package/src/cli/combo-result.ts +1 -1
- package/src/cli/combo.ts +4 -0
- package/src/cli/config-command.ts +16 -2
- package/src/cli/connect.ts +12 -10
- package/src/cli/dispatch.ts +4 -0
- package/src/cli/doctor.ts +11 -0
- package/src/cli/help.ts +1 -0
- package/src/cli/init.ts +4 -0
- package/src/cli/link-enrollment.ts +152 -25
- package/src/cli/link-local-control.ts +49 -0
- package/src/cli/link.ts +37 -10
- package/src/cli/provider-lifecycle-runtime.ts +4 -2
- package/src/cli/registry.ts +13 -0
- package/src/cli/star-prompt.ts +2 -0
- package/src/cli/status.ts +1 -1
- package/src/cli/telemetry-profile.ts +206 -0
- package/src/cli/telemetry.ts +260 -0
- package/src/client/hub-relay.ts +26 -0
- package/src/client/machine-api.ts +3 -0
- package/src/clients/config-export/t3code.ts +44 -5
- package/src/clients/omo-role-models.ts +1 -1
- package/src/codex/catalog/bundled.ts +44 -2
- package/src/codex/runtime.ts +12 -0
- package/src/codex/shim-restore-lock.ts +67 -30
- package/src/combos/types.ts +21 -1
- package/src/config/atomic-write.ts +24 -1
- package/src/config/diagnostics.ts +15 -0
- package/src/config/live-reconcile.ts +17 -0
- package/src/config/load-degrade.ts +16 -0
- package/src/config/provider-validation.ts +29 -0
- package/src/config/schema/config-schema.ts +24 -0
- package/src/config/schema/leaf-validators.ts +17 -0
- package/src/config/telemetry-carry.ts +30 -0
- package/src/config.ts +13 -1
- package/src/generated/compatibility-version.json +405 -153
- package/src/generated/model-metadata.ts +3 -3
- package/src/integrations/aside-profiles.ts +7 -1
- package/src/integrations/owned-refresh.ts +9 -2
- package/src/lab/artifacts/secure-fs.ts +12 -11
- package/src/lab/fabric/scratch.ts +11 -9
- package/src/lab/public/community.ts +2 -3
- package/src/lab/public/file-safety.ts +18 -20
- package/src/lab/public/mutation-lock.ts +109 -337
- package/src/lab/public/private-file.ts +4 -3
- package/src/lib/bun-runtime-preflight.ts +1 -1
- package/src/lib/codex-catalog-read-only.ts +122 -0
- package/src/lib/config-ownership.ts +49 -18
- package/src/lib/errors.ts +11 -0
- package/src/lib/gcp-adc.ts +167 -48
- package/src/lib/jsonc.ts +4 -2
- package/src/lib/local-control-socket.ts +88 -0
- package/src/lib/privacy.ts +15 -0
- package/src/lib/provider-client-headers.ts +3 -0
- package/src/lib/token-estimate.ts +13 -2
- package/src/link/child-binary-source.ts +228 -0
- package/src/link/ephemeral-child.ts +45 -0
- package/src/link/remote-plant.ts +306 -0
- package/src/link/remote-windows.ts +59 -5
- package/src/link/ssh-argv.ts +71 -8
- package/src/link/status-projection.ts +2 -0
- package/src/link/store.ts +5 -1
- package/src/oauth/claude-keychain-read.ts +165 -0
- package/src/oauth/local-token-detect.ts +14 -17
- package/src/protocols/envelope.ts +20 -2
- package/src/providers/api-key-selection.ts +8 -2
- package/src/providers/claude-cli-usage-state.ts +93 -0
- package/src/providers/github-copilot-auto.ts +25 -5
- package/src/providers/meta-muse-plan-refusal.ts +94 -0
- package/src/providers/model-rename-fields.ts +3 -0
- package/src/providers/quota/vendor-probes-key.ts +23 -4
- package/src/providers/quota/vendor-probes-oauth.ts +59 -0
- package/src/providers/quota/workbuddy.ts +2 -2
- package/src/providers/quota.ts +25 -18
- package/src/providers/registry/entries-core.ts +7 -4
- package/src/providers/registry/entries-extended.ts +39 -7
- package/src/providers/registry/model-seeds.ts +10 -2
- package/src/providers/resolved-model-policy.ts +4 -0
- package/src/providers/routed-config.ts +30 -3
- package/src/providers/stale-no-vision-seed-migration.ts +7 -0
- package/src/providers/vertex-environment-authority.ts +50 -0
- package/src/providers/workbuddy-credit-balance.ts +38 -6
- package/src/quota-pool/assemble.ts +14 -6
- package/src/router.ts +1 -0
- package/src/server/adapter-resolve.ts +11 -6
- package/src/server/audio-dictation.ts +21 -7
- package/src/server/auth-cors.ts +19 -0
- package/src/server/background-lifecycle.ts +71 -0
- package/src/server/chat-completions.ts +2 -0
- package/src/server/claude-messages.ts +30 -3
- package/src/server/index/live-sideband.ts +15 -1
- package/src/server/index/local-control-listener.ts +515 -0
- package/src/server/index/optional-listeners.ts +21 -1
- package/src/server/index/serve-options.ts +4 -3
- package/src/server/index/websocket-handler.ts +2 -0
- package/src/server/index.ts +3 -0
- package/src/server/management/combo-routes.ts +6 -1
- package/src/server/management/context.ts +1 -0
- package/src/server/management/link-child-preflight.ts +74 -0
- package/src/server/management/link-enrollment.ts +190 -25
- package/src/server/management/link-route-state.ts +6 -1
- package/src/server/management/link-routes.ts +27 -9
- package/src/server/management/model-rows.ts +4 -0
- package/src/server/management/oauth-account-routes.ts +9 -4
- package/src/server/management/provider-overwrite-carry.ts +9 -2
- package/src/server/management/provider-routes.ts +14 -0
- package/src/server/management/provider-thinking-display-patch.ts +54 -0
- package/src/server/management/route-registry.ts +7 -0
- package/src/server/management/session-routes.ts +4 -1
- package/src/server/management/telemetry-routes.ts +37 -0
- package/src/server/management/usage-aggregate-cache.ts +290 -67
- package/src/server/management-api.ts +11 -2
- package/src/server/messages-native-preflight.ts +166 -0
- package/src/server/messages-native.ts +143 -43
- package/src/server/messages-shared.ts +14 -0
- package/src/server/proxy-liveness.ts +99 -6
- package/src/server/request-log-conversation.ts +18 -2
- package/src/server/request-log.ts +7 -2
- package/src/server/responses/adapter-continuation.ts +1 -0
- package/src/server/responses/adapter-dispatch.ts +11 -28
- package/src/server/responses/adapter-failure-message.ts +68 -0
- package/src/server/responses/combo-stream-preflight.ts +14 -2
- package/src/server/responses/core-auth.ts +26 -5
- package/src/server/responses/core-codex-account.ts +8 -1
- package/src/server/responses/core-combo-failure.ts +6 -1
- package/src/server/responses/core-combo-native.ts +150 -37
- package/src/server/responses/core-combo.ts +15 -9
- package/src/server/responses/core-normalize.ts +1 -0
- package/src/server/responses/core-options.ts +7 -1
- package/src/server/responses/core-pipeline.ts +21 -1
- package/src/server/responses/fetch-helpers.ts +5 -0
- package/src/server/responses/opengateway-completion-guard.ts +7 -3
- package/src/server/responses/passthrough-delivery.ts +8 -1
- package/src/server/responses/passthrough-error.ts +22 -0
- package/src/server/responses/request-prepare.ts +7 -1
- package/src/server/responses/request-transport.ts +28 -5
- package/src/server/responses/run-turn-execution.ts +13 -0
- package/src/server/sse-frame-buffer.ts +5 -0
- package/src/server/ws-bridge.ts +10 -0
- package/src/service/launchd.ts +78 -4
- package/src/telemetry/aggregate.ts +348 -0
- package/src/telemetry/consent.ts +128 -0
- package/src/telemetry/contract.ts +403 -0
- package/src/telemetry/disclosure.ts +19 -0
- package/src/telemetry/install-id.ts +226 -0
- package/src/telemetry/payload.ts +129 -0
- package/src/telemetry/profile/client.ts +150 -0
- package/src/telemetry/profile/contract.ts +288 -0
- package/src/telemetry/profile/disclosure.ts +14 -0
- package/src/telemetry/profile/paths.ts +13 -0
- package/src/telemetry/profile/routes.ts +99 -0
- package/src/telemetry/profile/service.ts +234 -0
- package/src/telemetry/profile/state.ts +189 -0
- package/src/telemetry/profile/sync.ts +276 -0
- package/src/telemetry/routes.ts +183 -0
- package/src/telemetry/scheduler.ts +386 -0
- package/src/telemetry/sender.ts +97 -0
- package/src/telemetry/state.ts +334 -0
- package/src/types/combo-jev-effort-mode.ts +11 -0
- package/src/types/config.ts +3 -0
- package/src/types/provider.ts +11 -1
- package/src/types/request.ts +2 -0
- package/src/types/telemetry.ts +15 -0
- package/src/types/wire.ts +10 -0
- package/src/types.ts +5 -0
- package/src/update/index.ts +7 -1
- package/src/update/minisign.d.mts +17 -0
- package/src/update/minisign.mjs +88 -0
- package/src/update/signed-supply.d.mts +59 -0
- package/src/update/signed-supply.mjs +289 -0
- package/src/update/trust-root.json +38 -0
- package/src/update/update-bounded-json.d.mts +4 -0
- package/src/update/update-bounded-json.mjs +113 -0
- package/src/update/update-bytes.d.mts +2 -0
- package/src/update/update-bytes.mjs +23 -0
- package/src/update/update-candidate-id.d.mts +2 -0
- package/src/update/update-candidate-id.mjs +23 -0
- package/src/update/update-checkpoint-codec.d.mts +33 -0
- package/src/update/update-checkpoint-codec.mjs +217 -0
- package/src/update/update-contracts.d.mts +24 -0
- package/src/update/update-contracts.mjs +207 -0
- package/src/update/update-signature-threshold.d.mts +12 -0
- package/src/update/update-signature-threshold.mjs +47 -0
- package/src/update/update-signed-version.d.mts +5 -0
- package/src/update/update-signed-version.mjs +25 -0
- package/src/update/update-trust-anchor.d.mts +2 -0
- package/src/update/update-trust-anchor.mjs +2 -0
- package/src/usage/cache-diagnostic.ts +195 -50
- package/src/usage/expected-prices.ts +20 -2
- package/src/usage/filter-index.ts +1469 -0
- package/src/usage/ledger-scanner.ts +245 -4
- package/src/usage/source-filter.ts +10 -2
- package/src/usage/summary.ts +16 -0
- package/src/web-search/parse.ts +244 -38
- package/gui/dist/assets/App-CWz0areF.js +0 -59
- package/gui/dist/assets/Tray-BosDoskm.js +0 -1
- package/gui/dist/assets/index-CAxjGWkm.js +0 -94
- package/gui/dist/assets/index-bnUWypyz.css +0 -2
- package/gui/dist/classic/assets/App-BOEsFmiN.js +0 -58
- package/gui/dist/classic/assets/index-ClZ_zZ77.js +0 -94
- package/gui/dist/classic/assets/quota-summary-xwvDtvu7.js +0 -1
|
@@ -0,0 +1,1469 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Derived candidate index for filtered usage queries (#7846).
|
|
3
|
+
*
|
|
4
|
+
* `usage.jsonl` stays the only authority. This module maintains a rebuildable SQLite
|
|
5
|
+
* projection that records, per complete ledger row, its byte offset and the few fields a
|
|
6
|
+
* filtered `GET /api/usage` selects on: timestamp, API-key id, client/surface/admission
|
|
7
|
+
* labels, and one row per provider/model attribution the row can match. A filtered query
|
|
8
|
+
* asks the index which rows qualify, then re-reads and normalizes only those rows.
|
|
9
|
+
*
|
|
10
|
+
* The alternative was a per-filter full scan: 143k rows cost roughly 25 seconds each time a
|
|
11
|
+
* new provider/model/key/window combination is requested, and the work was repeated for
|
|
12
|
+
* every distinct filter. Here the expensive pass happens once -- the initial build, and
|
|
13
|
+
* again only when the ledger is replaced -- and every later query, including a brand-new
|
|
14
|
+
* filter, reads a bounded candidate set.
|
|
15
|
+
*
|
|
16
|
+
* Crash and multi-process contract:
|
|
17
|
+
* - Rows belong to a generation. Meta names the one published generation; readers only ever
|
|
18
|
+
* select rows of that generation, bounded by the published byte offset.
|
|
19
|
+
* - A rebuild stages a fresh generation and publishes it in one short transaction that
|
|
20
|
+
* re-checks the generation it started from. Nothing deletes another writer's staged rows
|
|
21
|
+
* while it is still ingesting; only a successful publish prunes other generations. An
|
|
22
|
+
* interrupted rebuild therefore leaves the previously published generation intact, so a
|
|
23
|
+
* crash can never turn a complete projection into a complete-looking zero.
|
|
24
|
+
* - An append writes into the published generation and publishes its new offset, digest,
|
|
25
|
+
* counts and span in the same transaction. A crash mid-append leaves rows beyond the
|
|
26
|
+
* published offset, which readers ignore and the next append replays idempotently.
|
|
27
|
+
* - Two writers that stage concurrently get distinct generations, so neither can mix the
|
|
28
|
+
* other's rows. The loser's publish is fenced out and it retries against the winner.
|
|
29
|
+
* - Metadata is written by one transaction; there is no window where the cursor, digest,
|
|
30
|
+
* counts and span describe different scans.
|
|
31
|
+
*
|
|
32
|
+
* Bounds and identity:
|
|
33
|
+
* - Ingest reuses `scanUsageLedgerCooperatively`, so the 1 MiB row ceiling, the 1 MiB
|
|
34
|
+
* chunk, the cooperative yield, the opened-EOF snapshot boundary, and the
|
|
35
|
+
* path/dev/inode/birthtime identity with its 64 KiB boundary digest all stay owned by the
|
|
36
|
+
* ledger scanner rather than being re-implemented here.
|
|
37
|
+
* - A replaced, truncated, or same-size rewritten ledger rebuilds the projection. Rows are
|
|
38
|
+
* only ever appended to the projection; the ledger is never truncated or rewritten.
|
|
39
|
+
* - An index that disagrees with the ledger (a recorded offset no longer frames a complete
|
|
40
|
+
* row) is discarded and rebuilt rather than served partially.
|
|
41
|
+
* - A file at the index path that is not this projection is never modified, and a passive
|
|
42
|
+
* management read never creates, writes or registers anything.
|
|
43
|
+
*
|
|
44
|
+
* Rebuild cost: the initial build normalizes every row once. It is cooperative, so the
|
|
45
|
+
* event loop yields between chunks, and it is persisted, so a restart does not repeat it.
|
|
46
|
+
* The projection stores offsets and keys, not row bodies, which keeps it far smaller than
|
|
47
|
+
* the ledger it indexes. A nonselective filter (no provider/model/key, `client=all`,
|
|
48
|
+
* `device=all`) still has to normalize every row in its window; the index makes the
|
|
49
|
+
* selection exact, not sublinear.
|
|
50
|
+
*/
|
|
51
|
+
import { constants as sqliteConstants, Database } from "bun:sqlite";
|
|
52
|
+
import { randomUUID } from "node:crypto";
|
|
53
|
+
import { chmodSync, existsSync } from "node:fs";
|
|
54
|
+
import { getConfigDir } from "../config";
|
|
55
|
+
import { recordOwnedConfigPath } from "../lib/config-ownership";
|
|
56
|
+
import {
|
|
57
|
+
preparePrivateSqliteDatabase,
|
|
58
|
+
SQLITE_SIDECAR_SUFFIXES,
|
|
59
|
+
UnsafeSqlitePathError,
|
|
60
|
+
} from "../lib/private-sqlite-path";
|
|
61
|
+
import { hardenSecretPath } from "../lib/windows-secret-acl";
|
|
62
|
+
import { baseProviderLabel } from "../providers/label";
|
|
63
|
+
import {
|
|
64
|
+
currentUsageLogRevision,
|
|
65
|
+
usageLogIdentityKey,
|
|
66
|
+
usageLogRevisionKey,
|
|
67
|
+
type PersistedUsageEntry,
|
|
68
|
+
type UsageLogRevision,
|
|
69
|
+
} from "./log";
|
|
70
|
+
import {
|
|
71
|
+
readUsageLedgerRowsAtOffsets,
|
|
72
|
+
scanUsageLedgerCooperatively,
|
|
73
|
+
UsageLedgerRebuildRequiredError,
|
|
74
|
+
type UsageLedgerScanResult,
|
|
75
|
+
} from "./ledger-scanner";
|
|
76
|
+
import { USAGE_CODEX_CLIENT_LABELS, type UsageSourceFilter } from "./source-filter";
|
|
77
|
+
import { usageAttributions } from "./summary";
|
|
78
|
+
import type { UsageTimeWindow } from "./time-range";
|
|
79
|
+
|
|
80
|
+
export const USAGE_FILTER_INDEX_SCHEMA_VERSION = 4;
|
|
81
|
+
export const USAGE_FILTER_INDEX_DB_FILENAME = "usage-filter-index.sqlite";
|
|
82
|
+
/** Rows per SQLite transaction during ingest; also the peak pending-row allocation. */
|
|
83
|
+
export const USAGE_FILTER_INDEX_INSERT_BATCH = 500;
|
|
84
|
+
const META_TABLE = "usage_filter_index_meta";
|
|
85
|
+
const MAX_REFRESH_ATTEMPTS = 3;
|
|
86
|
+
// SQLite's busy handler blocks the thread, so the wait stays short and contention is retried
|
|
87
|
+
// asynchronously instead of parking the event loop behind another writer. A single ingest batch
|
|
88
|
+
// or publish holds the write lock for a few milliseconds.
|
|
89
|
+
const SQLITE_BUSY_TIMEOUT_MS = 250;
|
|
90
|
+
/** Bound on the WAL kept between checkpoints; it is truncated to empty on the last close. */
|
|
91
|
+
const RETAINED_WAL_LIMIT_BYTES = 4 * 1024 * 1024;
|
|
92
|
+
const BATCH_BUSY_RETRIES = 3;
|
|
93
|
+
const BATCH_BUSY_RETRY_MS = 5;
|
|
94
|
+
const BUSY_RETRY_DELAYS_MS = [10, 25, 50] as const;
|
|
95
|
+
|
|
96
|
+
const SCHEMA_DDL = `
|
|
97
|
+
CREATE TABLE IF NOT EXISTS ${META_TABLE} (
|
|
98
|
+
key TEXT PRIMARY KEY,
|
|
99
|
+
value TEXT NOT NULL
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
CREATE TABLE IF NOT EXISTS usage_rows (
|
|
103
|
+
generation TEXT NOT NULL,
|
|
104
|
+
byte_offset INTEGER NOT NULL,
|
|
105
|
+
timestamp INTEGER,
|
|
106
|
+
api_key_id TEXT,
|
|
107
|
+
client TEXT,
|
|
108
|
+
surface TEXT,
|
|
109
|
+
admission_kind TEXT,
|
|
110
|
+
unattributable INTEGER NOT NULL DEFAULT 0,
|
|
111
|
+
PRIMARY KEY (generation, byte_offset)
|
|
112
|
+
) WITHOUT ROWID;
|
|
113
|
+
|
|
114
|
+
CREATE INDEX IF NOT EXISTS idx_usage_rows_ts ON usage_rows(generation, timestamp, byte_offset);
|
|
115
|
+
CREATE INDEX IF NOT EXISTS idx_usage_rows_key ON usage_rows(generation, api_key_id, timestamp, byte_offset);
|
|
116
|
+
CREATE INDEX IF NOT EXISTS idx_usage_rows_client ON usage_rows(generation, client, timestamp, byte_offset);
|
|
117
|
+
CREATE INDEX IF NOT EXISTS idx_usage_rows_admission ON usage_rows(generation, admission_kind, api_key_id, byte_offset);
|
|
118
|
+
CREATE INDEX IF NOT EXISTS idx_usage_rows_unattributable ON usage_rows(generation, unattributable);
|
|
119
|
+
|
|
120
|
+
CREATE TABLE IF NOT EXISTS usage_attributions (
|
|
121
|
+
generation TEXT NOT NULL,
|
|
122
|
+
byte_offset INTEGER NOT NULL,
|
|
123
|
+
provider TEXT NOT NULL,
|
|
124
|
+
model TEXT NOT NULL,
|
|
125
|
+
PRIMARY KEY (generation, byte_offset, provider, model)
|
|
126
|
+
) WITHOUT ROWID;
|
|
127
|
+
|
|
128
|
+
CREATE INDEX IF NOT EXISTS idx_usage_attr_provider ON usage_attributions(generation, provider, model, byte_offset);
|
|
129
|
+
CREATE INDEX IF NOT EXISTS idx_usage_attr_model ON usage_attributions(generation, model, provider, byte_offset);
|
|
130
|
+
|
|
131
|
+
CREATE TABLE IF NOT EXISTS usage_filter_index_writers (
|
|
132
|
+
generation TEXT PRIMARY KEY,
|
|
133
|
+
pid INTEGER NOT NULL,
|
|
134
|
+
started_at INTEGER NOT NULL,
|
|
135
|
+
heartbeat_at INTEGER NOT NULL
|
|
136
|
+
);
|
|
137
|
+
|
|
138
|
+
CREATE TABLE IF NOT EXISTS usage_filter_index_generations (
|
|
139
|
+
generation TEXT PRIMARY KEY,
|
|
140
|
+
created_at INTEGER NOT NULL
|
|
141
|
+
);
|
|
142
|
+
`;
|
|
143
|
+
|
|
144
|
+
const META_KEYS = {
|
|
145
|
+
schemaVersion: "schema_version",
|
|
146
|
+
generation: "generation",
|
|
147
|
+
sourcePath: "source_path",
|
|
148
|
+
sourceDev: "source_dev",
|
|
149
|
+
sourceIno: "source_ino",
|
|
150
|
+
sourceBirthtimeMs: "source_birthtime_ms",
|
|
151
|
+
sourceSize: "source_size",
|
|
152
|
+
sourceMtimeMs: "source_mtime_ms",
|
|
153
|
+
sourceCtimeMs: "source_ctime_ms",
|
|
154
|
+
indexedOffset: "indexed_offset",
|
|
155
|
+
boundaryDigest: "boundary_digest",
|
|
156
|
+
indexedRows: "indexed_rows",
|
|
157
|
+
invalidRows: "invalid_rows",
|
|
158
|
+
oversizedRows: "oversized_rows",
|
|
159
|
+
minTimestamp: "min_timestamp",
|
|
160
|
+
maxTimestamp: "max_timestamp",
|
|
161
|
+
} as const;
|
|
162
|
+
|
|
163
|
+
/** The file at the index path is not this projection. It is left untouched. */
|
|
164
|
+
export class UsageFilterIndexForeignError extends Error {
|
|
165
|
+
constructor() {
|
|
166
|
+
super("usage filter index path holds a file this projection does not own");
|
|
167
|
+
this.name = "UsageFilterIndexForeignError";
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Reading `usage.jsonl` itself failed; this is not an index problem and must not be hidden. */
|
|
172
|
+
export class UsageFilterIndexSourceError extends Error {
|
|
173
|
+
constructor(readonly sourceError: unknown) {
|
|
174
|
+
super("usage ledger could not be read for the filtered usage index");
|
|
175
|
+
this.name = "UsageFilterIndexSourceError";
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** The config directory changed while a refresh was running; the projection must not mix homes. */
|
|
180
|
+
export class UsageFilterIndexHomeChangedError extends Error {
|
|
181
|
+
constructor() {
|
|
182
|
+
super("usage filter index home changed during a refresh");
|
|
183
|
+
this.name = "UsageFilterIndexHomeChangedError";
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* A retained filtered projection read this generation. A full rebuild replaces every offset,
|
|
189
|
+
* so a projection from an older generation must be discarded even when the file identity and
|
|
190
|
+
* size are unchanged.
|
|
191
|
+
*/
|
|
192
|
+
export interface UsageFilterIndexSnapshot {
|
|
193
|
+
revision: UsageLogRevision | null;
|
|
194
|
+
identityKey: string;
|
|
195
|
+
generation: string;
|
|
196
|
+
indexedOffset: number;
|
|
197
|
+
processedThroughDigest: string;
|
|
198
|
+
indexedRows: number;
|
|
199
|
+
/** Whole-ledger diagnostics, monotonic across appends and reset by a rebuild. */
|
|
200
|
+
invalidRows: number;
|
|
201
|
+
oversizedRows: number;
|
|
202
|
+
minTimestamp: number | null;
|
|
203
|
+
maxTimestamp: number | null;
|
|
204
|
+
/** Full-file ingest passes this process has run for this home; the initial build counts. */
|
|
205
|
+
ingestPasses: number;
|
|
206
|
+
/** Rows this call ingested; a rebuild reports every row. */
|
|
207
|
+
ingestedRows: number;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
export interface UsageFilterIndexQuery {
|
|
211
|
+
provider: string | null;
|
|
212
|
+
model: string | null;
|
|
213
|
+
apiKeyId: string | null;
|
|
214
|
+
sourceFilter?: UsageSourceFilter;
|
|
215
|
+
configuredApiKeyIds?: readonly string[];
|
|
216
|
+
window?: UsageTimeWindow;
|
|
217
|
+
/** Inclusive lower bound: the byte after the last row already folded in. */
|
|
218
|
+
afterOffset?: number;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export interface UsageFilterCandidateRead {
|
|
222
|
+
snapshot: UsageFilterIndexSnapshot;
|
|
223
|
+
/** Ascending row-start offsets selected by the index. */
|
|
224
|
+
offsets: number[];
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Counts a caller can assert on.
|
|
229
|
+
*
|
|
230
|
+
* `candidateRows` is the number of ledger rows this query selected and normalized -- not the
|
|
231
|
+
* number of SQL index entries SQLite examined. For a selective filter it is far below
|
|
232
|
+
* `totalRows`; for a nonselective one it is every row in scope, which is inherent.
|
|
233
|
+
*/
|
|
234
|
+
export interface UsageFilterQueryStats {
|
|
235
|
+
totalRows: number;
|
|
236
|
+
/** Rows selected for this call; an append selects only the rows after its retained cursor. */
|
|
237
|
+
candidateRows: number;
|
|
238
|
+
/** Whether those rows are the whole candidate set or only the appended suffix. */
|
|
239
|
+
scope: "rebuild" | "append" | "unchanged";
|
|
240
|
+
ingestedRows: number;
|
|
241
|
+
ingestPasses: number;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export interface UsageFilterIndexTestHooks {
|
|
245
|
+
/**
|
|
246
|
+
* Runs after a rebuild has staged its rows and before the publish transaction. Throwing
|
|
247
|
+
* simulates a crash at that boundary; the hook also lets a test publish from a second
|
|
248
|
+
* connection to exercise the generation fence.
|
|
249
|
+
*/
|
|
250
|
+
beforePublish?: (info: {
|
|
251
|
+
dbPath: string;
|
|
252
|
+
stagedGeneration: string;
|
|
253
|
+
publishedGeneration: string;
|
|
254
|
+
}) => void | Promise<void>;
|
|
255
|
+
/** Runs after a publish committed, before the caller reads the projection. */
|
|
256
|
+
afterPublish?: (info: { dbPath: string; generation: string }) => void | Promise<void>;
|
|
257
|
+
/** Runs after a staging generation registered its lease, before its first row. */
|
|
258
|
+
afterStagingRegistered?: (info: { dbPath: string; generation: string }) => void | Promise<void>;
|
|
259
|
+
/** Runs inside the staging transaction, between the registry and lease writes. */
|
|
260
|
+
betweenStagingWrites?: (info: { dbPath: string; generation: string }) => void;
|
|
261
|
+
/** Runs after a reclamation pre-check, before its write transaction takes the lock. */
|
|
262
|
+
betweenReclaimCheckAndLock?: (info: { dbPath: string; candidates: string[] }) => void;
|
|
263
|
+
/** Runs after a refresh, before the candidate select reads the projection. */
|
|
264
|
+
afterRefresh?: (info: { dbPath: string }) => void;
|
|
265
|
+
/** Runs when a read met another writer's lock, before its back-off and retry. */
|
|
266
|
+
beforeBusyRetry?: (info: { dbPath: string; attempt: number }) => void;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
interface IndexMeta {
|
|
270
|
+
schemaVersion: number;
|
|
271
|
+
generation: string;
|
|
272
|
+
revision: UsageLogRevision | null;
|
|
273
|
+
indexedOffset: number;
|
|
274
|
+
digest: string;
|
|
275
|
+
indexedRows: number;
|
|
276
|
+
invalidRows: number;
|
|
277
|
+
oversizedRows: number;
|
|
278
|
+
minTimestamp: number | null;
|
|
279
|
+
maxTimestamp: number | null;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
let ingestFlight: { path: string; promise: Promise<UsageFilterIndexSnapshot> } | null = null;
|
|
283
|
+
const ingestPassesByPath = new Map<string, number>();
|
|
284
|
+
const rebuildRequestedPaths = new Set<string>();
|
|
285
|
+
/** Index paths whose kept sidecars this process has already registered for uninstall. */
|
|
286
|
+
const sidecarOwnershipChecked = new Set<string>();
|
|
287
|
+
let testHooks: UsageFilterIndexTestHooks | null = null;
|
|
288
|
+
|
|
289
|
+
export function usageFilterIndexPath(configDir: string): string {
|
|
290
|
+
return `${configDir.replace(/[\\/]+$/, "")}/${USAGE_FILTER_INDEX_DB_FILENAME}`;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function indexPathForCurrentHome(): string {
|
|
294
|
+
return usageFilterIndexPath(getConfigDir());
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function errorCode(error: unknown): unknown {
|
|
298
|
+
return error !== null && typeof error === "object" && "code" in error
|
|
299
|
+
? (error as { code?: unknown }).code
|
|
300
|
+
: undefined;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function isBusyError(error: unknown): boolean {
|
|
304
|
+
const code = errorCode(error);
|
|
305
|
+
if (code === "SQLITE_BUSY" || code === "SQLITE_LOCKED") return true;
|
|
306
|
+
const message = error instanceof Error ? error.message : "";
|
|
307
|
+
return /database is locked|database table is locked/i.test(message);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function isCorruptError(error: unknown): boolean {
|
|
311
|
+
const code = errorCode(error);
|
|
312
|
+
return code === "SQLITE_CORRUPT" || code === "SQLITE_NOTADB";
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function metaText(db: Database, key: string): string {
|
|
316
|
+
const row = db.query(`SELECT value FROM ${META_TABLE} WHERE key = ?`).get(key) as
|
|
317
|
+
| { value: string }
|
|
318
|
+
| undefined;
|
|
319
|
+
return row?.value ?? "";
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function metaNumber(db: Database, key: string): number {
|
|
323
|
+
const value = metaText(db, key);
|
|
324
|
+
if (value === "") return 0;
|
|
325
|
+
const parsed = Number(value);
|
|
326
|
+
return Number.isFinite(parsed) ? parsed : 0;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function setMeta(db: Database, key: string, value: string | number): void {
|
|
330
|
+
db.query(
|
|
331
|
+
`INSERT INTO ${META_TABLE} (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
|
332
|
+
).run(key, String(value));
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function readMeta(db: Database): IndexMeta {
|
|
336
|
+
const path = metaText(db, META_KEYS.sourcePath);
|
|
337
|
+
const revision: UsageLogRevision | null = path
|
|
338
|
+
? {
|
|
339
|
+
path,
|
|
340
|
+
dev: metaText(db, META_KEYS.sourceDev),
|
|
341
|
+
ino: metaText(db, META_KEYS.sourceIno),
|
|
342
|
+
birthtimeMs: metaNumber(db, META_KEYS.sourceBirthtimeMs),
|
|
343
|
+
size: metaNumber(db, META_KEYS.sourceSize),
|
|
344
|
+
mtimeMs: metaNumber(db, META_KEYS.sourceMtimeMs),
|
|
345
|
+
ctimeMs: metaNumber(db, META_KEYS.sourceCtimeMs),
|
|
346
|
+
}
|
|
347
|
+
: null;
|
|
348
|
+
const minTimestamp = metaText(db, META_KEYS.minTimestamp);
|
|
349
|
+
const maxTimestamp = metaText(db, META_KEYS.maxTimestamp);
|
|
350
|
+
return {
|
|
351
|
+
schemaVersion: metaNumber(db, META_KEYS.schemaVersion),
|
|
352
|
+
generation: metaText(db, META_KEYS.generation),
|
|
353
|
+
revision,
|
|
354
|
+
indexedOffset: metaNumber(db, META_KEYS.indexedOffset),
|
|
355
|
+
digest: metaText(db, META_KEYS.boundaryDigest),
|
|
356
|
+
indexedRows: metaNumber(db, META_KEYS.indexedRows),
|
|
357
|
+
invalidRows: metaNumber(db, META_KEYS.invalidRows),
|
|
358
|
+
oversizedRows: metaNumber(db, META_KEYS.oversizedRows),
|
|
359
|
+
minTimestamp: minTimestamp === "" ? null : Number(minTimestamp),
|
|
360
|
+
maxTimestamp: maxTimestamp === "" ? null : Number(maxTimestamp),
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
function writeMetaStatements(db: Database, meta: IndexMeta): void {
|
|
365
|
+
setMeta(db, META_KEYS.schemaVersion, meta.schemaVersion);
|
|
366
|
+
setMeta(db, META_KEYS.generation, meta.generation);
|
|
367
|
+
setMeta(db, META_KEYS.sourcePath, meta.revision?.path ?? "");
|
|
368
|
+
setMeta(db, META_KEYS.sourceDev, meta.revision?.dev ?? "");
|
|
369
|
+
setMeta(db, META_KEYS.sourceIno, meta.revision?.ino ?? "");
|
|
370
|
+
setMeta(db, META_KEYS.sourceBirthtimeMs, meta.revision?.birthtimeMs ?? 0);
|
|
371
|
+
setMeta(db, META_KEYS.sourceSize, meta.revision?.size ?? 0);
|
|
372
|
+
setMeta(db, META_KEYS.sourceMtimeMs, meta.revision?.mtimeMs ?? 0);
|
|
373
|
+
setMeta(db, META_KEYS.sourceCtimeMs, meta.revision?.ctimeMs ?? 0);
|
|
374
|
+
setMeta(db, META_KEYS.indexedOffset, meta.indexedOffset);
|
|
375
|
+
setMeta(db, META_KEYS.boundaryDigest, meta.digest);
|
|
376
|
+
setMeta(db, META_KEYS.indexedRows, meta.indexedRows);
|
|
377
|
+
setMeta(db, META_KEYS.invalidRows, meta.invalidRows);
|
|
378
|
+
setMeta(db, META_KEYS.oversizedRows, meta.oversizedRows);
|
|
379
|
+
setMeta(db, META_KEYS.minTimestamp, meta.minTimestamp === null ? "" : meta.minTimestamp);
|
|
380
|
+
setMeta(db, META_KEYS.maxTimestamp, meta.maxTimestamp === null ? "" : meta.maxTimestamp);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function emptyMeta(generation: string): IndexMeta {
|
|
384
|
+
return {
|
|
385
|
+
schemaVersion: USAGE_FILTER_INDEX_SCHEMA_VERSION,
|
|
386
|
+
generation,
|
|
387
|
+
revision: null,
|
|
388
|
+
indexedOffset: 0,
|
|
389
|
+
digest: "",
|
|
390
|
+
indexedRows: 0,
|
|
391
|
+
invalidRows: 0,
|
|
392
|
+
oversizedRows: 0,
|
|
393
|
+
minTimestamp: null,
|
|
394
|
+
maxTimestamp: null,
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
function tableNames(db: Database): string[] {
|
|
399
|
+
const rows = db.query(
|
|
400
|
+
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
|
|
401
|
+
).all() as Array<{ name: string }>;
|
|
402
|
+
return rows.map(row => row.name);
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* `fresh` is an empty database, `ours` carries this projection's meta table, anything else is
|
|
407
|
+
* foreign. Contention is not an answer: a lock held by another writer rethrows, so the caller
|
|
408
|
+
* retries instead of classifying a projection it simply could not read yet as someone else's.
|
|
409
|
+
*/
|
|
410
|
+
function classifyDatabase(db: Database): "fresh" | "ours" | "foreign" {
|
|
411
|
+
let names: string[];
|
|
412
|
+
try {
|
|
413
|
+
names = tableNames(db);
|
|
414
|
+
} catch (error) {
|
|
415
|
+
if (isBusyError(error)) throw error;
|
|
416
|
+
return "foreign";
|
|
417
|
+
}
|
|
418
|
+
if (names.length === 0) return "fresh";
|
|
419
|
+
// Require a data table as well as the meta table: a foreign database that happens to use the
|
|
420
|
+
// meta table's name is not this projection and must never have its tables dropped.
|
|
421
|
+
const ours = names.includes(META_TABLE)
|
|
422
|
+
&& (names.includes("usage_rows") || names.includes("usage_attributions"));
|
|
423
|
+
return ours ? "ours" : "foreign";
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Create the schema in one transaction, deciding under the write lock.
|
|
428
|
+
*
|
|
429
|
+
* Two processes can both see the empty file. Created table by table, the second could observe the
|
|
430
|
+
* meta table without the data tables and classify a half-built projection as foreign, giving up
|
|
431
|
+
* the index for that request. One transaction makes the schema appear whole, and re-reading the
|
|
432
|
+
* table list under the lock adopts a schema another process finished first.
|
|
433
|
+
*/
|
|
434
|
+
function createSchema(db: Database): void {
|
|
435
|
+
db.exec("BEGIN IMMEDIATE");
|
|
436
|
+
try {
|
|
437
|
+
const classification = classifyDatabase(db);
|
|
438
|
+
if (classification === "foreign") throw new UsageFilterIndexForeignError();
|
|
439
|
+
if (classification === "fresh") {
|
|
440
|
+
db.exec(SCHEMA_DDL);
|
|
441
|
+
const generation = randomUUID();
|
|
442
|
+
writeMetaStatements(db, emptyMeta(generation));
|
|
443
|
+
db.query("INSERT OR REPLACE INTO usage_filter_index_generations (generation, created_at) VALUES (?, ?)")
|
|
444
|
+
.run(generation, Date.now());
|
|
445
|
+
}
|
|
446
|
+
db.exec("COMMIT");
|
|
447
|
+
} catch (error) {
|
|
448
|
+
try { db.exec("ROLLBACK"); } catch { /* the close releases the lock */ }
|
|
449
|
+
throw error;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/**
|
|
454
|
+
* Recreate the projection's own tables. Only called for a database already classified as
|
|
455
|
+
* ours, so a file this module did not create is never dropped.
|
|
456
|
+
*
|
|
457
|
+
* `onlyIfStale` re-reads the schema under the write lock first: two processes that both saw an
|
|
458
|
+
* outdated schema must not have the second drop the tables (and any staged rows) the first just
|
|
459
|
+
* recreated.
|
|
460
|
+
*/
|
|
461
|
+
function recreateSchema(db: Database, options: { onlyIfStale?: boolean } = {}): void {
|
|
462
|
+
db.exec("BEGIN IMMEDIATE");
|
|
463
|
+
try {
|
|
464
|
+
if (options.onlyIfStale && schemaIsCurrent(db)) {
|
|
465
|
+
db.exec("COMMIT");
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
db.exec("DROP TABLE IF EXISTS usage_rows");
|
|
469
|
+
db.exec("DROP TABLE IF EXISTS usage_attributions");
|
|
470
|
+
db.exec("DROP TABLE IF EXISTS usage_filter_index_writers");
|
|
471
|
+
db.exec("DROP TABLE IF EXISTS usage_filter_index_generations");
|
|
472
|
+
db.exec(`DROP TABLE IF EXISTS ${META_TABLE}`);
|
|
473
|
+
db.exec(SCHEMA_DDL);
|
|
474
|
+
const generation = randomUUID();
|
|
475
|
+
writeMetaStatements(db, emptyMeta(generation));
|
|
476
|
+
db.query("INSERT OR REPLACE INTO usage_filter_index_generations (generation, created_at) VALUES (?, ?)")
|
|
477
|
+
.run(generation, Date.now());
|
|
478
|
+
db.exec("COMMIT");
|
|
479
|
+
} catch (error) {
|
|
480
|
+
try { db.exec("ROLLBACK"); } catch { /* the close below releases the lock */ }
|
|
481
|
+
throw error;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
function schemaIsCurrent(db: Database): boolean {
|
|
486
|
+
const tables = new Set(tableNames(db));
|
|
487
|
+
return metaNumber(db, META_KEYS.schemaVersion) === USAGE_FILTER_INDEX_SCHEMA_VERSION
|
|
488
|
+
&& tables.has("usage_rows")
|
|
489
|
+
&& tables.has("usage_attributions")
|
|
490
|
+
&& tables.has("usage_filter_index_writers")
|
|
491
|
+
&& tables.has("usage_filter_index_generations");
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
function ensureSchema(db: Database, classification: "fresh" | "ours"): void {
|
|
495
|
+
if (classification === "fresh") {
|
|
496
|
+
createSchema(db);
|
|
497
|
+
// Another process may have created the schema first; verify it like any existing one.
|
|
498
|
+
}
|
|
499
|
+
if (schemaIsCurrent(db)) return;
|
|
500
|
+
// A schema change can alter column shape, so the derived tables are recreated rather than
|
|
501
|
+
// migrated. The ledger is untouched and the next refresh rebuilds from it.
|
|
502
|
+
recreateSchema(db, { onlyIfStale: true });
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* Keep the WAL and shared-memory sidecars after the last connection closes.
|
|
507
|
+
*
|
|
508
|
+
* SQLite's default deletes both on the final close, so every filtered query created and removed two
|
|
509
|
+
* entries in the config directory. On Windows that churn is not free: the config directory's ACL
|
|
510
|
+
* memo is keyed on its change time, so the next config load re-ran icacls over the directory, the
|
|
511
|
+
* inherited-ACE rewrite moved usage.jsonl's ChangeTime, and both the usage aggregate and this
|
|
512
|
+
* index then rebuilt from the whole ledger. macOS's system SQLite already persists WAL; this makes
|
|
513
|
+
* every platform behave that way. Read-only connections never delete the sidecars, so only the
|
|
514
|
+
* connections that can write set it, and only after the file classified as this projection: a
|
|
515
|
+
* foreign WAL database keeps SQLite's default and leaves nothing behind.
|
|
516
|
+
*
|
|
517
|
+
* The writer also checkpoints with TRUNCATE before it closes (see refreshUsageFilterIndex), so a
|
|
518
|
+
* clean close leaves the kept WAL empty on every SQLite build: a large rebuild does not leave its
|
|
519
|
+
* bytes on disk, and no stale frames remain to replay into whatever file later sits at this path.
|
|
520
|
+
* The size limit bounds the WAL between checkpoints.
|
|
521
|
+
*/
|
|
522
|
+
function keepWalSidecars(db: Database): void {
|
|
523
|
+
db.fileControl(sqliteConstants.SQLITE_FCNTL_PERSIST_WAL, 1);
|
|
524
|
+
db.exec(`PRAGMA journal_size_limit = ${RETAINED_WAL_LIMIT_BYTES}`);
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* Register the WAL sidecars that now persist beside this projection.
|
|
529
|
+
*
|
|
530
|
+
* Creation registers the sidecar names up front, but an existing file skips that: a 0-byte
|
|
531
|
+
* projection left by a crash between the private create and the first open, or one whose names
|
|
532
|
+
* were never recorded. Its sidecars now persist and would otherwise survive uninstall. Called only
|
|
533
|
+
* after the file classified as this projection and WAL mode is on, so every sidecar present is
|
|
534
|
+
* this database's own. Sidecars appear only once SQLite has used the WAL, so the path is
|
|
535
|
+
* remembered only when both exist and were offered for registration; until then each open checks.
|
|
536
|
+
*/
|
|
537
|
+
function registerKeptSidecars(dir: string, path: string): void {
|
|
538
|
+
if (sidecarOwnershipChecked.has(path)) return;
|
|
539
|
+
// The same crash leaves the projection file itself unrecorded; it is this projection now.
|
|
540
|
+
recordOwnedConfigPath(dir, path);
|
|
541
|
+
let present = 0;
|
|
542
|
+
for (const suffix of ["-wal", "-shm"] as const) {
|
|
543
|
+
const sidecar = `${path}${suffix}`;
|
|
544
|
+
if (!existsSync(sidecar)) continue;
|
|
545
|
+
recordOwnedConfigPath(dir, sidecar);
|
|
546
|
+
present += 1;
|
|
547
|
+
}
|
|
548
|
+
if (present === 2) sidecarOwnershipChecked.add(path);
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
function openIndexForWrite(dir: string, path: string): Database {
|
|
552
|
+
const existed = existsSync(path);
|
|
553
|
+
preparePrivateSqliteDatabase(path);
|
|
554
|
+
const db = new Database(path, { create: true });
|
|
555
|
+
try {
|
|
556
|
+
// Before the first read: another process creating or writing the projection holds its lock
|
|
557
|
+
// for a moment, and that must wait (then retry), never read as a foreign file.
|
|
558
|
+
db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
|
|
559
|
+
const classification = classifyDatabase(db);
|
|
560
|
+
if (classification === "foreign") throw new UsageFilterIndexForeignError();
|
|
561
|
+
keepWalSidecars(db);
|
|
562
|
+
if (!existed) {
|
|
563
|
+
// Register only names this module is about to create; an existing sidecar may belong to
|
|
564
|
+
// something else and is never adopted.
|
|
565
|
+
recordOwnedConfigPath(dir, path);
|
|
566
|
+
for (const suffix of SQLITE_SIDECAR_SUFFIXES) {
|
|
567
|
+
const sidecar = `${path}${suffix}`;
|
|
568
|
+
if (!existsSync(sidecar)) recordOwnedConfigPath(dir, sidecar);
|
|
569
|
+
}
|
|
570
|
+
if (process.platform !== "win32") {
|
|
571
|
+
try { chmodSync(path, 0o600); } catch { /* platform may ignore chmod */ }
|
|
572
|
+
}
|
|
573
|
+
hardenSecretPath(path, { required: false });
|
|
574
|
+
}
|
|
575
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
576
|
+
registerKeptSidecars(dir, path);
|
|
577
|
+
ensureSchema(db, classification);
|
|
578
|
+
return db;
|
|
579
|
+
} catch (error) {
|
|
580
|
+
try { db.close(); } catch { /* already closed */ }
|
|
581
|
+
throw error;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/** Sidecars inherit the database's mode on creation; re-assert it after the first write. */
|
|
586
|
+
function hardenSidecars(path: string): void {
|
|
587
|
+
if (process.platform === "win32") return;
|
|
588
|
+
for (const suffix of SQLITE_SIDECAR_SUFFIXES) {
|
|
589
|
+
try { chmodSync(`${path}${suffix}`, 0o600); } catch { /* not created yet */ }
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function openIndexForRead(path: string): Database {
|
|
594
|
+
try {
|
|
595
|
+
return new Database(path, { readonly: true });
|
|
596
|
+
} catch (error) {
|
|
597
|
+
// A WAL left without its index (a crash, or a writer that exited abruptly) cannot be
|
|
598
|
+
// opened read-only; the read-write handle recovers it on open. Never create a database
|
|
599
|
+
// that vanished: a missing file means the projection is gone, not that it should be
|
|
600
|
+
// recreated without the private-path and ownership checks.
|
|
601
|
+
if (!existsSync(path)) throw error;
|
|
602
|
+
const db = new Database(path, { create: false });
|
|
603
|
+
try {
|
|
604
|
+
if (classifyDatabase(db) !== "ours") throw new UsageFilterIndexForeignError();
|
|
605
|
+
keepWalSidecars(db);
|
|
606
|
+
return db;
|
|
607
|
+
} catch (inner) {
|
|
608
|
+
try { db.close(); } catch { /* already closed */ }
|
|
609
|
+
throw inner;
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
interface IngestResult {
|
|
615
|
+
scan: UsageLedgerScanResult;
|
|
616
|
+
rows: number;
|
|
617
|
+
minTimestamp: number | null;
|
|
618
|
+
maxTimestamp: number | null;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
async function ingest(
|
|
622
|
+
db: Database,
|
|
623
|
+
generation: string,
|
|
624
|
+
startAtBytes: number,
|
|
625
|
+
expected: { identityKey: string; digest: string } | null,
|
|
626
|
+
onBatch?: () => void,
|
|
627
|
+
): Promise<IngestResult> {
|
|
628
|
+
const insertRow = db.prepare(
|
|
629
|
+
"INSERT OR REPLACE INTO usage_rows (generation, byte_offset, timestamp, api_key_id, client, surface, admission_kind, unattributable) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
630
|
+
);
|
|
631
|
+
const insertAttribution = db.prepare(
|
|
632
|
+
"INSERT OR REPLACE INTO usage_attributions (generation, byte_offset, provider, model) VALUES (?, ?, ?, ?)",
|
|
633
|
+
);
|
|
634
|
+
const writeBatch = (batch: Array<{ offset: number; entry: PersistedUsageEntry }>): void => {
|
|
635
|
+
for (const { offset, entry } of batch) {
|
|
636
|
+
// Normalization preserves hand-edited rows with a missing or non-numeric timestamp and
|
|
637
|
+
// a missing model. A null timestamp is selected without a window and excluded by one,
|
|
638
|
+
// exactly as the accumulator treats it.
|
|
639
|
+
const timestamp = Number.isFinite(entry.timestamp) ? entry.timestamp : null;
|
|
640
|
+
let attributions: ReturnType<typeof usageAttributions> = [];
|
|
641
|
+
try {
|
|
642
|
+
attributions = usageAttributions(entry);
|
|
643
|
+
} catch {
|
|
644
|
+
attributions = [];
|
|
645
|
+
}
|
|
646
|
+
const attributable = attributions.filter(attribution =>
|
|
647
|
+
typeof attribution.provider === "string" && typeof attribution.model === "string");
|
|
648
|
+
insertRow.run(
|
|
649
|
+
generation,
|
|
650
|
+
offset,
|
|
651
|
+
timestamp,
|
|
652
|
+
entry.apiKeyId ?? null,
|
|
653
|
+
entry.client ?? null,
|
|
654
|
+
entry.surface ?? null,
|
|
655
|
+
entry.admissionKind ?? null,
|
|
656
|
+
attributable.length === 0 ? 1 : 0,
|
|
657
|
+
);
|
|
658
|
+
for (const attribution of attributable) {
|
|
659
|
+
// Keys are derived from the same attribution helper the accumulator filters with, so
|
|
660
|
+
// the index can over-select but never reject a row the exact filter would keep.
|
|
661
|
+
insertAttribution.run(
|
|
662
|
+
generation,
|
|
663
|
+
offset,
|
|
664
|
+
baseProviderLabel(attribution.provider).toLowerCase(),
|
|
665
|
+
attribution.model.toLowerCase(),
|
|
666
|
+
);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
};
|
|
670
|
+
/**
|
|
671
|
+
* Commit one batch under `BEGIN IMMEDIATE`.
|
|
672
|
+
*
|
|
673
|
+
* A deferred `BEGIN` that later upgrades can fail with SQLITE_BUSY without consulting the
|
|
674
|
+
* busy handler, which is exactly the shape two concurrent rebuilds produce. Taking the write
|
|
675
|
+
* lock up front makes the handler apply, and the short synchronous retry rides out a batch
|
|
676
|
+
* another process is committing.
|
|
677
|
+
*/
|
|
678
|
+
const commit = (batch: Array<{ offset: number; entry: PersistedUsageEntry }>): void => {
|
|
679
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
680
|
+
db.exec("BEGIN IMMEDIATE");
|
|
681
|
+
try {
|
|
682
|
+
writeBatch(batch);
|
|
683
|
+
db.exec("COMMIT");
|
|
684
|
+
onBatch?.();
|
|
685
|
+
return;
|
|
686
|
+
} catch (error) {
|
|
687
|
+
try { db.exec("ROLLBACK"); } catch { /* the transaction never started */ }
|
|
688
|
+
if (!isBusyError(error) || attempt >= BATCH_BUSY_RETRIES) throw error;
|
|
689
|
+
Bun.sleepSync(BATCH_BUSY_RETRY_MS);
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
};
|
|
693
|
+
let pending: Array<{ offset: number; entry: PersistedUsageEntry }> = [];
|
|
694
|
+
let rows = 0;
|
|
695
|
+
let minTimestamp: number | null = null;
|
|
696
|
+
let maxTimestamp: number | null = null;
|
|
697
|
+
let callbackError: unknown = null;
|
|
698
|
+
try {
|
|
699
|
+
let scan: UsageLedgerScanResult;
|
|
700
|
+
try {
|
|
701
|
+
scan = await scanUsageLedgerCooperatively({
|
|
702
|
+
startAtBytes,
|
|
703
|
+
...(expected
|
|
704
|
+
? { expectedIdentityKey: expected.identityKey, expectedProcessedThroughDigest: expected.digest }
|
|
705
|
+
: {}),
|
|
706
|
+
onEntry(entry, offset) {
|
|
707
|
+
try {
|
|
708
|
+
pending.push({ offset, entry });
|
|
709
|
+
rows += 1;
|
|
710
|
+
if (Number.isFinite(entry.timestamp)) {
|
|
711
|
+
if (minTimestamp === null || entry.timestamp < minTimestamp) minTimestamp = entry.timestamp;
|
|
712
|
+
if (maxTimestamp === null || entry.timestamp > maxTimestamp) maxTimestamp = entry.timestamp;
|
|
713
|
+
}
|
|
714
|
+
if (pending.length >= USAGE_FILTER_INDEX_INSERT_BATCH) {
|
|
715
|
+
commit(pending);
|
|
716
|
+
pending = [];
|
|
717
|
+
}
|
|
718
|
+
} catch (error) {
|
|
719
|
+
// An index write failure is not a ledger read failure; keep it classified as one.
|
|
720
|
+
callbackError = error;
|
|
721
|
+
throw error;
|
|
722
|
+
}
|
|
723
|
+
},
|
|
724
|
+
});
|
|
725
|
+
} catch (error) {
|
|
726
|
+
if (error instanceof UsageLedgerRebuildRequiredError) throw error;
|
|
727
|
+
if (callbackError !== null) throw callbackError;
|
|
728
|
+
// A ledger read failure is the authoritative read failing, not the index: surface it.
|
|
729
|
+
throw new UsageFilterIndexSourceError(error);
|
|
730
|
+
}
|
|
731
|
+
if (pending.length > 0) commit(pending);
|
|
732
|
+
return { scan, rows, minTimestamp, maxTimestamp };
|
|
733
|
+
} finally {
|
|
734
|
+
insertRow.finalize();
|
|
735
|
+
insertAttribution.finalize();
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
function ingestPassesFor(path: string): number {
|
|
740
|
+
return ingestPassesByPath.get(path) ?? 0;
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
function bumpIngestPasses(path: string): number {
|
|
744
|
+
const next = ingestPassesFor(path) + 1;
|
|
745
|
+
ingestPassesByPath.set(path, next);
|
|
746
|
+
return next;
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
function snapshotFrom(meta: IndexMeta, ingestedRows: number, ingestPasses: number): UsageFilterIndexSnapshot {
|
|
750
|
+
return {
|
|
751
|
+
revision: meta.revision,
|
|
752
|
+
identityKey: usageLogIdentityKey(meta.revision),
|
|
753
|
+
generation: meta.generation,
|
|
754
|
+
indexedOffset: meta.indexedOffset,
|
|
755
|
+
processedThroughDigest: meta.digest,
|
|
756
|
+
indexedRows: meta.indexedRows,
|
|
757
|
+
invalidRows: meta.invalidRows,
|
|
758
|
+
oversizedRows: meta.oversizedRows,
|
|
759
|
+
minTimestamp: meta.minTimestamp,
|
|
760
|
+
maxTimestamp: meta.maxTimestamp,
|
|
761
|
+
ingestPasses,
|
|
762
|
+
ingestedRows,
|
|
763
|
+
};
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
function earlier(left: number | null, right: number | null): number | null {
|
|
767
|
+
if (left === null) return right;
|
|
768
|
+
if (right === null) return left;
|
|
769
|
+
return Math.min(left, right);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
function later(left: number | null, right: number | null): number | null {
|
|
773
|
+
if (left === null) return right;
|
|
774
|
+
if (right === null) return left;
|
|
775
|
+
return Math.max(left, right);
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
function discardGeneration(db: Database, generation: string): void {
|
|
779
|
+
db.query("DELETE FROM usage_rows WHERE generation = ?").run(generation);
|
|
780
|
+
db.query("DELETE FROM usage_attributions WHERE generation = ?").run(generation);
|
|
781
|
+
db.query("DELETE FROM usage_filter_index_writers WHERE generation = ?").run(generation);
|
|
782
|
+
db.query("DELETE FROM usage_filter_index_generations WHERE generation = ?").run(generation);
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
/** Whether the process that staged a generation is still running. EPERM means it exists. */
|
|
786
|
+
function processAlive(pid: number): boolean {
|
|
787
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
788
|
+
if (pid === process.pid) return true;
|
|
789
|
+
try {
|
|
790
|
+
process.kill(pid, 0);
|
|
791
|
+
return true;
|
|
792
|
+
} catch (error) {
|
|
793
|
+
return errorCode(error) === "EPERM";
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
/**
|
|
798
|
+
* Record the writer that owns a staging generation.
|
|
799
|
+
*
|
|
800
|
+
* The lease is written before the first staged row and removed with the rows, so a generation
|
|
801
|
+
* with rows but no lease cannot belong to a live writer. Reclamation uses the recorded pid as
|
|
802
|
+
* owner evidence; a timestamp alone never deletes a generation whose writer is alive.
|
|
803
|
+
*/
|
|
804
|
+
function beginStaging(db: Database, path: string, generation: string): void {
|
|
805
|
+
// Registry and lease must become visible together: a reclaimer that saw the registry row
|
|
806
|
+
// without the lease would treat a live writer's generation as abandoned.
|
|
807
|
+
db.exec("BEGIN IMMEDIATE");
|
|
808
|
+
try {
|
|
809
|
+
const now = Date.now();
|
|
810
|
+
db.query("INSERT OR REPLACE INTO usage_filter_index_generations (generation, created_at) VALUES (?, ?)")
|
|
811
|
+
.run(generation, now);
|
|
812
|
+
testHooks?.betweenStagingWrites?.({ dbPath: path, generation });
|
|
813
|
+
db.query(
|
|
814
|
+
"INSERT OR REPLACE INTO usage_filter_index_writers (generation, pid, started_at, heartbeat_at) VALUES (?, ?, ?, ?)",
|
|
815
|
+
).run(generation, process.pid, now, now);
|
|
816
|
+
db.exec("COMMIT");
|
|
817
|
+
} catch (error) {
|
|
818
|
+
try { db.exec("ROLLBACK"); } catch { /* the close releases the lock */ }
|
|
819
|
+
throw error;
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
function heartbeatLease(db: Database, generation: string): void {
|
|
824
|
+
db.query("UPDATE usage_filter_index_writers SET heartbeat_at = ? WHERE generation = ?")
|
|
825
|
+
.run(Date.now(), generation);
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
function endLease(db: Database, generation: string): void {
|
|
829
|
+
db.query("DELETE FROM usage_filter_index_writers WHERE generation = ?").run(generation);
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* Remove staging generations left behind by writers that are gone.
|
|
834
|
+
*
|
|
835
|
+
* Every staging generation writes its lease before its first row and removes it with the rows,
|
|
836
|
+
* so the lease table is the complete set of in-flight generations. A generation is reclaimed
|
|
837
|
+
* only when it is not published and its recorded writer pid is dead; a live writer's staging is
|
|
838
|
+
* never touched, which is what keeps this from repeating the multi-writer row loss it cleans up
|
|
839
|
+
* after. The published generation never has a lease and is never a candidate.
|
|
840
|
+
*/
|
|
841
|
+
function reclaimAbandonedStaging(db: Database, path: string, published: string): void {
|
|
842
|
+
// Cheap pre-check without the write lock: most refreshes have nothing to reclaim.
|
|
843
|
+
const generations = db.query(
|
|
844
|
+
"SELECT g.generation AS generation, w.pid AS pid FROM usage_filter_index_generations g LEFT JOIN usage_filter_index_writers w ON w.generation = g.generation",
|
|
845
|
+
).all() as Array<{ generation: string; pid: number | null }>;
|
|
846
|
+
const maybe = new Set<string>();
|
|
847
|
+
for (const entry of generations) {
|
|
848
|
+
if (entry.generation === published) continue;
|
|
849
|
+
// No lease means no live writer ever owned it; a dead pid means its writer is gone.
|
|
850
|
+
if (entry.pid === null || !processAlive(entry.pid)) maybe.add(entry.generation);
|
|
851
|
+
}
|
|
852
|
+
// A lease whose generation is no longer registered cannot belong to a live writer, because
|
|
853
|
+
// registration and the lease commit together.
|
|
854
|
+
const orphanLease = db.query(
|
|
855
|
+
"SELECT 1 AS present FROM usage_filter_index_writers WHERE generation NOT IN (SELECT generation FROM usage_filter_index_generations) LIMIT 1",
|
|
856
|
+
).get() !== null;
|
|
857
|
+
if (maybe.size === 0 && !orphanLease) return;
|
|
858
|
+
testHooks?.betweenReclaimCheckAndLock?.({ dbPath: path, candidates: [...maybe] });
|
|
859
|
+
db.exec("BEGIN IMMEDIATE");
|
|
860
|
+
try {
|
|
861
|
+
// Re-read under the write lock: a writer may have registered its lease, or a publish may
|
|
862
|
+
// have moved the watermark, since the pre-check. Eligibility is decided here, not outside.
|
|
863
|
+
const currentPublished = metaText(db, META_KEYS.generation);
|
|
864
|
+
for (const generation of maybe) {
|
|
865
|
+
if (generation === currentPublished) continue;
|
|
866
|
+
const lease = db.query("SELECT pid FROM usage_filter_index_writers WHERE generation = ?")
|
|
867
|
+
.get(generation) as { pid: number } | null;
|
|
868
|
+
if (lease && processAlive(lease.pid)) continue;
|
|
869
|
+
db.query("DELETE FROM usage_rows WHERE generation = ?").run(generation);
|
|
870
|
+
db.query("DELETE FROM usage_attributions WHERE generation = ?").run(generation);
|
|
871
|
+
db.query("DELETE FROM usage_filter_index_writers WHERE generation = ?").run(generation);
|
|
872
|
+
db.query("DELETE FROM usage_filter_index_generations WHERE generation = ?").run(generation);
|
|
873
|
+
}
|
|
874
|
+
// A lease whose generation is no longer registered cannot belong to a live writer:
|
|
875
|
+
// registration is atomic, so this is a leftover record rather than an active one.
|
|
876
|
+
db.query(
|
|
877
|
+
"DELETE FROM usage_filter_index_writers WHERE generation NOT IN (SELECT generation FROM usage_filter_index_generations)",
|
|
878
|
+
).run();
|
|
879
|
+
db.exec("COMMIT");
|
|
880
|
+
} catch (error) {
|
|
881
|
+
try { db.exec("ROLLBACK"); } catch { /* the close releases the lock */ }
|
|
882
|
+
throw error;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/** The watermark a publish must still be standing on: the generation and its cursor. */
|
|
887
|
+
interface PublishFence {
|
|
888
|
+
generation: string;
|
|
889
|
+
indexedOffset: number;
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
/**
|
|
893
|
+
* Publish one consistent meta snapshot, fenced on the watermark the writer started from.
|
|
894
|
+
*
|
|
895
|
+
* The fence covers the generation and its cursor: a stale append that started at an older offset
|
|
896
|
+
* must not publish over a newer one, because that would regress the cursor and hide the rows in
|
|
897
|
+
* between until the next read. Returns false when another writer moved the watermark first; the
|
|
898
|
+
* caller then discards its staging set and retries. Pruning other generations happens here,
|
|
899
|
+
* inside the fence, and never while they may still be the rows another writer is ingesting.
|
|
900
|
+
*/
|
|
901
|
+
function publish(
|
|
902
|
+
db: Database,
|
|
903
|
+
fence: PublishFence,
|
|
904
|
+
next: IndexMeta,
|
|
905
|
+
): boolean {
|
|
906
|
+
db.exec("BEGIN IMMEDIATE");
|
|
907
|
+
try {
|
|
908
|
+
if (metaText(db, META_KEYS.generation) !== fence.generation
|
|
909
|
+
|| metaNumber(db, META_KEYS.indexedOffset) !== fence.indexedOffset) {
|
|
910
|
+
db.exec("ROLLBACK");
|
|
911
|
+
return false;
|
|
912
|
+
}
|
|
913
|
+
writeMetaStatements(db, next);
|
|
914
|
+
// A published staging generation no longer needs its writer lease.
|
|
915
|
+
endLease(db, next.generation);
|
|
916
|
+
// Delete retired generations by equality. The registry is small and known, so this never
|
|
917
|
+
// scans the row table under the write lock.
|
|
918
|
+
const retired = db.query(
|
|
919
|
+
"SELECT generation FROM usage_filter_index_generations WHERE generation <> ?",
|
|
920
|
+
).all(next.generation) as Array<{ generation: string }>;
|
|
921
|
+
for (const { generation } of retired) {
|
|
922
|
+
if (generation === "") continue;
|
|
923
|
+
// A live writer still owns its staging generation. Its own publish will fail the fence
|
|
924
|
+
// and retry, so leaving it here is what keeps its rows from being deleted mid-ingest.
|
|
925
|
+
const liveLease = db.query("SELECT pid FROM usage_filter_index_writers WHERE generation = ?")
|
|
926
|
+
.get(generation) as { pid: number } | null;
|
|
927
|
+
if (liveLease && processAlive(liveLease.pid)) continue;
|
|
928
|
+
db.query("DELETE FROM usage_rows WHERE generation = ?").run(generation);
|
|
929
|
+
db.query("DELETE FROM usage_attributions WHERE generation = ?").run(generation);
|
|
930
|
+
db.query("DELETE FROM usage_filter_index_generations WHERE generation = ?").run(generation);
|
|
931
|
+
// Drop a dead writer's record with its generation; a live writer keeps its lease so the
|
|
932
|
+
// fence can make it retry rather than look abandoned.
|
|
933
|
+
db.query("DELETE FROM usage_filter_index_writers WHERE generation = ?").run(generation);
|
|
934
|
+
}
|
|
935
|
+
db.exec("COMMIT");
|
|
936
|
+
return true;
|
|
937
|
+
} catch (error) {
|
|
938
|
+
try { db.exec("ROLLBACK"); } catch { /* the close releases the lock */ }
|
|
939
|
+
throw error;
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
function assertHomeUnchanged(dir: string): void {
|
|
944
|
+
if (getConfigDir() !== dir) throw new UsageFilterIndexHomeChangedError();
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
async function rebuild(
|
|
948
|
+
db: Database,
|
|
949
|
+
dir: string,
|
|
950
|
+
path: string,
|
|
951
|
+
prior: IndexMeta,
|
|
952
|
+
): Promise<UsageFilterIndexSnapshot | null> {
|
|
953
|
+
const staging = randomUUID();
|
|
954
|
+
beginStaging(db, path, staging);
|
|
955
|
+
let published = false;
|
|
956
|
+
try {
|
|
957
|
+
await testHooks?.afterStagingRegistered?.({ dbPath: path, generation: staging });
|
|
958
|
+
const result = await ingest(db, staging, 0, null, () => heartbeatLease(db, staging));
|
|
959
|
+
const next: IndexMeta = {
|
|
960
|
+
schemaVersion: USAGE_FILTER_INDEX_SCHEMA_VERSION,
|
|
961
|
+
generation: staging,
|
|
962
|
+
revision: result.scan.revision,
|
|
963
|
+
indexedOffset: result.scan.processedThroughBytes,
|
|
964
|
+
digest: result.scan.processedThroughDigest,
|
|
965
|
+
indexedRows: result.rows,
|
|
966
|
+
invalidRows: result.scan.invalidRows,
|
|
967
|
+
oversizedRows: result.scan.oversizedRows,
|
|
968
|
+
minTimestamp: result.minTimestamp,
|
|
969
|
+
maxTimestamp: result.maxTimestamp,
|
|
970
|
+
};
|
|
971
|
+
await testHooks?.beforePublish?.({
|
|
972
|
+
dbPath: path,
|
|
973
|
+
stagedGeneration: staging,
|
|
974
|
+
publishedGeneration: prior.generation,
|
|
975
|
+
});
|
|
976
|
+
// The ledger this ingest read is the live home's; publishing it under another home would
|
|
977
|
+
// mix two installations.
|
|
978
|
+
assertHomeUnchanged(dir);
|
|
979
|
+
if (!publish(db, { generation: prior.generation, indexedOffset: prior.indexedOffset }, next)) {
|
|
980
|
+
return null;
|
|
981
|
+
}
|
|
982
|
+
published = true;
|
|
983
|
+
await testHooks?.afterPublish?.({ dbPath: path, generation: staging });
|
|
984
|
+
hardenSidecars(path);
|
|
985
|
+
return snapshotFrom(next, result.rows, bumpIngestPasses(path));
|
|
986
|
+
} finally {
|
|
987
|
+
if (!published) {
|
|
988
|
+
// A failed or crashed attempt must not leave its staging behind.
|
|
989
|
+
discardGeneration(db, staging);
|
|
990
|
+
endLease(db, staging);
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
async function append(
|
|
996
|
+
db: Database,
|
|
997
|
+
dir: string,
|
|
998
|
+
path: string,
|
|
999
|
+
meta: IndexMeta,
|
|
1000
|
+
): Promise<UsageFilterIndexSnapshot | null> {
|
|
1001
|
+
const result = await ingest(db, meta.generation, meta.indexedOffset, {
|
|
1002
|
+
identityKey: usageLogIdentityKey(meta.revision),
|
|
1003
|
+
digest: meta.digest,
|
|
1004
|
+
});
|
|
1005
|
+
const next: IndexMeta = {
|
|
1006
|
+
...meta,
|
|
1007
|
+
revision: result.scan.revision,
|
|
1008
|
+
indexedOffset: result.scan.processedThroughBytes,
|
|
1009
|
+
digest: result.scan.processedThroughDigest,
|
|
1010
|
+
indexedRows: meta.indexedRows + result.rows,
|
|
1011
|
+
invalidRows: meta.invalidRows + result.scan.invalidRows,
|
|
1012
|
+
oversizedRows: meta.oversizedRows + result.scan.oversizedRows,
|
|
1013
|
+
minTimestamp: earlier(meta.minTimestamp, result.minTimestamp),
|
|
1014
|
+
maxTimestamp: later(meta.maxTimestamp, result.maxTimestamp),
|
|
1015
|
+
};
|
|
1016
|
+
await testHooks?.beforePublish?.({
|
|
1017
|
+
dbPath: path,
|
|
1018
|
+
stagedGeneration: meta.generation,
|
|
1019
|
+
publishedGeneration: meta.generation,
|
|
1020
|
+
});
|
|
1021
|
+
assertHomeUnchanged(dir);
|
|
1022
|
+
if (!publish(db, { generation: meta.generation, indexedOffset: meta.indexedOffset }, next)) {
|
|
1023
|
+
return null;
|
|
1024
|
+
}
|
|
1025
|
+
await testHooks?.afterPublish?.({ dbPath: path, generation: meta.generation });
|
|
1026
|
+
hardenSidecars(path);
|
|
1027
|
+
return snapshotFrom(next, result.rows, ingestPassesFor(path));
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
/**
|
|
1031
|
+
* Bring the projection up to the ledger's current EOF, appending only new complete rows.
|
|
1032
|
+
*
|
|
1033
|
+
* A same-size file whose revision metadata moved is a rewrite or replacement, not an append,
|
|
1034
|
+
* so it rebuilds. An append is verified by the scanner's boundary digest before any of its
|
|
1035
|
+
* rows are retained. Returns null when another writer won the publish race.
|
|
1036
|
+
*/
|
|
1037
|
+
async function refreshOnce(
|
|
1038
|
+
db: Database,
|
|
1039
|
+
dir: string,
|
|
1040
|
+
path: string,
|
|
1041
|
+
): Promise<UsageFilterIndexSnapshot | null> {
|
|
1042
|
+
assertHomeUnchanged(dir);
|
|
1043
|
+
const meta = readMeta(db);
|
|
1044
|
+
reclaimAbandonedStaging(db, path, meta.generation);
|
|
1045
|
+
const revision = currentUsageLogRevision();
|
|
1046
|
+
if (meta.schemaVersion !== USAGE_FILTER_INDEX_SCHEMA_VERSION
|
|
1047
|
+
|| !meta.generation
|
|
1048
|
+
|| usageLogIdentityKey(meta.revision) !== usageLogIdentityKey(revision)) {
|
|
1049
|
+
return rebuildOrRetry(db, dir, path, meta);
|
|
1050
|
+
}
|
|
1051
|
+
if (revision === null) return snapshotFrom(meta, 0, ingestPassesFor(path));
|
|
1052
|
+
// Growth is measured against the physical size the snapshot recorded, not the processed
|
|
1053
|
+
// boundary: a torn suffix leaves the file larger than the boundary without being new content,
|
|
1054
|
+
// and treating that as growth would let a same-size rewrite masquerade as an append.
|
|
1055
|
+
const recordedSize = meta.revision?.size ?? 0;
|
|
1056
|
+
if (revision.size < recordedSize) return rebuildOrRetry(db, dir, path, meta);
|
|
1057
|
+
if (revision.size === recordedSize) {
|
|
1058
|
+
if (meta.revision && usageLogRevisionKey(meta.revision) === usageLogRevisionKey(revision)) {
|
|
1059
|
+
return snapshotFrom(meta, 0, ingestPassesFor(path));
|
|
1060
|
+
}
|
|
1061
|
+
return rebuildOrRetry(db, dir, path, meta);
|
|
1062
|
+
}
|
|
1063
|
+
try {
|
|
1064
|
+
return await append(db, dir, path, meta);
|
|
1065
|
+
} catch (error) {
|
|
1066
|
+
// The append suffix failed the boundary digest: the ledger was rewritten under the
|
|
1067
|
+
// projection, so the answer is a full rebuild, not another attempt at the same suffix.
|
|
1068
|
+
if (error instanceof UsageLedgerRebuildRequiredError) {
|
|
1069
|
+
return rebuildOrRetry(db, dir, path, meta);
|
|
1070
|
+
}
|
|
1071
|
+
throw error;
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
async function rebuildOrRetry(
|
|
1076
|
+
db: Database,
|
|
1077
|
+
dir: string,
|
|
1078
|
+
path: string,
|
|
1079
|
+
meta: IndexMeta,
|
|
1080
|
+
): Promise<UsageFilterIndexSnapshot | null> {
|
|
1081
|
+
try {
|
|
1082
|
+
return await rebuild(db, dir, path, meta);
|
|
1083
|
+
} catch (error) {
|
|
1084
|
+
if (error instanceof UsageLedgerRebuildRequiredError) return null;
|
|
1085
|
+
throw error;
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
async function refreshWithRetry(
|
|
1090
|
+
db: Database,
|
|
1091
|
+
dir: string,
|
|
1092
|
+
path: string,
|
|
1093
|
+
): Promise<UsageFilterIndexSnapshot> {
|
|
1094
|
+
let lastError: unknown;
|
|
1095
|
+
for (let attempt = 0; attempt < MAX_REFRESH_ATTEMPTS; attempt += 1) {
|
|
1096
|
+
try {
|
|
1097
|
+
const snapshot = await refreshOnce(db, dir, path);
|
|
1098
|
+
if (snapshot) return snapshot;
|
|
1099
|
+
} catch (error) {
|
|
1100
|
+
if (error instanceof UsageLedgerRebuildRequiredError) {
|
|
1101
|
+
lastError = error;
|
|
1102
|
+
continue;
|
|
1103
|
+
}
|
|
1104
|
+
// A corrupt file this module owns is derived data: recreate its own tables and retry.
|
|
1105
|
+
if (isCorruptError(error) && attempt === 0) {
|
|
1106
|
+
try {
|
|
1107
|
+
recreateSchema(db);
|
|
1108
|
+
continue;
|
|
1109
|
+
} catch {
|
|
1110
|
+
throw error;
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
throw error;
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
throw lastError ?? new Error("usage filter index refresh did not settle");
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
async function refreshUsageFilterIndex(
|
|
1120
|
+
dir: string,
|
|
1121
|
+
path: string,
|
|
1122
|
+
): Promise<UsageFilterIndexSnapshot> {
|
|
1123
|
+
if (ingestFlight?.path === path) return ingestFlight.promise;
|
|
1124
|
+
const promise = (async () => {
|
|
1125
|
+
const db = openIndexForWrite(dir, path);
|
|
1126
|
+
try {
|
|
1127
|
+
if (rebuildRequestedPaths.delete(path)) {
|
|
1128
|
+
const rebuilt = await rebuild(db, dir, path, readMeta(db));
|
|
1129
|
+
return rebuilt ?? await refreshWithRetry(db, dir, path);
|
|
1130
|
+
}
|
|
1131
|
+
return await refreshWithRetry(db, dir, path);
|
|
1132
|
+
} finally {
|
|
1133
|
+
// A reader in another process can still pin the WAL; then this checkpoint is partial (SQLite
|
|
1134
|
+
// reports that as a result row, not an error) and a later writer's TRUNCATE empties it once
|
|
1135
|
+
// no reader pins it. The close itself never empties a kept WAL that still has frames.
|
|
1136
|
+
try { db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); } catch { /* the close below still runs */ }
|
|
1137
|
+
db.close();
|
|
1138
|
+
}
|
|
1139
|
+
})();
|
|
1140
|
+
ingestFlight = { path, promise };
|
|
1141
|
+
try {
|
|
1142
|
+
return await promise;
|
|
1143
|
+
} finally {
|
|
1144
|
+
if (ingestFlight?.promise === promise) ingestFlight = null;
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
function buildCandidateSelect(
|
|
1149
|
+
query: UsageFilterIndexQuery,
|
|
1150
|
+
snapshot: UsageFilterIndexSnapshot,
|
|
1151
|
+
): { sql: string; values: Array<string | number> } {
|
|
1152
|
+
const attrClauses: string[] = [];
|
|
1153
|
+
const attrValues: Array<string | number> = [];
|
|
1154
|
+
const rowClauses: string[] = [];
|
|
1155
|
+
const rowValues: Array<string | number> = [];
|
|
1156
|
+
const usesAttributions = query.provider !== null || query.model !== null;
|
|
1157
|
+
if (query.provider !== null) {
|
|
1158
|
+
attrClauses.push("a.provider = ?");
|
|
1159
|
+
attrValues.push(query.provider);
|
|
1160
|
+
}
|
|
1161
|
+
if (query.model !== null) {
|
|
1162
|
+
attrClauses.push("a.model = ?");
|
|
1163
|
+
attrValues.push(query.model);
|
|
1164
|
+
}
|
|
1165
|
+
if (query.apiKeyId !== null) {
|
|
1166
|
+
rowClauses.push("r.api_key_id = ?");
|
|
1167
|
+
rowValues.push(query.apiKeyId);
|
|
1168
|
+
}
|
|
1169
|
+
const device = query.sourceFilter?.device;
|
|
1170
|
+
if (device?.kind === "no-key") rowClauses.push("r.api_key_id IS NULL");
|
|
1171
|
+
else if (device?.kind === "key") {
|
|
1172
|
+
rowClauses.push("r.api_key_id = ?");
|
|
1173
|
+
rowValues.push(device.id);
|
|
1174
|
+
} else if (device?.kind === "unconfigured") {
|
|
1175
|
+
rowClauses.push("r.admission_kind = 'configured' AND r.api_key_id IS NOT NULL AND r.api_key_id <> ''");
|
|
1176
|
+
const ids = query.configuredApiKeyIds ?? [];
|
|
1177
|
+
if (ids.length > 0) {
|
|
1178
|
+
rowClauses.push(`r.api_key_id NOT IN (${ids.map(() => "?").join(", ")})`);
|
|
1179
|
+
rowValues.push(...ids);
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
const client = query.sourceFilter?.client;
|
|
1183
|
+
// Each alternative is its own index-friendly predicate. Keeping the Codex and unknown
|
|
1184
|
+
// expansions as an OR inside one scan made SQLite walk the whole generation; as separate
|
|
1185
|
+
// branches each one searches the client index.
|
|
1186
|
+
const clientAlternatives: Array<{ clauses: string[]; values: Array<string | number> }> = [];
|
|
1187
|
+
if (client === "codex") {
|
|
1188
|
+
clientAlternatives.push({
|
|
1189
|
+
clauses: [`r.client IN (${USAGE_CODEX_CLIENT_LABELS.map(() => "?").join(", ")})`],
|
|
1190
|
+
values: [...USAGE_CODEX_CLIENT_LABELS],
|
|
1191
|
+
});
|
|
1192
|
+
// Mirrors matchesUsageSource: legacy Codex-surface rows are included too.
|
|
1193
|
+
clientAlternatives.push({ clauses: ["r.client IS NULL AND r.surface IS NULL"], values: [] });
|
|
1194
|
+
} else if (client === "unknown") {
|
|
1195
|
+
clientAlternatives.push({ clauses: ["r.client = ?"], values: ["unknown"] });
|
|
1196
|
+
// Mirrors matchesUsageSource: attributed-but-unclassified rows are unknown too.
|
|
1197
|
+
clientAlternatives.push({ clauses: ["r.client IS NULL AND r.surface IS NOT NULL"], values: [] });
|
|
1198
|
+
} else if (client !== undefined && client !== "all") {
|
|
1199
|
+
clientAlternatives.push({ clauses: ["r.client = ?"], values: [client] });
|
|
1200
|
+
} else {
|
|
1201
|
+
clientAlternatives.push({ clauses: [], values: [] });
|
|
1202
|
+
}
|
|
1203
|
+
if (query.window) {
|
|
1204
|
+
// A row without a usable timestamp is excluded, exactly as the accumulator skips it.
|
|
1205
|
+
rowClauses.push("r.timestamp >= ? AND r.timestamp <= ?");
|
|
1206
|
+
rowValues.push(query.window.since, query.window.until);
|
|
1207
|
+
}
|
|
1208
|
+
if (query.afterOffset !== undefined) {
|
|
1209
|
+
rowClauses.push("r.byte_offset >= ?");
|
|
1210
|
+
rowValues.push(query.afterOffset);
|
|
1211
|
+
}
|
|
1212
|
+
rowClauses.push("r.byte_offset <= ?");
|
|
1213
|
+
rowValues.push(snapshot.indexedOffset);
|
|
1214
|
+
const alternativeSql = clientAlternatives
|
|
1215
|
+
.map(alternative => alternative.clauses.length > 0 ? `(${alternative.clauses.join(" AND ")})` : "")
|
|
1216
|
+
.filter(clause => clause !== "")
|
|
1217
|
+
.join(" OR ");
|
|
1218
|
+
const rowSql = [rowClauses.join(" AND "), alternativeSql === "" ? "" : `(${alternativeSql})`]
|
|
1219
|
+
.filter(clause => clause !== "")
|
|
1220
|
+
.join(" AND ");
|
|
1221
|
+
if (!usesAttributions) {
|
|
1222
|
+
// Force the matching index for a selective row predicate. Without it SQLite's default
|
|
1223
|
+
// estimate prefers the row primary key, which examines every row in the ledger even when
|
|
1224
|
+
// one API key or client matches a handful.
|
|
1225
|
+
const baseIndex = rowAccessIndexWithoutClient(query);
|
|
1226
|
+
const branches = clientAlternatives.map(alternative => {
|
|
1227
|
+
const index = alternative.clauses.length > 0 ? "idx_usage_rows_client" : baseIndex;
|
|
1228
|
+
const clauses = [...rowClauses, ...alternative.clauses];
|
|
1229
|
+
return {
|
|
1230
|
+
sql: `SELECT r.byte_offset AS byteOffset FROM usage_rows r${index ? ` INDEXED BY ${index}` : ""} WHERE r.generation = ? AND ${clauses.join(" AND ")}`,
|
|
1231
|
+
values: [snapshot.generation, ...rowValues, ...alternative.values],
|
|
1232
|
+
};
|
|
1233
|
+
});
|
|
1234
|
+
return {
|
|
1235
|
+
sql: `${branches.map(branch => branch.sql).join(" UNION ")} ORDER BY byteOffset`,
|
|
1236
|
+
values: branches.flatMap(branch => branch.values),
|
|
1237
|
+
};
|
|
1238
|
+
}
|
|
1239
|
+
// Drive from the attribution index rather than scanning the row range: SQLite's default
|
|
1240
|
+
// estimate otherwise prefers the row primary key, which examines every row in the ledger.
|
|
1241
|
+
const attrIndex = query.provider === null ? "idx_usage_attr_model" : "idx_usage_attr_provider";
|
|
1242
|
+
// Rows with no usable attribution are selected too: the accumulator is what decides, and a
|
|
1243
|
+
// hand-edited row with a missing model must fail a provider/model filter exactly as the
|
|
1244
|
+
// streaming read does rather than disappear from the result.
|
|
1245
|
+
return {
|
|
1246
|
+
sql: `SELECT DISTINCT a.byte_offset AS byteOffset FROM usage_attributions a INDEXED BY ${attrIndex} WHERE a.generation = ? AND ${attrClauses.join(" AND ")} AND a.byte_offset <= ? AND EXISTS (SELECT 1 FROM usage_rows r WHERE r.generation = a.generation AND r.byte_offset = a.byte_offset AND ${rowSql}) UNION SELECT r.byte_offset AS byteOffset FROM usage_rows r INDEXED BY idx_usage_rows_unattributable WHERE r.generation = ? AND r.unattributable = 1 AND ${rowSql} ORDER BY byteOffset`,
|
|
1247
|
+
values: [
|
|
1248
|
+
snapshot.generation,
|
|
1249
|
+
...attrValues,
|
|
1250
|
+
snapshot.indexedOffset,
|
|
1251
|
+
...rowValues,
|
|
1252
|
+
...clientAlternatives.flatMap(alternative => alternative.values),
|
|
1253
|
+
snapshot.generation,
|
|
1254
|
+
...rowValues,
|
|
1255
|
+
...clientAlternatives.flatMap(alternative => alternative.values),
|
|
1256
|
+
],
|
|
1257
|
+
};
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
/** The index that answers the non-client part of a row-only filter. */
|
|
1261
|
+
function rowAccessIndexWithoutClient(query: UsageFilterIndexQuery): string | null {
|
|
1262
|
+
if (query.apiKeyId !== null) return "idx_usage_rows_key";
|
|
1263
|
+
const device = query.sourceFilter?.device;
|
|
1264
|
+
if (device?.kind === "no-key" || device?.kind === "key") return "idx_usage_rows_key";
|
|
1265
|
+
if (device?.kind === "unconfigured") return "idx_usage_rows_admission";
|
|
1266
|
+
if (query.window) return "idx_usage_rows_ts";
|
|
1267
|
+
return null;
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
function selectCandidates(
|
|
1271
|
+
db: Database,
|
|
1272
|
+
snapshot: UsageFilterIndexSnapshot,
|
|
1273
|
+
query: UsageFilterIndexQuery,
|
|
1274
|
+
): number[] {
|
|
1275
|
+
const { sql, values } = buildCandidateSelect(query, snapshot);
|
|
1276
|
+
const rows = db.query(sql).all(...values) as Array<{ byteOffset: number }>;
|
|
1277
|
+
return rows.map(row => row.byteOffset);
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
/** A published generation with a nonzero row count must actually hold rows. */
|
|
1281
|
+
function hasPublishedRows(db: Database, generation: string): boolean {
|
|
1282
|
+
return db.query("SELECT 1 AS present FROM usage_rows WHERE generation = ? LIMIT 1")
|
|
1283
|
+
.get(generation) !== null;
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
async function readActive(
|
|
1287
|
+
dir: string,
|
|
1288
|
+
path: string,
|
|
1289
|
+
query: UsageFilterIndexQuery,
|
|
1290
|
+
): Promise<UsageFilterCandidateRead> {
|
|
1291
|
+
const refreshed = await refreshUsageFilterIndex(dir, path);
|
|
1292
|
+
testHooks?.afterRefresh?.({ dbPath: path });
|
|
1293
|
+
const db = openIndexForRead(path);
|
|
1294
|
+
try {
|
|
1295
|
+
db.exec("BEGIN");
|
|
1296
|
+
try {
|
|
1297
|
+
const meta = readMeta(db);
|
|
1298
|
+
if (meta.schemaVersion !== USAGE_FILTER_INDEX_SCHEMA_VERSION
|
|
1299
|
+
|| meta.generation !== refreshed.generation) {
|
|
1300
|
+
throw new UsageLedgerRebuildRequiredError("content_changed");
|
|
1301
|
+
}
|
|
1302
|
+
const snapshot: UsageFilterIndexSnapshot = {
|
|
1303
|
+
...snapshotFrom(meta, 0, ingestPassesFor(path)),
|
|
1304
|
+
ingestedRows: refreshed.ingestedRows,
|
|
1305
|
+
ingestPasses: refreshed.ingestPasses,
|
|
1306
|
+
};
|
|
1307
|
+
const offsets = selectCandidates(db, snapshot, query);
|
|
1308
|
+
if (offsets.length === 0 && meta.indexedRows > 0 && !hasPublishedRows(db, meta.generation)) {
|
|
1309
|
+
// The published generation claims rows it no longer has. Never serve that as a
|
|
1310
|
+
// complete zero; drop the projection and rebuild. The refresh alone cannot see this
|
|
1311
|
+
// (the cursor and identity still match), so the rebuild must be requested explicitly.
|
|
1312
|
+
requestUsageFilterIndexRebuild();
|
|
1313
|
+
throw new UsageLedgerRebuildRequiredError("content_changed");
|
|
1314
|
+
}
|
|
1315
|
+
db.exec("COMMIT");
|
|
1316
|
+
return { snapshot, offsets };
|
|
1317
|
+
} catch (error) {
|
|
1318
|
+
try { db.exec("ROLLBACK"); } catch { /* nothing to roll back */ }
|
|
1319
|
+
throw error;
|
|
1320
|
+
}
|
|
1321
|
+
} finally {
|
|
1322
|
+
db.close();
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
/**
|
|
1327
|
+
* Refresh the projection, then select the candidate offsets for one filter.
|
|
1328
|
+
*
|
|
1329
|
+
* Returns null when the projection cannot be used -- a foreign file at its path, an unsafe or
|
|
1330
|
+
* unwritable path, a full disk, or sustained write contention -- so the caller falls back to
|
|
1331
|
+
* the streaming read. A failure to read `usage.jsonl` itself is rethrown: that is the
|
|
1332
|
+
* authoritative source failing, not the acceleration.
|
|
1333
|
+
*/
|
|
1334
|
+
export async function readUsageFilterCandidates(
|
|
1335
|
+
query: UsageFilterIndexQuery,
|
|
1336
|
+
): Promise<UsageFilterCandidateRead | null> {
|
|
1337
|
+
// Capture the home once: every open and ownership registration for this call must target the
|
|
1338
|
+
// directory the flight was keyed on, even if the environment changes mid-flight.
|
|
1339
|
+
const dir = getConfigDir();
|
|
1340
|
+
const path = usageFilterIndexPath(dir);
|
|
1341
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
1342
|
+
try {
|
|
1343
|
+
return await readActive(dir, path, query);
|
|
1344
|
+
} catch (error) {
|
|
1345
|
+
if (error instanceof UsageFilterIndexSourceError) throw error;
|
|
1346
|
+
if (error instanceof UsageFilterIndexForeignError
|
|
1347
|
+
|| error instanceof UnsafeSqlitePathError
|
|
1348
|
+
|| error instanceof UsageFilterIndexHomeChangedError) {
|
|
1349
|
+
return null;
|
|
1350
|
+
}
|
|
1351
|
+
if (error instanceof UsageLedgerRebuildRequiredError) {
|
|
1352
|
+
const delay = BUSY_RETRY_DELAYS_MS[attempt];
|
|
1353
|
+
if (delay === undefined) return null;
|
|
1354
|
+
await new Promise<void>(resolve => setTimeout(resolve, delay));
|
|
1355
|
+
continue;
|
|
1356
|
+
}
|
|
1357
|
+
if (isBusyError(error)) {
|
|
1358
|
+
// Contention, including a peer still creating the projection, is waited out rather than
|
|
1359
|
+
// read as a foreign file. The bound is four busy timeouts (the fourth attempt has no
|
|
1360
|
+
// back-off) plus these delays, about 1.1 s, before the request falls back to the
|
|
1361
|
+
// streaming read.
|
|
1362
|
+
const delay = BUSY_RETRY_DELAYS_MS[attempt];
|
|
1363
|
+
if (delay === undefined) return null;
|
|
1364
|
+
testHooks?.beforeBusyRetry?.({ dbPath: path, attempt });
|
|
1365
|
+
await new Promise<void>(resolve => setTimeout(resolve, delay));
|
|
1366
|
+
continue;
|
|
1367
|
+
}
|
|
1368
|
+
// Any other index-side failure (permissions, disk full, a corrupt file) degrades to the
|
|
1369
|
+
// streaming read rather than failing a request the authoritative scan can answer.
|
|
1370
|
+
return null;
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* Read the projection without writing anything, for a passive management read.
|
|
1377
|
+
*
|
|
1378
|
+
* Returns null unless the projection is current for the exact ledger revision, in which case
|
|
1379
|
+
* the caller keeps its streaming read. No database, WAL or ownership write happens here.
|
|
1380
|
+
*/
|
|
1381
|
+
export async function readUsageFilterCandidatesPassive(
|
|
1382
|
+
query: UsageFilterIndexQuery,
|
|
1383
|
+
): Promise<UsageFilterCandidateRead | null> {
|
|
1384
|
+
const path = indexPathForCurrentHome();
|
|
1385
|
+
if (!existsSync(path)) return null;
|
|
1386
|
+
let db: Database;
|
|
1387
|
+
try {
|
|
1388
|
+
db = new Database(path, { readonly: true });
|
|
1389
|
+
} catch {
|
|
1390
|
+
return null;
|
|
1391
|
+
}
|
|
1392
|
+
try {
|
|
1393
|
+
// A foreign SQLite file that happens to carry a meta table is not this projection.
|
|
1394
|
+
const tables = new Set(tableNames(db));
|
|
1395
|
+
if (!tables.has(META_TABLE)
|
|
1396
|
+
|| !tables.has("usage_rows")
|
|
1397
|
+
|| !tables.has("usage_attributions")
|
|
1398
|
+
|| !tables.has("usage_filter_index_writers")) {
|
|
1399
|
+
return null;
|
|
1400
|
+
}
|
|
1401
|
+
db.exec("BEGIN");
|
|
1402
|
+
try {
|
|
1403
|
+
const meta = readMeta(db);
|
|
1404
|
+
const revision = currentUsageLogRevision();
|
|
1405
|
+
if (meta.schemaVersion !== USAGE_FILTER_INDEX_SCHEMA_VERSION || !meta.generation) return null;
|
|
1406
|
+
if (usageLogIdentityKey(meta.revision) !== usageLogIdentityKey(revision)) return null;
|
|
1407
|
+
if (!revision || meta.indexedOffset !== revision.size) return null;
|
|
1408
|
+
if (usageLogRevisionKey(meta.revision) !== usageLogRevisionKey(revision)) return null;
|
|
1409
|
+
const snapshot = snapshotFrom(meta, 0, 0);
|
|
1410
|
+
// The exact revision match above already proves the ledger is the one indexed; the
|
|
1411
|
+
// empty read keeps the passive path on the same validation as the active one.
|
|
1412
|
+
await readUsageLedgerRowsAtOffsets({
|
|
1413
|
+
offsets: [],
|
|
1414
|
+
expectedIdentityKey: snapshot.identityKey,
|
|
1415
|
+
expectedRevisionKey: usageLogRevisionKey(snapshot.revision),
|
|
1416
|
+
expectedProcessedThroughBytes: snapshot.indexedOffset,
|
|
1417
|
+
expectedProcessedThroughDigest: snapshot.processedThroughDigest,
|
|
1418
|
+
expectedFileSize: snapshot.revision?.size,
|
|
1419
|
+
});
|
|
1420
|
+
const offsets = selectCandidates(db, snapshot, query);
|
|
1421
|
+
if (offsets.length === 0 && meta.indexedRows > 0 && !hasPublishedRows(db, meta.generation)) {
|
|
1422
|
+
// Same guard as the active read: a published generation that lost its rows must not
|
|
1423
|
+
// be reported as a complete zero.
|
|
1424
|
+
return null;
|
|
1425
|
+
}
|
|
1426
|
+
return { snapshot, offsets };
|
|
1427
|
+
} finally {
|
|
1428
|
+
try { db.exec("COMMIT"); } catch { /* read-only transaction */ }
|
|
1429
|
+
}
|
|
1430
|
+
} catch {
|
|
1431
|
+
return null;
|
|
1432
|
+
} finally {
|
|
1433
|
+
db.close();
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
/** Test seam: the SQLite plan for one filter's candidate select. */
|
|
1438
|
+
export function usageFilterCandidatePlan(query: UsageFilterIndexQuery): string[] {
|
|
1439
|
+
const path = indexPathForCurrentHome();
|
|
1440
|
+
const db = new Database(path, { readonly: true });
|
|
1441
|
+
try {
|
|
1442
|
+
const meta = readMeta(db);
|
|
1443
|
+
const snapshot = snapshotFrom(meta, 0, 0);
|
|
1444
|
+
const { sql, values } = buildCandidateSelect(query, snapshot);
|
|
1445
|
+
const rows = db.query(`EXPLAIN QUERY PLAN ${sql}`).all(...values) as Array<{ detail: string }>;
|
|
1446
|
+
return rows.map(row => row.detail);
|
|
1447
|
+
} finally {
|
|
1448
|
+
db.close();
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
/** Test seam: simulate a crash or a competing writer at the publish boundary. */
|
|
1453
|
+
export function setUsageFilterIndexTestHooksForTests(hooks: UsageFilterIndexTestHooks | null): void {
|
|
1454
|
+
testHooks = hooks;
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
/** Discard the projection on the next refresh; used when an offset no longer frames a row. */
|
|
1458
|
+
export function requestUsageFilterIndexRebuild(): void {
|
|
1459
|
+
rebuildRequestedPaths.add(indexPathForCurrentHome());
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
/** Test-only process-state reset for isolated harnesses. */
|
|
1463
|
+
export function resetUsageFilterIndexForTests(): void {
|
|
1464
|
+
ingestFlight = null;
|
|
1465
|
+
ingestPassesByPath.clear();
|
|
1466
|
+
rebuildRequestedPaths.clear();
|
|
1467
|
+
sidecarOwnershipChecked.clear();
|
|
1468
|
+
testHooks = null;
|
|
1469
|
+
}
|