@agent-watch-ai/edge 0.2.5 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +512 -0
- package/README.md +217 -96
- package/SECURITY.md +69 -0
- package/dist/billing/session-billing.d.ts +19 -0
- package/dist/billing/session-billing.js +52 -0
- package/dist/cli/args.d.ts +10 -0
- package/dist/cli/args.js +20 -0
- package/dist/cli/constants/cli.constants.d.ts +50 -4
- package/dist/cli/constants/cli.constants.js +54 -5
- package/dist/cli/context.d.ts +33 -2
- package/dist/cli/context.js +84 -14
- package/dist/cli/doctor.js +309 -32
- package/dist/cli/hook-check.d.ts +23 -0
- package/dist/cli/hook-check.js +114 -0
- package/dist/cli/hook.js +118 -7
- package/dist/cli/index.d.ts +3 -1
- package/dist/cli/index.js +3 -1
- package/dist/cli/misc.d.ts +11 -0
- package/dist/cli/misc.js +80 -4
- package/dist/cli/npx-install.d.ts +37 -0
- package/dist/cli/npx-install.js +79 -0
- package/dist/cli/open-browser.d.ts +39 -0
- package/dist/cli/open-browser.js +62 -0
- package/dist/cli/setup.d.ts +17 -8
- package/dist/cli/setup.js +439 -98
- package/dist/cli/status.js +82 -13
- package/dist/cli/toggle.d.ts +9 -0
- package/dist/cli/toggle.js +113 -0
- package/dist/cli/types/cli.types.d.ts +37 -3
- package/dist/cli/ui.d.ts +7 -0
- package/dist/cli/ui.js +10 -1
- package/dist/cli/uninstall.js +39 -2
- package/dist/cli/update.d.ts +17 -0
- package/dist/cli/update.js +76 -0
- package/dist/cli.js +100 -13
- package/dist/config/config-store.d.ts +57 -0
- package/dist/config/config-store.js +250 -12
- package/dist/config/config.d.ts +53 -6
- package/dist/config/config.js +66 -6
- package/dist/config/constants/config.constants.d.ts +113 -6
- package/dist/config/constants/config.constants.js +126 -8
- package/dist/config/destination.d.ts +24 -0
- package/dist/config/destination.js +33 -0
- package/dist/config/index.d.ts +7 -4
- package/dist/config/index.js +5 -3
- package/dist/config/repo-config.d.ts +11 -7
- package/dist/config/repo-config.js +64 -15
- package/dist/config/root-config.d.ts +72 -0
- package/dist/config/root-config.js +170 -0
- package/dist/config/schemas/config.schema.d.ts +721 -84
- package/dist/config/schemas/config.schema.js +388 -15
- package/dist/config/types/config.types.d.ts +15 -1
- package/dist/config/types/destination.types.d.ts +8 -0
- package/dist/core/async.d.ts +36 -0
- package/dist/core/async.js +68 -0
- package/dist/core/index.d.ts +1 -1
- package/dist/core/index.js +1 -1
- package/dist/core/object.d.ts +5 -0
- package/dist/core/object.js +9 -4
- package/dist/core/pipe.d.ts +0 -13
- package/dist/core/pipe.js +1 -9
- package/dist/core/types/core.types.d.ts +9 -0
- package/dist/core/types/core.types.js +0 -3
- package/dist/core/which.d.ts +14 -0
- package/dist/core/which.js +43 -3
- package/dist/credentials/budget.d.ts +9 -0
- package/dist/credentials/budget.js +16 -0
- package/dist/credentials/command-runner.d.ts +16 -0
- package/dist/credentials/command-runner.js +37 -0
- package/dist/credentials/constants/credentials.constants.d.ts +111 -0
- package/dist/credentials/constants/credentials.constants.js +112 -0
- package/dist/credentials/credential-status.d.ts +14 -0
- package/dist/credentials/credential-status.js +34 -0
- package/dist/credentials/credentials.d.ts +72 -0
- package/dist/credentials/credentials.js +253 -0
- package/dist/credentials/device-api.d.ts +64 -0
- package/dist/credentials/device-api.js +150 -0
- package/dist/credentials/device-state.d.ts +31 -0
- package/dist/credentials/device-state.js +51 -0
- package/dist/credentials/identity.d.ts +62 -0
- package/dist/credentials/identity.js +90 -0
- package/dist/credentials/index.d.ts +7 -0
- package/dist/credentials/index.js +5 -0
- package/dist/credentials/migration.d.ts +46 -0
- package/dist/credentials/migration.js +164 -0
- package/dist/credentials/schemas/device-api.schema.d.ts +77 -0
- package/dist/credentials/schemas/device-api.schema.js +33 -0
- package/dist/credentials/schemas/device-state.schema.d.ts +51 -0
- package/dist/credentials/schemas/device-state.schema.js +20 -0
- package/dist/credentials/secret-store.d.ts +54 -0
- package/dist/credentials/secret-store.js +217 -0
- package/dist/credentials/sign-in.d.ts +18 -0
- package/dist/credentials/sign-in.js +86 -0
- package/dist/credentials/token-refresher.d.ts +39 -0
- package/dist/credentials/token-refresher.js +185 -0
- package/dist/credentials/types/credentials.types.d.ts +195 -0
- package/dist/credentials/types/credentials.types.js +1 -0
- package/dist/enforcement/constants/enforcement.constants.d.ts +47 -1
- package/dist/enforcement/constants/enforcement.constants.js +56 -0
- package/dist/enforcement/decision-cache.d.ts +20 -1
- package/dist/enforcement/decision-cache.js +65 -2
- package/dist/enforcement/decision-client.d.ts +9 -8
- package/dist/enforcement/decision-client.js +62 -16
- package/dist/enforcement/enforcement.d.ts +23 -2
- package/dist/enforcement/enforcement.js +103 -15
- package/dist/enforcement/index.d.ts +4 -3
- package/dist/enforcement/index.js +3 -2
- package/dist/enforcement/types/enforcement.types.d.ts +64 -0
- package/dist/enrollment/constants/enrollment.constants.d.ts +2 -6
- package/dist/enrollment/constants/enrollment.constants.js +2 -6
- package/dist/enrollment/enrollment.d.ts +16 -0
- package/dist/enrollment/enrollment.js +54 -0
- package/dist/enrollment/index.d.ts +3 -7
- package/dist/enrollment/index.js +1 -1
- package/dist/enrollment/types/enrollment.types.d.ts +9 -13
- package/dist/events/constants/enrich.constants.d.ts +15 -0
- package/dist/events/constants/enrich.constants.js +15 -0
- package/dist/events/constants/events.constants.d.ts +6 -2
- package/dist/events/constants/events.constants.js +5 -1
- package/dist/events/enrich.js +92 -10
- package/dist/events/event-id.d.ts +0 -8
- package/dist/events/event-id.js +0 -10
- package/dist/events/index.d.ts +2 -2
- package/dist/events/index.js +1 -1
- package/dist/events/product-event.d.ts +9 -6
- package/dist/events/types/repo-snapshot.types.d.ts +7 -0
- package/dist/events/types/session-billing.types.d.ts +16 -0
- package/dist/events/types/session-billing.types.js +1 -0
- package/dist/feature/constants/feature.constants.d.ts +7 -0
- package/dist/feature/constants/feature.constants.js +7 -0
- package/dist/feature/index.d.ts +1 -1
- package/dist/feature/index.js +1 -1
- package/dist/feature/ticket-candidates.d.ts +14 -0
- package/dist/feature/ticket-candidates.js +26 -8
- package/dist/git/blob-hash.d.ts +21 -0
- package/dist/git/blob-hash.js +69 -0
- package/dist/git/checkout-root.d.ts +27 -0
- package/dist/git/checkout-root.js +75 -0
- package/dist/git/constants/git.constants.d.ts +160 -0
- package/dist/git/constants/git.constants.js +181 -0
- package/dist/git/fingerprint.d.ts +74 -0
- package/dist/git/fingerprint.js +219 -0
- package/dist/git/gate-checkout.d.ts +44 -0
- package/dist/git/gate-checkout.js +201 -0
- package/dist/git/git-context.d.ts +45 -0
- package/dist/git/git-context.js +61 -3
- package/dist/git/index.d.ts +4 -2
- package/dist/git/index.js +2 -1
- package/dist/git/manifest-repository.d.ts +13 -0
- package/dist/git/manifest-repository.js +92 -0
- package/dist/git/repo-snapshot.d.ts +29 -3
- package/dist/git/repo-snapshot.js +110 -7
- package/dist/git/repository-root.d.ts +39 -0
- package/dist/git/repository-root.js +131 -0
- package/dist/git/types/git.types.d.ts +55 -0
- package/dist/git/types/repository-root.types.d.ts +11 -0
- package/dist/git/types/repository-root.types.js +1 -0
- package/dist/git/types/snapshot.types.d.ts +9 -0
- package/dist/pipeline/constants/pipeline.constants.d.ts +7 -0
- package/dist/pipeline/constants/pipeline.constants.js +7 -0
- package/dist/pipeline/hook-pipeline.js +360 -32
- package/dist/pipeline/types/pipeline.types.d.ts +39 -4
- package/dist/privacy/constants/privacy.constants.js +15 -1
- package/dist/privacy/index.d.ts +1 -0
- package/dist/privacy/index.js +1 -0
- package/dist/privacy/product-capture.d.ts +12 -0
- package/dist/privacy/product-capture.js +73 -0
- package/dist/privacy/sanitizer.d.ts +21 -3
- package/dist/privacy/sanitizer.js +52 -8
- package/dist/process/ancestors.d.ts +39 -0
- package/dist/process/ancestors.js +121 -0
- package/dist/process/constants/process.constants.d.ts +34 -0
- package/dist/process/constants/process.constants.js +34 -0
- package/dist/process/index.d.ts +7 -0
- package/dist/process/index.js +2 -0
- package/dist/process/types/process.types.d.ts +23 -0
- package/dist/process/types/process.types.js +1 -0
- package/dist/providers/antigravity/antigravity.adapter.d.ts +12 -1
- package/dist/providers/antigravity/antigravity.adapter.js +21 -3
- package/dist/providers/antigravity/antigravity.detect.d.ts +0 -9
- package/dist/providers/antigravity/antigravity.detect.js +1 -1
- package/dist/providers/antigravity/antigravity.provider.js +3 -2
- package/dist/providers/antigravity/constants/antigravity.constants.d.ts +2 -0
- package/dist/providers/antigravity/constants/antigravity.constants.js +2 -0
- package/dist/providers/claude/claude.adapter.d.ts +11 -1
- package/dist/providers/claude/claude.adapter.js +26 -5
- package/dist/providers/claude/claude.otel.d.ts +32 -18
- package/dist/providers/claude/claude.otel.js +81 -6
- package/dist/providers/claude/claude.provider.js +5 -2
- package/dist/providers/claude/constants/claude.constants.d.ts +2 -0
- package/dist/providers/claude/constants/claude.constants.js +2 -0
- package/dist/providers/claude/constants/claude.otel.constants.d.ts +4 -0
- package/dist/providers/claude/constants/claude.otel.constants.js +8 -0
- package/dist/providers/codex/codex.adapter.d.ts +15 -1
- package/dist/providers/codex/codex.adapter.js +29 -5
- package/dist/providers/codex/codex.otel.d.ts +24 -2
- package/dist/providers/codex/codex.otel.js +147 -30
- package/dist/providers/codex/codex.provider.js +5 -2
- package/dist/providers/codex/constants/codex.constants.d.ts +2 -0
- package/dist/providers/codex/constants/codex.constants.js +16 -0
- package/dist/providers/codex/constants/codex.otel.constants.d.ts +5 -0
- package/dist/providers/codex/constants/codex.otel.constants.js +5 -0
- package/dist/providers/constants/provider.constants.d.ts +0 -3
- package/dist/providers/constants/provider.constants.js +0 -3
- package/dist/providers/cursor/constants/cursor.constants.d.ts +7 -0
- package/dist/providers/cursor/constants/cursor.constants.js +7 -0
- package/dist/providers/cursor/cursor.adapter.d.ts +33 -1
- package/dist/providers/cursor/cursor.adapter.js +52 -3
- package/dist/providers/cursor/cursor.detect.d.ts +0 -7
- package/dist/providers/cursor/cursor.detect.js +1 -1
- package/dist/providers/cursor/cursor.provider.js +5 -2
- package/dist/providers/gemini/constants/gemini.constants.d.ts +9 -0
- package/dist/providers/gemini/constants/gemini.constants.js +9 -0
- package/dist/providers/gemini/constants/gemini.otel.constants.d.ts +3 -1
- package/dist/providers/gemini/constants/gemini.otel.constants.js +3 -0
- package/dist/providers/gemini/gemini.adapter.d.ts +14 -1
- package/dist/providers/gemini/gemini.adapter.js +25 -5
- package/dist/providers/gemini/gemini.detect.d.ts +0 -7
- package/dist/providers/gemini/gemini.detect.js +1 -1
- package/dist/providers/gemini/gemini.otel.d.ts +19 -20
- package/dist/providers/gemini/gemini.otel.js +121 -17
- package/dist/providers/gemini/gemini.provider.js +2 -1
- package/dist/providers/index.d.ts +1 -1
- package/dist/providers/index.js +1 -1
- package/dist/providers/loaders.d.ts +10 -0
- package/dist/providers/loaders.js +40 -0
- package/dist/providers/provider.d.ts +11 -0
- package/dist/providers/provider.js +1 -1
- package/dist/providers/registry.d.ts +11 -8
- package/dist/providers/registry.js +11 -15
- package/dist/providers/shared/constants/hook-refusal.constants.d.ts +7 -0
- package/dist/providers/shared/constants/hook-refusal.constants.js +7 -0
- package/dist/providers/shared/constants/shell-directories.constants.d.ts +30 -0
- package/dist/providers/shared/constants/shell-directories.constants.js +39 -0
- package/dist/providers/shared/constants/tooling.constants.d.ts +18 -2
- package/dist/providers/shared/constants/tooling.constants.js +19 -3
- package/dist/providers/shared/event-builder.d.ts +12 -14
- package/dist/providers/shared/event-builder.js +20 -14
- package/dist/providers/shared/exporter-allowlist.d.ts +20 -0
- package/dist/providers/shared/exporter-allowlist.js +49 -0
- package/dist/providers/shared/hook-config.d.ts +2 -21
- package/dist/providers/shared/hook-config.js +5 -4
- package/dist/providers/shared/hook-refusal.d.ts +12 -0
- package/dist/providers/shared/hook-refusal.js +17 -1
- package/dist/providers/shared/install-record.d.ts +12 -2
- package/dist/providers/shared/install-record.js +5 -2
- package/dist/providers/shared/shell-directories.d.ts +24 -0
- package/dist/providers/shared/shell-directories.js +90 -0
- package/dist/providers/shared/tooling.d.ts +12 -1
- package/dist/providers/shared/tooling.js +18 -0
- package/dist/providers/shared/types/adapter.types.d.ts +1 -2
- package/dist/providers/types/provider.types.d.ts +57 -1
- package/dist/snapshot/constants/snapshot.constants.d.ts +32 -0
- package/dist/snapshot/constants/snapshot.constants.js +33 -0
- package/dist/snapshot/snapshot-event.js +52 -11
- package/dist/snapshot/snapshot-pipeline.js +1 -1
- package/dist/snapshot/types/snapshot.types.d.ts +2 -0
- package/dist/storage/atomic-file.d.ts +7 -0
- package/dist/storage/atomic-file.js +1 -1
- package/dist/storage/constants/storage.constants.d.ts +17 -0
- package/dist/storage/constants/storage.constants.js +17 -0
- package/dist/storage/disabled.d.ts +13 -0
- package/dist/storage/disabled.js +24 -0
- package/dist/storage/index.d.ts +2 -0
- package/dist/storage/index.js +2 -0
- package/dist/storage/paths.js +3 -2
- package/dist/storage/schemas/storage.schema.d.ts +72 -0
- package/dist/storage/schemas/storage.schema.js +6 -0
- package/dist/storage/sweep-marker.d.ts +19 -0
- package/dist/storage/sweep-marker.js +55 -0
- package/dist/storage/types/storage.types.d.ts +2 -0
- package/dist/storage/uninstalled.d.ts +58 -0
- package/dist/storage/uninstalled.js +107 -0
- package/dist/transport/auth-block.d.ts +66 -0
- package/dist/transport/auth-block.js +129 -0
- package/dist/transport/constants/transport.constants.d.ts +82 -3
- package/dist/transport/constants/transport.constants.js +83 -4
- package/dist/transport/cooldown.js +8 -1
- package/dist/transport/delivery.d.ts +14 -6
- package/dist/transport/delivery.js +80 -55
- package/dist/transport/http-transport.d.ts +1 -0
- package/dist/transport/http-transport.js +52 -10
- package/dist/transport/index.d.ts +6 -3
- package/dist/transport/index.js +3 -0
- package/dist/transport/queue-partition.d.ts +91 -0
- package/dist/transport/queue-partition.js +143 -0
- package/dist/transport/queue.d.ts +99 -20
- package/dist/transport/queue.js +210 -69
- package/dist/transport/response-body.d.ts +25 -0
- package/dist/transport/response-body.js +64 -0
- package/dist/transport/send.d.ts +11 -0
- package/dist/transport/send.js +17 -0
- package/dist/transport/types/delivery-flow.types.d.ts +19 -0
- package/dist/transport/types/delivery-flow.types.js +1 -0
- package/dist/transport/types/transport.types.d.ts +45 -0
- package/dist/turns/claude-transcript.d.ts +19 -1
- package/dist/turns/claude-transcript.js +46 -1
- package/dist/turns/codex-originator.d.ts +11 -0
- package/dist/turns/codex-originator.js +31 -0
- package/dist/turns/codex-transcript.d.ts +30 -0
- package/dist/turns/codex-transcript.js +88 -0
- package/dist/turns/constants/turns.constants.d.ts +141 -5
- package/dist/turns/constants/turns.constants.js +155 -5
- package/dist/turns/file-identity.d.ts +27 -0
- package/dist/turns/file-identity.js +48 -0
- package/dist/turns/gemini-transcript.d.ts +30 -0
- package/dist/turns/gemini-transcript.js +96 -0
- package/dist/turns/index.d.ts +3 -3
- package/dist/turns/index.js +1 -1
- package/dist/turns/inherited-checkout.d.ts +54 -0
- package/dist/turns/inherited-checkout.js +155 -0
- package/dist/turns/native-session-titles.d.ts +26 -0
- package/dist/turns/native-session-titles.js +102 -0
- package/dist/turns/transcript.d.ts +3 -2
- package/dist/turns/transcript.js +5 -1
- package/dist/turns/turn-state.d.ts +224 -3
- package/dist/turns/turn-state.js +456 -21
- package/dist/turns/turn-summary.d.ts +1 -14
- package/dist/turns/turn-summary.js +87 -51
- package/dist/turns/turn-tracker.d.ts +21 -2
- package/dist/turns/turn-tracker.js +379 -56
- package/dist/turns/types/transcript.types.d.ts +5 -0
- package/dist/turns/types/turn-state.types.d.ts +116 -5
- package/dist/turns/types/turn-summary.types.d.ts +64 -14
- package/dist/turns/types/turn-tracker.types.d.ts +27 -1
- package/dist/turns/work-checkout.d.ts +65 -0
- package/dist/turns/work-checkout.js +666 -0
- package/dist/untraced/constants/untraced.constants.d.ts +31 -0
- package/dist/untraced/constants/untraced.constants.js +63 -0
- package/dist/untraced/untraced.d.ts +35 -0
- package/dist/untraced/untraced.js +105 -0
- package/dist/update/constants/update.constants.d.ts +18 -0
- package/dist/update/constants/update.constants.js +20 -0
- package/dist/update/update-check.d.ts +52 -0
- package/dist/update/update-check.js +113 -0
- package/docs/AUDIT_PROMPT.md +129 -0
- package/docs/DATA_HANDLING.md +648 -0
- package/docs/ENTERPRISE_DEPLOYMENT.md +179 -0
- package/docs/PERFORMANCE.md +81 -0
- package/package.json +14 -4
- package/dist/enrollment/enrollment-provider.d.ts +0 -5
- package/dist/enrollment/manual-enrollment.d.ts +0 -32
- package/dist/enrollment/manual-enrollment.js +0 -72
- package/example/README.md +0 -43
- package/example/server.mjs +0 -240
- /package/dist/{enrollment/enrollment-provider.js → config/types/destination.types.js} +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type ProductEvent } from '../events/product-event.js';
|
|
2
|
+
import type { BackendAuthBlock } from './auth-block.js';
|
|
2
3
|
import type { DrainStats, DrainStatsRecorder, EventTransport, QueueOptions } from './types/transport.types.js';
|
|
3
4
|
export { ANY_DESTINATION } from './constants/transport.constants.js';
|
|
4
5
|
export type { DrainStats, DrainStatsRecorder, QueueEntry, QueueOptions } from './types/transport.types.js';
|
|
@@ -50,26 +51,29 @@ export declare class EventQueue {
|
|
|
50
51
|
* @param transport - Where to send.
|
|
51
52
|
* @param maxBatch - Ceiling on events sent in one pass.
|
|
52
53
|
* @param statsRecorder - Optional sink for what this pass lost.
|
|
54
|
+
* @param authBlock - Optional sink for a refused credential, so a rejected
|
|
55
|
+
* bearer suspends sending instead of spending the backlog's attempts on it.
|
|
53
56
|
* @returns What the pass sent, failed, dropped and had rejected.
|
|
54
57
|
*/
|
|
55
|
-
drain(transport: EventTransport, maxBatch: number, statsRecorder?: DrainStatsRecorder): Promise<DrainStats>;
|
|
58
|
+
drain(transport: EventTransport, maxBatch: number, statsRecorder?: DrainStatsRecorder, authBlock?: BackendAuthBlock): Promise<DrainStats>;
|
|
56
59
|
/**
|
|
57
|
-
*
|
|
60
|
+
* Enforce retention across the whole partition, without sending anything.
|
|
58
61
|
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
62
|
+
* `drain` sweeps too, but every early return in `deliverEvents` skips a drain
|
|
63
|
+
* — a tripped cooldown, and a credential the backend is refusing — and those
|
|
64
|
+
* are precisely the multi-day outages the age bound exists for. A block has no
|
|
65
|
+
* timer at all, so a revoked token used to mean nothing aged out until someone
|
|
66
|
+
* fixed the credential, while `enforceBound` quietly shed the oldest records
|
|
67
|
+
* at the ceiling. Sharing the marker with the drain's own sweep, so a hook
|
|
68
|
+
* that does reach a drain still pays for only one pass.
|
|
64
69
|
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
70
|
+
* Takes no lock: every removal is idempotent, which is the same reason
|
|
71
|
+
* `claimSweep` is unlocked.
|
|
67
72
|
*
|
|
68
|
-
* @param
|
|
69
|
-
* @
|
|
70
|
-
* @returns False when the lock never freed and nothing was re-pinned.
|
|
73
|
+
* @param statsRecorder - Optional sink for what aged out.
|
|
74
|
+
* @returns How many entries were removed.
|
|
71
75
|
*/
|
|
72
|
-
|
|
76
|
+
sweep(statsRecorder?: DrainStatsRecorder): Promise<number>;
|
|
73
77
|
/**
|
|
74
78
|
* Entries pinned to exactly this destination; legacy and ANY entries are not
|
|
75
79
|
* counted, because they are not stranded.
|
|
@@ -81,17 +85,84 @@ export declare class EventQueue {
|
|
|
81
85
|
/**
|
|
82
86
|
* Entries due for this transport, dropping the ones that can never be sent.
|
|
83
87
|
*
|
|
88
|
+
* Reads at most `maxBatch` entries, whatever the backlog holds and whatever it
|
|
89
|
+
* finds in them. Reading and zod-parsing the whole partition to find the few
|
|
90
|
+
* entries a pass may send is what made an outage cost the developer up to
|
|
91
|
+
* `maxQueueEvents` file reads on every hook, which AGENTS.md §3 forbids
|
|
92
|
+
* outright. The bound is on entries *read*, not on entries found due: a
|
|
93
|
+
* backlog where nothing is due — every entry in backoff, or every entry
|
|
94
|
+
* pinned to a destination this identity no longer sends to, which is what a
|
|
95
|
+
* changed endpoint leaves behind for a week — is exactly the case that would
|
|
96
|
+
* otherwise walk all of it and send nothing.
|
|
97
|
+
*
|
|
98
|
+
* A hard read bound needs the scan to move, or the head of the sorted list
|
|
99
|
+
* would be re-read on every pass and the tail never reached. So the scan
|
|
100
|
+
* starts where the last one stopped and wraps around: every entry is examined
|
|
101
|
+
* within `ceil(backlog / maxBatch)` passes, which is what FR-009 asks for —
|
|
102
|
+
* no entry deferred forever. Losing the cursor costs one pass starting at the
|
|
103
|
+
* head again.
|
|
104
|
+
*
|
|
105
|
+
* The batch is therefore the oldest of what this pass *scanned*, not of the
|
|
106
|
+
* whole backlog. Global oldest-first would need a due-time index, which is
|
|
107
|
+
* separate work.
|
|
108
|
+
*
|
|
84
109
|
* @param transport - Where the pass will send.
|
|
85
110
|
* @param nowMs - This pass's clock reading.
|
|
111
|
+
* @param maxBatch - Entries this pass may read; also its ceiling on sends.
|
|
86
112
|
* @returns The due entries and what collecting them dropped.
|
|
87
113
|
*/
|
|
88
114
|
private collectDue;
|
|
115
|
+
/**
|
|
116
|
+
* Index the next bounded scan starts at.
|
|
117
|
+
*
|
|
118
|
+
* The first name *after* the one last scanned, so an entry that was sent and
|
|
119
|
+
* deleted in the meantime does not send the scan back to the head.
|
|
120
|
+
*
|
|
121
|
+
* @param names - This pass's sorted entry names.
|
|
122
|
+
* @returns The starting index; zero when there is no cursor to resume from.
|
|
123
|
+
*/
|
|
124
|
+
private scanStart;
|
|
125
|
+
/**
|
|
126
|
+
* The entry name the previous pass stopped on.
|
|
127
|
+
*
|
|
128
|
+
* @returns The name, or undefined when nothing has scanned yet.
|
|
129
|
+
*/
|
|
130
|
+
private lastScanned;
|
|
131
|
+
/**
|
|
132
|
+
* Record where this pass stopped.
|
|
133
|
+
*
|
|
134
|
+
* @param name - The last entry name it read.
|
|
135
|
+
*/
|
|
136
|
+
private rememberScan;
|
|
137
|
+
/**
|
|
138
|
+
* Where the scan cursor lives.
|
|
139
|
+
*
|
|
140
|
+
* Not a `.json` name, so the entry listing never sees it.
|
|
141
|
+
*
|
|
142
|
+
* @returns Absolute path to the cursor file.
|
|
143
|
+
*/
|
|
144
|
+
private cursorFile;
|
|
145
|
+
/**
|
|
146
|
+
* Remove everything that can never be sent, wherever it sits in the partition.
|
|
147
|
+
*
|
|
148
|
+
* The bounded scan only ever sees the head of the backlog, so the retention
|
|
149
|
+
* bound needs a pass that sees all of it. Throttled by a marker, because that
|
|
150
|
+
* pass is the expensive one — and it is an improvement on what it replaces:
|
|
151
|
+
* expiry used to be enforced only inside a drain, and a drain is skipped for
|
|
152
|
+
* the whole cooldown window, so during a backend outage nothing aged out at
|
|
153
|
+
* all. That was precisely the week-long case the bound exists for.
|
|
154
|
+
*
|
|
155
|
+
* @param nowMs - This pass's clock reading.
|
|
156
|
+
* @returns What the sweep dropped, or nothing when it was not due.
|
|
157
|
+
*/
|
|
158
|
+
private sweepUnsendable;
|
|
89
159
|
/**
|
|
90
160
|
* Send one batch, isolating a poison entry when the backend refuses it.
|
|
91
161
|
*
|
|
92
162
|
* @param transport - Where to send.
|
|
93
163
|
* @param batch - Entries to send.
|
|
94
164
|
* @param nowMs - This pass's clock reading.
|
|
165
|
+
* @param authBlock - Optional sink for a refused credential.
|
|
95
166
|
* @returns What the send accomplished and cost.
|
|
96
167
|
*/
|
|
97
168
|
private sendBatch;
|
|
@@ -107,15 +178,10 @@ export declare class EventQueue {
|
|
|
107
178
|
* @param transport - Where to send.
|
|
108
179
|
* @param batch - Entries the batch send refused.
|
|
109
180
|
* @param nowMs - This pass's clock reading.
|
|
181
|
+
* @param authBlock - Persisted refusal, including one received by a single probe.
|
|
110
182
|
* @returns What the probes accomplished and cost.
|
|
111
183
|
*/
|
|
112
184
|
private isolateBatch;
|
|
113
|
-
/**
|
|
114
|
-
* Acquire the drain lock, waiting a bounded time.
|
|
115
|
-
*
|
|
116
|
-
* @returns The release function, or undefined on timeout.
|
|
117
|
-
*/
|
|
118
|
-
private waitForDrainLock;
|
|
119
185
|
/**
|
|
120
186
|
* Back a failed entry off, or drop it once its budget is spent.
|
|
121
187
|
*
|
|
@@ -162,6 +228,11 @@ export declare class EventQueue {
|
|
|
162
228
|
*
|
|
163
229
|
* The oldest are the least likely to still be deliverable, and an unbounded
|
|
164
230
|
* queue on a developer machine is a disk-space bug.
|
|
231
|
+
*
|
|
232
|
+
* Reported through `options.stats`, like every other permanent loss: these
|
|
233
|
+
* deletions used to be the one kind that left `totalDropped` at zero, so a
|
|
234
|
+
* machine steadily shedding records at the bound looked, in `status`, exactly
|
|
235
|
+
* like one that had never lost any.
|
|
165
236
|
*/
|
|
166
237
|
private enforceBound;
|
|
167
238
|
/**
|
|
@@ -175,3 +246,11 @@ export declare class EventQueue {
|
|
|
175
246
|
*/
|
|
176
247
|
private queuedAt;
|
|
177
248
|
}
|
|
249
|
+
/**
|
|
250
|
+
* Queue entry filenames directly in a directory, sorted. Sub-directories are
|
|
251
|
+
* other identities' partitions and never entries.
|
|
252
|
+
*
|
|
253
|
+
* @param dir - Directory to list.
|
|
254
|
+
* @returns The names, or an empty list when the directory is absent.
|
|
255
|
+
*/
|
|
256
|
+
export declare function listQueueFiles(dir: string): Promise<string[]>;
|
package/dist/transport/queue.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import fs from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import { pollUntil } from '../core/async.js';
|
|
4
3
|
import { debugLog } from '../core/logger.js';
|
|
5
|
-
import {
|
|
4
|
+
import { isProductEvent } from '../events/product-event.js';
|
|
6
5
|
import { writeFileAtomic } from '../storage/atomic-file.js';
|
|
7
6
|
import { acquireLock } from '../storage/lock.js';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
7
|
+
import { claimSweep } from '../storage/sweep-marker.js';
|
|
8
|
+
import { SECRET_FILE_MODE, SWEEP_MARKER_FILE } from '../storage/constants/storage.constants.js';
|
|
9
|
+
import { ANY_DESTINATION, AUTH_REJECTED_STATUSES, BACKOFF_BASE_MS, BACKOFF_JITTER_MIN, BACKOFF_JITTER_RANGE, BACKOFF_MAX_MS, MAX_ISOLATION_SENDS, MS_PER_DAY, QUEUE_DRAIN_LOCK, QUEUE_FILE_SUFFIX, QUEUE_SCAN_CURSOR_FILE, QUEUE_SWEEP_INTERVAL_MS, RE_UNSAFE_QUEUE_NAME } from './constants/transport.constants.js';
|
|
10
|
+
import { sendEvents } from './send.js';
|
|
10
11
|
import { queueEntrySchema } from './schemas/queue.schema.js';
|
|
11
12
|
export { ANY_DESTINATION } from './constants/transport.constants.js';
|
|
12
13
|
/**
|
|
@@ -43,7 +44,7 @@ export class EventQueue {
|
|
|
43
44
|
if (await exists(file))
|
|
44
45
|
continue; // dedup by deterministic id
|
|
45
46
|
const at = this.now().toISOString();
|
|
46
|
-
// 0600: a queued turn summary holds
|
|
47
|
+
// 0600: a queued turn summary holds developer identity, branches and file paths.
|
|
47
48
|
await writeFileAtomic(file, JSON.stringify({ event, attempts: 0, firstQueuedAt: at, nextAttemptAt: at, destination }), SECRET_FILE_MODE);
|
|
48
49
|
}
|
|
49
50
|
await this.enforceBound();
|
|
@@ -88,23 +89,26 @@ export class EventQueue {
|
|
|
88
89
|
* @param transport - Where to send.
|
|
89
90
|
* @param maxBatch - Ceiling on events sent in one pass.
|
|
90
91
|
* @param statsRecorder - Optional sink for what this pass lost.
|
|
92
|
+
* @param authBlock - Optional sink for a refused credential, so a rejected
|
|
93
|
+
* bearer suspends sending instead of spending the backlog's attempts on it.
|
|
91
94
|
* @returns What the pass sent, failed, dropped and had rejected.
|
|
92
95
|
*/
|
|
93
|
-
async drain(transport, maxBatch, statsRecorder) {
|
|
96
|
+
async drain(transport, maxBatch, statsRecorder, authBlock) {
|
|
94
97
|
const release = await acquireLock(this.options.locksDir, QUEUE_DRAIN_LOCK, this.now);
|
|
95
98
|
if (!release)
|
|
96
99
|
return { sent: 0, failed: 0, dropped: 0, rejected: 0, skipped: true };
|
|
97
100
|
try {
|
|
98
101
|
const nowMs = this.now().getTime();
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
//
|
|
102
|
+
const swept = await this.sweepUnsendable(nowMs);
|
|
103
|
+
const collected = await this.collectDue(transport, nowMs, maxBatch);
|
|
104
|
+
// Oldest first among what was scanned: hash-ordered filenames would
|
|
105
|
+
// otherwise let a large backlog defer the same late-sorting entries.
|
|
102
106
|
const batch = collected.due
|
|
103
107
|
.slice()
|
|
104
108
|
.sort((a, b) => Date.parse(a.entry.firstQueuedAt) - Date.parse(b.entry.firstQueuedAt))
|
|
105
109
|
.slice(0, maxBatch);
|
|
106
|
-
const sent = batch.length === 0 ? EMPTY_DELTA : await this.sendBatch(transport, batch, nowMs);
|
|
107
|
-
const stats = { ...mergeDeltas(collected.delta, sent), skipped: false };
|
|
110
|
+
const sent = batch.length === 0 ? EMPTY_DELTA : await this.sendBatch(transport, batch, nowMs, authBlock);
|
|
111
|
+
const stats = { ...mergeDeltas(mergeDeltas(swept, collected.delta), sent), skipped: false };
|
|
108
112
|
await report(statsRecorder, stats);
|
|
109
113
|
return stats;
|
|
110
114
|
}
|
|
@@ -113,38 +117,27 @@ export class EventQueue {
|
|
|
113
117
|
}
|
|
114
118
|
}
|
|
115
119
|
/**
|
|
116
|
-
*
|
|
120
|
+
* Enforce retention across the whole partition, without sending anything.
|
|
117
121
|
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
122
|
+
* `drain` sweeps too, but every early return in `deliverEvents` skips a drain
|
|
123
|
+
* — a tripped cooldown, and a credential the backend is refusing — and those
|
|
124
|
+
* are precisely the multi-day outages the age bound exists for. A block has no
|
|
125
|
+
* timer at all, so a revoked token used to mean nothing aged out until someone
|
|
126
|
+
* fixed the credential, while `enforceBound` quietly shed the oldest records
|
|
127
|
+
* at the ceiling. Sharing the marker with the drain's own sweep, so a hook
|
|
128
|
+
* that does reach a drain still pays for only one pass.
|
|
123
129
|
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
130
|
+
* Takes no lock: every removal is idempotent, which is the same reason
|
|
131
|
+
* `claimSweep` is unlocked.
|
|
126
132
|
*
|
|
127
|
-
* @param
|
|
128
|
-
* @
|
|
129
|
-
* @returns False when the lock never freed and nothing was re-pinned.
|
|
133
|
+
* @param statsRecorder - Optional sink for what aged out.
|
|
134
|
+
* @returns How many entries were removed.
|
|
130
135
|
*/
|
|
131
|
-
async
|
|
132
|
-
const
|
|
133
|
-
if (
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
for (const name of await this.listFiles()) {
|
|
137
|
-
const file = path.join(this.options.queueDir, name);
|
|
138
|
-
const entry = await this.readEntry(file);
|
|
139
|
-
if (!entry || entry.destination !== previousDestination)
|
|
140
|
-
continue;
|
|
141
|
-
await writeFileAtomic(file, JSON.stringify({ ...entry, destination }), SECRET_FILE_MODE);
|
|
142
|
-
}
|
|
143
|
-
return true;
|
|
144
|
-
}
|
|
145
|
-
finally {
|
|
146
|
-
await release();
|
|
147
|
-
}
|
|
136
|
+
async sweep(statsRecorder) {
|
|
137
|
+
const { dropped } = await this.sweepUnsendable(this.now().getTime());
|
|
138
|
+
if (dropped > 0)
|
|
139
|
+
await statsRecorder?.recordDropped(dropped);
|
|
140
|
+
return dropped;
|
|
148
141
|
}
|
|
149
142
|
/**
|
|
150
143
|
* Entries pinned to exactly this destination; legacy and ANY entries are not
|
|
@@ -165,20 +158,46 @@ export class EventQueue {
|
|
|
165
158
|
/**
|
|
166
159
|
* Entries due for this transport, dropping the ones that can never be sent.
|
|
167
160
|
*
|
|
161
|
+
* Reads at most `maxBatch` entries, whatever the backlog holds and whatever it
|
|
162
|
+
* finds in them. Reading and zod-parsing the whole partition to find the few
|
|
163
|
+
* entries a pass may send is what made an outage cost the developer up to
|
|
164
|
+
* `maxQueueEvents` file reads on every hook, which AGENTS.md §3 forbids
|
|
165
|
+
* outright. The bound is on entries *read*, not on entries found due: a
|
|
166
|
+
* backlog where nothing is due — every entry in backoff, or every entry
|
|
167
|
+
* pinned to a destination this identity no longer sends to, which is what a
|
|
168
|
+
* changed endpoint leaves behind for a week — is exactly the case that would
|
|
169
|
+
* otherwise walk all of it and send nothing.
|
|
170
|
+
*
|
|
171
|
+
* A hard read bound needs the scan to move, or the head of the sorted list
|
|
172
|
+
* would be re-read on every pass and the tail never reached. So the scan
|
|
173
|
+
* starts where the last one stopped and wraps around: every entry is examined
|
|
174
|
+
* within `ceil(backlog / maxBatch)` passes, which is what FR-009 asks for —
|
|
175
|
+
* no entry deferred forever. Losing the cursor costs one pass starting at the
|
|
176
|
+
* head again.
|
|
177
|
+
*
|
|
178
|
+
* The batch is therefore the oldest of what this pass *scanned*, not of the
|
|
179
|
+
* whole backlog. Global oldest-first would need a due-time index, which is
|
|
180
|
+
* separate work.
|
|
181
|
+
*
|
|
168
182
|
* @param transport - Where the pass will send.
|
|
169
183
|
* @param nowMs - This pass's clock reading.
|
|
184
|
+
* @param maxBatch - Entries this pass may read; also its ceiling on sends.
|
|
170
185
|
* @returns The due entries and what collecting them dropped.
|
|
171
186
|
*/
|
|
172
|
-
async collectDue(transport, nowMs) {
|
|
187
|
+
async collectDue(transport, nowMs, maxBatch) {
|
|
188
|
+
const names = await this.listFiles();
|
|
189
|
+
const start = await this.scanStart(names);
|
|
190
|
+
const scan = [...names.slice(start), ...names.slice(0, start)].slice(0, maxBatch);
|
|
173
191
|
const due = [];
|
|
174
192
|
let dropped = 0;
|
|
175
|
-
for (const name of
|
|
193
|
+
for (const name of scan) {
|
|
176
194
|
const file = path.join(this.options.queueDir, name);
|
|
177
195
|
const entry = await this.readEntry(file);
|
|
178
|
-
if (!entry || this.isExpired(entry, nowMs) || !
|
|
196
|
+
if (!entry || this.isExpired(entry, nowMs) || !isProductEvent(entry.event)) {
|
|
179
197
|
// Unreadable, aged out, or — for a backlog written by a pre-product
|
|
180
198
|
// release — an internal lifecycle event the backend does not accept.
|
|
181
|
-
// Draining one of those would poison every batch it rides in.
|
|
199
|
+
// Draining one of those would poison every batch it rides in. Entries
|
|
200
|
+
// past the scan are the throttled sweep's business, not this pass's.
|
|
182
201
|
await fs.rm(file, { force: true });
|
|
183
202
|
dropped += 1;
|
|
184
203
|
continue;
|
|
@@ -188,25 +207,136 @@ export class EventQueue {
|
|
|
188
207
|
if (Date.parse(entry.nextAttemptAt) <= nowMs)
|
|
189
208
|
due.push({ file, entry });
|
|
190
209
|
}
|
|
210
|
+
const last = scan.at(-1);
|
|
211
|
+
if (last !== undefined)
|
|
212
|
+
await this.rememberScan(last);
|
|
191
213
|
return { due, delta: { ...EMPTY_DELTA, dropped } };
|
|
192
214
|
}
|
|
215
|
+
/**
|
|
216
|
+
* Index the next bounded scan starts at.
|
|
217
|
+
*
|
|
218
|
+
* The first name *after* the one last scanned, so an entry that was sent and
|
|
219
|
+
* deleted in the meantime does not send the scan back to the head.
|
|
220
|
+
*
|
|
221
|
+
* @param names - This pass's sorted entry names.
|
|
222
|
+
* @returns The starting index; zero when there is no cursor to resume from.
|
|
223
|
+
*/
|
|
224
|
+
async scanStart(names) {
|
|
225
|
+
const last = await this.lastScanned();
|
|
226
|
+
if (last === undefined)
|
|
227
|
+
return 0;
|
|
228
|
+
const next = names.findIndex((name) => name > last);
|
|
229
|
+
return next === -1 ? 0 : next;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* The entry name the previous pass stopped on.
|
|
233
|
+
*
|
|
234
|
+
* @returns The name, or undefined when nothing has scanned yet.
|
|
235
|
+
*/
|
|
236
|
+
async lastScanned() {
|
|
237
|
+
try {
|
|
238
|
+
return (await fs.readFile(this.cursorFile(), 'utf8')) || undefined;
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
// No cursor, or one we cannot read: start at the head. This is a hint for
|
|
242
|
+
// fairness, not state anything depends on.
|
|
243
|
+
return undefined;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Record where this pass stopped.
|
|
248
|
+
*
|
|
249
|
+
* @param name - The last entry name it read.
|
|
250
|
+
*/
|
|
251
|
+
async rememberScan(name) {
|
|
252
|
+
try {
|
|
253
|
+
// The same write as every other file in the partition, for the same
|
|
254
|
+
// reason: it renames a 0600 temp file over the target, so a cursor
|
|
255
|
+
// somebody pre-created as a symlink is replaced rather than followed.
|
|
256
|
+
await writeFileAtomic(this.cursorFile(), name, SECRET_FILE_MODE);
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
// Same rule as the sweep marker: the drain runs on the hook path, and
|
|
260
|
+
// failing to remember a position must never fail the agent's turn.
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Where the scan cursor lives.
|
|
265
|
+
*
|
|
266
|
+
* Not a `.json` name, so the entry listing never sees it.
|
|
267
|
+
*
|
|
268
|
+
* @returns Absolute path to the cursor file.
|
|
269
|
+
*/
|
|
270
|
+
cursorFile() {
|
|
271
|
+
return path.join(this.options.queueDir, QUEUE_SCAN_CURSOR_FILE);
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Remove everything that can never be sent, wherever it sits in the partition.
|
|
275
|
+
*
|
|
276
|
+
* The bounded scan only ever sees the head of the backlog, so the retention
|
|
277
|
+
* bound needs a pass that sees all of it. Throttled by a marker, because that
|
|
278
|
+
* pass is the expensive one — and it is an improvement on what it replaces:
|
|
279
|
+
* expiry used to be enforced only inside a drain, and a drain is skipped for
|
|
280
|
+
* the whole cooldown window, so during a backend outage nothing aged out at
|
|
281
|
+
* all. That was precisely the week-long case the bound exists for.
|
|
282
|
+
*
|
|
283
|
+
* @param nowMs - This pass's clock reading.
|
|
284
|
+
* @returns What the sweep dropped, or nothing when it was not due.
|
|
285
|
+
*/
|
|
286
|
+
async sweepUnsendable(nowMs) {
|
|
287
|
+
// `Date.now()`, not `nowMs`: the throttle is a marker file's mtime, which is
|
|
288
|
+
// real wall-clock time, and `nowMs` comes from the injectable clock. Compared
|
|
289
|
+
// against each other a fixed test clock — or a backward NTP step in
|
|
290
|
+
// production — makes `nowMs - last` negative forever, which reads as "not
|
|
291
|
+
// due" and silences the retention pass for good. `TurnStateStore.sweep`
|
|
292
|
+
// passes the wall clock here for the same reason. Expiry below still uses
|
|
293
|
+
// `nowMs`, because that is a judgement about the entries, not about when this
|
|
294
|
+
// process last swept.
|
|
295
|
+
if (!(await claimSweep(path.join(this.options.queueDir, SWEEP_MARKER_FILE), QUEUE_SWEEP_INTERVAL_MS, Date.now())))
|
|
296
|
+
return EMPTY_DELTA;
|
|
297
|
+
let dropped = 0;
|
|
298
|
+
for (const name of await this.listFiles()) {
|
|
299
|
+
const file = path.join(this.options.queueDir, name);
|
|
300
|
+
const entry = await this.readEntry(file);
|
|
301
|
+
if (entry && !this.isExpired(entry, nowMs) && isProductEvent(entry.event))
|
|
302
|
+
continue;
|
|
303
|
+
await fs.rm(file, { force: true });
|
|
304
|
+
dropped += 1;
|
|
305
|
+
}
|
|
306
|
+
return { ...EMPTY_DELTA, dropped };
|
|
307
|
+
}
|
|
193
308
|
/**
|
|
194
309
|
* Send one batch, isolating a poison entry when the backend refuses it.
|
|
195
310
|
*
|
|
196
311
|
* @param transport - Where to send.
|
|
197
312
|
* @param batch - Entries to send.
|
|
198
313
|
* @param nowMs - This pass's clock reading.
|
|
314
|
+
* @param authBlock - Optional sink for a refused credential.
|
|
199
315
|
* @returns What the send accomplished and cost.
|
|
200
316
|
*/
|
|
201
|
-
async sendBatch(transport, batch, nowMs) {
|
|
202
|
-
const result = await transport
|
|
317
|
+
async sendBatch(transport, batch, nowMs, authBlock) {
|
|
318
|
+
const result = await sendEvents(transport, batch.map(({ entry }) => entry.event));
|
|
203
319
|
if (result.ok) {
|
|
204
320
|
await Promise.all(batch.map(({ file }) => fs.rm(file, { force: true })));
|
|
321
|
+
// A 2xx is the backend accepting the credential: lift a standing block, or
|
|
322
|
+
// a `status` drain delivers one batch and leaves the hooks suspended.
|
|
323
|
+
await authBlock?.clear();
|
|
205
324
|
return { ...EMPTY_DELTA, sent: batch.length, rejected: reportRejected(result.counters?.rejected, 'the drained batch') };
|
|
206
325
|
}
|
|
326
|
+
if (result.deferred)
|
|
327
|
+
return { ...EMPTY_DELTA, failed: batch.length };
|
|
207
328
|
debugLog('queue drain failed', result.error ?? `status ${result.status}`);
|
|
329
|
+
if (result.status !== undefined && AUTH_REJECTED_STATUSES.has(result.status)) {
|
|
330
|
+
// The credential was refused, so nothing in this batch is at fault and
|
|
331
|
+
// nothing about it may change: an attempt spent here would walk the whole
|
|
332
|
+
// backlog to maxAttempts and delete it, which is the one thing the queue
|
|
333
|
+
// exists to prevent. Raise the block instead and leave every entry due.
|
|
334
|
+
if (authBlock && transport.destination)
|
|
335
|
+
await authBlock.raise(transport.destination, result.status, result.refusal);
|
|
336
|
+
return { ...EMPTY_DELTA, failed: batch.length };
|
|
337
|
+
}
|
|
208
338
|
if (!result.retryable && batch.length > 1)
|
|
209
|
-
return this.isolateBatch(transport, batch, nowMs);
|
|
339
|
+
return this.isolateBatch(transport, batch, nowMs, authBlock);
|
|
210
340
|
let delta = EMPTY_DELTA;
|
|
211
341
|
for (const { file, entry } of batch) {
|
|
212
342
|
delta = mergeDeltas(delta, await this.recordFailure(file, entry, nowMs));
|
|
@@ -225,9 +355,10 @@ export class EventQueue {
|
|
|
225
355
|
* @param transport - Where to send.
|
|
226
356
|
* @param batch - Entries the batch send refused.
|
|
227
357
|
* @param nowMs - This pass's clock reading.
|
|
358
|
+
* @param authBlock - Persisted refusal, including one received by a single probe.
|
|
228
359
|
* @returns What the probes accomplished and cost.
|
|
229
360
|
*/
|
|
230
|
-
async isolateBatch(transport, batch, nowMs) {
|
|
361
|
+
async isolateBatch(transport, batch, nowMs, authBlock) {
|
|
231
362
|
let delta = EMPTY_DELTA;
|
|
232
363
|
let probes = 0;
|
|
233
364
|
for (const { file, entry } of batch) {
|
|
@@ -236,24 +367,22 @@ export class EventQueue {
|
|
|
236
367
|
continue;
|
|
237
368
|
}
|
|
238
369
|
probes += 1;
|
|
239
|
-
const single = await transport
|
|
370
|
+
const single = await sendEvents(transport, [entry.event]);
|
|
371
|
+
if (single.deferred || (single.status !== undefined && AUTH_REJECTED_STATUSES.has(single.status))) {
|
|
372
|
+
if (!single.deferred && single.status !== undefined && authBlock && transport.destination)
|
|
373
|
+
await authBlock.raise(transport.destination, single.status, single.refusal);
|
|
374
|
+
return mergeDeltas(delta, { ...EMPTY_DELTA, failed: batch.length - probes + 1 });
|
|
375
|
+
}
|
|
240
376
|
if (!single.ok) {
|
|
241
377
|
delta = mergeDeltas(delta, await this.recordFailure(file, entry, nowMs));
|
|
242
378
|
continue;
|
|
243
379
|
}
|
|
244
380
|
await fs.rm(file, { force: true });
|
|
381
|
+
await authBlock?.clear();
|
|
245
382
|
delta = mergeDeltas(delta, { ...EMPTY_DELTA, sent: 1, rejected: reportRejected(single.counters?.rejected, 'an isolation probe') });
|
|
246
383
|
}
|
|
247
384
|
return delta;
|
|
248
385
|
}
|
|
249
|
-
/**
|
|
250
|
-
* Acquire the drain lock, waiting a bounded time.
|
|
251
|
-
*
|
|
252
|
-
* @returns The release function, or undefined on timeout.
|
|
253
|
-
*/
|
|
254
|
-
waitForDrainLock() {
|
|
255
|
-
return pollUntil(() => acquireLock(this.options.locksDir, QUEUE_DRAIN_LOCK, this.now), RETARGET_LOCK_WAIT_MS, RETARGET_LOCK_POLL_MS);
|
|
256
|
-
}
|
|
257
386
|
/**
|
|
258
387
|
* Back a failed entry off, or drop it once its budget is spent.
|
|
259
388
|
*
|
|
@@ -293,14 +422,8 @@ export class EventQueue {
|
|
|
293
422
|
*
|
|
294
423
|
* @returns The names, or an empty list when the directory is absent.
|
|
295
424
|
*/
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
const names = await fs.readdir(this.options.queueDir);
|
|
299
|
-
return names.filter((name) => name.endsWith(QUEUE_FILE_SUFFIX)).sort();
|
|
300
|
-
}
|
|
301
|
-
catch {
|
|
302
|
-
return [];
|
|
303
|
-
}
|
|
425
|
+
listFiles() {
|
|
426
|
+
return listQueueFiles(this.options.queueDir);
|
|
304
427
|
}
|
|
305
428
|
/**
|
|
306
429
|
* Read and validate one entry.
|
|
@@ -332,6 +455,11 @@ export class EventQueue {
|
|
|
332
455
|
*
|
|
333
456
|
* The oldest are the least likely to still be deliverable, and an unbounded
|
|
334
457
|
* queue on a developer machine is a disk-space bug.
|
|
458
|
+
*
|
|
459
|
+
* Reported through `options.stats`, like every other permanent loss: these
|
|
460
|
+
* deletions used to be the one kind that left `totalDropped` at zero, so a
|
|
461
|
+
* machine steadily shedding records at the bound looked, in `status`, exactly
|
|
462
|
+
* like one that had never lost any.
|
|
335
463
|
*/
|
|
336
464
|
async enforceBound() {
|
|
337
465
|
const files = await this.listFiles();
|
|
@@ -349,6 +477,8 @@ export class EventQueue {
|
|
|
349
477
|
for (const { full } of doomed) {
|
|
350
478
|
await fs.rm(full, { force: true });
|
|
351
479
|
}
|
|
480
|
+
if (doomed.length > 0)
|
|
481
|
+
await this.options.stats?.recordDropped(doomed.length);
|
|
352
482
|
}
|
|
353
483
|
/**
|
|
354
484
|
* When an entry was first queued, falling back to the file clock.
|
|
@@ -421,7 +551,7 @@ async function report(recorder, stats) {
|
|
|
421
551
|
* Whether an entry may be sent to this transport.
|
|
422
552
|
*
|
|
423
553
|
* Entries pinned to another backend are never re-routed at drain time; they wait
|
|
424
|
-
* for their own backend
|
|
554
|
+
* for their own backend.
|
|
425
555
|
* Legacy entries without destination metadata predate pinning, so they flow to
|
|
426
556
|
* the first configured backend just like ANY_DESTINATION entries.
|
|
427
557
|
*
|
|
@@ -440,10 +570,6 @@ function matchesDestination(entry, transport) {
|
|
|
440
570
|
* @param entry - The entry.
|
|
441
571
|
* @returns True for an llm.call or turn.summary.
|
|
442
572
|
*/
|
|
443
|
-
function isProductEntry(entry) {
|
|
444
|
-
const type = entry.event.event?.type;
|
|
445
|
-
return typeof type === 'string' && PRODUCT_EVENT_TYPE_SET.has(type);
|
|
446
|
-
}
|
|
447
573
|
/**
|
|
448
574
|
* Backoff for the nth attempt, with jitter.
|
|
449
575
|
*
|
|
@@ -472,3 +598,18 @@ async function exists(file) {
|
|
|
472
598
|
return false;
|
|
473
599
|
}
|
|
474
600
|
}
|
|
601
|
+
/**
|
|
602
|
+
* Queue entry filenames directly in a directory, sorted. Sub-directories are
|
|
603
|
+
* other identities' partitions and never entries.
|
|
604
|
+
*
|
|
605
|
+
* @param dir - Directory to list.
|
|
606
|
+
* @returns The names, or an empty list when the directory is absent.
|
|
607
|
+
*/
|
|
608
|
+
export async function listQueueFiles(dir) {
|
|
609
|
+
try {
|
|
610
|
+
return (await fs.readdir(dir)).filter((name) => name.endsWith(QUEUE_FILE_SUFFIX)).sort();
|
|
611
|
+
}
|
|
612
|
+
catch {
|
|
613
|
+
return [];
|
|
614
|
+
}
|
|
615
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read a small JSON body, refusing one too large to be the answer.
|
|
3
|
+
*
|
|
4
|
+
* Both bodies this guards — the batch's per-event counters and the enforcement
|
|
5
|
+
* decision — are a handful of integers, and both are decoded inside the coding
|
|
6
|
+
* agent's hook process. A compromised, misconfigured or proxy-intercepted
|
|
7
|
+
* endpoint can answer 200 with an arbitrarily large body, and the send timeout
|
|
8
|
+
* bounds the *request*, not the decode: without a cap the hook allocates
|
|
9
|
+
* whatever it is sent.
|
|
10
|
+
*
|
|
11
|
+
* `content-length` is checked first because it is free, and the stream is
|
|
12
|
+
* capped as well because a chunked response declares no length. Throwing is the
|
|
13
|
+
* right failure here: both call sites already treat an unreadable body as "no
|
|
14
|
+
* answer" and fail open.
|
|
15
|
+
*
|
|
16
|
+
* @param response - The response to decode.
|
|
17
|
+
* @returns The parsed body.
|
|
18
|
+
*/
|
|
19
|
+
export declare function readCappedJson(response: Response): Promise<unknown>;
|
|
20
|
+
/**
|
|
21
|
+
* Release unread responses without making cleanup part of the hook's latency.
|
|
22
|
+
* @param response - A response whose body the caller will not consume.
|
|
23
|
+
* @returns Nothing; cancellation errors cannot affect delivery or enforcement.
|
|
24
|
+
*/
|
|
25
|
+
export declare function discardResponseBody(response: Response): void;
|