@agent-watch-ai/edge 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +238 -7
- package/README.md +308 -8
- package/dist/billing/billing-mode.d.ts +37 -5
- package/dist/billing/billing-mode.js +28 -19
- package/dist/billing/constants/billing.constants.d.ts +0 -2
- package/dist/billing/constants/billing.constants.js +0 -2
- package/dist/billing/index.d.ts +2 -1
- package/dist/billing/index.js +1 -1
- package/dist/billing/session-billing.d.ts +3 -2
- package/dist/billing/session-billing.js +8 -8
- package/dist/billing/types/billing.types.d.ts +7 -0
- package/dist/capture/capture-env.d.ts +147 -0
- package/dist/capture/capture-env.js +301 -0
- package/dist/capture/captured-step.d.ts +88 -0
- package/dist/capture/captured-step.js +479 -0
- package/dist/capture/constants/capture.constants.d.ts +70 -0
- package/dist/capture/constants/capture.constants.js +72 -0
- package/dist/capture/launched-runs.d.ts +16 -0
- package/dist/capture/launched-runs.js +151 -0
- package/dist/capture/node.cjs +697 -0
- package/dist/capture/node.d.cts +32 -0
- package/dist/capture/python/sitecustomize.py +823 -0
- package/dist/capture/schemas/capture.schema.d.ts +35 -0
- package/dist/capture/schemas/capture.schema.js +31 -0
- package/dist/capture/types/capture.types.d.ts +20 -0
- package/dist/cli/constants/cli.constants.d.ts +2 -6
- package/dist/cli/constants/cli.constants.js +3 -7
- package/dist/cli/context.d.ts +19 -22
- package/dist/cli/context.js +23 -54
- package/dist/cli/doctor-run-capture.d.ts +13 -0
- package/dist/cli/doctor-run-capture.js +19 -0
- package/dist/cli/doctor.js +16 -67
- package/dist/cli/hook.d.ts +14 -0
- package/dist/cli/hook.js +40 -0
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +1 -1
- package/dist/cli/misc.js +10 -2
- package/dist/cli/replay.d.ts +33 -0
- package/dist/cli/replay.js +193 -0
- package/dist/cli/setup.d.ts +10 -2
- package/dist/cli/setup.js +154 -26
- package/dist/cli/status.js +45 -5
- package/dist/cli/toggle.js +10 -0
- package/dist/cli/types/cli.types.d.ts +15 -1
- package/dist/cli/uninstall.js +8 -22
- package/dist/cli.js +104 -17
- package/dist/config/config.d.ts +0 -2
- package/dist/config/config.js +0 -1
- package/dist/config/constants/config.constants.js +2 -0
- package/dist/config/events-transport.d.ts +15 -0
- package/dist/config/events-transport.js +29 -0
- package/dist/config/index.d.ts +2 -1
- package/dist/config/index.js +1 -0
- package/dist/config/root-config.d.ts +12 -0
- package/dist/config/root-config.js +24 -0
- package/dist/config/schemas/config.schema.d.ts +13 -0
- package/dist/config/schemas/config.schema.js +42 -6
- package/dist/config/types/config.types.d.ts +7 -0
- package/dist/core/index.d.ts +1 -1
- package/dist/core/index.js +1 -1
- package/dist/core/version.d.ts +9 -0
- package/dist/core/version.js +14 -0
- package/dist/core/which.d.ts +11 -0
- package/dist/core/which.js +18 -0
- package/dist/credentials/cloud-claim.d.ts +32 -0
- package/dist/credentials/cloud-claim.js +42 -0
- package/dist/credentials/constants/credentials.constants.d.ts +8 -1
- package/dist/credentials/constants/credentials.constants.js +9 -2
- package/dist/credentials/device-api.d.ts +15 -1
- package/dist/credentials/device-api.js +12 -1
- package/dist/credentials/sign-in.d.ts +27 -0
- package/dist/credentials/sign-in.js +31 -1
- package/dist/credentials/token-refresher.d.ts +5 -1
- package/dist/credentials/token-refresher.js +10 -2
- package/dist/events/constants/events.constants.d.ts +8 -2
- package/dist/events/constants/events.constants.js +7 -1
- package/dist/events/index.d.ts +2 -0
- package/dist/events/product-event.d.ts +10 -4
- package/dist/events/types/events.types.d.ts +6 -0
- package/dist/events/types/replay-progress.types.d.ts +26 -0
- package/dist/events/types/replay-progress.types.js +1 -0
- package/dist/events/types/repo-snapshot.types.d.ts +1 -4
- package/dist/events/types/repo-tree.types.d.ts +25 -0
- package/dist/events/types/repo-tree.types.js +1 -0
- package/dist/feature/ticket-candidates.d.ts +1 -1
- package/dist/git/checkout-root.d.ts +15 -0
- package/dist/git/checkout-root.js +18 -0
- package/dist/git/code-state.d.ts +2 -16
- package/dist/git/code-state.js +11 -12
- package/dist/git/constants/git.constants.d.ts +31 -11
- package/dist/git/constants/git.constants.js +36 -12
- package/dist/git/constants/tree.constants.d.ts +71 -0
- package/dist/git/constants/tree.constants.js +92 -0
- package/dist/git/fingerprint.d.ts +3 -6
- package/dist/git/fingerprint.js +5 -8
- package/dist/git/git-context.d.ts +1 -0
- package/dist/git/git-context.js +8 -3
- package/dist/git/index.d.ts +1 -2
- package/dist/git/index.js +1 -1
- package/dist/git/repo-snapshot.d.ts +13 -4
- package/dist/git/repo-snapshot.js +47 -9
- package/dist/git/repo-tree.d.ts +69 -0
- package/dist/git/repo-tree.js +130 -0
- package/dist/git/types/git.types.d.ts +5 -2
- package/dist/git/types/snapshot.types.d.ts +14 -0
- package/dist/pipeline/constants/pipeline.constants.d.ts +3 -0
- package/dist/pipeline/constants/pipeline.constants.js +3 -0
- package/dist/pipeline/hook-pipeline.js +112 -72
- package/dist/privacy/product-capture.js +36 -5
- package/dist/providers/antigravity/antigravity.capture.d.ts +2 -0
- package/dist/providers/antigravity/antigravity.capture.js +5 -0
- package/dist/providers/antigravity/antigravity.detect.js +8 -22
- package/dist/providers/antigravity/antigravity.provider.js +10 -3
- package/dist/providers/antigravity/constants/antigravity.constants.d.ts +1 -0
- package/dist/providers/antigravity/constants/antigravity.constants.js +1 -0
- package/dist/providers/claude/claude.adapter.js +9 -6
- package/dist/providers/claude/claude.detect.js +2 -17
- package/dist/providers/claude/claude.otel.d.ts +21 -3
- package/dist/providers/claude/claude.otel.js +216 -61
- package/dist/providers/claude/claude.provider.js +17 -1
- package/dist/providers/claude/claude.version.d.ts +9 -0
- package/dist/providers/claude/claude.version.js +28 -0
- package/dist/providers/claude/constants/claude.constants.d.ts +27 -0
- package/dist/providers/claude/constants/claude.constants.js +27 -0
- package/dist/providers/claude/constants/claude.otel.constants.d.ts +27 -0
- package/dist/providers/claude/constants/claude.otel.constants.js +45 -5
- package/dist/providers/claude/types/claude.types.d.ts +13 -0
- package/dist/providers/codex/codex.capture.d.ts +76 -0
- package/dist/providers/codex/codex.capture.js +454 -0
- package/dist/providers/codex/codex.detect.js +2 -4
- package/dist/providers/codex/codex.otel.d.ts +38 -2
- package/dist/providers/codex/codex.otel.js +50 -6
- package/dist/providers/codex/codex.provider.js +8 -0
- package/dist/providers/codex/codex.surface.d.ts +9 -0
- package/dist/{turns/codex-originator.js → providers/codex/codex.surface.js} +17 -3
- package/dist/providers/codex/codex.titles.d.ts +14 -0
- package/dist/providers/codex/codex.titles.js +35 -0
- package/dist/providers/codex/constants/codex.constants.d.ts +4 -0
- package/dist/providers/codex/constants/codex.constants.js +4 -0
- package/dist/providers/cursor/constants/cursor.constants.d.ts +17 -4
- package/dist/providers/cursor/constants/cursor.constants.js +25 -5
- package/dist/providers/cursor/cursor-mcp-memo.d.ts +34 -0
- package/dist/providers/cursor/cursor-mcp-memo.js +94 -0
- package/dist/providers/cursor/cursor.adapter.js +20 -82
- package/dist/providers/cursor/cursor.capture.d.ts +1 -0
- package/dist/providers/cursor/cursor.capture.js +4 -0
- package/dist/providers/cursor/cursor.detect.d.ts +5 -1
- package/dist/providers/cursor/cursor.detect.js +11 -26
- package/dist/providers/cursor/cursor.hooks.js +2 -2
- package/dist/providers/cursor/cursor.provider.js +16 -7
- package/dist/providers/cursor/cursor.titles.d.ts +14 -0
- package/dist/providers/cursor/cursor.titles.js +20 -0
- package/dist/providers/cursor/schemas/cursor.schema.d.ts +3 -3
- package/dist/providers/cursor/schemas/cursor.schema.js +5 -3
- package/dist/providers/gemini/constants/gemini.constants.d.ts +16 -0
- package/dist/providers/gemini/constants/gemini.constants.js +17 -0
- package/dist/providers/gemini/gemini.capture.d.ts +81 -0
- package/dist/providers/gemini/gemini.capture.js +210 -0
- package/dist/providers/gemini/gemini.detect.js +57 -24
- package/dist/providers/gemini/gemini.otel.d.ts +38 -2
- package/dist/providers/gemini/gemini.otel.js +56 -125
- package/dist/providers/gemini/gemini.provider.js +5 -0
- package/dist/providers/index.d.ts +1 -0
- package/dist/providers/index.js +1 -0
- package/dist/providers/provider.d.ts +8 -0
- package/dist/providers/provider.js +9 -0
- package/dist/providers/registry.d.ts +14 -0
- package/dist/providers/registry.js +14 -0
- package/dist/providers/shared/constants/hook-config.constants.d.ts +2 -0
- package/dist/providers/shared/constants/hook-config.constants.js +2 -0
- package/dist/providers/shared/constants/terminal-capture.constants.d.ts +6 -0
- package/dist/providers/shared/constants/terminal-capture.constants.js +6 -0
- package/dist/providers/shared/editor-paths.d.ts +8 -0
- package/dist/providers/shared/editor-paths.js +18 -0
- package/dist/providers/shared/hook-config.d.ts +5 -0
- package/dist/providers/shared/hook-config.js +39 -1
- package/dist/providers/shared/hooks-marker.d.ts +14 -0
- package/dist/providers/shared/hooks-marker.js +23 -0
- package/dist/providers/shared/install-record.d.ts +29 -0
- package/dist/providers/shared/install-record.js +52 -0
- package/dist/providers/shared/session-title-store.d.ts +23 -0
- package/dist/providers/shared/session-title-store.js +50 -0
- package/dist/providers/shared/terminal-capture.d.ts +41 -0
- package/dist/providers/shared/terminal-capture.js +109 -0
- package/dist/providers/shared/tooling.d.ts +21 -0
- package/dist/providers/shared/tooling.js +36 -1
- package/dist/providers/types/provider.types.d.ts +57 -4
- package/dist/replay/checkout-resolver.d.ts +29 -0
- package/dist/replay/checkout-resolver.js +69 -0
- package/dist/replay/codex-turns.d.ts +24 -0
- package/dist/replay/codex-turns.js +221 -0
- package/dist/replay/constants/replay.constants.d.ts +99 -0
- package/dist/replay/constants/replay.constants.js +102 -0
- package/dist/replay/describe.d.ts +10 -0
- package/dist/replay/describe.js +31 -0
- package/dist/replay/gemini-turns.d.ts +36 -0
- package/dist/replay/gemini-turns.js +192 -0
- package/dist/replay/machine.d.ts +34 -0
- package/dist/replay/machine.js +68 -0
- package/dist/replay/progress-event.d.ts +21 -0
- package/dist/replay/progress-event.js +45 -0
- package/dist/replay/replay-runner.d.ts +58 -0
- package/dist/replay/replay-runner.js +529 -0
- package/dist/replay/replay-scan.d.ts +83 -0
- package/dist/replay/replay-scan.js +170 -0
- package/dist/replay/replay-state.d.ts +108 -0
- package/dist/replay/replay-state.js +199 -0
- package/dist/replay/resume.d.ts +18 -0
- package/dist/replay/resume.js +34 -0
- package/dist/replay/start.d.ts +23 -0
- package/dist/replay/start.js +39 -0
- package/dist/replay/throttle.d.ts +35 -0
- package/dist/replay/throttle.js +41 -0
- package/dist/replay/transcript-turns.d.ts +35 -0
- package/dist/replay/transcript-turns.js +215 -0
- package/dist/replay/turn-event.d.ts +38 -0
- package/dist/replay/turn-event.js +116 -0
- package/dist/replay/types/replay.types.d.ts +162 -0
- package/dist/replay/types/replay.types.js +1 -0
- package/dist/replay/window.d.ts +28 -0
- package/dist/replay/window.js +41 -0
- package/dist/snapshot/branch-selection.d.ts +40 -23
- package/dist/snapshot/branch-selection.js +74 -42
- package/dist/snapshot/budget.js +2 -2
- package/dist/snapshot/constants/snapshot-stages.constants.d.ts +10 -0
- package/dist/snapshot/constants/snapshot-stages.constants.js +10 -0
- package/dist/snapshot/constants/snapshot.constants.d.ts +37 -4
- package/dist/snapshot/constants/snapshot.constants.js +37 -4
- package/dist/snapshot/index.d.ts +6 -1
- package/dist/snapshot/index.js +3 -1
- package/dist/snapshot/snapshot-event.d.ts +7 -0
- package/dist/snapshot/snapshot-event.js +8 -5
- package/dist/snapshot/snapshot-pipeline.js +89 -26
- package/dist/snapshot/snapshot-state.js +24 -2
- package/dist/snapshot/tree-event.d.ts +14 -0
- package/dist/snapshot/tree-event.js +91 -0
- package/dist/snapshot/tree-pipeline.d.ts +12 -0
- package/dist/snapshot/tree-pipeline.js +172 -0
- package/dist/snapshot/types/snapshot.types.d.ts +62 -2
- package/dist/storage/schemas/storage.schema.d.ts +2 -0
- package/dist/storage/schemas/storage.schema.js +6 -0
- package/dist/transport/constants/transport.constants.d.ts +5 -0
- package/dist/transport/constants/transport.constants.js +5 -0
- package/dist/transport/delivery-stats.d.ts +13 -1
- package/dist/transport/delivery-stats.js +18 -2
- package/dist/transport/delivery.d.ts +4 -3
- package/dist/transport/delivery.js +21 -7
- package/dist/transport/identity-delivery.d.ts +20 -0
- package/dist/transport/identity-delivery.js +41 -0
- package/dist/transport/index.d.ts +2 -0
- package/dist/transport/index.js +1 -0
- package/dist/transport/queue.js +40 -6
- package/dist/transport/types/delivery-flow.types.d.ts +7 -0
- package/dist/turns/claude-transcript.js +4 -12
- package/dist/turns/codex-transcript.d.ts +9 -0
- package/dist/turns/codex-transcript.js +27 -19
- package/dist/turns/constants/turns.constants.d.ts +13 -12
- package/dist/turns/constants/turns.constants.js +15 -19
- package/dist/turns/gemini-transcript.d.ts +19 -1
- package/dist/turns/gemini-transcript.js +29 -18
- package/dist/turns/index.d.ts +3 -4
- package/dist/turns/index.js +2 -3
- package/dist/turns/run-state.d.ts +71 -0
- package/dist/turns/run-state.js +114 -0
- package/dist/turns/tool-events.d.ts +11 -1
- package/dist/turns/tool-events.js +32 -9
- package/dist/turns/transcript.d.ts +15 -14
- package/dist/turns/transcript.js +23 -4
- package/dist/turns/turn-state.d.ts +49 -7
- package/dist/turns/turn-state.js +78 -10
- package/dist/turns/turn-summary.js +10 -6
- package/dist/turns/turn-tracker.d.ts +9 -16
- package/dist/turns/turn-tracker.js +162 -86
- package/dist/turns/types/transcript.types.d.ts +11 -1
- package/dist/turns/types/turn-provider.types.d.ts +33 -0
- package/dist/turns/types/turn-provider.types.js +1 -0
- package/dist/turns/types/turn-state.types.d.ts +9 -0
- package/dist/turns/types/turn-summary.types.d.ts +20 -6
- package/dist/turns/types/turn-tracker.types.d.ts +3 -1
- package/dist/turns/work-checkout.js +1 -2
- package/docs/AUDIT_PROMPT.md +10 -0
- package/docs/DATA_HANDLING.md +269 -32
- package/docs/ENTERPRISE_DEPLOYMENT.md +1 -1
- package/docs/PERFORMANCE.md +27 -0
- package/package.json +3 -2
- package/dist/events/canonical-event.d.ts +0 -10
- package/dist/events/canonical-event.js +0 -9
- package/dist/git/repository-root.d.ts +0 -39
- package/dist/git/repository-root.js +0 -131
- package/dist/git/types/repository-root.types.d.ts +0 -11
- package/dist/transport/transport.d.ts +0 -6
- package/dist/turns/codex-originator.d.ts +0 -11
- package/dist/turns/cursor-transcript.d.ts +0 -21
- package/dist/turns/cursor-transcript.js +0 -66
- package/dist/turns/native-session-titles.d.ts +0 -26
- package/dist/turns/native-session-titles.js +0 -102
- /package/dist/{git/types/repository-root.types.js → billing/types/billing.types.js} +0 -0
- /package/dist/{transport/transport.js → capture/types/capture.types.js} +0 -0
|
@@ -0,0 +1,823 @@
|
|
|
1
|
+
"""Run capture for Python, loaded into a process a coding agent starts through PYTHONPATH.
|
|
2
|
+
|
|
3
|
+
It runs inside the customer's process, so it keeps four promises: it never opens a
|
|
4
|
+
socket, never reads a request body or a request header but the host, never raises
|
|
5
|
+
into the host, and never holds the process open (no threads, no timers; each call is
|
|
6
|
+
one synchronous append). A fault costs a record, never the run.
|
|
7
|
+
|
|
8
|
+
Stdlib only. Importing http.client or httpx here would cost every Python the agent
|
|
9
|
+
starts, so both are patched when something imports them. `src/capture/node.cts`
|
|
10
|
+
mirrors the table and the readers; `conformance/capture/cases.json` holds both to one
|
|
11
|
+
result.
|
|
12
|
+
|
|
13
|
+
Being named sitecustomize masks one further along sys.path, which the customer's
|
|
14
|
+
interpreter may rely on, so that one is run too.
|
|
15
|
+
|
|
16
|
+
It never sets sys.dont_write_bytecode: that would turn bytecode off for the customer's
|
|
17
|
+
whole process, and is too late for this file anyway, which is compiled and cached before
|
|
18
|
+
its first line runs. Its .pyc lands beside it, in the copy under the data dir.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import base64
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import sys
|
|
26
|
+
import time
|
|
27
|
+
import zlib
|
|
28
|
+
|
|
29
|
+
EXACT_HOSTS = {
|
|
30
|
+
"api.anthropic.com": "anthropic",
|
|
31
|
+
"api.openai.com": "openai",
|
|
32
|
+
"generativelanguage.googleapis.com": "google",
|
|
33
|
+
"aiplatform.googleapis.com": "vertex",
|
|
34
|
+
}
|
|
35
|
+
RE_VERTEX_HOST = re.compile(r"^[a-z0-9-]+-aiplatform\.googleapis\.com$")
|
|
36
|
+
RE_BEDROCK_HOST = re.compile(r"^bedrock-runtime(?:-fips)?\.[a-z0-9-]+\.amazonaws\.com(?:\.cn)?$")
|
|
37
|
+
|
|
38
|
+
REQUEST_ID_HEADERS = {
|
|
39
|
+
"anthropic": ("request-id", "x-request-id"),
|
|
40
|
+
"openai": ("x-request-id",),
|
|
41
|
+
"google": ("x-goog-request-id", "x-request-id"),
|
|
42
|
+
"vertex": ("request-id", "x-request-id", "x-goog-request-id"),
|
|
43
|
+
"bedrock": ("x-amzn-requestid", "x-amz-request-id"),
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
# Bedrock's plain invoke states its usage in response headers only.
|
|
47
|
+
BEDROCK_USAGE_HEADERS = (
|
|
48
|
+
("x-amzn-bedrock-input-token-count", "input_tokens"),
|
|
49
|
+
("x-amzn-bedrock-output-token-count", "output_tokens"),
|
|
50
|
+
("x-amzn-bedrock-cache-read-input-token-count", "cached_input_tokens"),
|
|
51
|
+
("x-amzn-bedrock-cache-write-input-token-count", "cache_creation_input_tokens"),
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
RE_URL_MODEL = re.compile(r"/models/([^/:]+):")
|
|
55
|
+
RE_BEDROCK_MODEL = re.compile(r"/model/([^/]+)/")
|
|
56
|
+
RE_ARN = re.compile(r"^arn(?::|%3a)", re.IGNORECASE)
|
|
57
|
+
# "bytes" is Bedrock's envelope, opened and asked this again.
|
|
58
|
+
RE_INTERESTING = re.compile(r'sage|Metrics|"bytes"')
|
|
59
|
+
|
|
60
|
+
# A coding agent's own process reports its own calls; capturing it would count them twice.
|
|
61
|
+
AGENT_ENTRIES = {"claude", "codex", "gemini", "agentwatch", "cursor", "cursor-agent", "antigravity", "agy"}
|
|
62
|
+
AGENT_PACKAGES = re.compile(r"[\\/](?:@anthropic-ai[\\/]claude-code|@anthropic-ai[\\/]claude-agent-sdk|@openai[\\/]codex|@google[\\/]gemini-cli|@agent-watch-ai[\\/]edge|claude[\\/]versions|cursor-agent[\\/]versions|Cursor\.app|Antigravity\.app|\.gemini[\\/]antigravity-cli)[\\/]")
|
|
63
|
+
# Absolute: a repository's folder on PATH must not be handed a `ps` to run.
|
|
64
|
+
PS_PATHS = ("/bin/ps", "/usr/bin/ps")
|
|
65
|
+
RE_WINDOWS_EXEC = re.compile(r"\.(?:exe|cmd)$", re.IGNORECASE)
|
|
66
|
+
|
|
67
|
+
SPOOL_DIR = "captured"
|
|
68
|
+
DISABLED_MARKER = "disabled.json"
|
|
69
|
+
MAX_LINES_PER_FILE = 10000
|
|
70
|
+
# Past this, a JSON or compressed body costs its usage rather than memory.
|
|
71
|
+
MAX_BODY_BYTES = 8 * 1024 * 1024
|
|
72
|
+
MAX_ANCESTORS = 16
|
|
73
|
+
PS_TIMEOUT_S = 2
|
|
74
|
+
RE_PS_ROW = re.compile(r"^\s*(\d+)\s+(\d+)\s+(\w{3} \w{3}\s+\d+ \d\d:\d\d:\d\d \d{4})")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _count(value):
|
|
78
|
+
return value if isinstance(value, (int, float)) and not isinstance(value, bool) and value >= 0 else None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _difference(whole, part):
|
|
82
|
+
return None if whole is None else max(0, whole - (part or 0))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _record(value):
|
|
86
|
+
return value if isinstance(value, dict) else {}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# Every reader answers in one convention: input_tokens excludes cache reads and
|
|
90
|
+
# output_tokens includes reasoning. Which of those the ledger wants on the wire depends
|
|
91
|
+
# on the launching agent, and the attach step decides that.
|
|
92
|
+
def _read_usage_object(usage):
|
|
93
|
+
if usage.get("prompt_tokens") is not None or usage.get("completion_tokens") is not None:
|
|
94
|
+
cached = _count(_record(usage.get("prompt_tokens_details")).get("cached_tokens"))
|
|
95
|
+
return {
|
|
96
|
+
"input_tokens": _difference(_count(usage.get("prompt_tokens")), cached),
|
|
97
|
+
"cached_input_tokens": cached,
|
|
98
|
+
"output_tokens": _count(usage.get("completion_tokens")),
|
|
99
|
+
"reasoning_output_tokens": _count(_record(usage.get("completion_tokens_details")).get("reasoning_tokens")),
|
|
100
|
+
"total_tokens": _count(usage.get("total_tokens")),
|
|
101
|
+
}
|
|
102
|
+
if usage.get("input_tokens_details") is not None or usage.get("output_tokens_details") is not None:
|
|
103
|
+
cached = _count(_record(usage.get("input_tokens_details")).get("cached_tokens"))
|
|
104
|
+
return {
|
|
105
|
+
"input_tokens": _difference(_count(usage.get("input_tokens")), cached),
|
|
106
|
+
"cached_input_tokens": cached,
|
|
107
|
+
"output_tokens": _count(usage.get("output_tokens")),
|
|
108
|
+
"reasoning_output_tokens": _count(_record(usage.get("output_tokens_details")).get("reasoning_tokens")),
|
|
109
|
+
"total_tokens": _count(usage.get("total_tokens")),
|
|
110
|
+
}
|
|
111
|
+
if usage.get("inputTokens") is not None or usage.get("outputTokens") is not None:
|
|
112
|
+
return {
|
|
113
|
+
"input_tokens": _count(usage.get("inputTokens")),
|
|
114
|
+
"cached_input_tokens": _count(usage.get("cacheReadInputTokens")),
|
|
115
|
+
"cache_creation_input_tokens": _count(usage.get("cacheWriteInputTokens")),
|
|
116
|
+
"output_tokens": _count(usage.get("outputTokens")),
|
|
117
|
+
"total_tokens": _count(usage.get("totalTokens")),
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
"input_tokens": _count(usage.get("input_tokens")),
|
|
121
|
+
"cached_input_tokens": _count(usage.get("cache_read_input_tokens")),
|
|
122
|
+
"cache_creation_input_tokens": _count(usage.get("cache_creation_input_tokens")),
|
|
123
|
+
"output_tokens": _count(usage.get("output_tokens")),
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _read_gemini_usage(usage):
|
|
128
|
+
cached = _count(usage.get("cachedContentTokenCount"))
|
|
129
|
+
reasoning = _count(usage.get("thoughtsTokenCount"))
|
|
130
|
+
candidates = _count(usage.get("candidatesTokenCount"))
|
|
131
|
+
return {
|
|
132
|
+
"input_tokens": _difference(_count(usage.get("promptTokenCount")), cached),
|
|
133
|
+
"cached_input_tokens": cached,
|
|
134
|
+
"output_tokens": None if candidates is None and reasoning is None else (candidates or 0) + (reasoning or 0),
|
|
135
|
+
"reasoning_output_tokens": reasoning,
|
|
136
|
+
"total_tokens": _count(usage.get("totalTokenCount")),
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _read_bedrock_metrics(metrics):
|
|
141
|
+
return {
|
|
142
|
+
"input_tokens": _count(metrics.get("inputTokenCount")),
|
|
143
|
+
"cached_input_tokens": _count(metrics.get("cacheReadInputTokenCount")),
|
|
144
|
+
"cache_creation_input_tokens": _count(metrics.get("cacheWriteInputTokenCount")),
|
|
145
|
+
"output_tokens": _count(metrics.get("outputTokenCount")),
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
# The usage readers, by the key each provider's usage sits under.
|
|
150
|
+
USAGE_READERS = (
|
|
151
|
+
("usage", _read_usage_object),
|
|
152
|
+
("usageMetadata", _read_gemini_usage),
|
|
153
|
+
("amazon-bedrock-invocationMetrics", _read_bedrock_metrics),
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class _ResponseTap:
|
|
158
|
+
"""One response being read: what it says so far, and its framing's leftovers."""
|
|
159
|
+
|
|
160
|
+
def __init__(self, meta, status, headers, on_finish, decoded):
|
|
161
|
+
self.call = {"provider": meta["provider"], "status": status}
|
|
162
|
+
if meta.get("model") is not None:
|
|
163
|
+
self.call["model"] = meta["model"]
|
|
164
|
+
request_id = _request_id_of(meta["provider"], headers)
|
|
165
|
+
if request_id is not None:
|
|
166
|
+
self.call["request_id"] = request_id
|
|
167
|
+
self.on_finish = on_finish
|
|
168
|
+
self.framing = _framing_of(headers("content-type"))
|
|
169
|
+
self.encoding = "" if decoded else (headers("content-encoding") or "").lower().strip()
|
|
170
|
+
self.carry = b""
|
|
171
|
+
self.overflowed = False
|
|
172
|
+
self.finished = False
|
|
173
|
+
self.body_request_id = None
|
|
174
|
+
if meta["provider"] == "bedrock":
|
|
175
|
+
self._absorb_headers(headers)
|
|
176
|
+
|
|
177
|
+
def _absorb_headers(self, headers):
|
|
178
|
+
usage = {}
|
|
179
|
+
for name, field in BEDROCK_USAGE_HEADERS:
|
|
180
|
+
value = headers(name)
|
|
181
|
+
if value is not None:
|
|
182
|
+
try:
|
|
183
|
+
usage[field] = int(value)
|
|
184
|
+
except ValueError:
|
|
185
|
+
pass
|
|
186
|
+
self._merge(usage)
|
|
187
|
+
|
|
188
|
+
def _merge(self, usage):
|
|
189
|
+
merged = dict(self.call.get("usage") or {})
|
|
190
|
+
for key, value in usage.items():
|
|
191
|
+
if value is not None:
|
|
192
|
+
merged[key] = value
|
|
193
|
+
if merged:
|
|
194
|
+
self.call["usage"] = merged
|
|
195
|
+
|
|
196
|
+
def _absorb(self, value):
|
|
197
|
+
if isinstance(value, list):
|
|
198
|
+
for item in value:
|
|
199
|
+
self._absorb(item)
|
|
200
|
+
return
|
|
201
|
+
event = _record(value)
|
|
202
|
+
# Anthropic's message_start and OpenAI's response.* nest the whole message.
|
|
203
|
+
if event.get("message") is not None:
|
|
204
|
+
self._absorb(event["message"])
|
|
205
|
+
if event.get("response") is not None:
|
|
206
|
+
self._absorb(event["response"])
|
|
207
|
+
if isinstance(event.get("model"), str):
|
|
208
|
+
self.call["model"] = _model_name(event["model"])
|
|
209
|
+
if isinstance(event.get("modelVersion"), str):
|
|
210
|
+
self.call["model"] = event["modelVersion"]
|
|
211
|
+
if isinstance(event.get("responseId"), str):
|
|
212
|
+
self.body_request_id = event["responseId"]
|
|
213
|
+
for key, read in USAGE_READERS:
|
|
214
|
+
if isinstance(event.get(key), dict):
|
|
215
|
+
self._merge(read(event[key]))
|
|
216
|
+
# A Bedrock stream chunk carries the provider's own event as base64.
|
|
217
|
+
if isinstance(event.get("bytes"), str):
|
|
218
|
+
self._absorb_text(base64.b64decode(event["bytes"]).decode("utf-8", "replace"))
|
|
219
|
+
|
|
220
|
+
# Only what can hold usage is parsed: a text delta is never looked into.
|
|
221
|
+
def _absorb_text(self, text):
|
|
222
|
+
if not RE_INTERESTING.search(text) and not ("model" not in self.call and '"model' in text) and "responseId" not in text:
|
|
223
|
+
return
|
|
224
|
+
try:
|
|
225
|
+
self._absorb(json.loads(text))
|
|
226
|
+
except ValueError:
|
|
227
|
+
pass # Not JSON, or cut short; it holds no usage we can read.
|
|
228
|
+
|
|
229
|
+
def _lines(self, flush):
|
|
230
|
+
start = 0
|
|
231
|
+
while True:
|
|
232
|
+
end = self.carry.find(b"\n", start)
|
|
233
|
+
if end == -1:
|
|
234
|
+
break
|
|
235
|
+
self._line(self.carry[start:end])
|
|
236
|
+
start = end + 1
|
|
237
|
+
self.carry = self.carry[start:]
|
|
238
|
+
if flush and self.carry:
|
|
239
|
+
self._line(self.carry)
|
|
240
|
+
|
|
241
|
+
def _line(self, raw):
|
|
242
|
+
text = raw.decode("utf-8", "replace").strip()
|
|
243
|
+
if text.startswith("data:"):
|
|
244
|
+
self._absorb_text(text[5:].strip())
|
|
245
|
+
|
|
246
|
+
def _frames(self):
|
|
247
|
+
while len(self.carry) >= 12:
|
|
248
|
+
total = int.from_bytes(self.carry[0:4], "big")
|
|
249
|
+
headers_length = int.from_bytes(self.carry[4:8], "big")
|
|
250
|
+
if total < 16 or total > MAX_BODY_BYTES or 12 + headers_length > total - 4:
|
|
251
|
+
self.overflowed = True
|
|
252
|
+
self.carry = b""
|
|
253
|
+
return
|
|
254
|
+
if len(self.carry) < total:
|
|
255
|
+
return
|
|
256
|
+
self._absorb_text(self.carry[12 + headers_length : total - 4].decode("utf-8", "replace"))
|
|
257
|
+
self.carry = self.carry[total:]
|
|
258
|
+
|
|
259
|
+
def feed(self, chunk):
|
|
260
|
+
"""Take one chunk of the body as the host read it."""
|
|
261
|
+
if self.overflowed or self.finished:
|
|
262
|
+
return
|
|
263
|
+
self.carry = bytes(chunk) if not self.carry else self.carry + bytes(chunk)
|
|
264
|
+
if len(self.carry) > MAX_BODY_BYTES:
|
|
265
|
+
self.overflowed = True
|
|
266
|
+
self.carry = b""
|
|
267
|
+
return
|
|
268
|
+
if self.encoding:
|
|
269
|
+
return
|
|
270
|
+
if self.framing == "sse":
|
|
271
|
+
self._lines(False)
|
|
272
|
+
if self.framing == "eventstream":
|
|
273
|
+
self._frames()
|
|
274
|
+
|
|
275
|
+
# Bounded: a small compressed body can expand to gigabytes in the host's memory.
|
|
276
|
+
def _inflate(self, wbits):
|
|
277
|
+
inflater = zlib.decompressobj(wbits)
|
|
278
|
+
body = inflater.decompress(self.carry, MAX_BODY_BYTES)
|
|
279
|
+
if inflater.unconsumed_tail:
|
|
280
|
+
raise ValueError("too large")
|
|
281
|
+
return body
|
|
282
|
+
|
|
283
|
+
def _decoded_body(self):
|
|
284
|
+
if self.encoding in ("gzip", "x-gzip"):
|
|
285
|
+
return self._inflate(47)
|
|
286
|
+
if self.encoding == "deflate":
|
|
287
|
+
return self._inflate(15)
|
|
288
|
+
if self.encoding:
|
|
289
|
+
# ponytail: no brotli or zstd in the stdlib, so such a body costs its usage.
|
|
290
|
+
raise ValueError("encoding")
|
|
291
|
+
return self.carry
|
|
292
|
+
|
|
293
|
+
def finish(self):
|
|
294
|
+
"""The body ended, failed or was abandoned: say what is known, once."""
|
|
295
|
+
if self.finished:
|
|
296
|
+
return
|
|
297
|
+
try:
|
|
298
|
+
if not self.overflowed:
|
|
299
|
+
self.carry = self._decoded_body()
|
|
300
|
+
if self.framing == "sse":
|
|
301
|
+
self._lines(True)
|
|
302
|
+
if self.framing == "eventstream":
|
|
303
|
+
self._frames()
|
|
304
|
+
if self.framing == "json" and self.carry:
|
|
305
|
+
self._absorb(json.loads(self.carry.decode("utf-8")))
|
|
306
|
+
except Exception:
|
|
307
|
+
pass # An unreadable body costs the usage, nothing more.
|
|
308
|
+
self.finished = True
|
|
309
|
+
self.carry = b""
|
|
310
|
+
if "request_id" not in self.call and self.body_request_id is not None:
|
|
311
|
+
self.call["request_id"] = self.body_request_id
|
|
312
|
+
self.on_finish(self.call)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _framing_of(content_type):
|
|
316
|
+
kind = (content_type or "").lower()
|
|
317
|
+
if "text/event-stream" in kind:
|
|
318
|
+
return "sse"
|
|
319
|
+
if "vnd.amazon.eventstream" in kind:
|
|
320
|
+
return "eventstream"
|
|
321
|
+
return "json"
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _request_id_of(provider, headers):
|
|
325
|
+
for name in REQUEST_ID_HEADERS.get(provider, ()):
|
|
326
|
+
value = headers(name)
|
|
327
|
+
if value:
|
|
328
|
+
return value
|
|
329
|
+
return None
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _provider_of_host(host):
|
|
333
|
+
name = re.sub(r":\d+$", "", (host or "").lower())
|
|
334
|
+
if name in EXACT_HOSTS:
|
|
335
|
+
return EXACT_HOSTS[name]
|
|
336
|
+
if RE_VERTEX_HOST.match(name):
|
|
337
|
+
return "vertex"
|
|
338
|
+
return "bedrock" if RE_BEDROCK_HOST.match(name) else None
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
# Everything after ? is cut first: Gemini's API key can travel there.
|
|
342
|
+
def _model_from_path(raw_path):
|
|
343
|
+
pathname = raw_path.split("?")[0]
|
|
344
|
+
match = RE_BEDROCK_MODEL.search(pathname) or RE_URL_MODEL.search(pathname)
|
|
345
|
+
if not match:
|
|
346
|
+
return None
|
|
347
|
+
from urllib.parse import unquote
|
|
348
|
+
|
|
349
|
+
model = _model_name(unquote(match.group(1)))
|
|
350
|
+
# An ARN that stopped at its first /, or was encoded twice, still names the account.
|
|
351
|
+
return None if RE_ARN.match(model) else model
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
# An inference profile's ARN names the customer's account; the last segment names the model.
|
|
355
|
+
def _model_name(model):
|
|
356
|
+
return model[model.rfind("/") + 1 :] if "/" in model else model
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def _call_meta(host, method, request_path):
|
|
360
|
+
"""What the shim knows of a request, from its host, method and URL path alone.
|
|
361
|
+
|
|
362
|
+
Only a POST is a model call: a model listing or a health check is not spend.
|
|
363
|
+
"""
|
|
364
|
+
provider = _provider_of_host(host)
|
|
365
|
+
if provider is None or str(method).upper() != "POST":
|
|
366
|
+
return None
|
|
367
|
+
meta = {"provider": provider}
|
|
368
|
+
model = _model_from_path(request_path or "")
|
|
369
|
+
if model is not None:
|
|
370
|
+
meta["model"] = model
|
|
371
|
+
return meta
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _worth_recording(call):
|
|
375
|
+
"""A failure always (status only), a success only when it carried usage.
|
|
376
|
+
|
|
377
|
+
A success with none is no model call (count_tokens), or a body the shim could not read.
|
|
378
|
+
"""
|
|
379
|
+
return call["status"] < 200 or call["status"] >= 300 or "usage" in call
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _is_agent_entry(entry):
|
|
383
|
+
if not entry:
|
|
384
|
+
return False
|
|
385
|
+
name = RE_WINDOWS_EXEC.sub("", os.path.basename(entry))
|
|
386
|
+
return name in AGENT_ENTRIES or AGENT_PACKAGES.search(entry) is not None
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _is_agent_process():
|
|
390
|
+
# sys.argv is not always there yet while site runs, but sys.orig_argv is from 3.10.
|
|
391
|
+
argv = getattr(sys, "argv", None) or []
|
|
392
|
+
orig = getattr(sys, "orig_argv", None) or []
|
|
393
|
+
entry = argv[0] if argv else None
|
|
394
|
+
if entry in (None, "") and len(orig) > 1 and not orig[1].startswith("-"):
|
|
395
|
+
entry = orig[1]
|
|
396
|
+
return _is_agent_entry(entry)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _data_dir(env):
|
|
400
|
+
if env.get("AGENTWATCH_DATA_DIR"):
|
|
401
|
+
return env["AGENTWATCH_DATA_DIR"]
|
|
402
|
+
if sys.platform == "win32" and env.get("LOCALAPPDATA"):
|
|
403
|
+
return os.path.join(env["LOCALAPPDATA"], "agentwatch")
|
|
404
|
+
if env.get("XDG_DATA_HOME"):
|
|
405
|
+
return os.path.join(env["XDG_DATA_HOME"], "agentwatch")
|
|
406
|
+
return os.path.join(env.get("HOME") or os.path.expanduser("~"), ".local", "share", "agentwatch")
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _iso_now():
|
|
410
|
+
now = time.time()
|
|
411
|
+
return time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(now)) + ".%03dZ" % int((now % 1) * 1000)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _walk_ancestors(listing, pid):
|
|
415
|
+
rows = {}
|
|
416
|
+
for line in listing.split("\n"):
|
|
417
|
+
match = RE_PS_ROW.match(line)
|
|
418
|
+
if match:
|
|
419
|
+
rows[int(match.group(1))] = (int(match.group(2)), match.group(3))
|
|
420
|
+
chain = []
|
|
421
|
+
seen = {pid}
|
|
422
|
+
current = rows.get(pid)
|
|
423
|
+
while current and current[0] > 1 and current[0] not in seen and len(chain) < MAX_ANCESTORS:
|
|
424
|
+
parent = rows.get(current[0])
|
|
425
|
+
if not parent:
|
|
426
|
+
break
|
|
427
|
+
chain.append({"pid": current[0], "started": parent[1]})
|
|
428
|
+
seen.add(current[0])
|
|
429
|
+
current = parent
|
|
430
|
+
return chain
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
class _Spool:
|
|
434
|
+
"""Appends records to this process's spool. Every method swallows its own faults."""
|
|
435
|
+
|
|
436
|
+
def __init__(self, env):
|
|
437
|
+
self.dir = os.path.join(_data_dir(env), SPOOL_DIR)
|
|
438
|
+
self.started = _iso_now()
|
|
439
|
+
self.ancestors = None
|
|
440
|
+
self.ancestors_pid = None
|
|
441
|
+
self.seq = 0
|
|
442
|
+
self.lines = 0
|
|
443
|
+
|
|
444
|
+
# Edge's hook creates the folder once it holds a credential. No folder, no capture.
|
|
445
|
+
# `agentwatch off` writes a marker beside it, and a session already running still has
|
|
446
|
+
# the shim loaded: asked per call, so nothing made while off is sent after `on`.
|
|
447
|
+
def armed(self):
|
|
448
|
+
return os.path.isdir(self.dir) and not os.path.exists(os.path.join(os.path.dirname(self.dir), DISABLED_MARKER))
|
|
449
|
+
|
|
450
|
+
def remember_ancestors(self):
|
|
451
|
+
"""Read the ancestry now, while the launching shell still stands.
|
|
452
|
+
|
|
453
|
+
ponytail: one `ps -A` per process, at its first provider call, so a process with
|
|
454
|
+
no such call pays nothing. A process already orphaned by then (`cmd &` in a
|
|
455
|
+
shell that exited) attaches to no session, and is dropped and counted.
|
|
456
|
+
"""
|
|
457
|
+
pid = os.getpid()
|
|
458
|
+
if self.ancestors is not None and self.ancestors_pid == pid:
|
|
459
|
+
return
|
|
460
|
+
self.ancestors_pid = pid
|
|
461
|
+
try:
|
|
462
|
+
import subprocess
|
|
463
|
+
|
|
464
|
+
listing = subprocess.run(
|
|
465
|
+
[next(path for path in PS_PATHS if os.path.exists(path)), "-A", "-o", "pid=,ppid=,lstart="],
|
|
466
|
+
stdin=subprocess.DEVNULL,
|
|
467
|
+
stdout=subprocess.PIPE,
|
|
468
|
+
stderr=subprocess.DEVNULL,
|
|
469
|
+
timeout=PS_TIMEOUT_S,
|
|
470
|
+
# No inherited environment: it holds the customer's keys. TZ stays, because a
|
|
471
|
+
# start time is compared as text with the one the hook's own ps printed.
|
|
472
|
+
env=dict({"LC_ALL": "C"}, **({"TZ": os.environ["TZ"]} if os.environ.get("TZ") else {})),
|
|
473
|
+
check=False,
|
|
474
|
+
)
|
|
475
|
+
self.ancestors = _walk_ancestors(listing.stdout.decode("utf-8", "replace"), pid)
|
|
476
|
+
except Exception:
|
|
477
|
+
self.ancestors = []
|
|
478
|
+
|
|
479
|
+
def _open(self, name):
|
|
480
|
+
flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
|
|
481
|
+
return os.open(os.path.join(self.dir, name), flags, 0o600)
|
|
482
|
+
|
|
483
|
+
def write(self, call):
|
|
484
|
+
"""One synchronous append, so a call made just before exit is on disk."""
|
|
485
|
+
try:
|
|
486
|
+
self.remember_ancestors()
|
|
487
|
+
pid = os.getpid()
|
|
488
|
+
seq = self.seq
|
|
489
|
+
self.seq = seq + 1
|
|
490
|
+
fields = {"pid": pid, "started": self.started, "ancestors": self.ancestors, "at": _iso_now(), "seq": seq}
|
|
491
|
+
entry = (getattr(sys, "argv", None) or [""])[0]
|
|
492
|
+
if entry and entry not in ("-c", "-m", "-"):
|
|
493
|
+
fields["script"] = os.path.basename(entry)
|
|
494
|
+
line = json.dumps(dict(fields, **call))
|
|
495
|
+
fd = self._open("%d.jsonl" % pid)
|
|
496
|
+
try:
|
|
497
|
+
# The hook renames a file to consume it, so an empty one is a new count.
|
|
498
|
+
if os.fstat(fd).st_size == 0:
|
|
499
|
+
self.lines = 0
|
|
500
|
+
if self.lines >= MAX_LINES_PER_FILE:
|
|
501
|
+
self._drop(pid)
|
|
502
|
+
return
|
|
503
|
+
os.write(fd, (line + "\n").encode("utf-8"))
|
|
504
|
+
self.lines += 1
|
|
505
|
+
finally:
|
|
506
|
+
os.close(fd)
|
|
507
|
+
except Exception:
|
|
508
|
+
pass # A fault costs a record, never the run.
|
|
509
|
+
|
|
510
|
+
# One byte per call past the bound: the file's size is the count the hook reads.
|
|
511
|
+
def _drop(self, pid):
|
|
512
|
+
fd = self._open("%d.dropped" % pid)
|
|
513
|
+
try:
|
|
514
|
+
os.write(fd, b".")
|
|
515
|
+
finally:
|
|
516
|
+
os.close(fd)
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _safely(work, *args):
|
|
520
|
+
try:
|
|
521
|
+
return work(*args)
|
|
522
|
+
except Exception:
|
|
523
|
+
return None
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
class _Open:
|
|
527
|
+
"""Responses still being read, so a process exiting mid-body still says what it saw."""
|
|
528
|
+
|
|
529
|
+
tapped = set()
|
|
530
|
+
|
|
531
|
+
@classmethod
|
|
532
|
+
def finish_all(cls):
|
|
533
|
+
for tap in list(cls.tapped):
|
|
534
|
+
_safely(tap.finish)
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _start_tap(meta, status, headers, spool, decoded):
|
|
538
|
+
def on_finish(call):
|
|
539
|
+
_Open.tapped.discard(tap)
|
|
540
|
+
if _worth_recording(call):
|
|
541
|
+
spool.write(call)
|
|
542
|
+
|
|
543
|
+
tap = _ResponseTap(meta, status, headers, on_finish, decoded)
|
|
544
|
+
_Open.tapped.add(tap)
|
|
545
|
+
return tap
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _patch_http_client(module, spool):
|
|
549
|
+
"""Wrap HTTPConnection.getresponse, which urllib, urllib3 and requests all reach."""
|
|
550
|
+
connection = module.HTTPConnection
|
|
551
|
+
original_put = connection.putrequest
|
|
552
|
+
original_get = connection.getresponse
|
|
553
|
+
|
|
554
|
+
def putrequest(self, method, url, *args, **kwargs):
|
|
555
|
+
try:
|
|
556
|
+
self._agentwatch_call = _call_meta(getattr(self, "_tunnel_host", None) or self.host, method, url)
|
|
557
|
+
if self._agentwatch_call is not None:
|
|
558
|
+
if spool.armed():
|
|
559
|
+
spool.remember_ancestors()
|
|
560
|
+
else:
|
|
561
|
+
self._agentwatch_call = None
|
|
562
|
+
except Exception:
|
|
563
|
+
self._agentwatch_call = None
|
|
564
|
+
return original_put(self, method, url, *args, **kwargs)
|
|
565
|
+
|
|
566
|
+
def getresponse(self):
|
|
567
|
+
response = original_get(self)
|
|
568
|
+
try:
|
|
569
|
+
meta = getattr(self, "_agentwatch_call", None)
|
|
570
|
+
if meta is not None:
|
|
571
|
+
self._agentwatch_call = None
|
|
572
|
+
_tap_http_response(response, meta, spool)
|
|
573
|
+
except Exception:
|
|
574
|
+
pass # A fault costs a record, never the run.
|
|
575
|
+
return response
|
|
576
|
+
|
|
577
|
+
connection.putrequest = putrequest
|
|
578
|
+
connection.getresponse = getresponse
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _tap_http_response(response, meta, spool):
|
|
582
|
+
def header(name):
|
|
583
|
+
return response.getheader(name)
|
|
584
|
+
|
|
585
|
+
tap = _start_tap(meta, response.status, header, spool, False)
|
|
586
|
+
if response.status < 200 or response.status >= 300:
|
|
587
|
+
_safely(tap.finish)
|
|
588
|
+
return
|
|
589
|
+
base = type(response)
|
|
590
|
+
|
|
591
|
+
# The class is swapped rather than the file object wrapped: below read() the bytes
|
|
592
|
+
# still carry chunked framing. read() can reach readinto() on some versions, hence
|
|
593
|
+
# the depth check, so one chunk is seen once.
|
|
594
|
+
class TappedResponse(base):
|
|
595
|
+
def _agentwatch_observe(self, method, *args, **kwargs):
|
|
596
|
+
self._agentwatch_depth = getattr(self, "_agentwatch_depth", 0) + 1
|
|
597
|
+
try:
|
|
598
|
+
data = getattr(base, method)(self, *args, **kwargs)
|
|
599
|
+
except BaseException:
|
|
600
|
+
self._agentwatch_depth -= 1
|
|
601
|
+
_safely(tap.finish)
|
|
602
|
+
raise
|
|
603
|
+
self._agentwatch_depth -= 1
|
|
604
|
+
if self._agentwatch_depth == 0:
|
|
605
|
+
_safely(self._agentwatch_saw, method, data, args)
|
|
606
|
+
return data
|
|
607
|
+
|
|
608
|
+
def _agentwatch_saw(self, method, data, args):
|
|
609
|
+
# urllib3 decodes chunked framing itself, through _safe_read, and reads each
|
|
610
|
+
# chunk's trailing CRLF the same way; inside a JSON string that would corrupt it.
|
|
611
|
+
if method == "_safe_read" and data == b"\r\n":
|
|
612
|
+
return
|
|
613
|
+
if method == "readinto":
|
|
614
|
+
tap.feed(bytes(memoryview(args[0])[: data or 0]))
|
|
615
|
+
elif data:
|
|
616
|
+
tap.feed(data)
|
|
617
|
+
if not data or self.isclosed():
|
|
618
|
+
tap.finish()
|
|
619
|
+
|
|
620
|
+
def read(self, *args, **kwargs):
|
|
621
|
+
return self._agentwatch_observe("read", *args, **kwargs)
|
|
622
|
+
|
|
623
|
+
def _safe_read(self, *args, **kwargs):
|
|
624
|
+
return self._agentwatch_observe("_safe_read", *args, **kwargs)
|
|
625
|
+
|
|
626
|
+
def read1(self, *args, **kwargs):
|
|
627
|
+
return self._agentwatch_observe("read1", *args, **kwargs)
|
|
628
|
+
|
|
629
|
+
def readline(self, *args, **kwargs):
|
|
630
|
+
return self._agentwatch_observe("readline", *args, **kwargs)
|
|
631
|
+
|
|
632
|
+
def readinto(self, *args, **kwargs):
|
|
633
|
+
return self._agentwatch_observe("readinto", *args, **kwargs)
|
|
634
|
+
|
|
635
|
+
def close(self):
|
|
636
|
+
try:
|
|
637
|
+
return base.close(self)
|
|
638
|
+
finally:
|
|
639
|
+
_safely(tap.finish)
|
|
640
|
+
|
|
641
|
+
response.__class__ = TappedResponse
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def _httpx_header(response):
|
|
645
|
+
def header(name):
|
|
646
|
+
return response.headers.get(name)
|
|
647
|
+
|
|
648
|
+
return header
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _meta_of_httpx(request, spool):
|
|
652
|
+
meta = _call_meta(request.url.host, request.method, request.url.path)
|
|
653
|
+
if meta is None or not spool.armed():
|
|
654
|
+
return None
|
|
655
|
+
spool.remember_ancestors()
|
|
656
|
+
return meta
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def _patch_httpx(module, spool):
|
|
660
|
+
"""Wrap Client.send and AsyncClient.send: httpx never touches http.client."""
|
|
661
|
+
|
|
662
|
+
def tap_of(response, meta):
|
|
663
|
+
return _start_tap(meta, response.status_code, _httpx_header(response), spool, False)
|
|
664
|
+
|
|
665
|
+
class TappedStream(module.SyncByteStream):
|
|
666
|
+
def __init__(self, inner, tap):
|
|
667
|
+
self.inner = inner
|
|
668
|
+
self.tap = tap
|
|
669
|
+
|
|
670
|
+
def __iter__(self):
|
|
671
|
+
try:
|
|
672
|
+
for chunk in self.inner:
|
|
673
|
+
_safely(self.tap.feed, chunk)
|
|
674
|
+
yield chunk
|
|
675
|
+
finally:
|
|
676
|
+
_safely(self.tap.finish)
|
|
677
|
+
|
|
678
|
+
def close(self):
|
|
679
|
+
try:
|
|
680
|
+
self.inner.close()
|
|
681
|
+
finally:
|
|
682
|
+
_safely(self.tap.finish)
|
|
683
|
+
|
|
684
|
+
class TappedAsyncStream(module.AsyncByteStream):
|
|
685
|
+
def __init__(self, inner, tap):
|
|
686
|
+
self.inner = inner
|
|
687
|
+
self.tap = tap
|
|
688
|
+
|
|
689
|
+
async def __aiter__(self):
|
|
690
|
+
try:
|
|
691
|
+
async for chunk in self.inner:
|
|
692
|
+
_safely(self.tap.feed, chunk)
|
|
693
|
+
yield chunk
|
|
694
|
+
finally:
|
|
695
|
+
_safely(self.tap.finish)
|
|
696
|
+
|
|
697
|
+
async def aclose(self):
|
|
698
|
+
try:
|
|
699
|
+
await self.inner.aclose()
|
|
700
|
+
finally:
|
|
701
|
+
_safely(self.tap.finish)
|
|
702
|
+
|
|
703
|
+
def attach(response, meta, stream_class):
|
|
704
|
+
tap = tap_of(response, meta)
|
|
705
|
+
if response.status_code < 200 or response.status_code >= 300:
|
|
706
|
+
_safely(tap.finish)
|
|
707
|
+
return
|
|
708
|
+
# stream=False: send() has already read the body, and decoded it.
|
|
709
|
+
if getattr(response, "_content", None) is not None:
|
|
710
|
+
tap.encoding = ""
|
|
711
|
+
tap.feed(response.content)
|
|
712
|
+
_safely(tap.finish)
|
|
713
|
+
return
|
|
714
|
+
response.stream = stream_class(response.stream, tap)
|
|
715
|
+
|
|
716
|
+
original_send = module.Client.send
|
|
717
|
+
original_async_send = module.AsyncClient.send
|
|
718
|
+
|
|
719
|
+
def send(self, request, *args, **kwargs):
|
|
720
|
+
meta = _safely(_meta_of_httpx, request, spool)
|
|
721
|
+
response = original_send(self, request, *args, **kwargs)
|
|
722
|
+
if meta is not None:
|
|
723
|
+
_safely(attach, response, meta, TappedStream)
|
|
724
|
+
return response
|
|
725
|
+
|
|
726
|
+
async def async_send(self, request, *args, **kwargs):
|
|
727
|
+
meta = _safely(_meta_of_httpx, request, spool)
|
|
728
|
+
response = await original_async_send(self, request, *args, **kwargs)
|
|
729
|
+
if meta is not None:
|
|
730
|
+
_safely(attach, response, meta, TappedAsyncStream)
|
|
731
|
+
return response
|
|
732
|
+
|
|
733
|
+
module.Client.send = send
|
|
734
|
+
module.AsyncClient.send = async_send
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
class _PatchOnImport:
|
|
738
|
+
"""Patch a module the moment something imports it, and cost nothing until then."""
|
|
739
|
+
|
|
740
|
+
def __init__(self, patches):
|
|
741
|
+
self.patches = patches
|
|
742
|
+
|
|
743
|
+
def find_spec(self, name, path=None, target=None):
|
|
744
|
+
if name not in self.patches:
|
|
745
|
+
return None
|
|
746
|
+
# Any fault here would raise into the host's import, so it falls to the normal path.
|
|
747
|
+
try:
|
|
748
|
+
return self._wrapped_spec(name, path, target)
|
|
749
|
+
except Exception:
|
|
750
|
+
return None
|
|
751
|
+
|
|
752
|
+
def _wrapped_spec(self, name, path, target):
|
|
753
|
+
# Kept until the module really executes: find_spec alone (a library asking
|
|
754
|
+
# whether httpx is importable) must not use the patch up.
|
|
755
|
+
for finder in sys.meta_path:
|
|
756
|
+
if finder is self or not hasattr(finder, "find_spec"):
|
|
757
|
+
continue
|
|
758
|
+
spec = finder.find_spec(name, path, target)
|
|
759
|
+
if spec is not None:
|
|
760
|
+
loader = spec.loader
|
|
761
|
+
if hasattr(loader, "exec_module"):
|
|
762
|
+
original = loader.exec_module
|
|
763
|
+
|
|
764
|
+
def exec_module(module):
|
|
765
|
+
original(module)
|
|
766
|
+
patch = self.patches.pop(name, None)
|
|
767
|
+
if patch is not None:
|
|
768
|
+
_safely(patch, module)
|
|
769
|
+
|
|
770
|
+
loader.exec_module = exec_module
|
|
771
|
+
return spec
|
|
772
|
+
return None
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def _install():
|
|
776
|
+
if os.environ.get("AGENTWATCH_CAPTURE") == "off" or _is_agent_process():
|
|
777
|
+
return
|
|
778
|
+
spool = _Spool(os.environ)
|
|
779
|
+
patches = {
|
|
780
|
+
"http.client": lambda module: _patch_http_client(module, spool),
|
|
781
|
+
"httpx": lambda module: _patch_httpx(module, spool),
|
|
782
|
+
}
|
|
783
|
+
for name in list(patches):
|
|
784
|
+
if name in sys.modules:
|
|
785
|
+
_safely(patches.pop(name), sys.modules[name])
|
|
786
|
+
sys.meta_path.insert(0, _PatchOnImport(patches))
|
|
787
|
+
import atexit
|
|
788
|
+
|
|
789
|
+
atexit.register(_Open.finish_all)
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def _run_next_sitecustomize():
|
|
793
|
+
own = os.path.dirname(os.path.abspath(__file__))
|
|
794
|
+
for entry in sys.path:
|
|
795
|
+
folder = os.path.abspath(entry or os.getcwd())
|
|
796
|
+
candidate = os.path.join(folder, "sitecustomize.py")
|
|
797
|
+
if folder != own and os.path.isfile(candidate):
|
|
798
|
+
with open(candidate, "rb") as source:
|
|
799
|
+
exec(compile(source.read(), candidate, "exec"), {"__name__": "sitecustomize", "__file__": candidate})
|
|
800
|
+
return
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def _replay(meta, status, headers, chunks):
|
|
804
|
+
"""For the conformance suite only."""
|
|
805
|
+
call = _call_meta(meta["host"], meta["method"], meta["path"])
|
|
806
|
+
if call is None:
|
|
807
|
+
return None
|
|
808
|
+
result = {}
|
|
809
|
+
|
|
810
|
+
def on_finish(finished):
|
|
811
|
+
if _worth_recording(finished):
|
|
812
|
+
result.update(finished)
|
|
813
|
+
|
|
814
|
+
tap = _ResponseTap(call, status, headers.get, on_finish, False)
|
|
815
|
+
for chunk in chunks:
|
|
816
|
+
tap.feed(chunk)
|
|
817
|
+
tap.finish()
|
|
818
|
+
return result or None
|
|
819
|
+
|
|
820
|
+
|
|
821
|
+
if __name__ == "sitecustomize":
|
|
822
|
+
_safely(_install)
|
|
823
|
+
_safely(_run_next_sitecustomize)
|