deepdebug-local-agent 1.8.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/context.md +2 -1
- package/AGENTS.md +1 -1
- package/CLAUDE.md +1 -1
- package/docs/la-jobs-and-llm.md +56 -0
- package/docs/la-protocol-v2.md +19 -2
- package/env.example +11 -0
- package/package.json +1 -1
- package/src/incident-detector/README.md +2 -1
- package/src/incident-detector/aggregator.js +8 -3
- package/src/incident-detector/detector.js +2 -1
- package/src/incident-detector/emitter.js +16 -6
- package/src/incident-detector/index.js +3 -3
- package/src/incident-detector/sources/webhook-source.js +109 -0
- package/src/la-protocol-v2/content-handlers/install.js +27 -1
- package/src/la-protocol-v2/contract/CONTRACT_SOURCE.json +26 -18
- package/src/la-protocol-v2/contract/README.md +127 -3
- package/src/la-protocol-v2/contract/config.update.schema.json +85 -1
- package/src/la-protocol-v2/contract/hello.schema.json +26 -1
- package/src/la-protocol-v2/contract/incident.candidate.schema.json +11 -0
- package/src/la-protocol-v2/contract/index.json +9 -2
- package/src/la-protocol-v2/contract/job.assign.schema.json +31 -0
- package/src/la-protocol-v2/contract/ready.schema.json +8 -0
- package/src/la-protocol-v2/contract/register.schema.json +26 -1
- package/src/la-protocol-v2/contract/webhook.forward.schema.json +72 -3
- package/src/la-protocol-v2/contract.js +45 -7
- package/src/la-protocol-v2/jobs/config.js +60 -0
- package/src/la-protocol-v2/jobs/events.js +106 -0
- package/src/la-protocol-v2/jobs/install.js +37 -0
- package/src/la-protocol-v2/jobs/manager.js +276 -0
- package/src/la-protocol-v2/jobs/options.js +56 -0
- package/src/la-protocol-v2/jobs/smoke-runner.js +39 -0
- package/src/la-protocol-v2/llm/client.js +311 -0
- package/src/la-protocol-v2/llm/config.js +61 -0
- package/src/la-protocol-v2/llm/install.js +17 -0
- package/src/la-protocol-v2/llm-key/config.js +29 -0
- package/src/la-protocol-v2/llm-key/handler.js +33 -0
- package/src/la-protocol-v2/llm-key/install.js +21 -0
- package/src/la-protocol-v2/llm-key/store.js +183 -0
- package/src/la-protocol-v2/runtime.js +71 -4
- package/src/la-protocol-v2/session.js +47 -13
- package/src/la-protocol-v2/webhooks/assembler.js +85 -0
- package/src/la-protocol-v2/webhooks/config.js +51 -0
- package/src/la-protocol-v2/webhooks/handler.js +125 -0
- package/src/la-protocol-v2/webhooks/install.js +22 -0
- package/test/contract/la-protocol-v2/vectors/ack.vectors.json +91 -0
- package/test/contract/la-protocol-v2/vectors/config.update.vectors.json +1339 -0
- package/test/contract/la-protocol-v2/vectors/hello.vectors.json +261 -0
- package/test/contract/la-protocol-v2/vectors/incident.candidate.vectors.json +623 -0
- package/test/contract/la-protocol-v2/vectors/job.assign.vectors.json +1538 -0
- package/test/contract/la-protocol-v2/vectors/ready.vectors.json +599 -0
- package/test/contract/la-protocol-v2/vectors/register.vectors.json +275 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-webhook-chunk-after-last.json +186 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-webhook-chunk-body-hash-changes.json +186 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-webhook-chunk-last-flag-wrong.json +186 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-webhook-chunk-non-last-too-short.json +152 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-webhook-chunk-out-of-order.json +152 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-webhook-chunk-sha-mismatch.json +186 -0
- package/test/contract/la-protocol-v2/vectors/traces/invalid-webhook-chunk-total-changes.json +186 -0
- package/test/contract/la-protocol-v2/vectors/traces/valid-webhook-chunks-in-order.json +218 -0
- package/test/contract/la-protocol-v2/vectors/webhook.forward.vectors.json +1263 -0
- package/test/incident-detector/size-model.test.js +127 -0
- package/test/la-protocol-v2/contract-v21.test.js +107 -0
- package/test/la-protocol-v2/contract.test.js +6 -4
- package/test/la-protocol-v2/helpers/fake-llm.js +64 -0
- package/test/la-protocol-v2/helpers/reference-cp.js +8 -4
- package/test/la-protocol-v2/jobs-e2e.test.js +405 -0
- package/test/la-protocol-v2/jobs-manager.test.js +370 -0
- package/test/la-protocol-v2/jobs-options.test.js +288 -0
- package/test/la-protocol-v2/llm-client.test.js +380 -0
- package/test/la-protocol-v2/llm-key.test.js +509 -0
- package/test/la-protocol-v2/traces.test.js +3 -3
- package/test/la-protocol-v2/webhook-receiver.test.js +702 -0
package/.claude/context.md
CHANGED
|
@@ -15,7 +15,8 @@ Cloud Run service (`deepdebug-local-agent-qa` / `deepdebug-local-agent`) for the
|
|
|
15
15
|
variant.
|
|
16
16
|
|
|
17
17
|
**Does not:** call LLMs or hold provider keys (the thinking happens in vh-agents via
|
|
18
|
-
vh-litellm)
|
|
18
|
+
vh-litellm). The one exception, behind `LOCAL_AGENT_LLM_KEY` (default off): the VIRTUAL key of the AI
|
|
19
|
+
gateway delivered by `config.update` (F3-09), kept in memory or a 0600 file, never logged; there is no LLM client yet, decide tenant policy, or merge anything.
|
|
19
20
|
|
|
20
21
|
## Neighbours
|
|
21
22
|
|
package/AGENTS.md
CHANGED
|
@@ -15,7 +15,7 @@ Claude Code also reads `CLAUDE.md`; the same rules apply to every agent.
|
|
|
15
15
|
## Always
|
|
16
16
|
|
|
17
17
|
- PRs target `main` (publishes the npm package); Cloud Run deploy is manual.
|
|
18
|
-
- Runs inside customer infrastructure: argument-array process execution, paths confined to the workspace, no LLM calls, no secrets in logs.
|
|
18
|
+
- Runs inside customer infrastructure: argument-array process execution, paths confined to the workspace, no LLM calls except the opt-in F3-10 client (exception approved by Edu 2026-11-03; `LOCAL_AGENT_LLM_CLIENT`: AI-gateway virtual key only, DLP enforce, see `docs/la-jobs-and-llm.md`), no secrets in logs.
|
|
19
19
|
- Never print, log or commit secrets. Provider keys exist only in vh-litellm.
|
|
20
20
|
- Verify on the deployed artefact (curl, revision logs, bundle grep), not only on the diff.
|
|
21
21
|
- Small PRs with one intent each. Open PRs; a human merges them.
|
package/CLAUDE.md
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
layers, simplicity, testing, review checklist, `node.md`).
|
|
8
8
|
3. `README.md`: features, legacy connectors, configuration.
|
|
9
9
|
|
|
10
|
-
Non-negotiables: no LLM calls and no provider keys here; exec with argument arrays only;
|
|
10
|
+
Non-negotiables: no LLM calls and no provider keys here (the one exception, approved by Edu 2026-11-03, is the VIRTUAL key of the AI gateway delivered by `config.update`, F3-09, `client` tenants only: flag `LOCAL_AGENT_LLM_KEY`, memory by default and a 0600 file in a 0700 dir only with `LOCAL_AGENT_LLM_KEY_FILE`, never in logs or acks. F3-10 adds the ONLY LLM client (exception approved by Edu 2026-11-03; the virtual key lives only in memory or a 0600 file and never in logs), flag `LOCAL_AGENT_LLM_CLIENT` (default off; needs v2, the key holder and `LOCAL_AGENT_DLP_MODE=enforce`; every prompt is DLP-masked before it leaves; it calls only the AI gateway endpoint the control plane delivered with the key; no prompt, answer or key in logs, audit or frames; see `docs/la-jobs-and-llm.md`); exec with argument arrays only;
|
|
11
11
|
paths confined to the workspace; never log secrets or raw customer log payloads beyond
|
|
12
12
|
what an incident needs; PRs target `main` (merging publishes the npm package).
|
|
13
13
|
Personal, uncommitted notes go in `CLAUDE.local.md` (gitignored).
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# LLM client and job.assign receiver (F3-10, flags default false)
|
|
2
|
+
|
|
3
|
+
Design: vault `10 Library/Plans/Fase 3 Desenho Tecnico 2026-10-28.md` (sec. 4.2, 5.1, 5.2), ADR-0017. Contract: `src/la-protocol-v2/contract` (`job.assign`, `job.cancel`,
|
|
4
|
+
`job.event`, `job.result`, `config.update.llmKey`). The only contract change F3-10 brought is the optional closed object `job.assign.payload.options` and the capability `job.options` (see below; the copy is
|
|
5
|
+
the gateway's, synced by `scripts/sync-la-protocol-v2-contract.mjs`). The gateway side of the dispatch (sending `job.assign`) is vh-api-gateway `deepdebug.jobs.dispatch-to-la.enabled`.
|
|
6
|
+
|
|
7
|
+
## Flags (all default false; the first three are needed for anything below)
|
|
8
|
+
`LOCAL_AGENT_PROTOCOL_V2=true`, `LOCAL_AGENT_DLP_MODE=enforce` (no opt-out, read at every call), `LOCAL_AGENT_LLM_KEY=true` (F3-09 holder), then
|
|
9
|
+
`LOCAL_AGENT_LLM_CLIENT=true` and/or `LOCAL_AGENT_JOBS=true` (+ a runner). `runtime.js` logs an ERROR and leaves the feature off when a prerequisite is missing. Everything off = register/hello/acks exactly as before.
|
|
10
|
+
|
|
11
|
+
## LLM client (`src/la-protocol-v2/llm/`, `link.llm.client.chat()`)
|
|
12
|
+
- OpenAI-compatible `POST <endpoint of the key><LOCAL_AGENT_LLM_PATH>` through `link.llm.request` (the holder: https only, `Authorization` set by the holder, redirects refused). The key is only a header.
|
|
13
|
+
- Order: shape -> DLP enforce -> key held and alias in the key's models -> mask EVERY message (blocked class / DLP failure / oversize = refused, nothing sent) -> size and token budget -> concurrency slot -> call.
|
|
14
|
+
- Limits: timeout per attempt, stream idle timeout, total deadline, <= 5 retries (network error, 408, 429, 5xx; `Retry-After` honoured, capped 10 s; never a 4xx, never after a delta was delivered), prompt bytes, output
|
|
15
|
+
tokens, response bytes, concurrent calls (`busy`). Streaming (`stream:true`, `onDelta`) with `stream_options.include_usage`.
|
|
16
|
+
- Tags: request `metadata.tags` = `vh-tenant:<id>`, `vh-la:<agent>`, `vh-job:<id>`, `vh-agent:<runner>` (ids only; `LOCAL_AGENT_LLM_TAGS=false` to omit). The key carries the same tenant/LA tags at the proxy. Whether LiteLLM merges request tags with key tags is NOT verified (fake server only).
|
|
17
|
+
- Errors are codes only (`LlmClientError.code`, closed list in `client.js`); the body of an error response is never read. Usage comes from `usage`; without it the tokens are estimated (bytes/3) so a budget still counts. Cost from `x-litellm-response-cost` when present.
|
|
18
|
+
- Local audit (ring, optional `LOCAL_AGENT_LLM_AUDIT_FILE`): alias, byte and token counts, per-rule redaction counts, short hash of the job id. Never a prompt, answer or key.
|
|
19
|
+
|
|
20
|
+
## job.assign receiver (`src/la-protocol-v2/jobs/`, `link.jobs`)
|
|
21
|
+
- The session has already checked schema and Ed25519 signature. The handler answers the ack: `accepted`; `rejected protocol_unsupported` (no runner, DLP not enforce, kind not served); `rejected unauthorized`
|
|
22
|
+
(workspace not served here / of another tenant); `rejected rate_limited` (concurrency + queue, memory reservation, RSS brake); `rejected schema_violation` (`options` malformed, second line behind the session's schema check).
|
|
23
|
+
A known job id (running, queued, finished) is `accepted` and not run again. Admission is synchronous (check and reservation in one step), so a burst of `job.assign` can never exceed concurrency + queue.
|
|
24
|
+
- Budget (`tokens`, `seconds`) is clamped to `LOCAL_AGENT_JOBS_MAX_TOKENS/_SECONDS`. Failures after the ack are `job.event failed` + `job.result failed` with an `errorCode` (no key / key mismatch = `llm_error`).
|
|
25
|
+
- `job.event`: progress is coalesced (<= 1 per `LOCAL_AGENT_JOBS_EVENT_MIN_MS`, last state wins, dropped when the send queue is full); the terminal `job.event` + `job.result` are retried and never dropped for backpressure.
|
|
26
|
+
Payloads are built from a whitelist (ids, enums, counters, cost, duration): no model name, prompt, answer, path or error text. `job.cancel` aborts the job's `AbortSignal`, calls `runner.cancel(jobId)` and abandons a runner
|
|
27
|
+
that has not stopped after `LOCAL_AGENT_JOBS_CANCEL_GRACE_MS`. Unknown/finished ids are acked `accepted` (no oracle) and cancelling twice is harmless. A `job.cancel` for an id never seen is remembered (256, bounded): a
|
|
28
|
+
`job.assign` that arrives later for it (an orphan the control plane already gave up) is acked `accepted` and NOT run. A cancel whose `jobId` is not a UUID is refused (`schema_violation`: by the session, which closes that
|
|
29
|
+
connection like any closed-schema breach; `rejected schema_violation` by the handler if called without it), with no lookup and no effect. After a cancel, a timeout or an abandonment `ctx.readFile` reads nothing (`status:"cancelled"`)
|
|
30
|
+
and the signal handed to the reader is aborted, so a read in flight releases its descriptor and its buffer is dropped.
|
|
31
|
+
- Memory: each admitted job reserves its inline bytes + `LOCAL_AGENT_JOBS_MEM_PER_JOB_BYTES` against `LOCAL_AGENT_JOBS_MAX_MEMORY_BYTES`; `LOCAL_AGENT_JOBS_MAX_RSS_BYTES` (>0) refuses new jobs while the process RSS is above it. Accounting plus a brake, not a hard cap: a runner is code in this process.
|
|
32
|
+
The inline inputs (transit content) are dropped when the job ends.
|
|
33
|
+
- Capability `job.assign` (covers `job.cancel`) is announced only when the receiver is installed (flag + runner); the contract README (section (f)) documents it. Capability `job.options` is announced together with it
|
|
34
|
+
only when the runner declared `supportsOptions: true` (the smoke runner does; an injected one must say so).
|
|
35
|
+
|
|
36
|
+
## `job.assign` options (contract 2.1, capability `job.options`)
|
|
37
|
+
Optional, closed: `ticketKey`, `branchName`, `createBranch`, `createPullRequest`, `dryRun`, `maxIterations` (1..100). The runner reads them as `job.options` (`jobs/options.js`): frozen, every field present
|
|
38
|
+
(defaults: branch and PR on, no dry run, `maxIterations` = `LOCAL_AGENT_JOBS_MAX_ITERATIONS`, default 25, 1..100; a job can only LOWER it) plus `effects` = what the runner may do outside: `dryRun` -> no branch, no push, no PR;
|
|
39
|
+
`createBranch:false` -> work on `repoRef.branch`, never push to it, no PR; `createPullRequest:false` -> no PR. A runner without `supportsOptions` gets a job that carries options refused (`protocol_unsupported`), so
|
|
40
|
+
`dryRun` is never ignored silently. Option values are not content, but they are not logged either (the audit lists the key names). The smoke runner never touches Git, so it honours them trivially; a dry run still asks its question.
|
|
41
|
+
|
|
42
|
+
## Runner interface (F3-08c plugs the packaged agent loop here)
|
|
43
|
+
```
|
|
44
|
+
Runner { name, kinds? (default fix|implement|quality), needsLlm? (default true), supportsOptions? (default false), start(job, ctx): Promise<Outcome>, cancel(jobId) }
|
|
45
|
+
job = { jobId, kind, workspaceId, incidentRef?, repoRef, policy, budget(clamped), llm{alias,keyRef}, inputs[], options{...} } // frozen
|
|
46
|
+
ctx = { signal, workspace{id}, llm.chat(opts), readFile(ref,{maxBytes}), progress({phase,iteration,filesRead,filesChanged}), budget() }
|
|
47
|
+
Outcome = { outcome: pr_opened|no_change|failed|cancelled, errorCode?, prUrl?, branch?, commitSha?, filesChangedCount?, diffSha256?, tests?, answer? }
|
|
48
|
+
```
|
|
49
|
+
`ctx.llm.chat` fills in the job's alias, signal, tags and remaining budget and counts tokens/cost into the job. `ctx.readFile` is the content handlers' gate (opaque id minted by this agent for this
|
|
50
|
+
workspace, deny list, root/symlink checks), so it needs `LOCAL_AGENT_CONTENT_HANDLERS=true`. `answer` is client content: it goes to the local `onResult` hook only, never on the wire. Install a runner with
|
|
51
|
+
`buildV2Link({ jobRunner })` (announces the capability) or `link.jobs.setRunner()`.
|
|
52
|
+
Reference runner `LOCAL_AGENT_JOBS_RUNNER=smoke`: reads the first `inputs[].ref` file, asks one fixed question, outcome `no_change`. For QA and tests, not for production.
|
|
53
|
+
|
|
54
|
+
## Git credentials
|
|
55
|
+
The control plane never sends a Git token: `job.assign` is a closed schema with no field for one (a test sends one and gets `schema_violation`). A runner that opens a PR uses what the machine already has: the
|
|
56
|
+
user's git credential helper, ssh-agent, or variables of the agent's own process environment. The `repoRef` only names provider/org/repo/branch.
|
package/docs/la-protocol-v2.md
CHANGED
|
@@ -146,10 +146,27 @@ the same rules as the Java monitor (errorHash, fingerprint, significance, noise;
|
|
|
146
146
|
(9 metadata fields, no text) per errorHash window. Announces the capability `incident.candidate`. The detail stays in a local JSONL store (0600, DLP-masked).
|
|
147
147
|
Needs `LOCAL_AGENT_PROTOCOL_V2=true`. Details, limits and hooks: `src/incident-detector/README.md`. The `incident.detail` content kind is NOT served yet.
|
|
148
148
|
|
|
149
|
+
## Contract 2.1 consumers (F3-09 / F3-12 / F3-11b, flags default false)
|
|
150
|
+
|
|
151
|
+
The contract copy is revision 2.1 (`src/la-protocol-v2/contract/CONTRACT_SOURCE.json` names the gateway commit and branch). With every flag off the agent announces exactly what it
|
|
152
|
+
announced before (`protocol.v2 v1.commands chunk.base64`); `contract.v2.1` is announced only when one of the three features below is on. `job.assign` / `job.options` (F3-10) are plain capabilities, announced with `LOCAL_AGENT_JOBS` (see `docs/la-jobs-and-llm.md`).
|
|
153
|
+
- **LLM key** (`LOCAL_AGENT_LLM_KEY`, capability `llm.key`; `src/la-protocol-v2/llm-key/`): `config.update.llmKey` `set` / `revoke`. The virtual key of the AI gateway is held in a
|
|
154
|
+
private field (memory) or, with `LOCAL_AGENT_LLM_KEY_FILE`, in a 0600 file outside every workspace; it is never in an ack, a log, the audit or a frame. A key whose id is not the
|
|
155
|
+
sha256 of the key, or that has expired, is `rejected schema_violation`. The only caller of `link.llm.request(path, init)` (https endpoint of the control
|
|
156
|
+
plane, Authorization set by the holder, redirects refused) is the F3-10 LLM client (`LOCAL_AGENT_LLM_CLIENT`, see `docs/la-jobs-and-llm.md`); without that flag nothing calls it. Memory by default; the file only with `LOCAL_AGENT_LLM_KEY_FILE` (file 0600, directory 0700;
|
|
157
|
+
exception to "no keys here" approved by Edu 2026-11-03, recorded in `CLAUDE.md`). `llm.key.held` (contract 2.1 (e)): at EVERY connect the agent adds this capability to `register` AND `hello`
|
|
158
|
+
only while the holder has a valid, unexpired key (never without `llm.key`); a revoked or expired key stops being announced on the next connection, so the control plane delivers a new one.
|
|
159
|
+
- **Webhooks** (`LOCAL_AGENT_WEBHOOK_FORWARD`, capabilities `webhook.forward` + `webhook.chunked`; `src/la-protocol-v2/webhooks/`): one frame (<= 9216 bytes) or chunks (16 KiB, <= 64, <= 1 MiB)
|
|
160
|
+
rebuilt IN MEMORY with the contract limits (30 s, 4 assemblies, 4 MiB), sha256 verified, handed to the incident detector (`src/incident-detector/sources/webhook-source.js`, then the
|
|
161
|
+
same pipeline and the DLP-masked local store). The ack of the last chunk (or of the single frame) is sent only after the local delivery finished, waiting up to `LOCAL_AGENT_WEBHOOK_DELIVERY_MS`
|
|
162
|
+
(default 10 s, max 14 s; the session does not ack `accepted` after 2 s for this type); past it the ack is `rejected timeout`. Delivered ids (+ sha256 of the body) are remembered in memory for 10 min (max 1024):
|
|
163
|
+
a resend (same id and body) is acked `accepted`, chunk by chunk, without being processed again, so the detector's count is not inflated. Needs `LOCAL_AGENT_INCIDENT_DETECTOR=true`.
|
|
164
|
+
- **Candidate size and model** (no new flag, part of `LOCAL_AGENT_INCIDENT_DETECTOR`): `incident.candidate` carries `stackBytes` and `model` only if `ready.capabilities` listed `incident.candidate.size-model`.
|
|
165
|
+
|
|
149
166
|
## Tests
|
|
150
167
|
|
|
151
|
-
`npm test` (node:test). The incident detector (F3-11b) is in `test/incident-detector/` (`parity.test.js` against `parity.vectors.json`, `units`, `sources`, `wiring`); the rest of v2 in `test/la-protocol-v2/`:
|
|
152
|
-
`contract.test.js` (all
|
|
168
|
+
`npm test` (node:test). Contract 2.1: `contract-v21.test.js`, `webhook-receiver.test.js` (assembler, handler, the 8 traces, end to end), `llm-key.test.js` (including the sweep for the key), `test/incident-detector/size-model.test.js`. The incident detector (F3-11b) is in `test/incident-detector/` (`parity.test.js` against `parity.vectors.json`, `units`, `sources`, `wiring`); the rest of v2 in `test/la-protocol-v2/`:
|
|
169
|
+
`contract.test.js` (all 294 frame vectors of F3-22 + 2.1, valid and invalid, the 54 `file.list` body vectors, plus the drift check), `traces.test.js` (the 34
|
|
153
170
|
traces), `session-inbound.test.js`, `session-outbound.test.js`, `link.test.js`, `tunnel-e2e.test.js` (the real
|
|
154
171
|
`src/server.js` against a reference CP; flag off = v1 frames), and for F3-07 `content-path-guard.test.js`,
|
|
155
172
|
`content-handlers.test.js`, `content-wiring.test.js` (flag, runtime wiring, real `src/server.js`), and for F3-07b
|
package/env.example
CHANGED
|
@@ -92,4 +92,15 @@ LOG_LEVEL=info
|
|
|
92
92
|
# LOCAL_AGENT_INCIDENT_STDIN=true read log lines from the agent's own stdin
|
|
93
93
|
# LOCAL_AGENT_INCIDENT_STORE_DIR=~/.deepdebug/incidents LOCAL_AGENT_INCIDENT_WINDOW_MS=300000 LOCAL_AGENT_INCIDENT_FLUSH_MS=30000
|
|
94
94
|
# LOCAL_AGENT_INCIDENT_MAX_EMITS_PER_MINUTE=60 LOCAL_AGENT_INCIDENT_MAX_PENDING=500 LOCAL_AGENT_INCIDENT_MAX_TRACKED=5000
|
|
95
|
+
# LOCAL_AGENT_WEBHOOK_FORWARD=true receives the webhook bodies the control plane strips and forwards (webhook.forward, one frame <= 9216 bytes or chunks up to 1 MiB),
|
|
96
|
+
# rebuilds them in memory, verifies the sha256 and hands them to the incident detector. Needs LOCAL_AGENT_PROTOCOL_V2=true AND
|
|
97
|
+
# LOCAL_AGENT_INCIDENT_DETECTOR=true (the local consumer). Tighten only: LOCAL_AGENT_WEBHOOK_MAX_BYTES, _MAX_OPEN, _MAX_BUFFERED_BYTES, _ASSEMBLY_MS, _MAX_ENTRIES. LOCAL_AGENT_WEBHOOK_DELIVERY_MS (default 10000, max 14000: how long the final ack waits for the local delivery), _DEDUPE_MS (600000) and _DEDUPE_MAX (1024): resends of a delivered webhookId are acked without reprocessing
|
|
98
|
+
# LOCAL_AGENT_LLM_KEY=true accepts the virtual key of the AI gateway (config.update llmKey): in memory only, never logged or returned in an ack. Needs LOCAL_AGENT_PROTOCOL_V2=true
|
|
99
|
+
# LOCAL_AGENT_LLM_KEY_FILE=/var/lib/deepdebug/llm-key.json optional: keep the key across restarts (mode 0600, directory 0700, MUST be outside every workspace)
|
|
100
|
+
# LOCAL_AGENT_LLM_CLIENT=true F3-10: chat/completions to the AI gateway named by the key (OpenAI-compatible, optional streaming). Needs LOCAL_AGENT_PROTOCOL_V2=true, LOCAL_AGENT_LLM_KEY=true
|
|
101
|
+
# AND LOCAL_AGENT_DLP_MODE=enforce (every prompt is masked before it leaves). Never logs a prompt, an answer or the key. Tighten/tune: LOCAL_AGENT_LLM_PATH (default
|
|
102
|
+
# /v1/chat/completions), _TIMEOUT_MS, _STREAM_IDLE_MS, _TOTAL_MS, _MAX_RETRIES (2), _MAX_PROMPT_BYTES, _MAX_OUTPUT_TOKENS, _MAX_RESPONSE_BYTES, _MAX_CONCURRENT, _TAGS, _AUDIT_FILE
|
|
103
|
+
# LOCAL_AGENT_JOBS=true F3-10: receives job.assign / job.cancel, runs the job with a runner, answers with job.event / job.result (metadata only). Needs v2, DLP enforce and a runner:
|
|
104
|
+
# LOCAL_AGENT_JOBS_RUNNER=smoke the reference runner (one fixed question about one allowed file; needs the LLM client). Limits: LOCAL_AGENT_JOBS_MAX_CONCURRENT (1), _MAX_QUEUED (4), _MAX_TOKENS,
|
|
105
|
+
# _MAX_SECONDS, _MAX_ITERATIONS (25; caps job.assign options.maxIterations), _MEM_PER_JOB_BYTES, _MAX_MEMORY_BYTES, _MAX_RSS_BYTES (0 = off), _EVENT_MIN_MS, _CANCEL_GRACE_MS, _AUDIT_FILE. Git credentials stay the machine's own (never sent by the control plane)
|
|
95
106
|
# LOCAL_AGENT_CONTENT_DENY_GLOBS=a/,*.x extra deny globs (they add to the built-in list: .env*, keys, credentials files, secrets/ ...)
|
package/package.json
CHANGED
|
@@ -26,7 +26,8 @@ Flag: `LOCAL_AGENT_INCIDENT_DETECTOR=true` (default **false**; needs `LOCAL_AGEN
|
|
|
26
26
|
|
|
27
27
|
Out: `errorHash` (32 hex = sha256 prefix of the normalised FIRST line, Java `computeGroupHash`), `fingerprint` (64 hex, Java
|
|
28
28
|
`IncidentFingerprintGenerator`), `severity`, `service`, `environment`, `count`, `firstSeen`, `lastSeen` (the window), `logCategory`.
|
|
29
|
-
|
|
29
|
+
Contract 2.1 adds two optional numbers/labels, sent ONLY when the CP listed `incident.candidate.size-model` in `ready.capabilities`: `stackBytes` (UTF-8 size of the biggest stack of the window,
|
|
30
|
+
saturating at 16 MiB) and `model` (`stack-detector-<agent version>`). There is still no field for text. The size is also in the local store (`stackBytes`).
|
|
30
31
|
Stays: message, stack, everything else, in `LOCAL_AGENT_INCIDENT_STORE_DIR` (default `~/.deepdebug/incidents`).
|
|
31
32
|
|
|
32
33
|
## Window semantics (for the control plane, F3-11c)
|
|
@@ -12,17 +12,20 @@
|
|
|
12
12
|
* bounds at most `maxTracked` hashes at once. Beyond it the least recently seen one is closed (final update if it has one
|
|
13
13
|
* pending) and counted as evicted. A hash that comes back after its window closed opens a new window.
|
|
14
14
|
* severity the highest seen in the window. fingerprint/service/logCategory: those of the first occurrence.
|
|
15
|
+
* stackBytes the biggest stack (UTF-8 bytes) seen in the window, saturating at 16 MiB (contract 2.1 (c)); 0 = no stack.
|
|
15
16
|
* Pure: no timers, no I/O. The detector calls tick() on an interval.
|
|
16
17
|
*/
|
|
17
18
|
import { maxSeverity } from "./categorize.js";
|
|
18
19
|
|
|
20
|
+
const MAX_STACK_BYTES = 16 * 1024 * 1024; // contract 2.1: incident.candidate.stackBytes saturates here
|
|
21
|
+
|
|
19
22
|
export function createAggregator({ windowMs, flushMs, maxTracked, now = Date.now, onCandidate }) {
|
|
20
23
|
const states = new Map(); // errorHash -> state (insertion order = least recently seen first)
|
|
21
24
|
const stats = { observed: 0, windows: 0, emitted: 0, evicted: 0 };
|
|
22
25
|
|
|
23
26
|
const snapshot = (s) => ({
|
|
24
27
|
errorHash: s.errorHash, fingerprint: s.fingerprint, severity: s.severity, service: s.service, environment: s.environment,
|
|
25
|
-
count: s.count, firstSeen: s.firstSeen, lastSeen: s.lastSeen, logCategory: s.logCategory, windowStart: s.windowStart,
|
|
28
|
+
count: s.count, firstSeen: s.firstSeen, lastSeen: s.lastSeen, logCategory: s.logCategory, windowStart: s.windowStart, stackBytes: s.stackBytes,
|
|
26
29
|
});
|
|
27
30
|
const send = (s, t) => { s.sentCount = s.count; s.lastEmitAt = t; stats.emitted++; onCandidate(snapshot(s)); };
|
|
28
31
|
const close = (s, t) => { states.delete(s.errorHash); if (s.count > s.sentCount) send(s, t); };
|
|
@@ -31,7 +34,8 @@ export function createAggregator({ windowMs, flushMs, maxTracked, now = Date.now
|
|
|
31
34
|
stats,
|
|
32
35
|
get tracked() { return states.size; },
|
|
33
36
|
/** @returns {{opened: boolean, windowStart: number}} opened = this occurrence opened a new window. */
|
|
34
|
-
observe({ errorHash, fingerprint, severity, service, environment, logCategory, tsMs }) {
|
|
37
|
+
observe({ errorHash, fingerprint, severity, service, environment, logCategory, tsMs, stackBytes = 0 }) {
|
|
38
|
+
const size = Number.isFinite(stackBytes) && stackBytes > 0 ? Math.min(Math.trunc(stackBytes), MAX_STACK_BYTES) : 0;
|
|
35
39
|
const t = now();
|
|
36
40
|
stats.observed++;
|
|
37
41
|
const seenAt = Number.isFinite(tsMs) && tsMs >= 0 ? Math.min(tsMs, t) : t; // never in the future, never before the epoch (the log line is untrusted)
|
|
@@ -43,7 +47,7 @@ export function createAggregator({ windowMs, flushMs, maxTracked, now = Date.now
|
|
|
43
47
|
stats.evicted++;
|
|
44
48
|
close(oldest, t);
|
|
45
49
|
}
|
|
46
|
-
s = { errorHash, fingerprint, severity, service, environment, logCategory, count: 0, sentCount: 0, firstSeen: seenAt, lastSeen: seenAt, windowStart: t, lastEmitAt: 0 };
|
|
50
|
+
s = { errorHash, fingerprint, severity, service, environment, logCategory, count: 0, sentCount: 0, stackBytes: size, firstSeen: seenAt, lastSeen: seenAt, windowStart: t, lastEmitAt: 0 };
|
|
47
51
|
stats.windows++;
|
|
48
52
|
s.count = 1;
|
|
49
53
|
states.set(errorHash, s);
|
|
@@ -54,6 +58,7 @@ export function createAggregator({ windowMs, flushMs, maxTracked, now = Date.now
|
|
|
54
58
|
states.set(errorHash, s);
|
|
55
59
|
s.count++;
|
|
56
60
|
s.severity = maxSeverity(s.severity, severity);
|
|
61
|
+
if (size > s.stackBytes) s.stackBytes = size; // the biggest stack seen in the window (contract 2.1 (c))
|
|
57
62
|
// firstSeen never changes inside a window (it is part of the window identity), even for an out-of-order line
|
|
58
63
|
if (seenAt > s.lastSeen) s.lastSeen = seenAt;
|
|
59
64
|
if (t - s.lastEmitAt >= flushMs) send(s, t);
|
|
@@ -58,7 +58,8 @@ export function createIncidentDetector({ config, emitter, store, audit = null, n
|
|
|
58
58
|
const environment = config.environment || inferEnvironment(service);
|
|
59
59
|
const logCategory = logCategoryOf(message, stack, extractErrorType(message, stack));
|
|
60
60
|
stats.detected++;
|
|
61
|
-
const
|
|
61
|
+
const stackBytes = typeof entry.stacktrace === "string" ? Buffer.byteLength(entry.stacktrace) : 0; // as the source gave it, before the entry cap
|
|
62
|
+
const { windowStart } = aggregator.observe({ errorHash, fingerprint, severity, service, environment, logCategory, tsMs: entry.tsMs, stackBytes });
|
|
62
63
|
// the detail of the occurrence, tied to its window (the store keeps a few samples per window); resolves when queued
|
|
63
64
|
if (await store.add({ errorHash, fingerprint, service, severity, windowStart, message, stack })) stats.stored++;
|
|
64
65
|
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* emitter.js - sends `incident.candidate` frames (F3-11b, contract F3-22: la->cp, metadata only).
|
|
3
3
|
*
|
|
4
|
-
* The payload has
|
|
5
|
-
* lastSeen, logCategory
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* The payload has the 9 contract fields: errorHash, fingerprint, severity, service, environment, count, firstSeen,
|
|
5
|
+
* lastSeen, logCategory and, ONLY when the CP announced `incident.candidate.size-model` in `ready.capabilities` (contract 2.1 (c)), two more
|
|
6
|
+
* that carry no text: `stackBytes` (UTF-8 size of the biggest stack of the window) and `model` (detector version, e.g. stack-detector-1.8.0). There is no field for text, a stack, a path or a message, and the session validates every frame against
|
|
7
|
+
* the closed schema before it is written, so even a bug here cannot add one. (window = firstSeen..lastSeen.)
|
|
8
8
|
*
|
|
9
9
|
* DLP service and environment are labels of the customer's own logs; they go through the DLP and a label it would change
|
|
10
10
|
* (or deny) is not sent at all: the candidate is dropped and counted (`dlp`).
|
|
@@ -27,7 +27,11 @@ const RETRY_MS = 5_000;
|
|
|
27
27
|
const MAX_ATTEMPTS = 5;
|
|
28
28
|
const NON_RETRYABLE_REFUSALS = new Set(["schema_violation", "frame_too_large", "payload_too_large", "unknown_type", "wrong_direction"]);
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
const SIZE_MODEL_CAPABILITY = "incident.candidate.size-model";
|
|
31
|
+
const MODEL = /^[A-Za-z0-9][A-Za-z0-9._:+-]{0,47}$/;
|
|
32
|
+
const MAX_STACK_BYTES = 16 * 1024 * 1024;
|
|
33
|
+
|
|
34
|
+
export function createCandidateEmitter({ session, config, workspaceId = "", model = null, env = process.env, audit = null, log = console, now = Date.now, timers = { setTimeout, clearTimeout } }) {
|
|
31
35
|
const warn = (m) => (log.warn || log.log).call(log, m);
|
|
32
36
|
const pending = new Map(); // `${errorHash}|${windowStart}` -> snapshot
|
|
33
37
|
const stats = { submitted: 0, sent: 0, coalesced: 0, droppedFull: 0, droppedDlp: 0, droppedInvalid: 0, droppedGaveUp: 0, droppedStopped: 0, rejected: 0, byCode: {} };
|
|
@@ -59,7 +63,13 @@ export function createCandidateEmitter({ session, config, workspaceId = "", env
|
|
|
59
63
|
const r = protectEgress(label, { env, source: "incident-candidate" });
|
|
60
64
|
if (r.denied || r.text !== label) return null;
|
|
61
65
|
}
|
|
62
|
-
|
|
66
|
+
const payload = { errorHash: s.errorHash, fingerprint: s.fingerprint, severity: s.severity, service, environment, count: s.count, firstSeen: iso(s.firstSeen), lastSeen: iso(s.lastSeen), logCategory: s.logCategory };
|
|
67
|
+
// contract 2.1 (c): only when the CP said (ready.capabilities) it takes them: a 2.0 CP answers the closed schema with schema_violation
|
|
68
|
+
if (typeof session.peerHas === "function" && session.peerHas(SIZE_MODEL_CAPABILITY)) {
|
|
69
|
+
if (Number.isFinite(s.stackBytes)) payload.stackBytes = Math.max(0, Math.min(Math.trunc(s.stackBytes), MAX_STACK_BYTES));
|
|
70
|
+
if (typeof model === "string" && MODEL.test(model)) payload.model = model;
|
|
71
|
+
}
|
|
72
|
+
return payload;
|
|
63
73
|
}
|
|
64
74
|
|
|
65
75
|
function schedule(ms) {
|
|
@@ -23,8 +23,8 @@ export function buildSources(config, { stdin = process.stdin } = {}) {
|
|
|
23
23
|
return sources;
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
export function installIncidentDetector({ link, workspaceId = "", config, env = process.env, log = console, sources = null }) {
|
|
27
|
-
const list = sources || buildSources(config);
|
|
26
|
+
export function installIncidentDetector({ link, workspaceId = "", config, env = process.env, log = console, sources = null, extraSources = [], model = null }) {
|
|
27
|
+
const list = sources || [...buildSources(config), ...extraSources]; // extraSources: push sources owned by someone else (the webhook receiver, F3-12 consumer)
|
|
28
28
|
if (list.length === 0) {
|
|
29
29
|
log.warn("[la-incidents] LOCAL_AGENT_INCIDENT_DETECTOR=true but no source is configured (LOCAL_AGENT_INCIDENT_LOG_FILES, LOCAL_AGENT_INCIDENT_STDIN or LOCAL_AGENT_INCIDENT_LOKI_URL): the detector does not start");
|
|
30
30
|
return null;
|
|
@@ -33,7 +33,7 @@ export function installIncidentDetector({ link, workspaceId = "", config, env =
|
|
|
33
33
|
const audit = createAudit({ file: config.auditFile });
|
|
34
34
|
const store = createDetailStore({ dir: config.storeDir, maxBytes: config.storeMaxBytes, maxQueue: config.storeMaxQueue, messageChars: config.storeMessageChars,
|
|
35
35
|
stackChars: config.storeStackChars, samplesPerWindow: config.samplesPerWindow, env });
|
|
36
|
-
const emitter = createCandidateEmitter({ session: link.session, config, workspaceId: ws, env, audit, log });
|
|
36
|
+
const emitter = createCandidateEmitter({ session: link.session, config, workspaceId: ws, model, env, audit, log });
|
|
37
37
|
const detector = createIncidentDetector({ config, emitter, store, audit, sources: list, log });
|
|
38
38
|
const closeLink = link.close.bind(link);
|
|
39
39
|
link.close = () => { void detector.stop().finally(() => { emitter.close(); }); closeLink(); };
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* webhook-source.js - the incident detector as the LOCAL consumer of the webhooks the control plane strips and forwards (F3-12 consumer).
|
|
3
|
+
*
|
|
4
|
+
* `webhook.forward` hands the (rebuilt, sha256-verified) body here. It is turned into the same entries every other source produces
|
|
5
|
+
* ({severity, message, stacktrace, service, tsMs}) and goes through the SAME pipeline (Java rules, DLP-masked local store, candidates with
|
|
6
|
+
* metadata only): the text of the webhook never leaves this process except into the local store. The ack of the webhook waits for `deliver()`.
|
|
7
|
+
*
|
|
8
|
+
* text/plain lines, joined into records like a log file (EntryAssembler)
|
|
9
|
+
* alertmanager `alerts[]` with status firing: "alertname: summary", severity label critical -> CRITICAL, error/high/major -> ERROR,
|
|
10
|
+
* anything else WARN (the detector ignores it)
|
|
11
|
+
* sentry `data.event` / `event` (+ `data.issue`): "Type: value" and a stack from the exception frames, `level` as severity,
|
|
12
|
+
* project as service
|
|
13
|
+
* other JSON one log-shaped object (message/msg/level/stack... as in parse.js); a body that does not look like one is `unmapped`
|
|
14
|
+
* Only SYNTHETIC payloads in the tests: the real shapes of each tool should be confirmed with captured webhooks before relying on them.
|
|
15
|
+
* Bounded: at most `maxEntries` entries and `maxBytes` of text per webhook. Counters only; never a word of the body in a log.
|
|
16
|
+
*/
|
|
17
|
+
import { EntryAssembler, parseRecord, parseTime } from "../parse.js";
|
|
18
|
+
|
|
19
|
+
const str = (v) => (typeof v === "string" && v !== "" ? v : undefined);
|
|
20
|
+
|
|
21
|
+
function fromAlertmanager(o, maxEntries) {
|
|
22
|
+
const out = [];
|
|
23
|
+
for (const a of Array.isArray(o.alerts) ? o.alerts : []) {
|
|
24
|
+
if (out.length >= maxEntries) break;
|
|
25
|
+
if (!a || typeof a !== "object" || (a.status && a.status !== "firing")) continue;
|
|
26
|
+
const labels = a.labels && typeof a.labels === "object" ? a.labels : {};
|
|
27
|
+
const ann = a.annotations && typeof a.annotations === "object" ? a.annotations : {};
|
|
28
|
+
const name = str(labels.alertname) || "alert";
|
|
29
|
+
const text = str(ann.summary) || str(ann.description) || "";
|
|
30
|
+
const sev = String(labels.severity ?? "").toLowerCase();
|
|
31
|
+
out.push({
|
|
32
|
+
severity: sev === "critical" ? "CRITICAL" : ["error", "high", "major", "page"].includes(sev) ? "ERROR" : "WARN",
|
|
33
|
+
message: text ? `${name}: ${text}` : name,
|
|
34
|
+
stacktrace: null,
|
|
35
|
+
service: str(labels.service) || str(labels.job) || str(labels.app) || str(labels.namespace) || null,
|
|
36
|
+
tsMs: parseTime(a.startsAt),
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
return out;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function sentryStack(exc) {
|
|
43
|
+
const values = exc && Array.isArray(exc.values) ? exc.values : [];
|
|
44
|
+
const v = values[values.length - 1];
|
|
45
|
+
const frames = v && v.stacktrace && Array.isArray(v.stacktrace.frames) ? v.stacktrace.frames : [];
|
|
46
|
+
if (frames.length === 0) return null;
|
|
47
|
+
return frames.slice().reverse().slice(0, 100).map((f) => ` at ${str(f && f.function) || "?"} (${str(f && (f.filename || f.module)) || "?"}:${Number.isFinite(f && f.lineno) ? f.lineno : 0})`).join("\n");
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function fromSentry(o) {
|
|
51
|
+
const data = o.data && typeof o.data === "object" ? o.data : o;
|
|
52
|
+
const ev = data.event && typeof data.event === "object" ? data.event : data.issue && typeof data.issue === "object" ? data.issue : null;
|
|
53
|
+
if (!ev) return [];
|
|
54
|
+
const exc = ev.exception && typeof ev.exception === "object" ? ev.exception : null;
|
|
55
|
+
const last = exc && Array.isArray(exc.values) ? exc.values[exc.values.length - 1] : null;
|
|
56
|
+
const head = last && str(last.type) ? `${last.type}${str(last.value) ? `: ${last.value}` : ""}` : str(ev.title) || str(ev.message) || str(ev.culprit);
|
|
57
|
+
if (!head) return [];
|
|
58
|
+
const proj = ev.project && typeof ev.project === "object" ? ev.project : null;
|
|
59
|
+
const ts = typeof ev.timestamp === "number" ? ev.timestamp : ev.datetime || ev.firstSeen;
|
|
60
|
+
return [{
|
|
61
|
+
severity: str(ev.level) ? String(ev.level).toUpperCase() : "ERROR",
|
|
62
|
+
message: head.split("\n")[0],
|
|
63
|
+
stacktrace: sentryStack(exc),
|
|
64
|
+
service: str(o.project_name) || str(proj && (proj.slug || proj.name)) || str(ev.project) || null,
|
|
65
|
+
tsMs: parseTime(ts),
|
|
66
|
+
}];
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** @returns {{entries:object[], unmapped:boolean}} */
|
|
70
|
+
export function entriesFromWebhook({ source, contentType, body }, { maxEntries = 200, maxBytes = 1048576, maxEntryBytes, maxStackLines } = {}) {
|
|
71
|
+
const text = new TextDecoder("utf-8").decode(body.length > maxBytes ? body.subarray(0, maxBytes) : body);
|
|
72
|
+
if (contentType === "text/plain") {
|
|
73
|
+
const asm = new EntryAssembler({ maxEntryBytes, maxStackLines });
|
|
74
|
+
const out = [];
|
|
75
|
+
for (const line of text.split("\n")) { out.push(...asm.push(line.replace(/\r$/, ""))); if (out.length >= maxEntries) break; }
|
|
76
|
+
out.push(...asm.flush());
|
|
77
|
+
const entries = out.slice(0, maxEntries);
|
|
78
|
+
return { entries, unmapped: entries.length === 0 };
|
|
79
|
+
}
|
|
80
|
+
let o;
|
|
81
|
+
try { o = JSON.parse(text); } catch { return { entries: [], unmapped: true }; }
|
|
82
|
+
if (!o || typeof o !== "object" || Array.isArray(o)) return { entries: [], unmapped: true };
|
|
83
|
+
let entries = [];
|
|
84
|
+
if (source === "alertmanager" || Array.isArray(o.alerts)) entries = fromAlertmanager(o, maxEntries);
|
|
85
|
+
else if (source === "sentry") entries = fromSentry(o);
|
|
86
|
+
else { const r = parseRecord(text); entries = r ? [r] : []; }
|
|
87
|
+
return { entries, unmapped: entries.length === 0 };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function createWebhookSource({ maxEntries = 200, maxBytes = 1048576, maxEntryBytes, maxStackLines, stats = {} } = {}) {
|
|
91
|
+
let consumer = null;
|
|
92
|
+
Object.assign(stats, { received: 0, entries: 0, unmapped: 0, ...stats });
|
|
93
|
+
return {
|
|
94
|
+
name: "webhook",
|
|
95
|
+
stats,
|
|
96
|
+
start(onEntries) { consumer = onEntries; },
|
|
97
|
+
stop() { consumer = null; },
|
|
98
|
+
/** The sink of the webhook receiver. Resolves when every entry went through the detector (its detail queued): the ack waits for it. */
|
|
99
|
+
async deliver(webhook) {
|
|
100
|
+
if (!consumer) { const e = new Error("webhook source is not started"); e.code = "internal"; throw e; }
|
|
101
|
+
stats.received++;
|
|
102
|
+
const { entries, unmapped } = entriesFromWebhook(webhook, { maxEntries, maxBytes, maxEntryBytes, maxStackLines });
|
|
103
|
+
if (unmapped) { stats.unmapped++; return { entries: 0 }; }
|
|
104
|
+
stats.entries += entries.length;
|
|
105
|
+
await consumer(entries);
|
|
106
|
+
return { entries: entries.length };
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
}
|
|
@@ -12,7 +12,8 @@ import { dlpConfigFromEnv } from "../../dlp/index.js";
|
|
|
12
12
|
import { ContentRefRegistry } from "./ref-registry.js";
|
|
13
13
|
import { createContentHandler, SERVED_KINDS } from "./handler.js";
|
|
14
14
|
import { createContentIndexer } from "./content-index.js";
|
|
15
|
-
import { sizeInsideSync } from "./path-guard.js";
|
|
15
|
+
import { sizeInsideSync, normalizeRel, resolveInside, readRegular } from "./path-guard.js";
|
|
16
|
+
import { decodeText } from "./egress.js";
|
|
16
17
|
|
|
17
18
|
export const CONTENT_CAPABILITIES = Object.freeze(SERVED_KINDS.map((k) => `content.${k}`));
|
|
18
19
|
|
|
@@ -58,7 +59,32 @@ export function installContentHandlers({ link, tenantId, wsManager, getWorkspace
|
|
|
58
59
|
const closeLink = link.close.bind(link);
|
|
59
60
|
link.close = () => { handler.cancelAll(); indexer.close(); registry.clear(); closeLink(); };
|
|
60
61
|
|
|
62
|
+
/**
|
|
63
|
+
* F3-10: the job runners read client files through the SAME gate as content.request kind `file`: an opaque id minted by this agent for this
|
|
64
|
+
* workspace (never a path), the deny list, the root and symlink checks, the size cap. Returns the UTF-8 text (a runner that sends it anywhere
|
|
65
|
+
* does so through the DLP: the LLM client masks every prompt) or a status: not_found | forbidden | too_large.
|
|
66
|
+
* @returns {Promise<{ok:true, text:string} | {ok:false, status:string}>}
|
|
67
|
+
*/
|
|
68
|
+
const readAllowedFile = async ({ workspaceId, ref, maxBytes, signal }) => {
|
|
69
|
+
try {
|
|
70
|
+
const ws = resolveWorkspace(workspaceId);
|
|
71
|
+
if (!ws) return { ok: false, status: "not_found" };
|
|
72
|
+
const target = registry.resolve(ref, { workspaceId, root: ws.root, kind: "file" });
|
|
73
|
+
const norm = target && normalizeRel(target.rel);
|
|
74
|
+
if (!norm) return { ok: false, status: "not_found" };
|
|
75
|
+
const resolved = await resolveInside(ws.root, norm, config.extraDenyGlobs, { allowHardlinks: config.allowHardlinks });
|
|
76
|
+
if (!resolved.ok) return { ok: false, status: resolved.status };
|
|
77
|
+
const read = await readRegular(resolved, Math.min(Number.isSafeInteger(maxBytes) && maxBytes > 0 ? maxBytes : config.maxBytes, config.maxBytes), signal || new AbortController().signal, { allowHardlinks: config.allowHardlinks });
|
|
78
|
+
if (!read.ok) return { ok: false, status: read.status };
|
|
79
|
+
const text = decodeText(read.buffer);
|
|
80
|
+
return text === null ? { ok: false, status: "forbidden" } : { ok: true, text };
|
|
81
|
+
} catch {
|
|
82
|
+
return { ok: false, status: "not_found" }; // never an exception with a path in it
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
|
|
61
86
|
link.content = {
|
|
87
|
+
readAllowedFile,
|
|
62
88
|
registry,
|
|
63
89
|
handler,
|
|
64
90
|
indexer,
|
|
@@ -2,15 +2,15 @@
|
|
|
2
2
|
"repo": "VISIONHAMMER/vh-api-gateway",
|
|
3
3
|
"path": "docs/contracts/la-protocol-v2",
|
|
4
4
|
"branch": "release/qa",
|
|
5
|
-
"commit": "
|
|
5
|
+
"commit": "b9621b2a30bb38521b9caab24a79e8bdf536a2d3",
|
|
6
6
|
"note": "Copied by scripts/sync-la-protocol-v2-contract.mjs. Do not edit: change the gateway folder and sync again.",
|
|
7
7
|
"files": {
|
|
8
|
-
"README.md": "
|
|
8
|
+
"README.md": "9a6f2f3adc098aba38c2da793c1c3db9bb67d1ec5343f74533dff667c2c1e0fb",
|
|
9
9
|
"ack.schema.json": "d6dfdaa8b1ec2dd47db9e858cd047ca860196ab6b0ad350f8ccb5d2596441ccf",
|
|
10
10
|
"alert.schema.json": "01a8a77840be8ab7901ed7d07c3d68a2295bfaeb638694564e19e60be1747054",
|
|
11
11
|
"approval.decision.schema.json": "a9c7bc74d146465afde1d5830839165a02f232b830b5f6cfd8c8e91fb6da1563",
|
|
12
12
|
"audit.event.schema.json": "e9ceb58ea00077151373955957b8319324de488c80da4063f6d0dcb40a8b3ed0",
|
|
13
|
-
"config.update.schema.json": "
|
|
13
|
+
"config.update.schema.json": "432438b0c93ddafa0741e47f99b7357ad5f2b06b606779bf846c282e2f8a5639",
|
|
14
14
|
"content.index.schema.json": "1abc24344435f0f392d5b626c3abb6879d8f9dd2c1d0463840a67634a5caa71e",
|
|
15
15
|
"content.request.schema.json": "440e3675246657e9aae23e51176168bc21c91283b5fcaf6782111a18110bf76e",
|
|
16
16
|
"content.response.schema.json": "c9a66ac9cbbcaec7fc182b3e3fa23b34877bd342592521c7e393779230332231",
|
|
@@ -19,24 +19,24 @@
|
|
|
19
19
|
"error.schema.json": "b1fd663d4a1ea73d88cb575422f15f8c7279001f9385ebb2d5e1889b5d5ef878",
|
|
20
20
|
"file.list.body.schema.json": "dd58ac2af91bbde53e0fd585dc55af0adb01380492d1604f21ed8e972d7ba8d1",
|
|
21
21
|
"heartbeat.schema.json": "8aff84a1ef3ca56bdff8819abb3b0ab01a2547814c2ea2f9ee893cb08f7a08f1",
|
|
22
|
-
"hello.schema.json": "
|
|
23
|
-
"incident.candidate.schema.json": "
|
|
24
|
-
"index.json": "
|
|
25
|
-
"job.assign.schema.json": "
|
|
22
|
+
"hello.schema.json": "a90a6bf0e9a8a20e4c6edcf2137931cf044846e7924977228275ef648a48934f",
|
|
23
|
+
"incident.candidate.schema.json": "ffec36aca6916079debc2c6060d093819ea3ea660b57c0c92ac3b541a0e99000",
|
|
24
|
+
"index.json": "de79c065217e1fa8430c4266ec19fd57df35fc4182eb7c7246b6f1ea03d2630c",
|
|
25
|
+
"job.assign.schema.json": "c61bf42bfd94003ca373f94f2f7adb93026b1b96079beab97662497c629cbec1",
|
|
26
26
|
"job.cancel.schema.json": "40ef64373283e17306d857c9a40081689786015294b0b88ab8f68f063bd2cfb8",
|
|
27
27
|
"job.event.schema.json": "493d08f9db4f7ffd4a97403ee33ca4a73a65dc8f774d8f18ef0a137fdb5fc470",
|
|
28
28
|
"job.result.schema.json": "9cbc61211482557283b8f4df44e79011ead1abbacdb2d986138199c747edc043",
|
|
29
29
|
"log.stats.schema.json": "4b33b111946a19793a60756793294e90911b06f7050e48821759f2497ec17641",
|
|
30
|
-
"ready.schema.json": "
|
|
31
|
-
"register.schema.json": "
|
|
30
|
+
"ready.schema.json": "02e51ba5aad3357c9840c14eb45cb86b7aab0c10e349c94c720cda1a0b5b8366",
|
|
31
|
+
"register.schema.json": "bec30062cf99657a1ecb94bad004f502615901e36bc2b0ba62351ed021e059ac",
|
|
32
32
|
"scan.upload.schema.json": "b1bba86ee362f73afc94c8c216977c12f67f770490e81002d14789397692f23b",
|
|
33
33
|
"scan.uploaded.schema.json": "f4c6aed7b9363b3ce1a9e14f9c4f4fb003bcf104b60acf74421dd0c75fc5d229",
|
|
34
|
-
"webhook.forward.schema.json": "
|
|
35
|
-
"vectors/ack.vectors.json": "
|
|
34
|
+
"webhook.forward.schema.json": "9ab66adf31386445c2b76930155ffa6874326395d968582d2345ee7fb5b77f82",
|
|
35
|
+
"vectors/ack.vectors.json": "1756c1169cf5f42c0ab6b1c094071d1f8ccf2fe9efdbc97101e402e93bd65a67",
|
|
36
36
|
"vectors/alert.vectors.json": "0bf179ff390bbecf07a117b80237cb7ac8778864e5780f36d68c8c9a64db8f04",
|
|
37
37
|
"vectors/approval.decision.vectors.json": "338115973bf2c4390d2bb33ce8f4079f3132cd739cd5505e1e873fdc252d2271",
|
|
38
38
|
"vectors/audit.event.vectors.json": "d05c1820103eef041a84eb8b13d9a67a8a5b54b9b54ee3b6e43fd4ef15ed7000",
|
|
39
|
-
"vectors/config.update.vectors.json": "
|
|
39
|
+
"vectors/config.update.vectors.json": "e61865fad9b4efa808e02877c4352d3720c8069a0b31261701b369d8ae11f9cf",
|
|
40
40
|
"vectors/content.index.vectors.json": "f167db30910989c46f317313d651ca59f9cd9d881d2ea81b84912155d18227b7",
|
|
41
41
|
"vectors/content.request.vectors.json": "fdf0f4d9e19f6480e20e40f4652bf8205da2b113620f8461dfaeb0d61add36a0",
|
|
42
42
|
"vectors/content.response.vectors.json": "cbb47cbf8a9512aa5360eea425f0cdd9064334311d37834dc75bddff07dadc5d",
|
|
@@ -44,15 +44,15 @@
|
|
|
44
44
|
"vectors/error.vectors.json": "f5fd7954f2c1736ee0943eb330c2f85b7fdd7a8959e4f043f9cc9e9bcda39aa5",
|
|
45
45
|
"vectors/file.list.body.vectors.json": "da1dcc5d4ec976853161bef4b3947572f36b5526cbe16b838ac4ef72ddded403",
|
|
46
46
|
"vectors/heartbeat.vectors.json": "cfd2e048139c67aa925f1fa94c05326e75248839645c1a1e4d6d3bb60cafa624",
|
|
47
|
-
"vectors/hello.vectors.json": "
|
|
48
|
-
"vectors/incident.candidate.vectors.json": "
|
|
49
|
-
"vectors/job.assign.vectors.json": "
|
|
47
|
+
"vectors/hello.vectors.json": "7a50c4e54e6a4ef1227fd7485071bca8ec7b797c3b2907fddda3d8836c3feab6",
|
|
48
|
+
"vectors/incident.candidate.vectors.json": "a78eb4b103f411f1ec64d9eeb4c39623851e710ac0cc20b4d78b2739b7ff78d5",
|
|
49
|
+
"vectors/job.assign.vectors.json": "0358a8e3bd94a332ef922abd80faf6e07bb8dbec0436cb463a4826a47e4d65e2",
|
|
50
50
|
"vectors/job.cancel.vectors.json": "0fc9f99c8662435e9425389068adf418273489dc73928a21eae1b283256b9bc5",
|
|
51
51
|
"vectors/job.event.vectors.json": "03a2a2435918ba561821916bf4e435f2780210900139ecf68091acac6abf200c",
|
|
52
52
|
"vectors/job.result.vectors.json": "05995fbe65e5357705d75e9b660b517f54c6902069d36b62190b1b79f283e7a5",
|
|
53
53
|
"vectors/log.stats.vectors.json": "a8a67e09ef0f87b81f4028b77427ed7afd809c3f8534b8423a712b2ceef9a27f",
|
|
54
|
-
"vectors/ready.vectors.json": "
|
|
55
|
-
"vectors/register.vectors.json": "
|
|
54
|
+
"vectors/ready.vectors.json": "16327f5e5eccc57a0ec635bcde96116feeeeec7ddbc4b621d4d070738cdc48d5",
|
|
55
|
+
"vectors/register.vectors.json": "b612e9462b1e2ac198442afccf450f1535daadc4c32d87f6a99b9752c343c486",
|
|
56
56
|
"vectors/scan.upload.vectors.json": "e8a6f73fe91ac58a3222b68cf28124c1254aa4f00c3ef8f472deb7de31cab76e",
|
|
57
57
|
"vectors/scan.uploaded.vectors.json": "7314ea15b1afa7b9625048fb7e245492d8910bc0dae8a52ea62afe2a1e240382",
|
|
58
58
|
"vectors/traces/invalid-agent-mismatch.json": "5f17b867de71c1bcc6c42723acb53a50338c0e491193499ea25c2220ea4fa558",
|
|
@@ -76,11 +76,19 @@
|
|
|
76
76
|
"vectors/traces/invalid-signature-warn-with-enforce-required.json": "71b5b3dc5dce52417173934db1e0de6542ecae5798e87124d54a8017493312e1",
|
|
77
77
|
"vectors/traces/invalid-tenant-mismatch.json": "dcc04ea423bddef20524ead7bfd146a5d9c42fe474c9216dc38c90544d291151",
|
|
78
78
|
"vectors/traces/invalid-type-outside-allowlist.json": "e492d326cfa044d3fc9f7e48dee63a52b715a62ba4867fc5ce21f1d86e3c435f",
|
|
79
|
+
"vectors/traces/invalid-webhook-chunk-after-last.json": "9a35faaeeafce045a887e643f603ca69950872d62d27d17086513103999aefd6",
|
|
80
|
+
"vectors/traces/invalid-webhook-chunk-body-hash-changes.json": "7ade576a8dc8e9671383e85fbe6aae414863cbb1cde38737a2cb78efce9c9528",
|
|
81
|
+
"vectors/traces/invalid-webhook-chunk-last-flag-wrong.json": "8505b5fbbc0a8dc97021a1c02f9883df95f9401bc18eaa2498331b0b6b7d155c",
|
|
82
|
+
"vectors/traces/invalid-webhook-chunk-non-last-too-short.json": "ff1d64b019c082e7a5c6537fdbf069025a532c764e62c532a24a3b7257f4b6ab",
|
|
83
|
+
"vectors/traces/invalid-webhook-chunk-out-of-order.json": "b6baa056b4e3347deaf0a4b901a126a3f93d20627faaa3951028bd47894715f4",
|
|
84
|
+
"vectors/traces/invalid-webhook-chunk-sha-mismatch.json": "1bb79d0e1794cd791625ed2a7eab154e20150c5d7bee7c86aa97f9bf4aaeab6d",
|
|
85
|
+
"vectors/traces/invalid-webhook-chunk-total-changes.json": "5edf3c1916927f94ef15061a13a6ded8a0168cb0492158f34db8160c0d6bdbe4",
|
|
79
86
|
"vectors/traces/valid-content-chunks-in-order.json": "c5fc301527b062b30c625da2798c7cfab3e3debac29a36f6ec7c699705267ab3",
|
|
80
87
|
"vectors/traces/valid-content-index.json": "9305dc955c4cca1f09158a9c8cf21d9e89d6a6b03646088c547f64a61cd59c30",
|
|
81
88
|
"vectors/traces/valid-handshake-and-job.json": "9aa3c423c496403ca77d63965f1805f0a57ea9f6f8a01351310515b265aeb9cc",
|
|
82
89
|
"vectors/traces/valid-resume-continues-seq.json": "8aa90f24598f8f2d87a5355ec58c3531d16fd11ad41c782f53eca063372b704a",
|
|
83
90
|
"vectors/traces/valid-retransmit-is-idempotent.json": "924e9d296a67f0006c289e7cc978b9529ae1aa03bec2cd331a62f31354a90257",
|
|
84
|
-
"vectors/webhook.
|
|
91
|
+
"vectors/traces/valid-webhook-chunks-in-order.json": "2e5e6313a7fe9b3f7fea23d6a0c928d6bfef69b2a7179f1578230425f451ada9",
|
|
92
|
+
"vectors/webhook.forward.vectors.json": "991a677d72a9c698ed3c8d960af532daf8f3158d77a4bea7d08750ccd8693250"
|
|
85
93
|
}
|
|
86
94
|
}
|