agenticreplay 0.1.0 → 0.1.2

package/packages/cli/dist/agent-spans.d.ts

@@ -0,0 +1,238 @@
+/**
+ * Getting the OpenAI Agents SDK's own run structure into a trace, without editing the agent.
+ *
+ * A proxy sees `POST /v1/responses` and cannot tell which agent sent it, that a handoff occurred, or
+ * that a guardrail ran at all. Measured on a two-agent run with both — the same script recorded
+ * twice, once with this layer and once without:
+ *
+ *     which agent a turn belonged to        trace: no    spans: yes
+ *     that a handoff happened, and from whom trace: no    spans: yes
+ *     that a guardrail ran                   trace: no    spans: yes
+ *
+ * The handoff row is the sharp one. The SDK implements a handoff as a function tool named
+ * `transfer_to_<agent>`, so a rule *could* guess one from the tool name — but a user tool may be
+ * called that too, and the agent it came **from** never reaches the wire. `agenticreplay graph` already
+ * separates what a trace records from what a rule infers; this moves handoffs into the first column.
+ *
+ * Installed the way the fetch hook is, and for the same reason: `sitecustomize.py` is imported by
+ * Python at startup from anywhere on `sys.path`, so putting one directory on `PYTHONPATH` attaches
+ * the layer to an agent nobody modified. Asking the user to call `set_trace_processors` would make
+ * this the SDK wrapper the README says agenticreplay does not need.
+ *
+ * The file is self-contained and imports the published package by name, never this repository — it
+ * runs inside the agent's interpreter, which resolves imports against its own environment. The
+ * shell shim and the fetch hook are written out for the same reason.
+ */
+export declare const SPANS_ENV = "AGENTICREPLAY_AGENT_SPANS";
+export declare const SPANS_FILENAME = "agent-spans.jsonl";
+export declare const SITECUSTOMIZE = "sitecustomize.py";
+/** A Python framework whose own run structure agenticreplay can read, and the package that reads it. */
+export interface PythonAdapter {
+    /** The framework, as `importlib.metadata` names the distribution that provides it. */
+    distribution: string;
+    /** The top-level module the agent imports to use it, which is how the bootstrap notices it. */
+    root: string;
+    /** The adapter's import name, for the bootstrap's `from … import install`. */
+    module: string;
+    /** The adapter's distribution name, as `pip install` takes it. */
+    package: string;
+    /** What the trace goes without when the framework is used and the adapter is not there. */
+    lost: string;
+}
+/**
+ * Every adapter the bootstrap installs — one declaration, three consumers.
+ *
+ * The bootstrap's Python is generated from this, {@link agentSpanLosses} reports against it, and
+ * the drain's warning reads its prose from it. Adding a row is the whole of adding an adapter to
+ * the record path, and nothing can be added to one of the three and forgotten in the others.
+ *
+ * They share a transport file on purpose. The reader recovers a record from a torn line by finding
+ * `{"kind":` in it, which is a property of the writers rather than of the file, so a second writer
+ * costs nothing; a second file would cost a second environment variable, a second drain and a
+ * second entry in the stale-transport sweep.
+ */
+export declare const PYTHON_ADAPTERS: readonly PythonAdapter[];
+/**
+ * The bootstrap, as it is written into the run directory.
+ *
+ * Every path through it ends in a return rather than a traceback. It runs before the agent's first
+ * statement, in *every* Python process the recording starts — including `python --version` — so a
+ * failure here is a failure of the run rather than of the capture. Absence of an adapter, absence
+ * of the framework and a framework whose interface moved are all the same non-event.
+ *
+ * The adapters come from {@link PYTHON_ADAPTERS}, so adding one is a row in that list rather than
+ * an edit to this string.
+ *
+ * It also chains to whatever `sitecustomize` it displaced. Ours arrives via `PYTHONPATH` and so wins
+ * over a site's or a virtualenv's own; silently disabling someone's startup hook to add a debugging
+ * layer would be a poor trade.
+ */
+export declare const SITECUSTOMIZE_SOURCE: string;
+export interface AgentSpanCapture {
+    /** Where the processor writes, and what the child is told. */
+    spansPath: string;
+    /** Prepended to PYTHONPATH so Python finds the bootstrap. */
+    pythonPath: string;
+    /** The directory holding {@link spansPath}, for {@link discardAgentSpanTransport}. */
+    transportDir: string;
+}
+/**
+ * Write the bootstrap into the run directory and say how to point a child at it.
+ *
+ * The spans file is **not** in the run directory, and that is the point. It is a transport: agenticreplay
+ * reads it once, turns what it needs into events, and those go to disk through `TraceWriter`, which
+ * redacts. The file itself is written by the agent's own interpreter, so nothing agenticreplay owns can
+ * redact it on the way in — and while it sat in the run directory that made it a sink the write path
+ * never touched, `agenticreplay scrub` never rewrote (it reaches `events.jsonl`, the manifest and the blobs,
+ * and nothing else), and nobody deleted, in a directory people share. `agenticreplay scrub --match <secret>`
+ * over such a run printed "nothing matched — the trace is unchanged" with the secret still beside
+ * the trace, which is the one failure mode SECURITY.md says a scrubber must not have.
+ *
+ * Somewhere agenticreplay owns fixes it. `mkdtemp` gives a directory only this user can enter, with a name
+ * nobody can guess ahead of it, and {@link discardAgentSpanTransport} takes the whole thing away
+ * when the run is done. The file is pre-created here rather than left to the child, so it carries
+ * the 0600 SECURITY.md promises instead of whatever the agent's umask happens to be — the same thing
+ * the shell shim does with its frames file, for the same reason.
+ */
+export declare function installAgentSpans(runDir: string): Promise<AgentSpanCapture>;
+/** Names the process that minted a transport, so a sweep can ask whether it is still running. */
+export declare const TRANSPORT_OWNER = "owner.pid";
+/**
+ * Say, for as long as this run lasts, that its transports are not orphans.
+ *
+ * {@link sweepStaleTransports} asks two questions and only deletes when both say abandoned. The
+ * second one — has anything in there been written in the last day — is the one that had to stand
+ * alone whenever the first could not be answered, which is the shared-temp-directory case: `/tmp`
+ * bind-mounted into a devcontainer, a CI job with a shared tmp volume, an NFS `/tmp`. And silence
+ * is not evidence of abandonment, because a transport is only written when a span *ends* or the
+ * agent *runs a command*. An agent that spends a day on one model turn, or on one long command,
+ * writes nothing for a day while being entirely alive.
+ *
+ * The consequence was not a stale directory: the next `agenticreplay record` anywhere on that machine would
+ * delete a live run's transport, and both writers swallow the `ENOENT` that follows by design
+ * (`shell-shim/src/runner.ts`, `processor.py`). Every remaining frame and span would be lost
+ * without a word, the trace would still report success, and `shell.ineffective` would go on to
+ * blame the harness for commands agenticreplay itself had deleted the evidence of.
+ *
+ * A heartbeat answers the question instead of widening the window it is asked in. While the owner
+ * lives it keeps saying so, in the one way a sweep in another pid namespace can still read; an
+ * orphan stops saying it the moment its owner dies, so the collection the sweep exists for is
+ * untouched. It is the owner file that gets touched, because that is the one entry that exists from
+ * the moment the transport does.
+ *
+ * `dirs` is read on every beat rather than copied, so a transport installed part-way into a run is
+ * covered from the next beat without the caller having to say so. Stopping is the caller's job and
+ * belongs on every exit path; the timer is `unref`ed as well, so that a stop missed on some future
+ * path can still never be what holds the process open — the hang `record-teardown.test.ts` exists
+ * to catch.
+ */
+export declare function touchTransports(dirs: readonly string[], everyMs?: number): () => void;
+/**
+ * Remove transports that outlived the run that made them.
+ *
+ * The drain removes one, and `recordCommand` removes one whose run threw. Neither runs when agenticreplay
+ * itself is killed — `SIGKILL`, a `taskkill`, a power cut — and then the directory stays, holding
+ * every command line the agent ran, un-redacted, under a name nobody was ever shown. Deleting
+ * harder at exit does not help, because nothing gets to run at exit; the only thing that can
+ * collect an orphan is the next run.
+ *
+ * **Whose it is, not how old it is.** Age was the first answer and it was wrong: a transport
+ * directory's mtime is set when `mkdtemp` creates it and never moves again, because everything
+ * after that is an *append* to the one file inside, and appending does not touch the containing
+ * directory. So "older than a day" meant "this recording started more than a day ago", and any
+ * other `agenticreplay record` on the machine would collect the live transport of a recording still in
+ * progress — after which that run silently captures nothing, because both writers swallow their
+ * errors by design.
+ *
+ * **Two signals, and deletion needs both.** The owner's pid comes first: while it resolves to a
+ * running process, nothing else matters. But a pid only means anything in the pid namespace it was
+ * written in, and a temp directory can be shared across them — `/tmp` bind-mounted into a
+ * devcontainer, a CI job with a shared tmp volume, an NFS `/tmp`. There the owner of a live
+ * recording is simply absent from this process's pid table, and `ESRCH` is indistinguishable from
+ * "it has exited". Reasoning that a pid fails safely is true of *reuse*, which leaves an orphan,
+ * and false of *unresolvable*, which was deleting a run's transport mid-recording.
+ *
+ * So when the owner cannot be accounted for, the transport has to have been silent as well. The
+ * mtime of the file inside is the right thing to read, and the directory's is not — appending moves
+ * the file's and never the directory's, which is the mistake the version before this one made.
+ *
+ * What is deliberately not written is a start time alongside the pid. It would only be useful for
+ * telling a reused pid from the original, and that needs the *other* process's start time, which
+ * Node cannot portably read. A field nothing can act on is how payload gets into a file.
+ *
+ * Two gates left one gap between them, and {@link touchTransports} closes it rather than accepting
+ * it: a recording in a shared temp directory that captured nothing for a day satisfied both at
+ * once, and a day of silence is what an agent thinking about one long turn looks like. Its owner
+ * now refreshes the transport for as long as the run lasts, so reaching this line means the owner
+ * has stopped saying anything, not that the agent had nothing to say.
+ *
+ * Best-effort throughout: a directory with no owner is left alone rather than guessed at, a
+ * directory that cannot be read is somebody else's problem, and nothing here is ever fatal.
+ *
+ * `root` is a parameter so a test can sweep a directory of its own rather than the machine's, which
+ * is also what makes `now` safe to vary — reaching forward in time inside somebody else's temp
+ * directory is how an earlier test took the live transports of everything running beside it.
+ */
+export declare function sweepStaleTransports(root?: string, now?: number): Promise<number>;
+/**
+ * Take the transport away once it has been read.
+ *
+ * The directory, not the file: the whole point of minting one was that nothing else is in it, and
+ * removing the directory also takes anything a stray writer created beside the file after the drain.
+ * Returns a message rather than throwing — a run that produced a trace must not fail at teardown.
+ */
+export declare function discardAgentSpanTransport(dir: string): Promise<string | undefined>;
+/** PYTHONPATH with our directory in front, keeping whatever was already there. */
+export declare function pythonPathWith(dir: string, existing: string | undefined, sep: string): string;
+/** One span, as the processor writes it. */
+export interface AgentSpan {
+    kind: string;
+    type?: string;
+    span_id?: string;
+    parent_id?: string;
+    trace_id?: string;
+    started_at?: string;
+    ended_at?: string;
+    error?: unknown;
+    data?: Record<string, unknown>;
+}
+/**
+ * Read what the processor wrote.
+ *
+ * A malformed line is skipped rather than thrown on: the file is appended to by another process
+ * that may have been killed mid-write, and losing the last span is a much better outcome than
+ * failing to seal a trace that is otherwise complete.
+ */
+export declare function readAgentSpans(path: string): Promise<AgentSpan[]>;
+/**
+ * What this layer captured nothing of, and why — the two ways it can come back empty.
+ *
+ * Both are read off the same records `readAgentSpans` already returns, because both are written by
+ * the only two things that know: the bootstrap, which finds out at import time that the adapter is
+ * missing, and the processor, which is the only party that saw the record it could not write.
+ *
+ * `eventForSpan` ignores them — it answers only to `kind === 'span'` — so neither reaches the trace
+ * as an event. They are for the operator, now, while the run is still on screen.
+ */
+export interface AgentSpanLosses {
+    /** The framework was used and its adapter was not there, so nothing structural was captured. */
+    unavailable: string[];
+    /** Records an adapter held and could not write: unserialisable, or the append failed. */
+    dropped: number;
+    /**
+     * Which adapters reported those drops, for the records that said.
+     *
+     * The count alone stopped being actionable when there was more than one adapter: several write
+     * to one transport, so a total says how much was lost and nothing about where to look. Read
+     * rather than merely written — `_dropped` in the first adapter was counted from the start and
+     * read by nothing, which is how a field becomes payload.
+     */
+    droppedBy: string[];
+}
+export declare function agentSpanLosses(spans: AgentSpan[]): AgentSpanLosses;
+/** A trace event, or undefined for a span that carries nothing the proxy lacks. */
+export declare function eventForSpan(span: AgentSpan): {
+    type: string;
+    attrs: Record<string, unknown>;
+} | undefined;
+//# sourceMappingURL=agent-spans.d.ts.map

package/packages/cli/dist/agent-spans.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-spans.d.ts","sourceRoot":"","sources":["../src/agent-spans.ts"],"names":[],"mappings":"AAYA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,eAAO,MAAM,SAAS,8BAA8B,CAAC;AACrD,eAAO,MAAM,cAAc,sBAAsB,CAAC;AAClD,eAAO,MAAM,aAAa,qBAAqB,CAAC;AAEhD,wGAAwG;AACxG,MAAM,WAAW,aAAa;IAC5B,sFAAsF;IACtF,YAAY,EAAE,MAAM,CAAC;IACrB,+FAA+F;IAC/F,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,MAAM,EAAE,MAAM,CAAC;IACf,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;IAChB,2FAA2F;IAC3F,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,eAAe,EAAE,SAAS,aAAa,EAenD,CAAC;AA2BF;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,oBAAoB,QA6IhC,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,8DAA8D;IAC9D,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAwCjF;AAKD,iGAAiG;AACjG,eAAO,MAAM,eAAe,cAAc,CAAC;AAoB3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,GAAE,MAA+B,GACvC,MAAM,IAAI,CAWZ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,SAAW,EAAE,GAAG,SAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CA6B7F;AA8CD;;;;;;GAMG;AACH,wBAAsB,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAOxF;AAED,kFAAkF;AAClF,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAI7F;AAED,4CAA4C;AAC5C,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAuCvE;AAWD;;;;;;;;;GASG;AACH,MAAM,WAAW,eAAe;IAC9B,gGAAgG;IAChG,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,yFAAyF;IACzF,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,eAAe,CA4BnE;AAED,mFAAmF;AACnF,wBAAgB,YAAY,CAC1B,IAAI,EAAE,SAAS,GACd;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAAG,SAAS,CAwE9D"}