@tangle-network/traces 0.8.0 → 0.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -1
- package/dist/{chunk-BUD6I3ZJ.js → chunk-PRRWLKSI.js} +1943 -323
- package/dist/cli.js +309 -12
- package/dist/index.d.ts +471 -27
- package/dist/index.js +71 -5
- package/package.json +12 -13
package/dist/index.d.ts
CHANGED
|
@@ -11,13 +11,12 @@ export { HostedClient, createHostedClient, hostedClientFromEnv } from '@tangle-n
|
|
|
11
11
|
/**
|
|
12
12
|
* OTLP-flat-line emitter — the canonical normalized trace shape.
|
|
13
13
|
*
|
|
14
|
-
* Every harness adapter projects its native session log onto `OtlpSpan[]
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* form, which `external.ts` `toCanonicalOpenInference` produces.
|
|
14
|
+
* Every harness adapter projects its native session log onto `OtlpSpan[]`.
|
|
15
|
+
* `serializeSpans` emits one COMPLETE OpenInference span per JSONL line
|
|
16
|
+
* (top-level `kind`, `resource`, `scope`, string `parent_span_id`, plus the
|
|
17
|
+
* `openinference.span.kind` attribute). One standard artifact feeds three
|
|
18
|
+
* consumers unchanged: `@tangle-network/agent-eval`'s `OtlpFileTraceStore`
|
|
19
|
+
* (which reads the attribute vocabulary), HALO, and any OpenInference tool.
|
|
21
20
|
*
|
|
22
21
|
* Attribute vocabulary the downstream analysts key off:
|
|
23
22
|
* - `openinference.span.kind` → span kind (AGENT | LLM | TOOL | CHAIN)
|
|
@@ -149,6 +148,18 @@ declare const ATTR: {
|
|
|
149
148
|
readonly INGEST_SOURCE: "tangle.ingest_source";
|
|
150
149
|
readonly HARNESS: "tangle.harness";
|
|
151
150
|
readonly CWD: "tangle.cwd";
|
|
151
|
+
/** THE per-session grouping key the spine derives subjects from
|
|
152
|
+
* (`deriveSubjectKey` reads `tangle.subject.key` first). Resolves to the git
|
|
153
|
+
* remote (host/owner/repo) when readable, else the cwd path basename. */
|
|
154
|
+
readonly SUBJECT_KEY: "tangle.subject.key";
|
|
155
|
+
/** Normalized git remote (e.g. `github.com/tangle-network/agent-dev-container`). */
|
|
156
|
+
readonly GIT_REPOSITORY: "git.repository";
|
|
157
|
+
/** Current branch at conversion time. */
|
|
158
|
+
readonly GIT_BRANCH_NAME: "git.branch";
|
|
159
|
+
/** HEAD short sha at conversion time. */
|
|
160
|
+
readonly GIT_COMMIT: "git.commit";
|
|
161
|
+
/** How traces resolved the session cwd: recorded cwd, repaired cwd, or span path. */
|
|
162
|
+
readonly REPO_RESOLUTION_SOURCE: "traces.repo_resolution_source";
|
|
152
163
|
readonly GIT_BRANCH: "tangle.git_branch";
|
|
153
164
|
readonly HOST: "tangle.host";
|
|
154
165
|
/** Basename of the session file. (Renamed from the ambiguous `tangle.source`,
|
|
@@ -201,6 +212,13 @@ interface AdapterSelection {
|
|
|
201
212
|
* A named harness that doesn't resolve throws rather than silently dropping. */
|
|
202
213
|
declare function selectAdapters(opts: AdapterSelection): HarnessTraceAdapter[];
|
|
203
214
|
|
|
215
|
+
/**
|
|
216
|
+
* Parse one session to spans and stamp per-session repo/git resource attrs
|
|
217
|
+
* (`tangle.subject.key` etc.) derived from the ref's cwd. Every OTLP-producing
|
|
218
|
+
* path funnels through here so the spine can group by repo. Repo resolution is
|
|
219
|
+
* computed ONCE per session; it is fail-safe and never throws.
|
|
220
|
+
*/
|
|
221
|
+
declare function parseSession(adapter: HarnessTraceAdapter, ref: SessionRef): Promise<OtlpSpan[]>;
|
|
204
222
|
interface ScanOptions extends AdapterSelection {
|
|
205
223
|
/** Filter by working directory (exact/prefix). */
|
|
206
224
|
cwd?: string;
|
|
@@ -223,6 +241,37 @@ interface ScannedSession {
|
|
|
223
241
|
* errors route to `onError` and skip that adapter/session, never aborting. */
|
|
224
242
|
declare function scanSessions(opts: ScanOptions): AsyncGenerator<ScannedSession>;
|
|
225
243
|
|
|
244
|
+
type RepoResolutionSource = 'none' | 'ref-cwd' | 'repaired-cwd' | 'span-path' | 'span-workdir';
|
|
245
|
+
/** Resource-attribute keys this resolver may stamp. */
|
|
246
|
+
type RepoAttrs = Partial<Record<typeof ATTR.SUBJECT_KEY | typeof ATTR.GIT_REPOSITORY | typeof ATTR.GIT_BRANCH_NAME | typeof ATTR.GIT_COMMIT | typeof ATTR.CWD, string>> & Partial<Record<typeof ATTR.REPO_RESOLUTION_SOURCE, RepoResolutionSource>>;
|
|
247
|
+
interface RepoResolution {
|
|
248
|
+
/** Resource attributes to stamp onto spans. */
|
|
249
|
+
attrs: RepoAttrs;
|
|
250
|
+
/** Best-known cwd/repo directory after repair or span-path inference. */
|
|
251
|
+
cwd: string | null;
|
|
252
|
+
/** Which signal produced the selected cwd. */
|
|
253
|
+
source: RepoResolutionSource;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Normalize a git remote url to `host/owner/repo`, dropping scheme, auth, the
|
|
257
|
+
* `.git` suffix, and any trailing slash. Handles both `https://` and the
|
|
258
|
+
* `git@host:owner/repo.git` SCP-style forms. Returns null when unparseable.
|
|
259
|
+
*/
|
|
260
|
+
declare function normalizeRemote(url: string): string | null;
|
|
261
|
+
/**
|
|
262
|
+
* Resolve per-session repo/git resource attributes from a session's cwd.
|
|
263
|
+
* Fail-safe and never throws; see the module doc for the derivation contract.
|
|
264
|
+
*/
|
|
265
|
+
declare function resolveRepoAttrs(cwd: string | null | undefined): Promise<RepoAttrs>;
|
|
266
|
+
declare function resolveSessionRepoAttrs(cwd: string | null | undefined, spans: readonly OtlpSpan[]): Promise<RepoResolution>;
|
|
267
|
+
/**
|
|
268
|
+
* Stamp resolved repo attrs onto every span's attributes (so they land in the
|
|
269
|
+
* OTLP resource attributes via `toOpenInferenceSpan`). Additive — existing
|
|
270
|
+
* `service.name` / `agent.name` are untouched. Mutates in place and returns the
|
|
271
|
+
* same array for ergonomic chaining. A no-op when `attrs` is empty.
|
|
272
|
+
*/
|
|
273
|
+
declare function stampRepoAttrs(spans: readonly OtlpSpan[], attrs: RepoAttrs): readonly OtlpSpan[];
|
|
274
|
+
|
|
226
275
|
/**
|
|
227
276
|
* Claude Code adapter — `~/.claude/projects/<encoded-cwd>/<session>.jsonl`.
|
|
228
277
|
*
|
|
@@ -314,6 +363,11 @@ declare class GeminiFamilyAdapter implements HarnessTraceAdapter {
|
|
|
314
363
|
private readonly homeDirName;
|
|
315
364
|
constructor(cfg: GeminiFamilyConfig);
|
|
316
365
|
private chatsRoot;
|
|
366
|
+
/** `<home>/projects.json` maps `{ <absolute-cwd>: <projectName> }`. The session
|
|
367
|
+
* dir name (`tmp/<projectHash>/`) is that projectName for registered projects,
|
|
368
|
+
* so inverting the map recovers cwd. (Unregistered/older dirs are opaque cwd
|
|
369
|
+
* digests and stay null.) Loaded once per locate(). */
|
|
370
|
+
private loadProjectCwds;
|
|
317
371
|
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
318
372
|
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
319
373
|
}
|
|
@@ -461,6 +515,20 @@ declare const CONTENT_CAP = 8000;
|
|
|
461
515
|
* visible `… [+N chars]` marker is appended so analysts (and readers) know the
|
|
462
516
|
* content was truncated rather than silently lost. */
|
|
463
517
|
declare function capText(raw: string): string;
|
|
518
|
+
/**
|
|
519
|
+
* Who produced a `user.prompt` turn. The trace records every "user" message,
|
|
520
|
+
* but only some are a real human typing — the rest are the harness or another
|
|
521
|
+
* agent feeding the model:
|
|
522
|
+
* - `human` a person typed it
|
|
523
|
+
* - `subagent-spawn` a parent agent spawned a subagent with this prompt
|
|
524
|
+
* - `injected` a synthetic harness prompt (benchmark task, memory, …)
|
|
525
|
+
* - `tool-result` a tool result surfaced as a user turn (rare)
|
|
526
|
+
* Analysts that measure human reactions must filter to `human` only.
|
|
527
|
+
*/
|
|
528
|
+
type Actor = 'human' | 'subagent-spawn' | 'injected' | 'tool-result';
|
|
529
|
+
/** Span attribute key for {@link Actor}. New (additive) — not part of the
|
|
530
|
+
* `tangle.sessionId`/`tangle.ingest_source` wire contract. */
|
|
531
|
+
declare const ACTOR_ATTR = "tangle.actor";
|
|
464
532
|
interface UserPromptInput {
|
|
465
533
|
traceId: string;
|
|
466
534
|
spanId: string;
|
|
@@ -471,12 +539,70 @@ interface UserPromptInput {
|
|
|
471
539
|
service?: string | null;
|
|
472
540
|
agent?: string | null;
|
|
473
541
|
step?: number;
|
|
542
|
+
/** Who produced this turn. Defaults to `human` when the adapter can't tell. */
|
|
543
|
+
actor?: Actor;
|
|
474
544
|
}
|
|
475
545
|
/** The canonical `user.prompt` span. Every adapter emits the human's turn the
|
|
476
546
|
* same way — a CHAIN span, because a user message is not an LLM call — so
|
|
477
|
-
* analysts see one consistent shape regardless of harness.
|
|
547
|
+
* analysts see one consistent shape regardless of harness. The `actor`
|
|
548
|
+
* attribute lets analysts separate real human turns from agent-to-agent or
|
|
549
|
+
* injected prompts. */
|
|
478
550
|
declare function userPromptSpan(o: UserPromptInput): OtlpSpan;
|
|
479
551
|
|
|
552
|
+
/**
|
|
553
|
+
* Actor + reaction classification — ported from the proven trace-audit
|
|
554
|
+
* `extract.py` classifier so the CLI and the offline audit agree on what
|
|
555
|
+
* counts as a real human turn and how a human reaction is labelled.
|
|
556
|
+
*
|
|
557
|
+
* Two jobs:
|
|
558
|
+
* 1. {@link claudeActor} / {@link textIsSynthetic} — decide whether a "user"
|
|
559
|
+
* turn is a real human or an injected / agent-spawned prompt. Adapters use
|
|
560
|
+
* the harness's structural signals first (sidechain / userType) and fall
|
|
561
|
+
* back to these text heuristics.
|
|
562
|
+
* 2. {@link classifyReaction} — label a human turn as
|
|
563
|
+
* correction / frustration / praise / jargon / structure complaint, using
|
|
564
|
+
* extract.py's regexes verbatim. The user-reaction analyst keys off this.
|
|
565
|
+
*/
|
|
566
|
+
|
|
567
|
+
/** Synthetic harness prompt? (Checks only the head, as extract.py does.) */
|
|
568
|
+
declare function textIsSynthetic(s: string): boolean;
|
|
569
|
+
/** First-turn heuristic: does this read like an agent/subagent spawn prompt
|
|
570
|
+
* (long, or starts with an imperative "you are…/your task…") rather than a
|
|
571
|
+
* person? Ported from extract.py looks_like_agent_prompt. */
|
|
572
|
+
declare function looksLikeAgentPrompt(s: string): boolean;
|
|
573
|
+
/**
|
|
574
|
+
* Derive the actor for a Claude Code user turn from the structural signals the
|
|
575
|
+
* adapter already parses, falling back to the text heuristics above.
|
|
576
|
+
*
|
|
577
|
+
* isSidechain === true → subagent-spawn
|
|
578
|
+
* userType present and !== external → injected
|
|
579
|
+
* synthetic markers in the text → injected
|
|
580
|
+
* first-turn agent-spawn prompt → subagent-spawn
|
|
581
|
+
* otherwise → human
|
|
582
|
+
*/
|
|
583
|
+
declare function claudeActor(args: {
|
|
584
|
+
text: string;
|
|
585
|
+
isSidechain?: boolean;
|
|
586
|
+
userType?: string | null;
|
|
587
|
+
isFirstUserTurn?: boolean;
|
|
588
|
+
}): Actor;
|
|
589
|
+
/**
|
|
590
|
+
* Derive the actor for a Codex user message. Codex has no sidechain/userType,
|
|
591
|
+
* so it's text-only: synthetic markers → injected, first-turn agent-spawn
|
|
592
|
+
* brief → injected, otherwise human.
|
|
593
|
+
*/
|
|
594
|
+
declare function codexActor(args: {
|
|
595
|
+
text: string;
|
|
596
|
+
isFirstUserTurn?: boolean;
|
|
597
|
+
}): Actor;
|
|
598
|
+
type Reaction = 'correction' | 'frustration' | 'praise' | 'jargon' | 'structure';
|
|
599
|
+
/** Label a human turn with every reaction signal it matches. Mirrors
|
|
600
|
+
* extract.py classify() — lowercased match, multi-label. */
|
|
601
|
+
declare function classifyReaction(text: string): Reaction[];
|
|
602
|
+
/** Reactions that signal the agent did something wrong (the numerator of the
|
|
603
|
+
* corrective-to-positive ratio). `praise` is the positive denominator. */
|
|
604
|
+
declare const CORRECTIVE_REACTIONS: readonly Reaction[];
|
|
605
|
+
|
|
480
606
|
/**
|
|
481
607
|
* Run agent-eval's shipped trace pipelines over normalized harness spans.
|
|
482
608
|
*
|
|
@@ -505,6 +631,113 @@ interface PipelineOptions {
|
|
|
505
631
|
}
|
|
506
632
|
declare function runPipelines(spans: readonly OtlpSpan[], opts?: PipelineOptions): Promise<PipelineReport>;
|
|
507
633
|
|
|
634
|
+
/**
|
|
635
|
+
* User-reaction analyst — deterministic, $0, no API key.
|
|
636
|
+
*
|
|
637
|
+
* For every real human turn (`actor === 'human'` on a `user.prompt` span) that
|
|
638
|
+
* FOLLOWS an assistant turn, classify the human's reaction (correction /
|
|
639
|
+
* frustration / praise / jargon / structure complaint) with the trace-audit
|
|
640
|
+
* classifier, then roll up:
|
|
641
|
+
* - per-session + corpus signal counts
|
|
642
|
+
* - the corrective-to-positive ratio (how often the human pushed back vs
|
|
643
|
+
* praised)
|
|
644
|
+
* - the top "trigger pairs": the assistant prose that drew the strongest
|
|
645
|
+
* human reaction
|
|
646
|
+
*
|
|
647
|
+
* Findings only consider `actor === 'human'` turns — that's why the actor tag
|
|
648
|
+
* (feature 1) is a hard dependency. An agent-to-agent or injected prompt is not
|
|
649
|
+
* a human reaction and must not pollute the counts.
|
|
650
|
+
*/
|
|
651
|
+
|
|
652
|
+
/** Reaction labels in stable render order. */
|
|
653
|
+
declare const REACTIONS: readonly Reaction[];
|
|
654
|
+
interface TriggerPair {
|
|
655
|
+
/** The assistant prose immediately preceding the human reaction. */
|
|
656
|
+
assistant: string;
|
|
657
|
+
/** The human's reacting turn. */
|
|
658
|
+
human: string;
|
|
659
|
+
/** Signals matched on the human turn. */
|
|
660
|
+
reactions: Reaction[];
|
|
661
|
+
sessionId: string;
|
|
662
|
+
}
|
|
663
|
+
interface ReactionSessionReport {
|
|
664
|
+
sessionId: string;
|
|
665
|
+
/** Human turns that followed an assistant turn (the denominator). */
|
|
666
|
+
humanReactionTurns: number;
|
|
667
|
+
signals: Record<Reaction, number>;
|
|
668
|
+
}
|
|
669
|
+
interface ReactionReport {
|
|
670
|
+
sessions: ReactionSessionReport[];
|
|
671
|
+
/** Corpus signal totals. */
|
|
672
|
+
signals: Record<Reaction, number>;
|
|
673
|
+
/** Human turns that followed an assistant turn, corpus-wide. */
|
|
674
|
+
humanReactionTurns: number;
|
|
675
|
+
/** corrective signals / praise signals. Infinity when praise is 0 but
|
|
676
|
+
* corrective > 0; NaN-safe (null) when there are no signals at all. */
|
|
677
|
+
correctiveToPositiveRatio: number | null;
|
|
678
|
+
/** Highest-signal assistant→reaction pairs, most-corrective first. */
|
|
679
|
+
triggerPairs: TriggerPair[];
|
|
680
|
+
}
|
|
681
|
+
/**
|
|
682
|
+
* Build the user-reaction report from normalized spans. Spans may span multiple
|
|
683
|
+
* sessions (corpus run); reactions are scoped per `trace_id`.
|
|
684
|
+
*/
|
|
685
|
+
declare function analyzeReactions(spans: readonly OtlpSpan[]): ReactionReport;
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Skill + subagent adoption metrics — deterministic.
|
|
689
|
+
*
|
|
690
|
+
* Reports three things:
|
|
691
|
+
* 1. skill penetration — % of sessions that invoked any skill
|
|
692
|
+
* 2. per-skill invocation frequency
|
|
693
|
+
* 3. subagent (Task/Agent) spawn frequency
|
|
694
|
+
*
|
|
695
|
+
* CRITICAL undercount fix: the trace-level `Skill` tool-use count only sees
|
|
696
|
+
* skills a session invoked *interactively*. Skills dispatched inside an
|
|
697
|
+
* automation loop (e.g. `/evolve`, `/converge`) are logged to
|
|
698
|
+
* `.evolve/skill-runs.jsonl` under the session's cwd and never surface as a
|
|
699
|
+
* `Skill` tool span — historically ~370× more runs than the trace count. So we
|
|
700
|
+
* read those files too and report "explicit invocations" and "loop-dispatched
|
|
701
|
+
* runs" SEPARATELY, so neither silently reads as zero.
|
|
702
|
+
*/
|
|
703
|
+
|
|
704
|
+
interface AdoptionReport {
|
|
705
|
+
sessionCount: number;
|
|
706
|
+
/** Sessions that invoked ≥1 skill (explicitly, via the Skill tool). */
|
|
707
|
+
sessionsWithSkill: number;
|
|
708
|
+
/** sessionsWithSkill / sessionCount (0 when no sessions). */
|
|
709
|
+
skillPenetration: number;
|
|
710
|
+
/** Explicit `Skill` tool invocations per skill name, corpus-wide. */
|
|
711
|
+
skillInvocations: Record<string, number>;
|
|
712
|
+
/** Total explicit `Skill` tool invocations. */
|
|
713
|
+
totalSkillInvocations: number;
|
|
714
|
+
/** Subagent spawns per subagent type (from Task/Agent tool spans). */
|
|
715
|
+
subagentSpawns: Record<string, number>;
|
|
716
|
+
/** Total subagent spawns. */
|
|
717
|
+
totalSubagentSpawns: number;
|
|
718
|
+
/** Sessions that spawned ≥1 subagent. */
|
|
719
|
+
sessionsWithSubagent: number;
|
|
720
|
+
/** Loop-dispatched skill runs per skill, read from `.evolve/skill-runs.jsonl`
|
|
721
|
+
* (NOT visible at trace level — counted separately to avoid silent zeros). */
|
|
722
|
+
loopDispatchedRuns: Record<string, number>;
|
|
723
|
+
/** Total loop-dispatched skill runs. */
|
|
724
|
+
totalLoopDispatchedRuns: number;
|
|
725
|
+
/** `.evolve/skill-runs.jsonl` files that were read. */
|
|
726
|
+
skillRunFilesRead: number;
|
|
727
|
+
}
|
|
728
|
+
/**
|
|
729
|
+
* Count loop-dispatched skill runs from a `.evolve/skill-runs.jsonl` file's raw
|
|
730
|
+
* contents. Each line records one or more skills under `skills` (array) or
|
|
731
|
+
* `skill` (string) — the schema varies across producers, so accept both.
|
|
732
|
+
*/
|
|
733
|
+
declare function countSkillRunsJsonl(raw: string): Record<string, number>;
|
|
734
|
+
interface AdoptionOptions {
|
|
735
|
+
/** Session cwds to probe for `.evolve/skill-runs.jsonl`. The CLI passes the
|
|
736
|
+
* cwds of the analyzed sessions. */
|
|
737
|
+
cwds?: readonly string[];
|
|
738
|
+
}
|
|
739
|
+
declare function analyzeAdoption(spans: readonly OtlpSpan[], opts?: AdoptionOptions): Promise<AdoptionReport>;
|
|
740
|
+
|
|
508
741
|
/**
|
|
509
742
|
* Project normalized `OtlpSpan[]` onto agent-eval's runtime trace store.
|
|
510
743
|
*
|
|
@@ -534,10 +767,9 @@ declare function toRuntimeStore(spans: readonly OtlpSpan[]): Promise<RuntimeTrac
|
|
|
534
767
|
* LLM, model-agnostic). Supply `ai` to add the agentic RLM kinds
|
|
535
768
|
* (failure-mode / knowledge-gap / knowledge-poisoning / improvement).
|
|
536
769
|
*
|
|
537
|
-
* The written
|
|
538
|
-
*
|
|
539
|
-
*
|
|
540
|
-
* never locked to one engine.
|
|
770
|
+
* The written file is canonical OpenInference (see otlp.ts), so it feeds our
|
|
771
|
+
* analysts AND external engines directly — `--analyzer halo` runs HALO over the
|
|
772
|
+
* same artifact, no conversion. Analysis is never locked to one engine.
|
|
541
773
|
*/
|
|
542
774
|
|
|
543
775
|
interface AnalyzeOptions {
|
|
@@ -565,6 +797,227 @@ interface AnalyzeResult {
|
|
|
565
797
|
}
|
|
566
798
|
declare function analyzeSpans(spans: readonly OtlpSpan[], opts?: AnalyzeOptions): Promise<AnalyzeResult>;
|
|
567
799
|
|
|
800
|
+
interface PolicyEvidenceToolSummary {
|
|
801
|
+
readonly name: string;
|
|
802
|
+
readonly calls: number;
|
|
803
|
+
readonly errors: number;
|
|
804
|
+
}
|
|
805
|
+
interface PolicyEvidenceLoopSummary {
|
|
806
|
+
readonly toolName: string;
|
|
807
|
+
readonly occurrences: number;
|
|
808
|
+
}
|
|
809
|
+
interface PolicyEvidenceRecord {
|
|
810
|
+
readonly schemaVersion: 1;
|
|
811
|
+
readonly kind: 'traces.policy_evidence.session';
|
|
812
|
+
readonly generatedAt: string;
|
|
813
|
+
readonly session: {
|
|
814
|
+
readonly harness: string;
|
|
815
|
+
readonly sessionId: string;
|
|
816
|
+
readonly path: string;
|
|
817
|
+
readonly cwd: string | null;
|
|
818
|
+
readonly mtimeMs: number;
|
|
819
|
+
};
|
|
820
|
+
readonly repo: {
|
|
821
|
+
readonly subjectKey?: string;
|
|
822
|
+
readonly repository?: string;
|
|
823
|
+
readonly branch?: string;
|
|
824
|
+
readonly commit?: string;
|
|
825
|
+
readonly cwd?: string;
|
|
826
|
+
readonly resolutionSource?: string;
|
|
827
|
+
};
|
|
828
|
+
readonly metrics: {
|
|
829
|
+
readonly spanCount: number;
|
|
830
|
+
readonly llmTurnCount: number;
|
|
831
|
+
readonly toolCallCount: number;
|
|
832
|
+
readonly erroredToolCallCount: number;
|
|
833
|
+
readonly inputTokens: number;
|
|
834
|
+
readonly outputTokens: number;
|
|
835
|
+
readonly models: readonly string[];
|
|
836
|
+
readonly tools: readonly PolicyEvidenceToolSummary[];
|
|
837
|
+
readonly firstSpanAt: string | null;
|
|
838
|
+
readonly lastSpanAt: string | null;
|
|
839
|
+
};
|
|
840
|
+
readonly signals: {
|
|
841
|
+
readonly stuckLoopCount: number;
|
|
842
|
+
readonly affectedRunRatio: number;
|
|
843
|
+
readonly stuckLoops: readonly PolicyEvidenceLoopSummary[];
|
|
844
|
+
readonly stuckLoopsOmitted: number;
|
|
845
|
+
readonly toolErrorRate: number;
|
|
846
|
+
};
|
|
847
|
+
readonly provenance: {
|
|
848
|
+
readonly source: 'traces';
|
|
849
|
+
readonly evidenceKind: 'session-summary';
|
|
850
|
+
readonly otlpPath?: string;
|
|
851
|
+
readonly notCampaignCell: true;
|
|
852
|
+
readonly note: string;
|
|
853
|
+
};
|
|
854
|
+
}
|
|
855
|
+
interface BuildPolicyEvidenceOptions {
|
|
856
|
+
readonly generatedAt?: string;
|
|
857
|
+
readonly minLoopOccurrences?: number;
|
|
858
|
+
readonly maxLoopExamples?: number;
|
|
859
|
+
readonly otlpPath?: string;
|
|
860
|
+
}
|
|
861
|
+
interface CollectPolicyEvidenceOptions extends ScanOptions, BuildPolicyEvidenceOptions {
|
|
862
|
+
}
|
|
863
|
+
declare function buildPolicyEvidenceRecord(ref: SessionRef, spans: readonly OtlpSpan[], opts?: BuildPolicyEvidenceOptions): Promise<PolicyEvidenceRecord>;
|
|
864
|
+
declare function collectPolicyEvidence(opts: CollectPolicyEvidenceOptions): Promise<PolicyEvidenceRecord[]>;
|
|
865
|
+
declare function serializePolicyEvidence(records: readonly PolicyEvidenceRecord[]): string;
|
|
866
|
+
declare function writePolicyEvidenceFile(records: readonly PolicyEvidenceRecord[], outPath?: string): Promise<string>;
|
|
867
|
+
|
|
868
|
+
interface TraceSessionIndexRow {
|
|
869
|
+
readonly session: PolicyEvidenceRecord['session'];
|
|
870
|
+
readonly repo: PolicyEvidenceRecord['repo'];
|
|
871
|
+
readonly time: {
|
|
872
|
+
readonly firstSpanAt: string | null;
|
|
873
|
+
readonly lastSpanAt: string | null;
|
|
874
|
+
};
|
|
875
|
+
readonly metrics: {
|
|
876
|
+
readonly spanCount: number;
|
|
877
|
+
readonly llmTurnCount: number;
|
|
878
|
+
readonly toolCallCount: number;
|
|
879
|
+
readonly erroredToolCallCount: number;
|
|
880
|
+
readonly inputTokens: number;
|
|
881
|
+
readonly outputTokens: number;
|
|
882
|
+
readonly toolErrorRate: number;
|
|
883
|
+
};
|
|
884
|
+
readonly models: readonly string[];
|
|
885
|
+
readonly tools: PolicyEvidenceRecord['metrics']['tools'];
|
|
886
|
+
readonly signals: PolicyEvidenceRecord['signals'];
|
|
887
|
+
}
|
|
888
|
+
interface TraceIndexedFile {
|
|
889
|
+
readonly kind: 'instruction-doc' | 'evolve-jsonl' | 'evolve-json' | 'reflection' | 'handoff' | 'other';
|
|
890
|
+
readonly path: string;
|
|
891
|
+
readonly bytes: number;
|
|
892
|
+
readonly mtimeMs: number;
|
|
893
|
+
readonly lines?: number;
|
|
894
|
+
readonly jsonlRows?: number;
|
|
895
|
+
readonly markdown?: {
|
|
896
|
+
readonly headings: number;
|
|
897
|
+
readonly hasToc: boolean;
|
|
898
|
+
};
|
|
899
|
+
readonly jsonl?: {
|
|
900
|
+
readonly rows: number;
|
|
901
|
+
readonly invalidRows: number;
|
|
902
|
+
readonly keys: Record<string, number>;
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
interface TraceContextRoot {
|
|
906
|
+
readonly root: string;
|
|
907
|
+
readonly files: readonly TraceIndexedFile[];
|
|
908
|
+
}
|
|
909
|
+
interface TraceContextIndex {
|
|
910
|
+
readonly totals: {
|
|
911
|
+
readonly roots: number;
|
|
912
|
+
readonly files: number;
|
|
913
|
+
readonly instructionDocs: number;
|
|
914
|
+
readonly evolveFiles: number;
|
|
915
|
+
readonly jsonlRows: number;
|
|
916
|
+
readonly invalidJsonlRows: number;
|
|
917
|
+
};
|
|
918
|
+
readonly roots: readonly TraceContextRoot[];
|
|
919
|
+
}
|
|
920
|
+
interface TraceSessionIndex {
|
|
921
|
+
readonly schemaVersion: 1;
|
|
922
|
+
readonly kind: 'traces.session_index';
|
|
923
|
+
readonly generatedAt: string;
|
|
924
|
+
readonly selection?: Record<string, unknown>;
|
|
925
|
+
readonly totals: {
|
|
926
|
+
readonly sessions: number;
|
|
927
|
+
readonly spans: number;
|
|
928
|
+
readonly llmTurns: number;
|
|
929
|
+
readonly toolCalls: number;
|
|
930
|
+
readonly erroredToolCalls: number;
|
|
931
|
+
readonly inputTokens: number;
|
|
932
|
+
readonly outputTokens: number;
|
|
933
|
+
readonly stuckLoopSessions: number;
|
|
934
|
+
readonly stuckLoops: number;
|
|
935
|
+
readonly harnesses: readonly string[];
|
|
936
|
+
readonly repos: readonly string[];
|
|
937
|
+
readonly models: readonly string[];
|
|
938
|
+
readonly tools: readonly string[];
|
|
939
|
+
};
|
|
940
|
+
readonly context?: TraceContextIndex;
|
|
941
|
+
readonly sessions: readonly TraceSessionIndexRow[];
|
|
942
|
+
}
|
|
943
|
+
interface BuildSessionIndexOptions extends Pick<BuildPolicyEvidenceOptions, 'generatedAt' | 'minLoopOccurrences'> {
|
|
944
|
+
readonly selection?: Record<string, unknown>;
|
|
945
|
+
/** Index nearby local docs/artifacts for joins. Defaults to true in scan paths. */
|
|
946
|
+
readonly includeContext?: boolean;
|
|
947
|
+
}
|
|
948
|
+
interface CollectSessionIndexOptions extends ScanOptions, BuildSessionIndexOptions {
|
|
949
|
+
}
|
|
950
|
+
declare function buildSessionIndex(records: readonly PolicyEvidenceRecord[], opts?: BuildSessionIndexOptions & {
|
|
951
|
+
context?: TraceContextIndex;
|
|
952
|
+
}): TraceSessionIndex;
|
|
953
|
+
declare function buildSessionIndexFromRows(rows: readonly {
|
|
954
|
+
ref: SessionRef;
|
|
955
|
+
spans: readonly OtlpSpan[];
|
|
956
|
+
}[], opts?: BuildSessionIndexOptions): Promise<TraceSessionIndex>;
|
|
957
|
+
declare function collectSessionIndex(opts?: CollectSessionIndexOptions): Promise<TraceSessionIndex>;
|
|
958
|
+
declare function serializeSessionIndex(index: TraceSessionIndex): string;
|
|
959
|
+
declare function writeSessionIndexFile(index: TraceSessionIndex, outPath?: string): Promise<string>;
|
|
960
|
+
|
|
961
|
+
type TraceInspectionSeverity = 'high' | 'medium' | 'low';
|
|
962
|
+
type TraceInspectionArea = 'index' | 'session' | 'repo' | 'context';
|
|
963
|
+
interface TraceInspectionFinding {
|
|
964
|
+
readonly id: string;
|
|
965
|
+
readonly severity: TraceInspectionSeverity;
|
|
966
|
+
readonly area: TraceInspectionArea;
|
|
967
|
+
readonly title: string;
|
|
968
|
+
readonly impact: number;
|
|
969
|
+
readonly evidence: readonly string[];
|
|
970
|
+
readonly next: string;
|
|
971
|
+
readonly refs?: readonly string[];
|
|
972
|
+
}
|
|
973
|
+
interface TraceInspectionReport {
|
|
974
|
+
readonly schemaVersion: 1;
|
|
975
|
+
readonly kind: 'traces.inspection_report';
|
|
976
|
+
readonly generatedAt: string;
|
|
977
|
+
readonly source: {
|
|
978
|
+
readonly indexGeneratedAt?: string;
|
|
979
|
+
readonly sessions: number;
|
|
980
|
+
readonly contextFiles: number;
|
|
981
|
+
};
|
|
982
|
+
readonly totals: {
|
|
983
|
+
readonly findings: number;
|
|
984
|
+
readonly high: number;
|
|
985
|
+
readonly medium: number;
|
|
986
|
+
readonly low: number;
|
|
987
|
+
};
|
|
988
|
+
readonly findings: readonly TraceInspectionFinding[];
|
|
989
|
+
}
|
|
990
|
+
interface InspectSessionIndexOptions {
|
|
991
|
+
readonly generatedAt?: string;
|
|
992
|
+
readonly maxFindings?: number;
|
|
993
|
+
}
|
|
994
|
+
declare function inspectSessionIndex(index: TraceSessionIndex, opts?: InspectSessionIndexOptions): TraceInspectionReport;
|
|
995
|
+
declare function renderInspectionReport(report: TraceInspectionReport): string;
|
|
996
|
+
declare function serializeInspectionReport(report: TraceInspectionReport): string;
|
|
997
|
+
declare function readSessionIndexFile(path: string): Promise<TraceSessionIndex>;
|
|
998
|
+
declare function writeInspectionReportFile(report: TraceInspectionReport, outPath?: string): Promise<string>;
|
|
999
|
+
|
|
1000
|
+
type JsonObject = Record<string, unknown>;
|
|
1001
|
+
type TraceEvidenceInputFormat = 'policy-evidence' | 'sandbox-events' | 'openinference';
|
|
1002
|
+
type TraceEvidenceFormatOption = TraceEvidenceInputFormat | 'auto';
|
|
1003
|
+
interface TraceEvidenceExportOptions {
|
|
1004
|
+
readonly format?: TraceEvidenceFormatOption;
|
|
1005
|
+
readonly sourcePath?: string;
|
|
1006
|
+
readonly attributes?: JsonObject;
|
|
1007
|
+
}
|
|
1008
|
+
interface TraceEvidenceExportResult {
|
|
1009
|
+
readonly format: TraceEvidenceInputFormat;
|
|
1010
|
+
readonly spans: OtlpSpan[];
|
|
1011
|
+
readonly redactionCount: number;
|
|
1012
|
+
readonly redactionsByRule: Record<string, number>;
|
|
1013
|
+
}
|
|
1014
|
+
declare function exportTraceEvidenceRows(rows: readonly unknown[], opts?: TraceEvidenceExportOptions, wrapper?: JsonObject): TraceEvidenceExportResult;
|
|
1015
|
+
declare function exportTraceEvidenceText(text: string, opts?: TraceEvidenceExportOptions): TraceEvidenceExportResult;
|
|
1016
|
+
declare function exportTraceEvidenceFile(inputPath: string, opts?: TraceEvidenceExportOptions): Promise<TraceEvidenceExportResult>;
|
|
1017
|
+
declare function writeTraceEvidenceExportFile(inputPath: string, outPath?: string, opts?: TraceEvidenceExportOptions): Promise<TraceEvidenceExportResult & {
|
|
1018
|
+
path: string;
|
|
1019
|
+
}>;
|
|
1020
|
+
|
|
568
1021
|
/**
|
|
569
1022
|
* External-tool adapters: run analysis engines and PII scrubbers that `traces`
|
|
570
1023
|
* does NOT bundle. Each is a thin command adapter over a shared subprocess
|
|
@@ -619,20 +1072,11 @@ declare function commandAnalyzer(spec: {
|
|
|
619
1072
|
parse?: (stdout: string) => AnalystFinding[] | undefined;
|
|
620
1073
|
timeoutMs?: number;
|
|
621
1074
|
}): ExternalAnalyzer;
|
|
622
|
-
/**
|
|
623
|
-
*
|
|
624
|
-
*
|
|
625
|
-
*
|
|
626
|
-
*
|
|
627
|
-
declare function toCanonicalOpenInference(otlpJsonl: string, opts?: {
|
|
628
|
-
service?: string;
|
|
629
|
-
scopeName?: string;
|
|
630
|
-
}): string;
|
|
631
|
-
/** HALO (github.com/context-labs/halo) over the OTLP artifact. HALO requires
|
|
632
|
-
* canonical OpenInference, so this converts our OTLP to a `<path>.halo.jsonl`
|
|
633
|
-
* first, then runs `halo <file> -p <prompt> [-m <model>]`. Install HALO yourself
|
|
634
|
-
* and configure its LLM provider (it uses the OpenAI client — `OPENAI_BASE_URL`
|
|
635
|
-
* / `OPENAI_API_KEY`); this just drives it. */
|
|
1075
|
+
/** HALO (github.com/context-labs/halo) over the OTLP artifact: `halo <file> -p
|
|
1076
|
+
* <prompt> [-m <model>]`. Our `serializeSpans` already emits canonical
|
|
1077
|
+
* OpenInference, so HALO reads the artifact directly — no conversion. Install
|
|
1078
|
+
* HALO yourself and configure its LLM provider (it uses the OpenAI client —
|
|
1079
|
+
* `OPENAI_BASE_URL` / `OPENAI_API_KEY`); this just drives it. */
|
|
636
1080
|
declare function haloAnalyzer(opts?: {
|
|
637
1081
|
command?: string;
|
|
638
1082
|
defaultPrompt?: string;
|
|
@@ -875,4 +1319,4 @@ interface ExecuteOptions {
|
|
|
875
1319
|
/** Send the NEW items (or, on dryRun, write the redacted OTLP that would send). */
|
|
876
1320
|
declare function executeUpload(plan: UploadPlan, opts?: ExecuteOptions): Promise<UploadResult>;
|
|
877
1321
|
|
|
878
|
-
export { ATTR, AmpAdapter, type AnalyzeOptions, type AnalyzeResult, CODING_REDACTION_RULES, CONTENT_CAP, ClaudeAdapter, CodexAdapter, type CollectOptions, CopilotAdapter, DEFAULT_HARNESS, type ExecuteOptions, type ExternalAnalysisResult, type ExternalAnalyzer, FactoryAdapter, ForgeAdapter, GeminiAdapter, type HarnessTraceAdapter, INGEST_SOURCE_CLI, type LocateOptions, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpSpan, type OtlpSpanKind, type OtlpStatusCode, PiAdapter, type PipelineOptions, type PipelineReport, type PlanOptions, QwenAdapter, type Redactor, type RunResult, type RuntimeTrace, type ScanOptions, type ScannedSession, type SessionBatch, type SessionRef, type SpanInput, type SpanRedaction, TRACES_REDACTION_RULES, type UploadBackend, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, alreadyUploaded, analyzeSpans, applyRedactor, capText, collectSessions, commandAnalyzer, commandRedactor, executeUpload, haloAnalyzer, knownHarnesses, listAdapters, loadState, parseIsoToEpochMs, parseSince, planUpload, redactSpans, resolveAdapter, runCommand, runExternalAnalyzers, runPipelines, saveState, scanSessions, selectAdapters, serializeSpans, sessionHash, span,
|
|
1322
|
+
export { ACTOR_ATTR, ATTR, type Actor, type AdoptionOptions, type AdoptionReport, AmpAdapter, type AnalyzeOptions, type AnalyzeResult, type BuildPolicyEvidenceOptions, type BuildSessionIndexOptions, CODING_REDACTION_RULES, CONTENT_CAP, CORRECTIVE_REACTIONS, ClaudeAdapter, CodexAdapter, type CollectOptions, type CollectPolicyEvidenceOptions, type CollectSessionIndexOptions, CopilotAdapter, DEFAULT_HARNESS, type ExecuteOptions, type ExternalAnalysisResult, type ExternalAnalyzer, FactoryAdapter, ForgeAdapter, GeminiAdapter, type HarnessTraceAdapter, INGEST_SOURCE_CLI, type InspectSessionIndexOptions, type LocateOptions, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpSpan, type OtlpSpanKind, type OtlpStatusCode, PiAdapter, type PipelineOptions, type PipelineReport, type PlanOptions, type PolicyEvidenceLoopSummary, type PolicyEvidenceRecord, type PolicyEvidenceToolSummary, QwenAdapter, REACTIONS, type Reaction, type ReactionReport, type ReactionSessionReport, type Redactor, type RepoAttrs, type RepoResolution, type RepoResolutionSource, type RunResult, type RuntimeTrace, type ScanOptions, type ScannedSession, type SessionBatch, type SessionRef, type SpanInput, type SpanRedaction, TRACES_REDACTION_RULES, type TraceContextIndex, type TraceContextRoot, type TraceEvidenceExportOptions, type TraceEvidenceExportResult, type TraceEvidenceFormatOption, type TraceEvidenceInputFormat, type TraceIndexedFile, type TraceInspectionArea, type TraceInspectionFinding, type TraceInspectionReport, type TraceInspectionSeverity, type TraceSessionIndex, type TraceSessionIndexRow, type TriggerPair, type UploadBackend, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, type UserPromptInput, alreadyUploaded, analyzeAdoption, analyzeReactions, analyzeSpans, applyRedactor, buildPolicyEvidenceRecord, buildSessionIndex, buildSessionIndexFromRows, capText, classifyReaction, claudeActor, codexActor, collectPolicyEvidence, collectSessionIndex, collectSessions, commandAnalyzer, commandRedactor, countSkillRunsJsonl, executeUpload, exportTraceEvidenceFile, exportTraceEvidenceRows, exportTraceEvidenceText, haloAnalyzer, inspectSessionIndex, knownHarnesses, listAdapters, loadState, looksLikeAgentPrompt, normalizeRemote, parseIsoToEpochMs, parseSession, parseSince, planUpload, readSessionIndexFile, redactSpans, renderInspectionReport, resolveAdapter, resolveRepoAttrs, resolveSessionRepoAttrs, runCommand, runExternalAnalyzers, runPipelines, saveState, scanSessions, selectAdapters, serializeInspectionReport, serializePolicyEvidence, serializeSessionIndex, serializeSpans, sessionHash, span, stampRepoAttrs, textIsSynthetic, toOpenInferenceSpan, toRuntimeStore, toTraceSpanEvents, uploadKey, userPromptSpan, watchSessions, writeInspectionReportFile, writeOtlpFile, writePolicyEvidenceFile, writeSessionIndexFile, writeTraceEvidenceExportFile };
|