@tangle-network/traces 0.8.0 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -1
- package/dist/{chunk-BUD6I3ZJ.js → chunk-D5OMTM4S.js} +1203 -314
- package/dist/cli.js +260 -12
- package/dist/index.d.ts +321 -27
- package/dist/index.js +49 -5
- package/package.json +12 -13
package/dist/index.d.ts
CHANGED
|
@@ -11,13 +11,12 @@ export { HostedClient, createHostedClient, hostedClientFromEnv } from '@tangle-n
|
|
|
11
11
|
/**
|
|
12
12
|
* OTLP-flat-line emitter — the canonical normalized trace shape.
|
|
13
13
|
*
|
|
14
|
-
* Every harness adapter projects its native session log onto `OtlpSpan[]
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* form, which `external.ts` `toCanonicalOpenInference` produces.
|
|
14
|
+
* Every harness adapter projects its native session log onto `OtlpSpan[]`.
|
|
15
|
+
* `serializeSpans` emits one COMPLETE OpenInference span per JSONL line
|
|
16
|
+
* (top-level `kind`, `resource`, `scope`, string `parent_span_id`, plus the
|
|
17
|
+
* `openinference.span.kind` attribute). One standard artifact feeds three
|
|
18
|
+
* consumers unchanged: `@tangle-network/agent-eval`'s `OtlpFileTraceStore`
|
|
19
|
+
* (which reads the attribute vocabulary), HALO, and any OpenInference tool.
|
|
21
20
|
*
|
|
22
21
|
* Attribute vocabulary the downstream analysts key off:
|
|
23
22
|
* - `openinference.span.kind` → span kind (AGENT | LLM | TOOL | CHAIN)
|
|
@@ -149,6 +148,16 @@ declare const ATTR: {
|
|
|
149
148
|
readonly INGEST_SOURCE: "tangle.ingest_source";
|
|
150
149
|
readonly HARNESS: "tangle.harness";
|
|
151
150
|
readonly CWD: "tangle.cwd";
|
|
151
|
+
/** THE per-session grouping key the spine derives subjects from
|
|
152
|
+
* (`deriveSubjectKey` reads `tangle.subject.key` first). Resolves to the git
|
|
153
|
+
* remote (host/owner/repo) when readable, else the cwd path basename. */
|
|
154
|
+
readonly SUBJECT_KEY: "tangle.subject.key";
|
|
155
|
+
/** Normalized git remote (e.g. `github.com/tangle-network/agent-dev-container`). */
|
|
156
|
+
readonly GIT_REPOSITORY: "git.repository";
|
|
157
|
+
/** Current branch at conversion time. */
|
|
158
|
+
readonly GIT_BRANCH_NAME: "git.branch";
|
|
159
|
+
/** HEAD short sha at conversion time. */
|
|
160
|
+
readonly GIT_COMMIT: "git.commit";
|
|
152
161
|
readonly GIT_BRANCH: "tangle.git_branch";
|
|
153
162
|
readonly HOST: "tangle.host";
|
|
154
163
|
/** Basename of the session file. (Renamed from the ambiguous `tangle.source`,
|
|
@@ -201,6 +210,13 @@ interface AdapterSelection {
|
|
|
201
210
|
* A named harness that doesn't resolve throws rather than silently dropping. */
|
|
202
211
|
declare function selectAdapters(opts: AdapterSelection): HarnessTraceAdapter[];
|
|
203
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Parse one session to spans and stamp per-session repo/git resource attrs
|
|
215
|
+
* (`tangle.subject.key` etc.) derived from the ref's cwd. Every OTLP-producing
|
|
216
|
+
* path funnels through here so the spine can group by repo. Repo resolution is
|
|
217
|
+
* computed ONCE per session; it is fail-safe and never throws.
|
|
218
|
+
*/
|
|
219
|
+
declare function parseSession(adapter: HarnessTraceAdapter, ref: SessionRef): Promise<OtlpSpan[]>;
|
|
204
220
|
interface ScanOptions extends AdapterSelection {
|
|
205
221
|
/** Filter by working directory (exact/prefix). */
|
|
206
222
|
cwd?: string;
|
|
@@ -223,6 +239,27 @@ interface ScannedSession {
|
|
|
223
239
|
* errors route to `onError` and skip that adapter/session, never aborting. */
|
|
224
240
|
declare function scanSessions(opts: ScanOptions): AsyncGenerator<ScannedSession>;
|
|
225
241
|
|
|
242
|
+
/** Resource-attribute keys this resolver may stamp. */
|
|
243
|
+
type RepoAttrs = Partial<Record<typeof ATTR.SUBJECT_KEY | typeof ATTR.GIT_REPOSITORY | typeof ATTR.GIT_BRANCH_NAME | typeof ATTR.GIT_COMMIT | typeof ATTR.CWD, string>>;
|
|
244
|
+
/**
|
|
245
|
+
* Normalize a git remote url to `host/owner/repo`, dropping scheme, auth, the
|
|
246
|
+
* `.git` suffix, and any trailing slash. Handles both `https://` and the
|
|
247
|
+
* `git@host:owner/repo.git` SCP-style forms. Returns null when unparseable.
|
|
248
|
+
*/
|
|
249
|
+
declare function normalizeRemote(url: string): string | null;
|
|
250
|
+
/**
|
|
251
|
+
* Resolve per-session repo/git resource attributes from a session's cwd.
|
|
252
|
+
* Fail-safe and never throws; see the module doc for the derivation contract.
|
|
253
|
+
*/
|
|
254
|
+
declare function resolveRepoAttrs(cwd: string | null | undefined): Promise<RepoAttrs>;
|
|
255
|
+
/**
|
|
256
|
+
* Stamp resolved repo attrs onto every span's attributes (so they land in the
|
|
257
|
+
* OTLP resource attributes via `toOpenInferenceSpan`). Additive — existing
|
|
258
|
+
* `service.name` / `agent.name` are untouched. Mutates in place and returns the
|
|
259
|
+
* same array for ergonomic chaining. A no-op when `attrs` is empty.
|
|
260
|
+
*/
|
|
261
|
+
declare function stampRepoAttrs(spans: readonly OtlpSpan[], attrs: RepoAttrs): readonly OtlpSpan[];
|
|
262
|
+
|
|
226
263
|
/**
|
|
227
264
|
* Claude Code adapter — `~/.claude/projects/<encoded-cwd>/<session>.jsonl`.
|
|
228
265
|
*
|
|
@@ -461,6 +498,20 @@ declare const CONTENT_CAP = 8000;
|
|
|
461
498
|
* visible `… [+N chars]` marker is appended so analysts (and readers) know the
|
|
462
499
|
* content was truncated rather than silently lost. */
|
|
463
500
|
declare function capText(raw: string): string;
|
|
501
|
+
/**
|
|
502
|
+
* Who produced a `user.prompt` turn. The trace records every "user" message,
|
|
503
|
+
* but only some are a real human typing — the rest are the harness or another
|
|
504
|
+
* agent feeding the model:
|
|
505
|
+
* - `human` a person typed it
|
|
506
|
+
* - `subagent-spawn` a parent agent spawned a subagent with this prompt
|
|
507
|
+
* - `injected` a synthetic harness prompt (benchmark task, memory, …)
|
|
508
|
+
* - `tool-result` a tool result surfaced as a user turn (rare)
|
|
509
|
+
* Analysts that measure human reactions must filter to `human` only.
|
|
510
|
+
*/
|
|
511
|
+
type Actor = 'human' | 'subagent-spawn' | 'injected' | 'tool-result';
|
|
512
|
+
/** Span attribute key for {@link Actor}. New (additive) — not part of the
|
|
513
|
+
* `tangle.sessionId`/`tangle.ingest_source` wire contract. */
|
|
514
|
+
declare const ACTOR_ATTR = "tangle.actor";
|
|
464
515
|
interface UserPromptInput {
|
|
465
516
|
traceId: string;
|
|
466
517
|
spanId: string;
|
|
@@ -471,12 +522,70 @@ interface UserPromptInput {
|
|
|
471
522
|
service?: string | null;
|
|
472
523
|
agent?: string | null;
|
|
473
524
|
step?: number;
|
|
525
|
+
/** Who produced this turn. Defaults to `human` when the adapter can't tell. */
|
|
526
|
+
actor?: Actor;
|
|
474
527
|
}
|
|
475
528
|
/** The canonical `user.prompt` span. Every adapter emits the human's turn the
|
|
476
529
|
* same way — a CHAIN span, because a user message is not an LLM call — so
|
|
477
|
-
* analysts see one consistent shape regardless of harness.
|
|
530
|
+
* analysts see one consistent shape regardless of harness. The `actor`
|
|
531
|
+
* attribute lets analysts separate real human turns from agent-to-agent or
|
|
532
|
+
* injected prompts. */
|
|
478
533
|
declare function userPromptSpan(o: UserPromptInput): OtlpSpan;
|
|
479
534
|
|
|
535
|
+
/**
|
|
536
|
+
* Actor + reaction classification — ported from the proven trace-audit
|
|
537
|
+
* `extract.py` classifier so the CLI and the offline audit agree on what
|
|
538
|
+
* counts as a real human turn and how a human reaction is labelled.
|
|
539
|
+
*
|
|
540
|
+
* Two jobs:
|
|
541
|
+
* 1. {@link claudeActor} / {@link textIsSynthetic} — decide whether a "user"
|
|
542
|
+
* turn is a real human or an injected / agent-spawned prompt. Adapters use
|
|
543
|
+
* the harness's structural signals first (sidechain / userType) and fall
|
|
544
|
+
* back to these text heuristics.
|
|
545
|
+
* 2. {@link classifyReaction} — label a human turn as
|
|
546
|
+
* correction / frustration / praise / jargon / structure complaint, using
|
|
547
|
+
* extract.py's regexes verbatim. The user-reaction analyst keys off this.
|
|
548
|
+
*/
|
|
549
|
+
|
|
550
|
+
/** Synthetic harness prompt? (Checks only the head, as extract.py does.) */
|
|
551
|
+
declare function textIsSynthetic(s: string): boolean;
|
|
552
|
+
/** First-turn heuristic: does this read like an agent/subagent spawn prompt
|
|
553
|
+
* (long, or starts with an imperative "you are…/your task…") rather than a
|
|
554
|
+
* person? Ported from extract.py looks_like_agent_prompt. */
|
|
555
|
+
declare function looksLikeAgentPrompt(s: string): boolean;
|
|
556
|
+
/**
|
|
557
|
+
* Derive the actor for a Claude Code user turn from the structural signals the
|
|
558
|
+
* adapter already parses, falling back to the text heuristics above.
|
|
559
|
+
*
|
|
560
|
+
* isSidechain === true → subagent-spawn
|
|
561
|
+
* userType present and !== external → injected
|
|
562
|
+
* synthetic markers in the text → injected
|
|
563
|
+
* first-turn agent-spawn prompt → subagent-spawn
|
|
564
|
+
* otherwise → human
|
|
565
|
+
*/
|
|
566
|
+
declare function claudeActor(args: {
|
|
567
|
+
text: string;
|
|
568
|
+
isSidechain?: boolean;
|
|
569
|
+
userType?: string | null;
|
|
570
|
+
isFirstUserTurn?: boolean;
|
|
571
|
+
}): Actor;
|
|
572
|
+
/**
|
|
573
|
+
* Derive the actor for a Codex user message. Codex has no sidechain/userType,
|
|
574
|
+
* so it's text-only: synthetic markers → injected, first-turn agent-spawn
|
|
575
|
+
* brief → injected, otherwise human.
|
|
576
|
+
*/
|
|
577
|
+
declare function codexActor(args: {
|
|
578
|
+
text: string;
|
|
579
|
+
isFirstUserTurn?: boolean;
|
|
580
|
+
}): Actor;
|
|
581
|
+
type Reaction = 'correction' | 'frustration' | 'praise' | 'jargon' | 'structure';
|
|
582
|
+
/** Label a human turn with every reaction signal it matches. Mirrors
|
|
583
|
+
* extract.py classify() — lowercased match, multi-label. */
|
|
584
|
+
declare function classifyReaction(text: string): Reaction[];
|
|
585
|
+
/** Reactions that signal the agent did something wrong (the numerator of the
|
|
586
|
+
* corrective-to-positive ratio). `praise` is the positive denominator. */
|
|
587
|
+
declare const CORRECTIVE_REACTIONS: readonly Reaction[];
|
|
588
|
+
|
|
480
589
|
/**
|
|
481
590
|
* Run agent-eval's shipped trace pipelines over normalized harness spans.
|
|
482
591
|
*
|
|
@@ -505,6 +614,113 @@ interface PipelineOptions {
|
|
|
505
614
|
}
|
|
506
615
|
declare function runPipelines(spans: readonly OtlpSpan[], opts?: PipelineOptions): Promise<PipelineReport>;
|
|
507
616
|
|
|
617
|
+
/**
|
|
618
|
+
* User-reaction analyst — deterministic, $0, no API key.
|
|
619
|
+
*
|
|
620
|
+
* For every real human turn (`actor === 'human'` on a `user.prompt` span) that
|
|
621
|
+
* FOLLOWS an assistant turn, classify the human's reaction (correction /
|
|
622
|
+
* frustration / praise / jargon / structure complaint) with the trace-audit
|
|
623
|
+
* classifier, then roll up:
|
|
624
|
+
* - per-session + corpus signal counts
|
|
625
|
+
* - the corrective-to-positive ratio (how often the human pushed back vs
|
|
626
|
+
* praised)
|
|
627
|
+
* - the top "trigger pairs": the assistant prose that drew the strongest
|
|
628
|
+
* human reaction
|
|
629
|
+
*
|
|
630
|
+
* Findings only consider `actor === 'human'` turns — that's why the actor tag
|
|
631
|
+
* (feature 1) is a hard dependency. An agent-to-agent or injected prompt is not
|
|
632
|
+
* a human reaction and must not pollute the counts.
|
|
633
|
+
*/
|
|
634
|
+
|
|
635
|
+
/** Reaction labels in stable render order. */
|
|
636
|
+
declare const REACTIONS: readonly Reaction[];
|
|
637
|
+
interface TriggerPair {
|
|
638
|
+
/** The assistant prose immediately preceding the human reaction. */
|
|
639
|
+
assistant: string;
|
|
640
|
+
/** The human's reacting turn. */
|
|
641
|
+
human: string;
|
|
642
|
+
/** Signals matched on the human turn. */
|
|
643
|
+
reactions: Reaction[];
|
|
644
|
+
sessionId: string;
|
|
645
|
+
}
|
|
646
|
+
interface ReactionSessionReport {
|
|
647
|
+
sessionId: string;
|
|
648
|
+
/** Human turns that followed an assistant turn (the denominator). */
|
|
649
|
+
humanReactionTurns: number;
|
|
650
|
+
signals: Record<Reaction, number>;
|
|
651
|
+
}
|
|
652
|
+
interface ReactionReport {
|
|
653
|
+
sessions: ReactionSessionReport[];
|
|
654
|
+
/** Corpus signal totals. */
|
|
655
|
+
signals: Record<Reaction, number>;
|
|
656
|
+
/** Human turns that followed an assistant turn, corpus-wide. */
|
|
657
|
+
humanReactionTurns: number;
|
|
658
|
+
/** corrective signals / praise signals. Infinity when praise is 0 but
|
|
659
|
+
* corrective > 0; NaN-safe (null) when there are no signals at all. */
|
|
660
|
+
correctiveToPositiveRatio: number | null;
|
|
661
|
+
/** Highest-signal assistant→reaction pairs, most-corrective first. */
|
|
662
|
+
triggerPairs: TriggerPair[];
|
|
663
|
+
}
|
|
664
|
+
/**
|
|
665
|
+
* Build the user-reaction report from normalized spans. Spans may span multiple
|
|
666
|
+
* sessions (corpus run); reactions are scoped per `trace_id`.
|
|
667
|
+
*/
|
|
668
|
+
declare function analyzeReactions(spans: readonly OtlpSpan[]): ReactionReport;
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* Skill + subagent adoption metrics — deterministic.
|
|
672
|
+
*
|
|
673
|
+
* Reports three things:
|
|
674
|
+
* 1. skill penetration — % of sessions that invoked any skill
|
|
675
|
+
* 2. per-skill invocation frequency
|
|
676
|
+
* 3. subagent (Task/Agent) spawn frequency
|
|
677
|
+
*
|
|
678
|
+
* CRITICAL undercount fix: the trace-level `Skill` tool-use count only sees
|
|
679
|
+
* skills a session invoked *interactively*. Skills dispatched inside an
|
|
680
|
+
* automation loop (e.g. `/evolve`, `/converge`) are logged to
|
|
681
|
+
* `.evolve/skill-runs.jsonl` under the session's cwd and never surface as a
|
|
682
|
+
* `Skill` tool span — historically ~370× more runs than the trace count. So we
|
|
683
|
+
* read those files too and report "explicit invocations" and "loop-dispatched
|
|
684
|
+
* runs" SEPARATELY, so neither silently reads as zero.
|
|
685
|
+
*/
|
|
686
|
+
|
|
687
|
+
interface AdoptionReport {
|
|
688
|
+
sessionCount: number;
|
|
689
|
+
/** Sessions that invoked ≥1 skill (explicitly, via the Skill tool). */
|
|
690
|
+
sessionsWithSkill: number;
|
|
691
|
+
/** sessionsWithSkill / sessionCount (0 when no sessions). */
|
|
692
|
+
skillPenetration: number;
|
|
693
|
+
/** Explicit `Skill` tool invocations per skill name, corpus-wide. */
|
|
694
|
+
skillInvocations: Record<string, number>;
|
|
695
|
+
/** Total explicit `Skill` tool invocations. */
|
|
696
|
+
totalSkillInvocations: number;
|
|
697
|
+
/** Subagent spawns per subagent type (from Task/Agent tool spans). */
|
|
698
|
+
subagentSpawns: Record<string, number>;
|
|
699
|
+
/** Total subagent spawns. */
|
|
700
|
+
totalSubagentSpawns: number;
|
|
701
|
+
/** Sessions that spawned ≥1 subagent. */
|
|
702
|
+
sessionsWithSubagent: number;
|
|
703
|
+
/** Loop-dispatched skill runs per skill, read from `.evolve/skill-runs.jsonl`
|
|
704
|
+
* (NOT visible at trace level — counted separately to avoid silent zeros). */
|
|
705
|
+
loopDispatchedRuns: Record<string, number>;
|
|
706
|
+
/** Total loop-dispatched skill runs. */
|
|
707
|
+
totalLoopDispatchedRuns: number;
|
|
708
|
+
/** `.evolve/skill-runs.jsonl` files that were read. */
|
|
709
|
+
skillRunFilesRead: number;
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* Count loop-dispatched skill runs from a `.evolve/skill-runs.jsonl` file's raw
|
|
713
|
+
* contents. Each line records one or more skills under `skills` (array) or
|
|
714
|
+
* `skill` (string) — the schema varies across producers, so accept both.
|
|
715
|
+
*/
|
|
716
|
+
declare function countSkillRunsJsonl(raw: string): Record<string, number>;
|
|
717
|
+
interface AdoptionOptions {
|
|
718
|
+
/** Session cwds to probe for `.evolve/skill-runs.jsonl`. The CLI passes the
|
|
719
|
+
* cwds of the analyzed sessions. */
|
|
720
|
+
cwds?: readonly string[];
|
|
721
|
+
}
|
|
722
|
+
declare function analyzeAdoption(spans: readonly OtlpSpan[], opts?: AdoptionOptions): Promise<AdoptionReport>;
|
|
723
|
+
|
|
508
724
|
/**
|
|
509
725
|
* Project normalized `OtlpSpan[]` onto agent-eval's runtime trace store.
|
|
510
726
|
*
|
|
@@ -534,10 +750,9 @@ declare function toRuntimeStore(spans: readonly OtlpSpan[]): Promise<RuntimeTrac
|
|
|
534
750
|
* LLM, model-agnostic). Supply `ai` to add the agentic RLM kinds
|
|
535
751
|
* (failure-mode / knowledge-gap / knowledge-poisoning / improvement).
|
|
536
752
|
*
|
|
537
|
-
* The written
|
|
538
|
-
*
|
|
539
|
-
*
|
|
540
|
-
* never locked to one engine.
|
|
753
|
+
* The written file is canonical OpenInference (see otlp.ts), so it feeds our
|
|
754
|
+
* analysts AND external engines directly — `--analyzer halo` runs HALO over the
|
|
755
|
+
* same artifact, no conversion. Analysis is never locked to one engine.
|
|
541
756
|
*/
|
|
542
757
|
|
|
543
758
|
interface AnalyzeOptions {
|
|
@@ -565,6 +780,94 @@ interface AnalyzeResult {
|
|
|
565
780
|
}
|
|
566
781
|
declare function analyzeSpans(spans: readonly OtlpSpan[], opts?: AnalyzeOptions): Promise<AnalyzeResult>;
|
|
567
782
|
|
|
783
|
+
interface PolicyEvidenceToolSummary {
|
|
784
|
+
readonly name: string;
|
|
785
|
+
readonly calls: number;
|
|
786
|
+
readonly errors: number;
|
|
787
|
+
}
|
|
788
|
+
interface PolicyEvidenceLoopSummary {
|
|
789
|
+
readonly toolName: string;
|
|
790
|
+
readonly occurrences: number;
|
|
791
|
+
}
|
|
792
|
+
interface PolicyEvidenceRecord {
|
|
793
|
+
readonly schemaVersion: 1;
|
|
794
|
+
readonly kind: 'traces.policy_evidence.session';
|
|
795
|
+
readonly generatedAt: string;
|
|
796
|
+
readonly session: {
|
|
797
|
+
readonly harness: string;
|
|
798
|
+
readonly sessionId: string;
|
|
799
|
+
readonly path: string;
|
|
800
|
+
readonly cwd: string | null;
|
|
801
|
+
readonly mtimeMs: number;
|
|
802
|
+
};
|
|
803
|
+
readonly repo: {
|
|
804
|
+
readonly subjectKey?: string;
|
|
805
|
+
readonly repository?: string;
|
|
806
|
+
readonly branch?: string;
|
|
807
|
+
readonly commit?: string;
|
|
808
|
+
readonly cwd?: string;
|
|
809
|
+
};
|
|
810
|
+
readonly metrics: {
|
|
811
|
+
readonly spanCount: number;
|
|
812
|
+
readonly llmTurnCount: number;
|
|
813
|
+
readonly toolCallCount: number;
|
|
814
|
+
readonly erroredToolCallCount: number;
|
|
815
|
+
readonly inputTokens: number;
|
|
816
|
+
readonly outputTokens: number;
|
|
817
|
+
readonly models: readonly string[];
|
|
818
|
+
readonly tools: readonly PolicyEvidenceToolSummary[];
|
|
819
|
+
readonly firstSpanAt: string | null;
|
|
820
|
+
readonly lastSpanAt: string | null;
|
|
821
|
+
};
|
|
822
|
+
readonly signals: {
|
|
823
|
+
readonly stuckLoopCount: number;
|
|
824
|
+
readonly affectedRunRatio: number;
|
|
825
|
+
readonly stuckLoops: readonly PolicyEvidenceLoopSummary[];
|
|
826
|
+
readonly stuckLoopsOmitted: number;
|
|
827
|
+
readonly toolErrorRate: number;
|
|
828
|
+
};
|
|
829
|
+
readonly provenance: {
|
|
830
|
+
readonly source: 'traces';
|
|
831
|
+
readonly evidenceKind: 'session-summary';
|
|
832
|
+
readonly otlpPath?: string;
|
|
833
|
+
readonly notCampaignCell: true;
|
|
834
|
+
readonly note: string;
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
interface BuildPolicyEvidenceOptions {
|
|
838
|
+
readonly generatedAt?: string;
|
|
839
|
+
readonly minLoopOccurrences?: number;
|
|
840
|
+
readonly maxLoopExamples?: number;
|
|
841
|
+
readonly otlpPath?: string;
|
|
842
|
+
}
|
|
843
|
+
interface CollectPolicyEvidenceOptions extends ScanOptions, BuildPolicyEvidenceOptions {
|
|
844
|
+
}
|
|
845
|
+
declare function buildPolicyEvidenceRecord(ref: SessionRef, spans: readonly OtlpSpan[], opts?: BuildPolicyEvidenceOptions): Promise<PolicyEvidenceRecord>;
|
|
846
|
+
declare function collectPolicyEvidence(opts: CollectPolicyEvidenceOptions): Promise<PolicyEvidenceRecord[]>;
|
|
847
|
+
declare function serializePolicyEvidence(records: readonly PolicyEvidenceRecord[]): string;
|
|
848
|
+
declare function writePolicyEvidenceFile(records: readonly PolicyEvidenceRecord[], outPath?: string): Promise<string>;
|
|
849
|
+
|
|
850
|
+
type JsonObject = Record<string, unknown>;
|
|
851
|
+
type TraceEvidenceInputFormat = 'policy-evidence' | 'sandbox-events' | 'openinference';
|
|
852
|
+
type TraceEvidenceFormatOption = TraceEvidenceInputFormat | 'auto';
|
|
853
|
+
interface TraceEvidenceExportOptions {
|
|
854
|
+
readonly format?: TraceEvidenceFormatOption;
|
|
855
|
+
readonly sourcePath?: string;
|
|
856
|
+
readonly attributes?: JsonObject;
|
|
857
|
+
}
|
|
858
|
+
interface TraceEvidenceExportResult {
|
|
859
|
+
readonly format: TraceEvidenceInputFormat;
|
|
860
|
+
readonly spans: OtlpSpan[];
|
|
861
|
+
readonly redactionCount: number;
|
|
862
|
+
readonly redactionsByRule: Record<string, number>;
|
|
863
|
+
}
|
|
864
|
+
declare function exportTraceEvidenceRows(rows: readonly unknown[], opts?: TraceEvidenceExportOptions, wrapper?: JsonObject): TraceEvidenceExportResult;
|
|
865
|
+
declare function exportTraceEvidenceText(text: string, opts?: TraceEvidenceExportOptions): TraceEvidenceExportResult;
|
|
866
|
+
declare function exportTraceEvidenceFile(inputPath: string, opts?: TraceEvidenceExportOptions): Promise<TraceEvidenceExportResult>;
|
|
867
|
+
declare function writeTraceEvidenceExportFile(inputPath: string, outPath?: string, opts?: TraceEvidenceExportOptions): Promise<TraceEvidenceExportResult & {
|
|
868
|
+
path: string;
|
|
869
|
+
}>;
|
|
870
|
+
|
|
568
871
|
/**
|
|
569
872
|
* External-tool adapters: run analysis engines and PII scrubbers that `traces`
|
|
570
873
|
* does NOT bundle. Each is a thin command adapter over a shared subprocess
|
|
@@ -619,20 +922,11 @@ declare function commandAnalyzer(spec: {
|
|
|
619
922
|
parse?: (stdout: string) => AnalystFinding[] | undefined;
|
|
620
923
|
timeoutMs?: number;
|
|
621
924
|
}): ExternalAnalyzer;
|
|
622
|
-
/**
|
|
623
|
-
*
|
|
624
|
-
*
|
|
625
|
-
*
|
|
626
|
-
*
|
|
627
|
-
declare function toCanonicalOpenInference(otlpJsonl: string, opts?: {
|
|
628
|
-
service?: string;
|
|
629
|
-
scopeName?: string;
|
|
630
|
-
}): string;
|
|
631
|
-
/** HALO (github.com/context-labs/halo) over the OTLP artifact. HALO requires
|
|
632
|
-
* canonical OpenInference, so this converts our OTLP to a `<path>.halo.jsonl`
|
|
633
|
-
* first, then runs `halo <file> -p <prompt> [-m <model>]`. Install HALO yourself
|
|
634
|
-
* and configure its LLM provider (it uses the OpenAI client — `OPENAI_BASE_URL`
|
|
635
|
-
* / `OPENAI_API_KEY`); this just drives it. */
|
|
925
|
+
/** HALO (github.com/context-labs/halo) over the OTLP artifact: `halo <file> -p
|
|
926
|
+
* <prompt> [-m <model>]`. Our `serializeSpans` already emits canonical
|
|
927
|
+
* OpenInference, so HALO reads the artifact directly — no conversion. Install
|
|
928
|
+
* HALO yourself and configure its LLM provider (it uses the OpenAI client —
|
|
929
|
+
* `OPENAI_BASE_URL` / `OPENAI_API_KEY`); this just drives it. */
|
|
636
930
|
declare function haloAnalyzer(opts?: {
|
|
637
931
|
command?: string;
|
|
638
932
|
defaultPrompt?: string;
|
|
@@ -875,4 +1169,4 @@ interface ExecuteOptions {
|
|
|
875
1169
|
/** Send the NEW items (or, on dryRun, write the redacted OTLP that would send). */
|
|
876
1170
|
declare function executeUpload(plan: UploadPlan, opts?: ExecuteOptions): Promise<UploadResult>;
|
|
877
1171
|
|
|
878
|
-
export { ATTR, AmpAdapter, type AnalyzeOptions, type AnalyzeResult, CODING_REDACTION_RULES, CONTENT_CAP, ClaudeAdapter, CodexAdapter, type CollectOptions, CopilotAdapter, DEFAULT_HARNESS, type ExecuteOptions, type ExternalAnalysisResult, type ExternalAnalyzer, FactoryAdapter, ForgeAdapter, GeminiAdapter, type HarnessTraceAdapter, INGEST_SOURCE_CLI, type LocateOptions, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpSpan, type OtlpSpanKind, type OtlpStatusCode, PiAdapter, type PipelineOptions, type PipelineReport, type PlanOptions, QwenAdapter, type Redactor, type RunResult, type RuntimeTrace, type ScanOptions, type ScannedSession, type SessionBatch, type SessionRef, type SpanInput, type SpanRedaction, TRACES_REDACTION_RULES, type UploadBackend, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, alreadyUploaded, analyzeSpans, applyRedactor, capText, collectSessions, commandAnalyzer, commandRedactor, executeUpload, haloAnalyzer, knownHarnesses, listAdapters, loadState, parseIsoToEpochMs, parseSince, planUpload, redactSpans, resolveAdapter, runCommand, runExternalAnalyzers, runPipelines, saveState, scanSessions, selectAdapters, serializeSpans, sessionHash, span,
|
|
1172
|
+
export { ACTOR_ATTR, ATTR, type Actor, type AdoptionOptions, type AdoptionReport, AmpAdapter, type AnalyzeOptions, type AnalyzeResult, type BuildPolicyEvidenceOptions, CODING_REDACTION_RULES, CONTENT_CAP, CORRECTIVE_REACTIONS, ClaudeAdapter, CodexAdapter, type CollectOptions, type CollectPolicyEvidenceOptions, CopilotAdapter, DEFAULT_HARNESS, type ExecuteOptions, type ExternalAnalysisResult, type ExternalAnalyzer, FactoryAdapter, ForgeAdapter, GeminiAdapter, type HarnessTraceAdapter, INGEST_SOURCE_CLI, type LocateOptions, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpSpan, type OtlpSpanKind, type OtlpStatusCode, PiAdapter, type PipelineOptions, type PipelineReport, type PlanOptions, type PolicyEvidenceLoopSummary, type PolicyEvidenceRecord, type PolicyEvidenceToolSummary, QwenAdapter, REACTIONS, type Reaction, type ReactionReport, type ReactionSessionReport, type Redactor, type RepoAttrs, type RunResult, type RuntimeTrace, type ScanOptions, type ScannedSession, type SessionBatch, type SessionRef, type SpanInput, type SpanRedaction, TRACES_REDACTION_RULES, type TraceEvidenceExportOptions, type TraceEvidenceExportResult, type TraceEvidenceFormatOption, type TraceEvidenceInputFormat, type TriggerPair, type UploadBackend, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, type UserPromptInput, alreadyUploaded, analyzeAdoption, analyzeReactions, analyzeSpans, applyRedactor, buildPolicyEvidenceRecord, capText, classifyReaction, claudeActor, codexActor, collectPolicyEvidence, collectSessions, commandAnalyzer, commandRedactor, countSkillRunsJsonl, executeUpload, exportTraceEvidenceFile, exportTraceEvidenceRows, exportTraceEvidenceText, haloAnalyzer, knownHarnesses, listAdapters, loadState, looksLikeAgentPrompt, normalizeRemote, parseIsoToEpochMs, parseSession, parseSince, planUpload, redactSpans, resolveAdapter, resolveRepoAttrs, runCommand, runExternalAnalyzers, runPipelines, saveState, scanSessions, selectAdapters, serializePolicyEvidence, serializeSpans, sessionHash, span, stampRepoAttrs, textIsSynthetic, toOpenInferenceSpan, toRuntimeStore, toTraceSpanEvents, uploadKey, userPromptSpan, watchSessions, writeOtlpFile, writePolicyEvidenceFile, writeTraceEvidenceExportFile };
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
|
+
ACTOR_ATTR,
|
|
2
3
|
ATTR,
|
|
3
4
|
AmpAdapter,
|
|
4
5
|
CODING_REDACTION_RULES,
|
|
5
6
|
CONTENT_CAP,
|
|
7
|
+
CORRECTIVE_REACTIONS,
|
|
6
8
|
ClaudeAdapter,
|
|
7
9
|
CodexAdapter,
|
|
8
10
|
CopilotAdapter,
|
|
@@ -14,41 +16,61 @@ import {
|
|
|
14
16
|
OpencodeAdapter,
|
|
15
17
|
PiAdapter,
|
|
16
18
|
QwenAdapter,
|
|
19
|
+
REACTIONS,
|
|
17
20
|
TRACES_REDACTION_RULES,
|
|
18
21
|
alreadyUploaded,
|
|
22
|
+
analyzeAdoption,
|
|
23
|
+
analyzeReactions,
|
|
19
24
|
analyzeSpans,
|
|
20
25
|
applyRedactor,
|
|
26
|
+
buildPolicyEvidenceRecord,
|
|
21
27
|
capText,
|
|
28
|
+
classifyReaction,
|
|
29
|
+
claudeActor,
|
|
30
|
+
codexActor,
|
|
31
|
+
collectPolicyEvidence,
|
|
22
32
|
commandAnalyzer,
|
|
23
33
|
commandRedactor,
|
|
34
|
+
countSkillRunsJsonl,
|
|
24
35
|
executeUpload,
|
|
36
|
+
exportTraceEvidenceFile,
|
|
37
|
+
exportTraceEvidenceRows,
|
|
38
|
+
exportTraceEvidenceText,
|
|
25
39
|
haloAnalyzer,
|
|
26
40
|
knownHarnesses,
|
|
27
41
|
listAdapters,
|
|
28
42
|
loadState,
|
|
43
|
+
looksLikeAgentPrompt,
|
|
44
|
+
normalizeRemote,
|
|
29
45
|
parseIsoToEpochMs,
|
|
46
|
+
parseSession,
|
|
30
47
|
parseSince,
|
|
31
48
|
planUpload,
|
|
32
49
|
redactSpans,
|
|
33
50
|
resolveAdapter,
|
|
51
|
+
resolveRepoAttrs,
|
|
34
52
|
runCommand,
|
|
35
53
|
runExternalAnalyzers,
|
|
36
54
|
runPipelines,
|
|
37
55
|
saveState,
|
|
38
56
|
scanSessions,
|
|
39
57
|
selectAdapters,
|
|
58
|
+
serializePolicyEvidence,
|
|
40
59
|
serializeSpans,
|
|
41
60
|
sessionHash,
|
|
42
61
|
span,
|
|
43
|
-
|
|
62
|
+
stampRepoAttrs,
|
|
63
|
+
textIsSynthetic,
|
|
44
64
|
toOpenInferenceSpan,
|
|
45
65
|
toRuntimeStore,
|
|
46
66
|
toTraceSpanEvents,
|
|
47
67
|
uploadKey,
|
|
48
68
|
userPromptSpan,
|
|
49
69
|
watchSessions,
|
|
50
|
-
writeOtlpFile
|
|
51
|
-
|
|
70
|
+
writeOtlpFile,
|
|
71
|
+
writePolicyEvidenceFile,
|
|
72
|
+
writeTraceEvidenceExportFile
|
|
73
|
+
} from "./chunk-D5OMTM4S.js";
|
|
52
74
|
|
|
53
75
|
// src/collect.ts
|
|
54
76
|
async function collectSessions(opts = {}) {
|
|
@@ -73,11 +95,13 @@ import {
|
|
|
73
95
|
import { createHostedClient, hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
|
|
74
96
|
import { DEFAULT_REDACTION_RULES, redactString, redactValue } from "@tangle-network/agent-eval/traces";
|
|
75
97
|
export {
|
|
98
|
+
ACTOR_ATTR,
|
|
76
99
|
ATTR,
|
|
77
100
|
AmpAdapter,
|
|
78
101
|
AnalystRegistry,
|
|
79
102
|
CODING_REDACTION_RULES,
|
|
80
103
|
CONTENT_CAP,
|
|
104
|
+
CORRECTIVE_REACTIONS,
|
|
81
105
|
ClaudeAdapter,
|
|
82
106
|
CodexAdapter,
|
|
83
107
|
CopilotAdapter,
|
|
@@ -90,45 +114,65 @@ export {
|
|
|
90
114
|
OpencodeAdapter,
|
|
91
115
|
PiAdapter,
|
|
92
116
|
QwenAdapter,
|
|
117
|
+
REACTIONS,
|
|
93
118
|
TRACES_REDACTION_RULES,
|
|
94
119
|
alreadyUploaded,
|
|
120
|
+
analyzeAdoption,
|
|
121
|
+
analyzeReactions,
|
|
95
122
|
analyzeSpans,
|
|
96
123
|
applyRedactor,
|
|
97
124
|
buildDefaultAnalystRegistry,
|
|
125
|
+
buildPolicyEvidenceRecord,
|
|
98
126
|
capText,
|
|
127
|
+
classifyReaction,
|
|
128
|
+
claudeActor,
|
|
129
|
+
codexActor,
|
|
130
|
+
collectPolicyEvidence,
|
|
99
131
|
collectSessions,
|
|
100
132
|
commandAnalyzer,
|
|
101
133
|
commandRedactor,
|
|
134
|
+
countSkillRunsJsonl,
|
|
102
135
|
createHostedClient,
|
|
103
136
|
executeUpload,
|
|
137
|
+
exportTraceEvidenceFile,
|
|
138
|
+
exportTraceEvidenceRows,
|
|
139
|
+
exportTraceEvidenceText,
|
|
104
140
|
haloAnalyzer,
|
|
105
141
|
hostedClientFromEnv,
|
|
106
142
|
knownHarnesses,
|
|
107
143
|
listAdapters,
|
|
108
144
|
loadState,
|
|
145
|
+
looksLikeAgentPrompt,
|
|
109
146
|
makeFinding,
|
|
147
|
+
normalizeRemote,
|
|
110
148
|
parseIsoToEpochMs,
|
|
149
|
+
parseSession,
|
|
111
150
|
parseSince,
|
|
112
151
|
planUpload,
|
|
113
152
|
redactSpans,
|
|
114
153
|
redactString,
|
|
115
154
|
redactValue,
|
|
116
155
|
resolveAdapter,
|
|
156
|
+
resolveRepoAttrs,
|
|
117
157
|
runCommand,
|
|
118
158
|
runExternalAnalyzers,
|
|
119
159
|
runPipelines,
|
|
120
160
|
saveState,
|
|
121
161
|
scanSessions,
|
|
122
162
|
selectAdapters,
|
|
163
|
+
serializePolicyEvidence,
|
|
123
164
|
serializeSpans,
|
|
124
165
|
sessionHash,
|
|
125
166
|
span,
|
|
126
|
-
|
|
167
|
+
stampRepoAttrs,
|
|
168
|
+
textIsSynthetic,
|
|
127
169
|
toOpenInferenceSpan,
|
|
128
170
|
toRuntimeStore,
|
|
129
171
|
toTraceSpanEvents,
|
|
130
172
|
uploadKey,
|
|
131
173
|
userPromptSpan,
|
|
132
174
|
watchSessions,
|
|
133
|
-
writeOtlpFile
|
|
175
|
+
writeOtlpFile,
|
|
176
|
+
writePolicyEvidenceFile,
|
|
177
|
+
writeTraceEvidenceExportFile
|
|
134
178
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/traces",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.5",
|
|
4
4
|
"description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -22,7 +22,6 @@
|
|
|
22
22
|
"llm",
|
|
23
23
|
"coding-agent"
|
|
24
24
|
],
|
|
25
|
-
"packageManager": "pnpm@10.33.4",
|
|
26
25
|
"main": "./dist/index.js",
|
|
27
26
|
"types": "./dist/index.d.ts",
|
|
28
27
|
"exports": {
|
|
@@ -42,23 +41,23 @@
|
|
|
42
41
|
"engines": {
|
|
43
42
|
"node": ">=22"
|
|
44
43
|
},
|
|
45
|
-
"scripts": {
|
|
46
|
-
"dev": "tsx src/cli.ts",
|
|
47
|
-
"build": "tsup src/index.ts src/cli.ts --format esm --target node22 --clean --dts",
|
|
48
|
-
"test": "vitest run",
|
|
49
|
-
"test:watch": "vitest",
|
|
50
|
-
"typecheck": "tsc --noEmit",
|
|
51
|
-
"prepublishOnly": "pnpm build"
|
|
52
|
-
},
|
|
53
44
|
"dependencies": {
|
|
54
45
|
"@ax-llm/ax": "^19.0.45",
|
|
55
|
-
"@tangle-network/agent-eval": "0.
|
|
46
|
+
"@tangle-network/agent-eval": "^0.99.0"
|
|
56
47
|
},
|
|
57
48
|
"devDependencies": {
|
|
58
49
|
"@types/node": "^22.0.0",
|
|
59
50
|
"tsup": "^8.0.0",
|
|
60
51
|
"tsx": "^4.19.0",
|
|
61
52
|
"typescript": "^5.7.0",
|
|
62
|
-
"
|
|
53
|
+
"vite": "^8.1.2",
|
|
54
|
+
"vitest": "^4.1.9"
|
|
55
|
+
},
|
|
56
|
+
"scripts": {
|
|
57
|
+
"dev": "tsx src/cli.ts",
|
|
58
|
+
"build": "tsup src/index.ts src/cli.ts --format esm --target node22 --clean --dts",
|
|
59
|
+
"test": "vitest run",
|
|
60
|
+
"test:watch": "vitest",
|
|
61
|
+
"typecheck": "tsc --noEmit"
|
|
63
62
|
}
|
|
64
|
-
}
|
|
63
|
+
}
|