@tangle-network/agent-knowledge 1.10.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +250 -74
- package/dist/benchmarks/index.d.ts +5 -0
- package/dist/benchmarks/index.js +52 -0
- package/dist/chunk-DQ3PDMDP.js +115 -0
- package/dist/chunk-DQ3PDMDP.js.map +1 -0
- package/dist/chunk-ULXZI235.js +1855 -0
- package/dist/chunk-ULXZI235.js.map +1 -0
- package/dist/{chunk-U6KQ4FS3.js → chunk-VN2OGUUP.js} +306 -40
- package/dist/chunk-VN2OGUUP.js.map +1 -0
- package/dist/{chunk-WWY5FTKQ.js → chunk-W6VWYTNH.js} +10 -115
- package/dist/chunk-W6VWYTNH.js.map +1 -0
- package/dist/chunk-XVU5FFQA.js +49 -0
- package/dist/chunk-XVU5FFQA.js.map +1 -0
- package/dist/cli.js +4 -2
- package/dist/cli.js.map +1 -1
- package/dist/index-Cj5jRXOz.d.ts +426 -0
- package/dist/index.d.ts +1036 -169
- package/dist/index.js +3372 -558
- package/dist/index.js.map +1 -1
- package/dist/memory/index.d.ts +5 -70
- package/dist/memory/index.js +20 -4
- package/dist/types-CjqPcTTK.d.ts +319 -0
- package/docs/architecture.md +7 -5
- package/docs/eval/investment-material-facts.md +252 -0
- package/docs/eval/rag-eval-roadmap.md +119 -0
- package/docs/results/investment-thesis.md +306 -0
- package/docs/results/research-driving.md +265 -0
- package/docs/two-agent-research-ab.md +121 -10
- package/package.json +8 -9
- package/dist/chunk-MU5CEOGE.js +0 -387
- package/dist/chunk-MU5CEOGE.js.map +0 -1
- package/dist/chunk-U6KQ4FS3.js.map +0 -1
- package/dist/chunk-WWY5FTKQ.js.map +0 -1
- package/dist/profiles/index.d.ts +0 -194
- package/dist/profiles/index.js +0 -11
- package/docs/recursive-research-leaf.md +0 -86
- package/docs/supervisor-profiles.md +0 -295
- /package/dist/{profiles → benchmarks}/index.js.map +0 -0
|
@@ -33,7 +33,15 @@ LLM verifier calls 76% and dollars 74%, recovering the de-dup half of the
|
|
|
33
33
|
verifier's cleanliness while honestly giving up the relevance-judgment half on a
|
|
34
34
|
source pool dominated by authoritative hosts. The verifier earns its dollar on
|
|
35
35
|
misattribution, not on de-duplication; the right production loop spends it only
|
|
36
|
-
where the cheap signals can't decide.
|
|
36
|
+
where the cheap signals can't decide. Finally we ask the harder question this whole
|
|
37
|
+
metric can't reach — a filter agent can only make the base *carry less*, never
|
|
38
|
+
*answer more* — by building the opposite agent (a **driving** driver that chases
|
|
39
|
+
depth and corroboration instead of pruning) and the opposite metric (a firewalled
|
|
40
|
+
exam of 20 held-out **deep questions**, $0-graded). It is an honest null: driving
|
|
41
|
+
does **not** reliably beat plain collection at answering hard questions, and costs
|
|
42
|
+
**12–16×** more; the verdict flips with the compute budget, the signature of web
|
|
43
|
+
variance rather than a real topology effect, and it still ties a blind worker even
|
|
44
|
+
when forced to run its full multi-round mechanism (§9).
|
|
37
45
|
|
|
38
46
|
## 1. Setup
|
|
39
47
|
|
|
@@ -93,10 +101,8 @@ proposed. It talks to the router directly through `createTangleRouterClient` —
|
|
|
93
101
|
claude-code / opencode / sandbox harness, and no dynamic harness selection. The
|
|
94
102
|
driver (`createVerifyingResearchDriver`) is one glm-5.2 chat call per source.
|
|
95
103
|
|
|
96
|
-
The
|
|
97
|
-
|
|
98
|
-
it for the direct pipeline. This is a deliberate shortcut (no harness to stand up,
|
|
99
|
-
~$0.20 to run) and also the loop's main simplification debt; see §7.
|
|
104
|
+
The package keeps the live research mechanics runner-agnostic.
|
|
105
|
+
Agent profiles and sandbox-backed runners now live in `agent-runtime`; this package owns the source, write, validation, and eval layer.
|
|
100
106
|
|
|
101
107
|
### 2.3 Equal compute
|
|
102
108
|
|
|
@@ -370,17 +376,109 @@ measured; this is what changed.
|
|
|
370
376
|
|
|
371
377
|
What is still **not** built remains the worker: the live worker is a ~500-line
|
|
372
378
|
hand-wired pipeline (query-gen, search, fetch, propose) against the router directly,
|
|
373
|
-
where the
|
|
379
|
+
where the runtime integration's pattern is to author an agent profile
|
|
374
380
|
and run it on a harness with a web-search tool — reusable and harness-agnostic. The
|
|
375
381
|
direct pipeline is cheaper to run today (no harness, no creds beyond the router) but it
|
|
376
382
|
is the loop's main remaining piece of duplication, and the obvious next step if this
|
|
377
383
|
loop graduates from experiment to production.
|
|
378
384
|
|
|
379
|
-
## 9.
|
|
385
|
+
## 9. From hygiene to depth — a research-DRIVING loop, measured on held-out deep questions
|
|
386
|
+
|
|
387
|
+
Everything above measures one thing: **source hygiene** — how *few* sources a verifier
|
|
388
|
+
admits at equal coverage. That is the right metric for a verifier whose only job is to
|
|
389
|
+
filter, and it is why the win turned out to be de-duplication (§4.1): the most a
|
|
390
|
+
filter-only verifier can do is reject. By construction it cannot make the knowledge base
|
|
391
|
+
*answer more*; it can only make it *carry less*. So "the verifier mostly deduplicates" is
|
|
392
|
+
not a disappointing finding about *this* verifier — it is the ceiling of what *any*
|
|
393
|
+
admit-or-reject step can do. To ask whether a second agent can improve the research
|
|
394
|
+
itself, you have to change both the agent and the metric.
|
|
395
|
+
|
|
396
|
+
So we built the opposite agent and gave it the opposite metric. The **driving driver**
|
|
397
|
+
(`src/research-driving-driver.ts`, `createResearchDrivingDriver`) does not filter. It
|
|
398
|
+
extracts each fetched source's claims, demands a second independent source for every
|
|
399
|
+
claim, generates comparative / mechanism / contradiction sub-questions, and steers the
|
|
400
|
+
worker to chase them in the next round. Its thesis is that *driving the research deeper*
|
|
401
|
+
— not pruning it — builds a knowledge base that answers harder questions. We measure it
|
|
402
|
+
not on admitted-count (which would score its whole point as a regression — it admits
|
|
403
|
+
*more*) but on a **firewalled exam of 20 deep questions across 5 ML topics**
|
|
404
|
+
(`tests/loops/held-out-exam.ts`), graded with a $0 deterministic substring grader the
|
|
405
|
+
loop never sees. The questions are depth questions by construction — the grader scores
|
|
406
|
+
**0/20** on a one-line topic definition and **20/20** on a mechanism-rich paragraph, so
|
|
407
|
+
a high score is reachable only by depth, not by grader slack. All three arms run the
|
|
408
|
+
same real web worker and differ only in the driver: (A) plain collection, (B)
|
|
409
|
+
verify/dedup, (C) driving.
|
|
410
|
+
|
|
411
|
+
**The honest verdict: driving does NOT reliably beat plain collection, and costs
|
|
412
|
+
~12–16× more.** The tell is that the winner flips with the compute budget, on the same
|
|
413
|
+
exam:
|
|
414
|
+
|
|
415
|
+
| arm | answered @ B=4 | answered @ B=6 | cost (5 topics) | tokens |
|
|
416
|
+
|---|---|---|---|---|
|
|
417
|
+
| single-agent (collect) | 13/20 | **15/20** | $0.005–0.007 | ~2.4–3.0k |
|
|
418
|
+
| verify/dedup | **15/20** | **15/20** | $0.031–0.027 | ~21k |
|
|
419
|
+
| **driving (deepen)** | **16/20** | 13/20 | **$0.089–0.084** | ~69–71k |
|
|
420
|
+
|
|
421
|
+
Driving "wins" at B=4 (16 > 15 > 13) and "loses" at B=6 (13 < 15), while the
|
|
422
|
+
single-agent arm itself swings 15→13 across the two budgets. At n=5 topics a ±1–3
|
|
423
|
+
question difference is inside the run-to-run web variance — the **within-arm swing is as
|
|
424
|
+
large as the between-arm gap**, which is the signature of a null. What is stable and
|
|
425
|
+
large is the cost: an order of magnitude more dollars and ~24× the tokens, for the
|
|
426
|
+
claim-extraction LLM call driving runs on every fetched source.
|
|
427
|
+
|
|
428
|
+
The autopsy explains it: every arm finished in **one effective round**
|
|
429
|
+
(`passes=2` on every topic at every budget). The generic readiness gate — "one source
|
|
430
|
+
closes the gap" — is met by the first fetch, so the loop stops *before* the driving
|
|
431
|
+
driver ever steers a second round, and its entire mechanism is multi-round. So we gave
|
|
432
|
+
it its fairest test: a controlled probe that *forces* three rounds so the driver
|
|
433
|
+
actually steers. It **still ties** a blind worker that just re-searches the same gaps —
|
|
434
|
+
**8/12 vs 8/12, at ~9× the cost**. Driving was better on RLHF (3 vs 1 — chasing
|
|
435
|
+
corroboration reached a page blind search missed) and worse on speculative decoding (2
|
|
436
|
+
vs 4 — steering pulled the worker *off* the pages that answered the exam); the two
|
|
437
|
+
cancel. The null survives the fix, so it is not an artifact of a permissive gate.
|
|
438
|
+
|
|
439
|
+
The durable output is the apparatus, not the agent: a firewalled deep-question exam with
|
|
440
|
+
a $0 grader that can tell depth from surface, reusable for any future research-quality
|
|
441
|
+
claim. Full result, per-topic tables, and the probe: [`docs/results/research-driving.md`](results/research-driving.md).
|
|
442
|
+
The two findings compose into one rule for "should I add a second research agent?":
|
|
443
|
+
a **filter** agent measured on hygiene buys de-dup cleanliness you can get cheaper from a
|
|
444
|
+
hash (§3–§4); a **driver** agent measured on depth buys nothing reliable over plain
|
|
445
|
+
collection at this n and worker, for 9–16× the cost (§9). Neither earns a blanket "yes";
|
|
446
|
+
both earn a narrow, cost-stratified one — the verifier on misattribution and the
|
|
447
|
+
off-scope tail (§5), the driver only where a richer worker makes "go corroborate this"
|
|
448
|
+
reach a page collection can't.
|
|
449
|
+
|
|
450
|
+
### 9.1 The domain was too easy — re-running the A/B where one search is not enough
|
|
451
|
+
|
|
452
|
+
The §9 null has a structural cause, not a measurement one: **on an ML topic a single
|
|
453
|
+
good search already collects the answer.** Every arm finished in one effective round
|
|
454
|
+
because the first fetch met the readiness gate, so the driving driver — whose mechanism
|
|
455
|
+
is steering a *second* round — never acted. When one search suffices, there is no
|
|
456
|
+
investigation for a smarter coordinator to do, and the metric can only reward
|
|
457
|
+
collection. To ask whether topology *can ever* beat blind collection, you have to move
|
|
458
|
+
to a domain where the answer is buried and a single fetch provably cannot surface it.
|
|
459
|
+
|
|
460
|
+
So we did. We ported the whole apparatus — firewalled checklist, `$0` model-free
|
|
461
|
+
grader, matched-compute 3-arm A/B — onto **investment research**: give a loop a company
|
|
462
|
+
+ ticker + an as-of cutoff and grade the thesis on the buried, material 10-K-footnote
|
|
463
|
+
facts a ticker search misses (an HTM mark the size of a bank's equity, a 97%-uninsured
|
|
464
|
+
deposit base, a negative per-unit margin). First we *calibrated* the new metric and
|
|
465
|
+
proved it discriminates depth on this harder domain — a shallow ticker summary scores
|
|
466
|
+
**1/27 (4%)**, a deep filings-grounded thesis **27/27 (100%)**, a **+96-point** gap.
|
|
467
|
+
Then the live 3-arm A/B over 5 held-out companies: **driving surfaced the most buried
|
|
468
|
+
facts (16/27, 59%) vs blind collection (11/27, 41%)** for ~1.9× the cost — the lift is
|
|
469
|
+
real and points the right way, but at n=5 the paired-bootstrap CI still crosses zero
|
|
470
|
+
(P(Δ≤0)=0.08), and verify did not beat collection (10/27). So the verdict survives the
|
|
471
|
+
domain change: **no topology *significantly* beats collection — but on a domain where
|
|
472
|
+
the answer must be investigated for, driving is the only arm that even leans positive,
|
|
473
|
+
and it does so suggestively, not significantly.** Full reframe, calibration, and
|
|
474
|
+
per-company A/B: [`docs/results/investment-thesis.md`](results/investment-thesis.md).
|
|
475
|
+
|
|
476
|
+
## 10. Reproduce
|
|
380
477
|
|
|
381
478
|
The loop, the worker, the verifier, the claim-grounding mode, the adaptive driver, the
|
|
382
|
-
cost instrumentation, and every A/B are all in
|
|
383
|
-
cheap one-call glm-5.2 smoke before any
|
|
479
|
+
driving driver, the held-out exam, the cost instrumentation, and every A/B are all in
|
|
480
|
+
this repository. Each live test gates a cheap one-call glm-5.2 smoke before any
|
|
481
|
+
multi-topic burn.
|
|
384
482
|
|
|
385
483
|
```bash
|
|
386
484
|
git clone https://github.com/tangle-network/agent-knowledge
|
|
@@ -410,6 +508,15 @@ AGENT_KNOWLEDGE_LIVE=1 TANGLE_API_KEY=<…> \
|
|
|
410
508
|
AGENT_KNOWLEDGE_LIVE=1 TANGLE_API_KEY=<…> \
|
|
411
509
|
ADAPTIVE_LIVE_GOALS="self-speculative decoding|rotary position embeddings|grouped-query attention|KV-cache quantization|LoRA fine-tuning" \
|
|
412
510
|
pnpm exec vitest run tests/loops/adaptive-ab.test.ts -t "three-topology"
|
|
511
|
+
|
|
512
|
+
# the research-DRIVING 3-arm A/B (§9) — collect / verify / drive, graded on the
|
|
513
|
+
# held-out deep-question exam; re-run at RQ_LIVE_BUDGET=6 to see the verdict flip
|
|
514
|
+
AGENT_KNOWLEDGE_LIVE=1 RQ_LIVE_BUDGET=4 TANGLE_API_KEY=<…> \
|
|
515
|
+
pnpm exec vitest run tests/loops/research-driving-ab.test.ts -t "3-arm A/B"
|
|
516
|
+
|
|
517
|
+
# the controlled multi-round probe — forces 3 rounds so the driver actually steers
|
|
518
|
+
AGENT_KNOWLEDGE_LIVE=1 RQ_PROBE=1 RQ_PROBE_ROUNDS=3 TANGLE_API_KEY=<…> \
|
|
519
|
+
pnpm exec vitest run tests/loops/research-driving-ab.test.ts -t "multi-round probe"
|
|
413
520
|
```
|
|
414
521
|
|
|
415
522
|
`AGENT_KNOWLEDGE_LIVE_GOALS` (and the per-result `*_LIVE_GOALS`) take a `|`-separated
|
|
@@ -420,9 +527,13 @@ bootstrap and per-arm cost.
|
|
|
420
527
|
the live worker + verifier + cost instrumentation — [`src/web-research-worker.ts`](../src/web-research-worker.ts);
|
|
421
528
|
the misattribution check — [`src/claim-grounding.ts`](../src/claim-grounding.ts);
|
|
422
529
|
the adaptive driver — [`src/adaptive-driver.ts`](../src/adaptive-driver.ts);
|
|
530
|
+
the driving driver — [`src/research-driving-driver.ts`](../src/research-driving-driver.ts);
|
|
531
|
+
the held-out exam + $0 grader — [`tests/loops/held-out-exam.ts`](../tests/loops/held-out-exam.ts);
|
|
423
532
|
the A/B harnesses — [`tests/loops/`](../tests/loops/).
|
|
424
533
|
Per-result detail: [`docs/results/cost-quality.md`](results/cost-quality.md),
|
|
425
534
|
[`docs/results/claim-grounding.md`](results/claim-grounding.md),
|
|
426
|
-
[`docs/results/adaptive.md`](results/adaptive.md)
|
|
535
|
+
[`docs/results/adaptive.md`](results/adaptive.md),
|
|
536
|
+
[`docs/results/research-driving.md`](results/research-driving.md),
|
|
537
|
+
[`docs/results/investment-thesis.md`](results/investment-thesis.md) (§9.1 — the domain reframe + calibration + 3-arm A/B).
|
|
427
538
|
</content>
|
|
428
539
|
</invoke>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-knowledge",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.11.0",
|
|
4
4
|
"description": "Source-grounded, eval-gated knowledge growth primitives for agents.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-knowledge#readme",
|
|
6
6
|
"repository": {
|
|
@@ -39,10 +39,10 @@
|
|
|
39
39
|
"import": "./dist/sources/index.js",
|
|
40
40
|
"default": "./dist/sources/index.js"
|
|
41
41
|
},
|
|
42
|
-
"./
|
|
43
|
-
"types": "./dist/
|
|
44
|
-
"import": "./dist/
|
|
45
|
-
"default": "./dist/
|
|
42
|
+
"./benchmarks": {
|
|
43
|
+
"types": "./dist/benchmarks/index.d.ts",
|
|
44
|
+
"import": "./dist/benchmarks/index.js",
|
|
45
|
+
"default": "./dist/benchmarks/index.js"
|
|
46
46
|
}
|
|
47
47
|
},
|
|
48
48
|
"bin": {
|
|
@@ -68,16 +68,16 @@
|
|
|
68
68
|
"format": "biome format --write src tests"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@tangle-network/agent-eval": "^0.
|
|
72
|
-
"@tangle-network/agent-runtime": "^0.77.0",
|
|
71
|
+
"@tangle-network/agent-eval": "^0.107.0",
|
|
73
72
|
"zod": "^4.3.6"
|
|
74
73
|
},
|
|
75
74
|
"devDependencies": {
|
|
76
75
|
"@biomejs/biome": "^2.4.15",
|
|
77
76
|
"@neo4j-labs/agent-memory": "0.4.0",
|
|
78
|
-
"@tangle-network/sandbox": "^0.
|
|
77
|
+
"@tangle-network/sandbox": "^0.9.7",
|
|
79
78
|
"@types/node": "^25.6.0",
|
|
80
79
|
"tsup": "^8.0.0",
|
|
80
|
+
"tsx": "^4.22.4",
|
|
81
81
|
"typescript": "^5.7.0",
|
|
82
82
|
"vitest": "^3.0.0"
|
|
83
83
|
},
|
|
@@ -85,7 +85,6 @@
|
|
|
85
85
|
"minimumReleaseAge": 4320,
|
|
86
86
|
"minimumReleaseAgeExclude": [
|
|
87
87
|
"@tangle-network/agent-eval",
|
|
88
|
-
"@tangle-network/agent-runtime",
|
|
89
88
|
"@tangle-network/sandbox"
|
|
90
89
|
]
|
|
91
90
|
},
|
package/dist/chunk-MU5CEOGE.js
DELETED
|
@@ -1,387 +0,0 @@
|
|
|
1
|
-
// src/profiles/researcher.ts
|
|
2
|
-
var DEFAULT_HARNESS = "opencode/zai-coding-plan/glm-5.1";
|
|
3
|
-
var DEFAULT_CITATION_DENSITY_MIN = 0.7;
|
|
4
|
-
function researcherProfile(options = {}) {
|
|
5
|
-
const harness = options.harness ?? DEFAULT_HARNESS;
|
|
6
|
-
const name = options.name ?? `researcher-${harness}`;
|
|
7
|
-
const systemPrompt = options.systemPrompt ?? DEFAULT_RESEARCHER_SYSTEM_PROMPT;
|
|
8
|
-
const citationDensityMin = options.citationDensityMin ?? DEFAULT_CITATION_DENSITY_MIN;
|
|
9
|
-
const profile = {
|
|
10
|
-
name,
|
|
11
|
-
description: "Source-grounded research agent. Propose-don't-apply.",
|
|
12
|
-
prompt: { systemPrompt },
|
|
13
|
-
model: options.model ? { default: options.model } : void 0,
|
|
14
|
-
tools: { web_search: true, fs: true, shell: true },
|
|
15
|
-
metadata: { backendType: harness, role: "researcher" }
|
|
16
|
-
};
|
|
17
|
-
const output = { parse: parseResearcherEvents };
|
|
18
|
-
const validator = options.task ? createResearcherValidator(options.task, { citationDensityMin }) : createResearcherValidator(
|
|
19
|
-
{ question: "", knowledgeNamespace: "" },
|
|
20
|
-
{ citationDensityMin, namespaceCheck: false }
|
|
21
|
-
);
|
|
22
|
-
const agentRunSpec = {
|
|
23
|
-
name,
|
|
24
|
-
profile,
|
|
25
|
-
taskToPrompt: formatResearcherPrompt
|
|
26
|
-
};
|
|
27
|
-
return { profile, taskToPrompt: formatResearcherPrompt, output, validator, agentRunSpec };
|
|
28
|
-
}
|
|
29
|
-
function multiHarnessResearcherFanout(options = {}) {
|
|
30
|
-
const harnesses = options.harnesses && options.harnesses.length > 0 ? options.harnesses : ["opencode/zai-coding-plan/glm-5.1", "claude-code", "codex"];
|
|
31
|
-
const models = options.models ?? [];
|
|
32
|
-
const agentRuns = harnesses.map((harness, i) => {
|
|
33
|
-
const { agentRunSpec } = researcherProfile({ harness, model: models[i] });
|
|
34
|
-
return agentRunSpec;
|
|
35
|
-
});
|
|
36
|
-
const { output, validator } = researcherProfile({
|
|
37
|
-
citationDensityMin: options.citationDensityMin,
|
|
38
|
-
task: options.task
|
|
39
|
-
});
|
|
40
|
-
const driver = {
|
|
41
|
-
name: "researcher-fanout",
|
|
42
|
-
async plan(task, history) {
|
|
43
|
-
return history.length === 0 ? Array.from({ length: harnesses.length }, () => task) : [];
|
|
44
|
-
},
|
|
45
|
-
// 'done' is a terminal decision in the kernel; the loop finalizes after
|
|
46
|
-
// the single fanout round and selects the winner via `defaultSelectWinner`.
|
|
47
|
-
decide() {
|
|
48
|
-
return "done";
|
|
49
|
-
},
|
|
50
|
-
describePlan() {
|
|
51
|
-
return { kind: "fanout" };
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
return { agentRuns, output, validator, driver };
|
|
55
|
-
}
|
|
56
|
-
function createResearcherValidator(task, config = {}) {
|
|
57
|
-
const citationDensityMin = config.citationDensityMin ?? DEFAULT_CITATION_DENSITY_MIN;
|
|
58
|
-
const namespaceCheck = config.namespaceCheck ?? true;
|
|
59
|
-
return {
|
|
60
|
-
async validate(output) {
|
|
61
|
-
const notes = [];
|
|
62
|
-
const scores = {};
|
|
63
|
-
let pass = true;
|
|
64
|
-
if (!Array.isArray(output.items) || output.items.length === 0) {
|
|
65
|
-
pass = false;
|
|
66
|
-
notes.push("no items");
|
|
67
|
-
scores.items = 0;
|
|
68
|
-
} else {
|
|
69
|
-
scores.items = 1;
|
|
70
|
-
}
|
|
71
|
-
const missingEvidence = (output.items ?? []).filter(
|
|
72
|
-
(item) => !Array.isArray(item.evidence) || item.evidence.length === 0
|
|
73
|
-
);
|
|
74
|
-
if (missingEvidence.length > 0) {
|
|
75
|
-
pass = false;
|
|
76
|
-
notes.push(`${missingEvidence.length} item(s) without evidence`);
|
|
77
|
-
scores.provenance = 0;
|
|
78
|
-
} else {
|
|
79
|
-
scores.provenance = 1;
|
|
80
|
-
}
|
|
81
|
-
if (namespaceCheck) {
|
|
82
|
-
const foreignItems = (output.items ?? []).filter(
|
|
83
|
-
(item) => item.namespace !== task.knowledgeNamespace
|
|
84
|
-
);
|
|
85
|
-
const foreignWrites = (output.proposedWrites ?? []).filter(
|
|
86
|
-
(write) => write.namespace !== task.knowledgeNamespace
|
|
87
|
-
);
|
|
88
|
-
if (foreignItems.length > 0 || foreignWrites.length > 0) {
|
|
89
|
-
pass = false;
|
|
90
|
-
notes.push(
|
|
91
|
-
`namespace violation: ${foreignItems.length} item(s) + ${foreignWrites.length} write(s) outside ${task.knowledgeNamespace}`
|
|
92
|
-
);
|
|
93
|
-
scores.namespace = 0;
|
|
94
|
-
} else {
|
|
95
|
-
scores.namespace = 1;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
const itemCount = Math.max(output.items?.length ?? 0, 1);
|
|
99
|
-
const citationsWithQuote = (output.citations ?? []).filter(
|
|
100
|
-
(citation) => typeof citation.quote === "string" && citation.quote.length > 0
|
|
101
|
-
).length;
|
|
102
|
-
const citationDensity = Math.min(1, citationsWithQuote / itemCount);
|
|
103
|
-
if (citationDensity < citationDensityMin) {
|
|
104
|
-
pass = false;
|
|
105
|
-
notes.push(
|
|
106
|
-
`citation density ${citationDensity.toFixed(2)} below floor ${citationDensityMin.toFixed(2)}`
|
|
107
|
-
);
|
|
108
|
-
}
|
|
109
|
-
scores.citation_density = citationDensity;
|
|
110
|
-
const sourceSet = /* @__PURE__ */ new Set();
|
|
111
|
-
for (const item of output.items ?? []) {
|
|
112
|
-
for (const evidence of item.evidence ?? []) {
|
|
113
|
-
if (evidence.source) sourceSet.add(evidence.source);
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
scores.source_diversity = Math.min(1, sourceSet.size / itemCount);
|
|
117
|
-
scores.recency_match = recencyMatchScore(output.items ?? [], task.recencyWindow);
|
|
118
|
-
const maxGaps = itemCount;
|
|
119
|
-
const gapCount = output.gaps?.length ?? 0;
|
|
120
|
-
scores.gap_coverage = Math.max(0, 1 - gapCount / maxGaps);
|
|
121
|
-
const score = 0.4 * scores.citation_density + 0.2 * scores.source_diversity + 0.2 * scores.recency_match + 0.2 * scores.gap_coverage;
|
|
122
|
-
const verdict = {
|
|
123
|
-
valid: pass,
|
|
124
|
-
score: Number.isFinite(score) ? score : 0,
|
|
125
|
-
scores
|
|
126
|
-
};
|
|
127
|
-
if (notes.length > 0) verdict.notes = notes.join("; ");
|
|
128
|
-
return verdict;
|
|
129
|
-
}
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
|
-
function recencyMatchScore(items, window) {
|
|
133
|
-
if (!window || window.since === void 0 && window.until === void 0) return 1;
|
|
134
|
-
if (items.length === 0) return 0;
|
|
135
|
-
const sinceMs = window.since?.getTime() ?? Number.NEGATIVE_INFINITY;
|
|
136
|
-
const untilMs = window.until?.getTime() ?? Number.POSITIVE_INFINITY;
|
|
137
|
-
let hits = 0;
|
|
138
|
-
let total = 0;
|
|
139
|
-
for (const item of items) {
|
|
140
|
-
for (const evidence of item.evidence ?? []) {
|
|
141
|
-
if (typeof evidence.capturedAt !== "number") continue;
|
|
142
|
-
total += 1;
|
|
143
|
-
if (evidence.capturedAt >= sinceMs && evidence.capturedAt <= untilMs) hits += 1;
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
return total === 0 ? 0 : hits / total;
|
|
147
|
-
}
|
|
148
|
-
var DEFAULT_RESEARCHER_SYSTEM_PROMPT = [
|
|
149
|
-
"You are a research agent. Your job is to answer a research question with",
|
|
150
|
-
"source-grounded knowledge items that the caller will choose whether to",
|
|
151
|
-
"persist to a multi-tenant knowledge base.",
|
|
152
|
-
"",
|
|
153
|
-
"Hard rules:",
|
|
154
|
-
" 1. Every item you emit MUST carry the task's knowledgeNamespace exactly.",
|
|
155
|
-
" Never write to a different namespace.",
|
|
156
|
-
" 2. Every item MUST carry at least one evidence entry with a source.",
|
|
157
|
-
" A quote + url is strongly preferred; capturedAt is unix ms.",
|
|
158
|
-
" 3. You propose writes \u2014 you do NOT apply them. The caller decides.",
|
|
159
|
-
" 4. Self-report confidence honestly in [0, 1]. Do not inflate.",
|
|
160
|
-
" 5. List what you could not answer in `gaps`. Better to admit a gap",
|
|
161
|
-
" than fabricate.",
|
|
162
|
-
"",
|
|
163
|
-
"When you finish, emit a single final structured message of the shape:",
|
|
164
|
-
" ```json",
|
|
165
|
-
' { "items": [{ "id": "...", "namespace": "...", "claim": "...",',
|
|
166
|
-
' "evidence": [{ "source": "...", "quote": "...",',
|
|
167
|
-
' "url": "...", "capturedAt": 0 }],',
|
|
168
|
-
' "confidence": 0.0,',
|
|
169
|
-
' "authoredBy": { "kind": "agent", "id": "..." } }],',
|
|
170
|
-
' "citations": [{ "url": "...", "quote": "...", "confidence": 0.0 }],',
|
|
171
|
-
' "proposedWrites": [{ "kind": "insert", "namespace": "...",',
|
|
172
|
-
' "item": { /* same shape as items[] */ } }],',
|
|
173
|
-
' "gaps": ["..."],',
|
|
174
|
-
' "notes": "free-form commentary" }',
|
|
175
|
-
" ```"
|
|
176
|
-
].join("\n");
|
|
177
|
-
function formatResearcherPrompt(task) {
|
|
178
|
-
const sources = task.sources?.length ? task.sources.join(", ") : "(no preference)";
|
|
179
|
-
const window = formatRecencyWindow(task.recencyWindow);
|
|
180
|
-
return [
|
|
181
|
-
`Question: ${task.question}`,
|
|
182
|
-
`Knowledge namespace (DO NOT cross): ${task.knowledgeNamespace}`,
|
|
183
|
-
`Scope: ${task.scope ?? "(unspecified)"}`,
|
|
184
|
-
`Preferred sources: ${sources}`,
|
|
185
|
-
`Recency window: ${window}`,
|
|
186
|
-
`Max items: ${task.maxItems ?? "(no cap)"}`,
|
|
187
|
-
`Per-item minimum confidence: ${task.minConfidence ?? "(no floor)"}`,
|
|
188
|
-
"",
|
|
189
|
-
"Produce knowledge items with provenance + citations + proposed writes.",
|
|
190
|
-
"List gaps for anything you could not answer. Emit the final JSON",
|
|
191
|
-
"result block exactly as instructed."
|
|
192
|
-
].join("\n");
|
|
193
|
-
}
|
|
194
|
-
function formatRecencyWindow(window) {
|
|
195
|
-
if (!window) return "(none)";
|
|
196
|
-
const since = window.since ? window.since.toISOString() : "-\u221E";
|
|
197
|
-
const until = window.until ? window.until.toISOString() : "now";
|
|
198
|
-
return `${since} .. ${until}`;
|
|
199
|
-
}
|
|
200
|
-
function parseResearcherEvents(events) {
|
|
201
|
-
for (let i = events.length - 1; i >= 0; i -= 1) {
|
|
202
|
-
const event = events[i];
|
|
203
|
-
if (!event) continue;
|
|
204
|
-
const type = String(event.type ?? "");
|
|
205
|
-
const data = isRecord(event.data) ? event.data : {};
|
|
206
|
-
if (type === "result" || type === "final" || type === "research.result") {
|
|
207
|
-
const direct = coerceResearchOutput(data.result ?? data.output ?? data);
|
|
208
|
-
const finalText = pickString(data.finalText);
|
|
209
|
-
const fenced = finalText ? extractFencedJson(finalText) : void 0;
|
|
210
|
-
const fromFinalText = fenced ? coerceResearchOutput(fenced) : void 0;
|
|
211
|
-
if (direct && !isEmptyOutput(direct)) return direct;
|
|
212
|
-
if (fromFinalText) return fromFinalText;
|
|
213
|
-
if (direct) return direct;
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
for (let i = events.length - 1; i >= 0; i -= 1) {
|
|
217
|
-
const event = events[i];
|
|
218
|
-
if (!event) continue;
|
|
219
|
-
const data = isRecord(event.data) ? event.data : {};
|
|
220
|
-
const text = pickString(data.text) ?? pickString(data.delta) ?? pickString(data.finalText);
|
|
221
|
-
if (!text) continue;
|
|
222
|
-
const fenced = extractFencedJson(text);
|
|
223
|
-
if (!fenced) continue;
|
|
224
|
-
const coerced = coerceResearchOutput(fenced);
|
|
225
|
-
if (coerced) return coerced;
|
|
226
|
-
}
|
|
227
|
-
return { items: [], citations: [], proposedWrites: [] };
|
|
228
|
-
}
|
|
229
|
-
function isRecord(value) {
|
|
230
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
231
|
-
}
|
|
232
|
-
function isEmptyOutput(o) {
|
|
233
|
-
return o.items.length === 0 && o.citations.length === 0 && o.proposedWrites.length === 0;
|
|
234
|
-
}
|
|
235
|
-
function pickString(value) {
|
|
236
|
-
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
237
|
-
}
|
|
238
|
-
function extractFencedJson(text) {
|
|
239
|
-
const match = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
|
240
|
-
if (!match) return void 0;
|
|
241
|
-
const body = (match[1] ?? "").trim();
|
|
242
|
-
if (!body) return void 0;
|
|
243
|
-
try {
|
|
244
|
-
return JSON.parse(body);
|
|
245
|
-
} catch {
|
|
246
|
-
return void 0;
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
function coerceResearchOutput(value) {
|
|
250
|
-
if (!isRecord(value)) return void 0;
|
|
251
|
-
const items = coerceItems(value.items);
|
|
252
|
-
const citations = coerceCitations(value.citations);
|
|
253
|
-
const proposedWrites = coerceProposedWrites(value.proposedWrites);
|
|
254
|
-
if (items === void 0 && citations === void 0 && proposedWrites === void 0) {
|
|
255
|
-
return void 0;
|
|
256
|
-
}
|
|
257
|
-
const output = {
|
|
258
|
-
items: items ?? [],
|
|
259
|
-
citations: citations ?? [],
|
|
260
|
-
proposedWrites: proposedWrites ?? []
|
|
261
|
-
};
|
|
262
|
-
if (Array.isArray(value.gaps)) {
|
|
263
|
-
output.gaps = value.gaps.filter((entry) => typeof entry === "string");
|
|
264
|
-
}
|
|
265
|
-
const notes = pickString(value.notes);
|
|
266
|
-
if (notes) output.notes = notes;
|
|
267
|
-
const known = /* @__PURE__ */ new Set(["items", "citations", "proposedWrites", "gaps", "notes"]);
|
|
268
|
-
const extras = {};
|
|
269
|
-
let extrasCount = 0;
|
|
270
|
-
for (const [key, val] of Object.entries(value)) {
|
|
271
|
-
if (known.has(key)) continue;
|
|
272
|
-
extras[key] = val;
|
|
273
|
-
extrasCount += 1;
|
|
274
|
-
}
|
|
275
|
-
if (extrasCount > 0) output.raw = extras;
|
|
276
|
-
return output;
|
|
277
|
-
}
|
|
278
|
-
function coerceItems(value) {
|
|
279
|
-
if (!Array.isArray(value)) return void 0;
|
|
280
|
-
const out = [];
|
|
281
|
-
for (const entry of value) {
|
|
282
|
-
if (!isRecord(entry)) continue;
|
|
283
|
-
const id = pickString(entry.id);
|
|
284
|
-
const namespace = pickString(entry.namespace);
|
|
285
|
-
const claim = pickString(entry.claim);
|
|
286
|
-
const evidence = coerceEvidence(entry.evidence);
|
|
287
|
-
const confidence = toFiniteNumber(entry.confidence);
|
|
288
|
-
const authoredBy = coerceAuthoredBy(entry.authoredBy);
|
|
289
|
-
if (!id || !namespace || !claim || !authoredBy) continue;
|
|
290
|
-
const item = {
|
|
291
|
-
id,
|
|
292
|
-
namespace,
|
|
293
|
-
claim,
|
|
294
|
-
evidence,
|
|
295
|
-
confidence: clamp01(confidence),
|
|
296
|
-
authoredBy
|
|
297
|
-
};
|
|
298
|
-
if (Array.isArray(entry.supersedes)) {
|
|
299
|
-
item.supersedes = entry.supersedes.filter((s) => typeof s === "string");
|
|
300
|
-
}
|
|
301
|
-
const retractedAt = toFiniteNumber(entry.retractedAt);
|
|
302
|
-
if (retractedAt > 0) item.retractedAt = retractedAt;
|
|
303
|
-
out.push(item);
|
|
304
|
-
}
|
|
305
|
-
return out;
|
|
306
|
-
}
|
|
307
|
-
function coerceEvidence(value) {
|
|
308
|
-
if (!Array.isArray(value)) return [];
|
|
309
|
-
const out = [];
|
|
310
|
-
for (const entry of value) {
|
|
311
|
-
if (!isRecord(entry)) continue;
|
|
312
|
-
const source = pickString(entry.source);
|
|
313
|
-
if (!source) continue;
|
|
314
|
-
const item = {
|
|
315
|
-
source,
|
|
316
|
-
capturedAt: toFiniteNumber(entry.capturedAt)
|
|
317
|
-
};
|
|
318
|
-
const quote = pickString(entry.quote);
|
|
319
|
-
if (quote) item.quote = quote;
|
|
320
|
-
const url = pickString(entry.url);
|
|
321
|
-
if (url) item.url = url;
|
|
322
|
-
out.push(item);
|
|
323
|
-
}
|
|
324
|
-
return out;
|
|
325
|
-
}
|
|
326
|
-
function coerceAuthoredBy(value) {
|
|
327
|
-
if (!isRecord(value)) return void 0;
|
|
328
|
-
const kind = value.kind === "human" || value.kind === "agent" ? value.kind : void 0;
|
|
329
|
-
const id = pickString(value.id);
|
|
330
|
-
if (!kind || !id) return void 0;
|
|
331
|
-
return { kind, id };
|
|
332
|
-
}
|
|
333
|
-
function coerceCitations(value) {
|
|
334
|
-
if (!Array.isArray(value)) return void 0;
|
|
335
|
-
const out = [];
|
|
336
|
-
for (const entry of value) {
|
|
337
|
-
if (!isRecord(entry)) continue;
|
|
338
|
-
const url = pickString(entry.url);
|
|
339
|
-
const quote = pickString(entry.quote);
|
|
340
|
-
if (!url || !quote) continue;
|
|
341
|
-
out.push({ url, quote, confidence: clamp01(toFiniteNumber(entry.confidence)) });
|
|
342
|
-
}
|
|
343
|
-
return out;
|
|
344
|
-
}
|
|
345
|
-
function coerceProposedWrites(value) {
|
|
346
|
-
if (!Array.isArray(value)) return void 0;
|
|
347
|
-
const out = [];
|
|
348
|
-
for (const entry of value) {
|
|
349
|
-
if (!isRecord(entry)) continue;
|
|
350
|
-
const namespace = pickString(entry.namespace);
|
|
351
|
-
if (!namespace) continue;
|
|
352
|
-
if (entry.kind === "insert") {
|
|
353
|
-
const items = coerceItems([entry.item]);
|
|
354
|
-
const item = items?.[0];
|
|
355
|
-
if (!item) continue;
|
|
356
|
-
out.push({ kind: "insert", namespace, item });
|
|
357
|
-
} else if (entry.kind === "supersede") {
|
|
358
|
-
const previousId = pickString(entry.previousId);
|
|
359
|
-
const items = coerceItems([entry.item]);
|
|
360
|
-
const item = items?.[0];
|
|
361
|
-
if (!previousId || !item) continue;
|
|
362
|
-
out.push({ kind: "supersede", namespace, previousId, item });
|
|
363
|
-
} else if (entry.kind === "retract") {
|
|
364
|
-
const itemId = pickString(entry.itemId);
|
|
365
|
-
const reason = pickString(entry.reason);
|
|
366
|
-
if (!itemId || !reason) continue;
|
|
367
|
-
out.push({ kind: "retract", namespace, itemId, reason });
|
|
368
|
-
}
|
|
369
|
-
}
|
|
370
|
-
return out;
|
|
371
|
-
}
|
|
372
|
-
function toFiniteNumber(value) {
|
|
373
|
-
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
374
|
-
}
|
|
375
|
-
function clamp01(value) {
|
|
376
|
-
if (!Number.isFinite(value)) return 0;
|
|
377
|
-
if (value < 0) return 0;
|
|
378
|
-
if (value > 1) return 1;
|
|
379
|
-
return value;
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
export {
|
|
383
|
-
researcherProfile,
|
|
384
|
-
multiHarnessResearcherFanout,
|
|
385
|
-
createResearcherValidator
|
|
386
|
-
};
|
|
387
|
-
//# sourceMappingURL=chunk-MU5CEOGE.js.map
|