@tangle-network/agent-knowledge 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +32 -18
- package/CHANGELOG.md +32 -0
- package/README.md +31 -19
- package/dist/benchmarks/index.d.ts +51 -3
- package/dist/benchmarks/index.js +1 -1
- package/dist/{chunk-EUAXSBMH.js → chunk-AKYJG2MR.js} +224 -224
- package/dist/chunk-AKYJG2MR.js.map +1 -0
- package/dist/{chunk-DW6APRTX.js → chunk-BBCIB4UL.js} +2486 -2457
- package/dist/chunk-BBCIB4UL.js.map +1 -0
- package/dist/{chunk-UWOTQNBI.js → chunk-CPMLJYA3.js} +1886 -1842
- package/dist/chunk-CPMLJYA3.js.map +1 -0
- package/dist/{chunk-WCYW2GDA.js → chunk-MYFM6LKH.js} +2 -2
- package/dist/chunk-MYFM6LKH.js.map +1 -0
- package/dist/cli.js +1 -1
- package/dist/index.d.ts +48 -52
- package/dist/index.js +2906 -2874
- package/dist/index.js.map +1 -1
- package/dist/memory/index.d.ts +129 -6
- package/dist/memory/index.js +2 -2
- package/dist/sources/index.d.ts +17 -33
- package/dist/sources/index.js +1 -1
- package/dist/{index-Bqg1mBPt.d.ts → types-DP38encz.d.ts} +127 -284
- package/docs/eval/investment-material-facts.md +28 -29
- package/docs/eval/rag-eval-roadmap.md +6 -5
- package/docs/results/adaptive.md +13 -13
- package/docs/results/claim-grounding.md +11 -11
- package/docs/results/cost-quality.md +4 -4
- package/docs/results/investment-thesis.md +56 -56
- package/docs/results/research-driving.md +54 -54
- package/docs/two-agent-research-ab.md +94 -94
- package/package.json +4 -7
- package/dist/chunk-DW6APRTX.js.map +0 -1
- package/dist/chunk-EUAXSBMH.js.map +0 -1
- package/dist/chunk-UWOTQNBI.js.map +0 -1
- package/dist/chunk-WCYW2GDA.js.map +0 -1
package/AGENTS.md
CHANGED
|
@@ -2,22 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
Use this package when an agent needs persistent, source-grounded knowledge that improves over time.
|
|
4
4
|
|
|
5
|
-
##
|
|
5
|
+
## Package layering
|
|
6
6
|
|
|
7
7
|
```
|
|
8
|
-
agent-
|
|
9
|
-
|
|
10
|
-
agent-
|
|
8
|
+
agent-runtime
|
|
9
|
+
|
|
|
10
|
+
agent-knowledge
|
|
11
|
+
/ \
|
|
12
|
+
agent-eval agent-interface
|
|
11
13
|
```
|
|
12
14
|
|
|
13
|
-
|
|
15
|
+
Imports flow downward in this diagram.
|
|
16
|
+
`agent-knowledge` may import `agent-eval` and `agent-interface`, but it must not import `agent-runtime`.
|
|
17
|
+
Agent-powered work enters this package through callbacks.
|
|
18
|
+
`agent-runtime` owns live agent execution and composes this package when a complete agent workflow is needed.
|
|
19
|
+
`agent-eval` must not import `agent-knowledge`.
|
|
20
|
+
Shared run and experiment types that knowledge needs live in `agent-eval`.
|
|
14
21
|
|
|
15
22
|
Types that stay in THIS repo because they're knowledge-domain-shaped:
|
|
16
23
|
- `KbStore`, `KnowledgeFragment`, `KnowledgeChange`
|
|
17
24
|
- `KnowledgeDiscoveryDispatcher`, source adapters (`createCornellLiiSource`, `createIrsPublicationsSource`)
|
|
18
25
|
- Freshness store + change-detection primitives
|
|
19
26
|
|
|
20
|
-
**The test for "where does a type live?"**
|
|
27
|
+
**The test for "where does a type live?"**
|
|
28
|
+
If the concept makes sense without persistent knowledge or sourced fragments, it belongs in `agent-eval` or `agent-interface`.
|
|
29
|
+
Otherwise, it stays in this package.
|
|
21
30
|
|
|
22
31
|
## Rules
|
|
23
32
|
|
|
@@ -72,36 +81,42 @@ Use `knowledgeReleaseReport()` before promotion. It folds the candidate and base
|
|
|
72
81
|
|
|
73
82
|
## Integration Boundaries
|
|
74
83
|
|
|
75
|
-
- Use `KbStore` for storage.
|
|
76
|
-
- Use `KnowledgeDiscoveryDispatcher` for research workers.
|
|
84
|
+
- Use `KbStore` for storage. Applications may provide any durable backend that implements it.
|
|
85
|
+
- Use `KnowledgeDiscoveryDispatcher` for research workers. Applications should connect it to their own runtime.
|
|
77
86
|
- Do not bypass `lint` or `validate` before using generated knowledge in an agent.
|
|
78
87
|
|
|
79
88
|
## Pluggable Sources + Freshness + Changes
|
|
80
89
|
|
|
81
90
|
Agents that need to stay current against external authorities should compose:
|
|
82
91
|
|
|
83
|
-
- `createCornellLiiSource({ selectors })
|
|
84
|
-
- `createIrsPublicationsSource({ publications, revenueProcedures })
|
|
85
|
-
- `createStateSosSource({ state, baseUrl, entities })
|
|
92
|
+
- `createCornellLiiSource({ selectors })`: US Code and Wex from law.cornell.edu.
|
|
93
|
+
- `createIrsPublicationsSource({ publications, revenueProcedures })`: IRS index and named publications.
|
|
94
|
+
- `createStateSosSource({ state, baseUrl, entities })`: generic state SOS adapter.
|
|
86
95
|
|
|
87
|
-
Every fetch returns `KnowledgeFragment[]` with `provenance.verifiable` indicating whether the
|
|
96
|
+
Every fetch returns `KnowledgeFragment[]` with `provenance.verifiable` indicating whether the configured URL returned an acceptable response and the expected content was extracted.
|
|
97
|
+
This flag does not authenticate the publisher or cryptographically prove the content.
|
|
98
|
+
Refuse to cite fragments with `verifiable: false`.
|
|
88
99
|
|
|
89
100
|
Track per-tenant freshness with `createFileSystemFreshnessStore({ root })` and re-fetch only when `stale({ workspaceId, sourceId, ttlMs })` returns true.
|
|
90
101
|
|
|
91
|
-
Diff snapshots with `detectChanges(prev, next)`.
|
|
102
|
+
Diff snapshots with `detectChanges(prev, next)`.
|
|
103
|
+
Each `KnowledgeChange` carries `affectedDimensions`; pass those to your eval scheduler to run only the relevant campaigns again.
|
|
92
104
|
|
|
93
105
|
## Authorship
|
|
94
106
|
|
|
95
|
-
Do not add `Co-Authored-By:` trailers
|
|
107
|
+
Do not add `Co-Authored-By:` trailers or other AI-attribution lines to commits, PR descriptions, or repository artifacts.
|
|
108
|
+
The author is the human running the session.
|
|
96
109
|
|
|
97
110
|
## Comment & doc discipline (no historical narrative)
|
|
98
111
|
|
|
99
|
-
Comments describe **what the code does and why
|
|
112
|
+
Comments describe **what the code does and why**.
|
|
113
|
+
They must not describe what code used to do, what it replaced, which audit found a bug, or what a prior version looked like.
|
|
114
|
+
History belongs in commit messages and PR descriptions.
|
|
100
115
|
|
|
101
116
|
- Bad: `// replaces the inline retry loop`, `// fix for the silent-zero bug`, `// the 2yr rewrite added this`, `// audit fix`
|
|
102
|
-
- Good: `// value
|
|
117
|
+
- Good: `// value is null when retries exhaust; callers must inspect succeeded`
|
|
103
118
|
|
|
104
|
-
|
|
119
|
+
This applies anywhere the repository carries prose.
|
|
105
120
|
|
|
106
121
|
## No fallbacks. Fail loud.
|
|
107
122
|
|
|
@@ -110,4 +125,3 @@ Sloppy fallbacks corrupt every signal downstream. No silent zeros, no `?? defaul
|
|
|
110
125
|
External-boundary calls (LLM, network, FS, subprocess) return *typed outcomes* (`{ succeeded, value, error }`). Callers MUST inspect `succeeded` before using `value`. Named, opted-in fallback rotations (`policy.fallbackModels: [...]`) are fine; deep `?? "kimi"` helpers are not.
|
|
111
126
|
|
|
112
127
|
Full doctrine: `~/dotfiles/claude/AGENTS.md` → "No fallbacks. Fail loud."
|
|
113
|
-
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,37 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 4.1.0
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- Added optional scenario input to `knowledgeReleaseReport()` so a required holdout can prove both scenario and run coverage.
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
- Split knowledge-base improvement and RAG evaluation internals into focused modules while preserving public exports and implementation behavior.
|
|
12
|
+
- Split the knowledge improvement tests into candidate, promotion, activation, and integrity suites with shared setup in one support module.
|
|
13
|
+
- Removed unused development dependencies and internal-only exports.
|
|
14
|
+
- Updated the test runner to Vitest 4 and Node type definitions to 26; TypeScript remains on 5.9 because tsup's declaration bundler does not yet support TypeScript 7.
|
|
15
|
+
- Declared Node types explicitly in TypeScript configuration instead of relying on ambient type discovery.
|
|
16
|
+
- Corrected the package dependency guide and RAG roadmap to reflect runtime-owned agent execution through `runKnowledgeImprovementJob()`.
|
|
17
|
+
- Removed historical commentary and em dashes from repository documentation.
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- Replaced the stale versioned HTTP user agent with a stable package identity and added request-header coverage.
|
|
22
|
+
|
|
23
|
+
## 4.0.1
|
|
24
|
+
|
|
25
|
+
### Changed
|
|
26
|
+
|
|
27
|
+
- Split benchmark, memory experiment, and memory improvement internals into focused modules while preserving every public export and signature.
|
|
28
|
+
- Split memory and benchmark tests by behavior, with shared controller and adapter fixtures kept in one test-support module.
|
|
29
|
+
- Moved retrieval holdout contracts into the memory type layer to remove the holdout/types import cycle.
|
|
30
|
+
|
|
31
|
+
### Fixed
|
|
32
|
+
|
|
33
|
+
- Made the Mem0 deletion-convergence test deterministic under file-level parallel execution.
|
|
34
|
+
|
|
3
35
|
## 4.0.0
|
|
4
36
|
|
|
5
37
|
### Breaking Changes
|
package/README.md
CHANGED
|
@@ -14,7 +14,7 @@ This package turns raw sources and generated markdown knowledge into a versionab
|
|
|
14
14
|
- [Benchmark runner](#benchmark-runner): BEIR/MTEB/qrels, RAG answer, hallucination, KB-improvement cases
|
|
15
15
|
- [Agent-Eval integration](#agent-eval-integration): retrieval eval, readiness bundles, and release reports
|
|
16
16
|
- [Memory adapters](#memory-adapters): Mem0, Graphiti, Neo4j, branching, and automatic improvement
|
|
17
|
-
- [Research
|
|
17
|
+
- [Research loops](#research-loop): direct and worker/reviewer knowledge growth
|
|
18
18
|
- [Runtime integration](#runtime-integration): how agent runners plug into the pure KB loop
|
|
19
19
|
- [Pluggable knowledge sources](#pluggable-knowledge-sources): live authorities → eval re-runs
|
|
20
20
|
|
|
@@ -31,7 +31,7 @@ Two ways in, depending on what you're doing:
|
|
|
31
31
|
- **Author / inspect a KB by hand** → the [CLI](#cli) (`init` → `source-add` → `index` → `search` → `lint`). Fastest way to see the shape on disk.
|
|
32
32
|
- **Drive it from an agent** → pick the primitive by intent:
|
|
33
33
|
- *"Does the agent have enough context to run?"* → [`buildEvalKnowledgeBundle`](#agent-eval-integration) (block / ask / acquire before execution).
|
|
34
|
-
- *"Grow the KB as a researcher"* → [`runKnowledgeResearchLoop`](#research-loop) (deterministic mechanics; your agent owns judgment) or [`
|
|
34
|
+
- *"Grow the KB as a researcher"* → [`runKnowledgeResearchLoop`](#research-loop) (deterministic mechanics; your agent owns judgment) or [`runVerifiedResearchLoop`](#verified-research-loop) (researcher proposes, reviewer checks and fills gaps).
|
|
35
35
|
- *"Spawn live agents to improve a KB"* → pass an `updateKnowledge` callback to `improveKnowledgeBase`; runtime-backed supervisors live in `@tangle-network/agent-runtime`.
|
|
36
36
|
- *"Tune retrieval for a knowledge base"* → `runRetrievalImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section.
|
|
37
37
|
- *"Run an operator-grade KB improvement cycle"* → `improveKnowledgeBase` in the [Agent-Eval integration](#agent-eval-integration) section.
|
|
@@ -419,6 +419,17 @@ To answer whether a candidate knowledge base improves agent task success, run an
|
|
|
419
419
|
|
|
420
420
|
Use `knowledgeReleaseReport()` before promotion: pass the candidate and baseline `RunRecord[]` (plus optional `ReleaseTraceEvidence` and the gate decision) and it folds them into a `ReleaseConfidenceScorecard` and a `KnowledgeRelease` using `agent-eval`'s release gates and `RunRecord` validation.
|
|
421
421
|
|
|
422
|
+
When holdout evidence is required, pass both the holdout-tagged run records and the scenario split:
|
|
423
|
+
|
|
424
|
+
```ts
|
|
425
|
+
const release = knowledgeReleaseReport({
|
|
426
|
+
candidateId,
|
|
427
|
+
candidateRuns,
|
|
428
|
+
scenarios,
|
|
429
|
+
hasHoldout: true,
|
|
430
|
+
})
|
|
431
|
+
```
|
|
432
|
+
|
|
422
433
|
Use `buildEvalKnowledgeBundle()` before execution when the question is whether
|
|
423
434
|
the agent has enough task-world context to run:
|
|
424
435
|
|
|
@@ -805,15 +816,13 @@ await runAgentControlLoop({
|
|
|
805
816
|
})
|
|
806
817
|
```
|
|
807
818
|
|
|
808
|
-
##
|
|
819
|
+
## Verified research loop
|
|
809
820
|
|
|
810
|
-
`
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
Both are yours (no creds). The loop owns the deterministic mechanics (indexing,
|
|
816
|
-
applying write blocks, scoring readiness) and stops once no blocking gap remains.
|
|
821
|
+
`runVerifiedResearchLoop()` coordinates a worker and reviewer over one knowledge base.
|
|
822
|
+
The worker discovers sources and proposes pages for open gaps.
|
|
823
|
+
The reviewer checks each source before admission and can run its own gap-filling pass with `driverResearches: true`.
|
|
824
|
+
Callers provide both roles and any credentials.
|
|
825
|
+
The loop owns indexing, write-block application, readiness scoring, and stopping when no blocking gap remains.
|
|
817
826
|
|
|
818
827
|
An equal-compute comparison across 9 ML topics using `glm-5.2` is documented in
|
|
819
828
|
[docs/two-agent-research-ab.md](docs/two-agent-research-ab.md).
|
|
@@ -824,10 +833,10 @@ The document includes limitations and reproduction steps.
|
|
|
824
833
|
```ts
|
|
825
834
|
import {
|
|
826
835
|
defineReadinessSpec,
|
|
827
|
-
|
|
836
|
+
runVerifiedResearchLoop,
|
|
828
837
|
} from '@tangle-network/agent-knowledge'
|
|
829
838
|
|
|
830
|
-
await
|
|
839
|
+
await runVerifiedResearchLoop({
|
|
831
840
|
root: './kb',
|
|
832
841
|
goal: 'Build a grounded onboarding wiki for billing support',
|
|
833
842
|
readinessSpecs: [defineReadinessSpec({
|
|
@@ -836,14 +845,14 @@ await runTwoAgentResearchLoop({
|
|
|
836
845
|
query: 'refund policy customer request',
|
|
837
846
|
requiredFor: ['support-agent'],
|
|
838
847
|
})],
|
|
839
|
-
//
|
|
848
|
+
// The worker researches the current gaps.
|
|
840
849
|
async worker({ gaps, index }) {
|
|
841
850
|
return {
|
|
842
851
|
sources: [/* AddSourceTextInput for the open gaps */],
|
|
843
852
|
proposalText: '/* ---FILE: knowledge/…--- write-protocol blocks */',
|
|
844
853
|
}
|
|
845
854
|
},
|
|
846
|
-
//
|
|
855
|
+
// The reviewer checks each candidate source before admission.
|
|
847
856
|
driver: {
|
|
848
857
|
verifySource(source, { gaps }) {
|
|
849
858
|
return source.uri ? { accept: true } : { accept: false, reason: 'no uri' }
|
|
@@ -915,11 +924,14 @@ three primitives that bridge "live authority" → "eval re-runs":
|
|
|
915
924
|
|
|
916
925
|
- `KnowledgeSource`: pluggable contract (`fetch(opts) → KnowledgeFragment[]`).
|
|
917
926
|
Every fragment carries `provenance` (URL, source-attested timestamp,
|
|
918
|
-
jurisdiction,
|
|
919
|
-
dimensions a change in this fragment should re-score).
|
|
927
|
+
jurisdiction, and fetch/extraction status) and `dimensionHints` (which eval
|
|
928
|
+
dimensions a change in this fragment should re-score). The `verifiable`
|
|
929
|
+
flag rejects failed, blocked, or malformed responses; it does not
|
|
930
|
+
cryptographically authenticate the publisher.
|
|
920
931
|
- `KnowledgeFreshnessStore`: per-`(workspaceId, sourceId)` last-refresh
|
|
921
|
-
tracker.
|
|
922
|
-
|
|
932
|
+
tracker. The package ships a filesystem implementation and
|
|
933
|
+
`createD1FreshnessStoreStub(adapter)`, a complete bridge for an
|
|
934
|
+
application-owned D1, PostgreSQL, SQLite, or equivalent adapter.
|
|
923
935
|
- `detectChanges(prev, next)`: diffs two fragment snapshots, emits
|
|
924
936
|
`KnowledgeChange[]` tagged with the affected eval dimensions so a cron
|
|
925
937
|
scheduler knows exactly which campaigns to re-run.
|
|
@@ -998,4 +1010,4 @@ async function tick({ workspaceId, prevSnapshots }: {
|
|
|
998
1010
|
Polite-by-default: every HTTP fetch carries the package User-Agent, is
|
|
999
1011
|
throttled to 1 req/sec/origin, caches successful responses to disk, and
|
|
1000
1012
|
marks `verifiable: false` on block pages / 4xx rather than promoting
|
|
1001
|
-
|
|
1013
|
+
unusable content. See `src/sources/http.ts` for the checks.
|
|
@@ -1,4 +1,52 @@
|
|
|
1
|
-
import '
|
|
2
|
-
export {
|
|
1
|
+
import { b as AgentMemoryAdapter, C as KnowledgeBenchmarkSpec, H as KnowledgeMemoryBenchmarkTaskKind, o as KnowledgeAnswerBenchmarkTaskKind, G as KnowledgeMemoryBenchmarkCase, q as KnowledgeBenchmarkCase, p as KnowledgeBenchmarkArtifact, ad as RunMemoryAdapterBenchmarkOptions, ae as RunMemoryAdapterBenchmarkResult, x as KnowledgeBenchmarkScenario, v as KnowledgeBenchmarkReport, B as BuildRetrievalBenchmarkCasesFromQrelsOptions, L as KnowledgeRetrievalBenchmarkCase, M as KnowledgeRetrievalBenchmarkQrel, t as KnowledgeBenchmarkEvaluation, D as KnowledgeBenchmarkSplit, ab as RunKnowledgeBenchmarkSuiteOptions, ac as RunKnowledgeBenchmarkSuiteResult } from '../types-DP38encz.js';
|
|
2
|
+
export { K as KnowledgeAnswerBenchmarkCase, r as KnowledgeBenchmarkCaseBase, s as KnowledgeBenchmarkDistribution, u as KnowledgeBenchmarkFamily, w as KnowledgeBenchmarkResponder, y as KnowledgeBenchmarkSliceSummary, z as KnowledgeBenchmarkSource, E as KnowledgeBenchmarkTaskKind, F as KnowledgeClaimMatcher, I as KnowledgeMemoryEvent, J as KnowledgeMemoryFactMatcher, N as KnowledgeRetrievalBenchmarkQuery, O as MemoryAdapterBenchmarkCandidate, P as MemoryAdapterBenchmarkRankingRow } from '../types-DP38encz.js';
|
|
3
|
+
import { CampaignResult, JudgeConfig } from '@tangle-network/agent-eval/campaign';
|
|
3
4
|
import '../types-6x0OpfW6.js';
|
|
4
|
-
|
|
5
|
+
|
|
6
|
+
declare function createNoopMemoryBenchmarkAdapter(id?: string): AgentMemoryAdapter;
|
|
7
|
+
declare function createInMemoryBenchmarkAdapter(options?: {
|
|
8
|
+
id?: string;
|
|
9
|
+
}): AgentMemoryAdapter;
|
|
10
|
+
|
|
11
|
+
declare const INDUSTRY_RAG_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & {
|
|
12
|
+
taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind;
|
|
13
|
+
})[];
|
|
14
|
+
declare const INDUSTRY_MEMORY_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & {
|
|
15
|
+
taskKind: KnowledgeMemoryBenchmarkTaskKind;
|
|
16
|
+
})[];
|
|
17
|
+
declare function buildIndustryRagBenchmarkSmokeCases(specs?: readonly (KnowledgeBenchmarkSpec & {
|
|
18
|
+
taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind;
|
|
19
|
+
})[]): KnowledgeBenchmarkCase[];
|
|
20
|
+
declare function respondToIndustryRagBenchmarkSmokeCase(input: {
|
|
21
|
+
case: KnowledgeBenchmarkCase;
|
|
22
|
+
}): KnowledgeBenchmarkArtifact;
|
|
23
|
+
declare function buildIndustryMemoryBenchmarkSmokeCases(specs?: readonly (KnowledgeBenchmarkSpec & {
|
|
24
|
+
taskKind: KnowledgeMemoryBenchmarkTaskKind;
|
|
25
|
+
})[]): KnowledgeMemoryBenchmarkCase[];
|
|
26
|
+
declare function respondToIndustryMemoryBenchmarkSmokeCase(input: {
|
|
27
|
+
case: KnowledgeMemoryBenchmarkCase;
|
|
28
|
+
}): KnowledgeBenchmarkArtifact;
|
|
29
|
+
declare function buildFirstPartyMemoryLifecycleBenchmarkCases(): KnowledgeMemoryBenchmarkCase[];
|
|
30
|
+
|
|
31
|
+
declare function runMemoryAdapterBenchmark(options: RunMemoryAdapterBenchmarkOptions): Promise<RunMemoryAdapterBenchmarkResult>;
|
|
32
|
+
|
|
33
|
+
declare function summarizeKnowledgeBenchmarkCampaign<TArtifact>(input: {
|
|
34
|
+
scenarios: readonly KnowledgeBenchmarkScenario[];
|
|
35
|
+
campaign: CampaignResult<TArtifact, KnowledgeBenchmarkScenario>;
|
|
36
|
+
}): KnowledgeBenchmarkReport;
|
|
37
|
+
|
|
38
|
+
declare function parseKnowledgeBenchmarkJsonl<T = unknown>(text: string): T[];
|
|
39
|
+
declare function parseKnowledgeBenchmarkQrels(text: string): KnowledgeRetrievalBenchmarkQrel[];
|
|
40
|
+
declare function buildRetrievalBenchmarkCasesFromQrels(options: BuildRetrievalBenchmarkCasesFromQrelsOptions): KnowledgeRetrievalBenchmarkCase[];
|
|
41
|
+
|
|
42
|
+
declare function scoreKnowledgeBenchmarkArtifact<TArtifact>(testCase: KnowledgeBenchmarkCase, artifact: TArtifact): KnowledgeBenchmarkEvaluation;
|
|
43
|
+
declare function scoreMemoryBenchmarkArtifact<TArtifact>(testCase: KnowledgeMemoryBenchmarkCase, artifact: TArtifact): KnowledgeBenchmarkEvaluation;
|
|
44
|
+
|
|
45
|
+
declare function runKnowledgeBenchmarkSuite<TArtifact = KnowledgeBenchmarkArtifact>(options: RunKnowledgeBenchmarkSuiteOptions<TArtifact>): Promise<RunKnowledgeBenchmarkSuiteResult<TArtifact>>;
|
|
46
|
+
declare function renderKnowledgeBenchmarkReportMarkdown(report: KnowledgeBenchmarkReport): string;
|
|
47
|
+
declare function buildKnowledgeBenchmarkScenarios(cases: readonly KnowledgeBenchmarkCase[], splits?: readonly KnowledgeBenchmarkSplit[]): KnowledgeBenchmarkScenario[];
|
|
48
|
+
declare function knowledgeBenchmarkJudge<TArtifact = KnowledgeBenchmarkArtifact>(): JudgeConfig<TArtifact, KnowledgeBenchmarkScenario>;
|
|
49
|
+
|
|
50
|
+
declare function isKnowledgeMemoryBenchmarkCase(testCase: KnowledgeBenchmarkCase): testCase is KnowledgeMemoryBenchmarkCase;
|
|
51
|
+
|
|
52
|
+
export { BuildRetrievalBenchmarkCasesFromQrelsOptions, INDUSTRY_MEMORY_BENCHMARKS, INDUSTRY_RAG_BENCHMARKS, KnowledgeAnswerBenchmarkTaskKind, KnowledgeBenchmarkArtifact, KnowledgeBenchmarkCase, KnowledgeBenchmarkEvaluation, KnowledgeBenchmarkReport, KnowledgeBenchmarkScenario, KnowledgeBenchmarkSpec, KnowledgeBenchmarkSplit, KnowledgeMemoryBenchmarkCase, KnowledgeMemoryBenchmarkTaskKind, KnowledgeRetrievalBenchmarkCase, KnowledgeRetrievalBenchmarkQrel, RunKnowledgeBenchmarkSuiteOptions, RunKnowledgeBenchmarkSuiteResult, RunMemoryAdapterBenchmarkOptions, RunMemoryAdapterBenchmarkResult, buildFirstPartyMemoryLifecycleBenchmarkCases, buildIndustryMemoryBenchmarkSmokeCases, buildIndustryRagBenchmarkSmokeCases, buildKnowledgeBenchmarkScenarios, buildRetrievalBenchmarkCasesFromQrels, createInMemoryBenchmarkAdapter, createNoopMemoryBenchmarkAdapter, isKnowledgeMemoryBenchmarkCase, knowledgeBenchmarkJudge, parseKnowledgeBenchmarkJsonl, parseKnowledgeBenchmarkQrels, renderKnowledgeBenchmarkReportMarkdown, respondToIndustryMemoryBenchmarkSmokeCase, respondToIndustryRagBenchmarkSmokeCase, runKnowledgeBenchmarkSuite, runMemoryAdapterBenchmark, scoreKnowledgeBenchmarkArtifact, scoreMemoryBenchmarkArtifact, summarizeKnowledgeBenchmarkCampaign };
|
package/dist/benchmarks/index.js
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
scoreKnowledgeBenchmarkArtifact,
|
|
21
21
|
scoreMemoryBenchmarkArtifact,
|
|
22
22
|
summarizeKnowledgeBenchmarkCampaign
|
|
23
|
-
} from "../chunk-
|
|
23
|
+
} from "../chunk-BBCIB4UL.js";
|
|
24
24
|
import "../chunk-DQ3PDMDP.js";
|
|
25
25
|
import "../chunk-YMKHCTS2.js";
|
|
26
26
|
export {
|