@tangle-network/agent-knowledge 4.0.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/benchmarks/index.d.ts +51 -3
- package/dist/benchmarks/index.js +1 -1
- package/dist/{chunk-UWOTQNBI.js → chunk-3CAAGJCE.js} +1886 -1842
- package/dist/chunk-3CAAGJCE.js.map +1 -0
- package/dist/{chunk-DW6APRTX.js → chunk-BBCIB4UL.js} +2486 -2457
- package/dist/chunk-BBCIB4UL.js.map +1 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +2 -2
- package/dist/memory/index.d.ts +132 -6
- package/dist/memory/index.js +2 -2
- package/dist/{index-Bqg1mBPt.d.ts → types-ZzY_x0r7.d.ts} +115 -276
- package/package.json +1 -1
- package/dist/chunk-DW6APRTX.js.map +0 -1
- package/dist/chunk-UWOTQNBI.js.map +0 -1
package/dist/memory/index.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { b as AgentMemoryAdapter, j as AgentMemorySearchOptions, d as AgentMemoryContext, f as AgentMemoryHit, i as AgentMemoryScope, k as AgentMemoryWriteInput, l as AgentMemoryWriteResult,
|
|
2
|
-
export { c as AgentMemoryBranchIsolation, g as AgentMemoryKind,
|
|
1
|
+
import { b as AgentMemoryAdapter, j as AgentMemorySearchOptions, d as AgentMemoryContext, f as AgentMemoryHit, i as AgentMemoryScope, k as AgentMemoryWriteInput, l as AgentMemoryWriteResult, u as KnowledgeBenchmarkFamily, D as KnowledgeBenchmarkSplit, H as KnowledgeMemoryBenchmarkTaskKind, J as KnowledgeMemoryFactMatcher, h as AgentMemoryRunLease, I as KnowledgeMemoryEvent, G as KnowledgeMemoryBenchmarkCase, e as AgentMemoryControllerMode, A as AgentMemoryAcquireRunLease, $ as RetrievalHoldoutConfig, _ as RetrievalHoldoutCallContext, a2 as RetrievalHoldoutResult, Z as RetrievalHoldoutBypassReason, a1 as RetrievalHoldoutEvent } from '../types-ZzY_x0r7.js';
|
|
2
|
+
export { c as AgentMemoryBranchIsolation, g as AgentMemoryKind, Q as OwnedAgentMemoryRunLease, a0 as RetrievalHoldoutEligibleItem, a3 as RetrievalHoldoutSessionState, af as acquireAgentMemoryRunLease } from '../types-ZzY_x0r7.js';
|
|
3
3
|
import { CampaignResult, Scenario, DispatchContext, CampaignStorage, CostLedgerHandle, JudgeConfig, HeldoutSignificance, Lineage, GovernorContext, GovernorOp, SurfaceProposer, LineageStore, HeldoutSignificanceOptions } from '@tangle-network/agent-eval/campaign';
|
|
4
|
+
import { OffPolicyTrajectory } from '@tangle-network/agent-eval/rl';
|
|
4
5
|
import { z } from 'zod';
|
|
5
6
|
import { S as SourceRecord } from '../types-6x0OpfW6.js';
|
|
6
|
-
import '@tangle-network/agent-eval/rl';
|
|
7
7
|
|
|
8
8
|
declare function defaultGetMemoryContext(adapter: Pick<AgentMemoryAdapter, 'search'>, query: string, options?: AgentMemorySearchOptions): Promise<AgentMemoryContext>;
|
|
9
9
|
declare function renderMemoryContext(hits: AgentMemoryHit[]): string;
|
|
@@ -257,13 +257,16 @@ interface RunAgentMemoryExperimentResult {
|
|
|
257
257
|
attemptLogPath: string;
|
|
258
258
|
recoveryLogPath: string;
|
|
259
259
|
}
|
|
260
|
+
|
|
260
261
|
/** Converts existing ordered memory benchmark cases into executable histories. */
|
|
262
|
+
|
|
261
263
|
declare function buildAgentMemorySequencesFromBenchmarkCases(cases: readonly KnowledgeMemoryBenchmarkCase[], options?: BuildAgentMemorySequencesFromBenchmarkCasesOptions): AgentMemorySequence[];
|
|
262
|
-
/** Runs ordered, branch-isolated memory histories across candidate systems. */
|
|
263
|
-
declare function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOptions): Promise<RunAgentMemoryExperimentResult>;
|
|
264
264
|
declare function buildAgentMemorySequenceScenarios(sequences: readonly AgentMemorySequence[], candidates: readonly Pick<AgentMemoryExperimentCandidate, 'id'>[]): AgentMemorySequenceScenario[];
|
|
265
265
|
declare function agentMemorySequenceJudge(): JudgeConfig<AgentMemorySequenceArtifact, AgentMemorySequenceScenario>;
|
|
266
266
|
|
|
267
|
+
/** Runs ordered, branch-isolated memory histories across candidate systems. */
|
|
268
|
+
declare function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOptions): Promise<RunAgentMemoryExperimentResult>;
|
|
269
|
+
|
|
267
270
|
interface GraphitiMcpClientLike {
|
|
268
271
|
callTool(request: {
|
|
269
272
|
name: string;
|
|
@@ -298,6 +301,128 @@ declare function graphitiMemoryAdapterIdentity(options: Pick<GraphitiMemoryAdapt
|
|
|
298
301
|
backendRef: string;
|
|
299
302
|
}): string;
|
|
300
303
|
|
|
304
|
+
/**
|
|
305
|
+
* Deterministic uniform [0,1) for assignment draws. The sha256 key derivation is ours — it makes
|
|
306
|
+
* every draw replayable from the logged keys alone (design rule D5) — while the generator core is
|
|
307
|
+
* the substrate's `mulberry32` (@tangle-network/agent-eval statistics vocabulary), seeded with the
|
|
308
|
+
* top 32 bits of the digest, so the statistical machinery is reused rather than forked.
|
|
309
|
+
*/
|
|
310
|
+
declare function deterministicRng(key: string): number;
|
|
311
|
+
/**
|
|
312
|
+
* Value-hash of the knobs that DEFINE the experiment (epsilon + watchlist, order-independent).
|
|
313
|
+
* Everything else on the config (callbacks, attribution stamps, privacy flags) does not change
|
|
314
|
+
* which assignments are drawn, so it stays out of the hash.
|
|
315
|
+
*/
|
|
316
|
+
declare function retrievalHoldoutConfigHash(config: Pick<RetrievalHoldoutConfig, 'epsilon' | 'watchlist'>): string;
|
|
317
|
+
/**
|
|
318
|
+
* Pure per-call holdout: takes post-filter hits, returns delivered hits plus the log event
|
|
319
|
+
* and the next session state. Suppression only removes items (no backfill), and it happens
|
|
320
|
+
* before rendering so a drop session's context is byte-identical to a natural smaller retrieval.
|
|
321
|
+
*/
|
|
322
|
+
declare function applyRetrievalHoldout(hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx?: RetrievalHoldoutCallContext): RetrievalHoldoutResult;
|
|
323
|
+
/**
|
|
324
|
+
* Logs a holdout event for adapter context paths that answer WITHOUT going through the
|
|
325
|
+
* search→render seam (Neo4j short-term conversation context, raw-string getContext results),
|
|
326
|
+
* so a consumer with a holdout configured still sees every call and the fraction-under-experiment
|
|
327
|
+
* denominator stays honest instead of silently losing these calls. No suppression is applied:
|
|
328
|
+
* dropping is only meaningful for retrieved memory hits, never for conversation context.
|
|
329
|
+
*/
|
|
330
|
+
declare function emitRetrievalHoldoutBypass(hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx: Omit<RetrievalHoldoutCallContext, 'session'>, bypassReason: RetrievalHoldoutBypassReason): RetrievalHoldoutEvent;
|
|
331
|
+
/**
|
|
332
|
+
* Clears all sticky-session state. For test isolation and experiment-epoch boundaries;
|
|
333
|
+
* sessions in flight afterwards re-draw deterministically (same rng keys), so with the default
|
|
334
|
+
* rng a reset is invisible in the logs unless eligibility sets changed in between.
|
|
335
|
+
*/
|
|
336
|
+
declare function resetRetrievalHoldoutRegistry(): void;
|
|
337
|
+
/**
|
|
338
|
+
* Convenience wrapper that threads session state internally, keyed by the VALUE of the
|
|
339
|
+
* experiment-defining knobs (configHash = epsilon + sorted watchlist) plus sessionId, so a fresh
|
|
340
|
+
* config object per call — the natural pattern when options are built inline — keeps full
|
|
341
|
+
* stickiness. Distinct experiments never share session state; two config objects with the same
|
|
342
|
+
* epsilon/watchlist are the same experiment by definition.
|
|
343
|
+
*/
|
|
344
|
+
declare function applySessionStickyRetrievalHoldout(hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx?: Omit<RetrievalHoldoutCallContext, 'session'>): RetrievalHoldoutResult;
|
|
345
|
+
/** Aggregated view of one session's holdout exposure — the randomization unit of this design. */
|
|
346
|
+
interface RetrievalHoldoutSessionSummary {
|
|
347
|
+
/** `${configHash}:${sessionIdHash}` — the trajectory's runId. */
|
|
348
|
+
runId: string;
|
|
349
|
+
configHash: string;
|
|
350
|
+
sessionIdHash: string;
|
|
351
|
+
sessionHoldout: boolean;
|
|
352
|
+
sessionTargetId: string | null;
|
|
353
|
+
/** Item actually suppressed in this session (equals sessionTargetId when a drop occurred). */
|
|
354
|
+
droppedId: string | null;
|
|
355
|
+
/** |watchlist ∩ E| at the session's first intersecting randomized call; 0 when never intersecting. */
|
|
356
|
+
firstCandidateCount: number;
|
|
357
|
+
/** Randomized retrieval calls observed for this session. */
|
|
358
|
+
callCount: number;
|
|
359
|
+
/** Adapter bypass calls observed for this session (logged, never randomized). */
|
|
360
|
+
bypassCallCount: number;
|
|
361
|
+
/** Session-level probability of the OBSERVED assignment under the behavior policy. */
|
|
362
|
+
behaviorProb: number;
|
|
363
|
+
/** True when the session's events disagree on arm or target (e.g. registry eviction re-draw). */
|
|
364
|
+
mixedExposure: boolean;
|
|
365
|
+
/** Present only on sessions surfaced in `excluded` rather than converted. */
|
|
366
|
+
exclusionReason?: 'mixed-exposure' | 'no-randomized-calls';
|
|
367
|
+
}
|
|
368
|
+
interface RetrievalHoldoutOffPolicyOptions {
|
|
369
|
+
/**
|
|
370
|
+
* Realized outcome PER SESSION, keyed by sessionIdHash — the session is the randomization
|
|
371
|
+
* unit, so rewards are per session, never per call. A missing entry for an included session
|
|
372
|
+
* throws: silently dropping unscored sessions would corrupt the estimator's denominator, so
|
|
373
|
+
* filter events to scored sessions upstream instead.
|
|
374
|
+
*/
|
|
375
|
+
rewards: Record<string, number>;
|
|
376
|
+
/**
|
|
377
|
+
* Target-policy probability of the session's observed assignment. Default models the
|
|
378
|
+
* always-deliver-in-full policy: 1 for full-delivery sessions, 0 for drop sessions.
|
|
379
|
+
*/
|
|
380
|
+
targetProb?: (session: RetrievalHoldoutSessionSummary) => number;
|
|
381
|
+
/** Per-session reward-model prediction enabling `doublyRobust`; when absent, qHat is null. */
|
|
382
|
+
qHat?: (session: RetrievalHoldoutSessionSummary) => number | null;
|
|
383
|
+
}
|
|
384
|
+
interface RetrievalHoldoutOffPolicyResult {
|
|
385
|
+
/** One trajectory per (configHash, sessionIdHash) — never per call. */
|
|
386
|
+
trajectories: OffPolicyTrajectory[];
|
|
387
|
+
/** 1:1 with `trajectories` (same order): the diagnostic view of each converted session. */
|
|
388
|
+
sessions: RetrievalHoldoutSessionSummary[];
|
|
389
|
+
/** Sessions surfaced but NOT converted (mixed exposure, bypass-only) — counted, never hidden. */
|
|
390
|
+
excluded: RetrievalHoldoutSessionSummary[];
|
|
391
|
+
/** Events with no sessionIdHash: outside session randomization, cannot join a reward. */
|
|
392
|
+
unattributableEvents: number;
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* Maps holdout logs onto agent-eval's `OffPolicyTrajectory`, ONE TRAJECTORY PER SESSION, so
|
|
396
|
+
* EXP-007's analysis consumes the substrate's `inverseProbabilityWeighting` /
|
|
397
|
+
* `selfNormalizedImportanceWeighting` / `doublyRobust` estimators directly. This matches the
|
|
398
|
+
* PREREG estimator (session-sticky self-normalized IPW): the design randomizes per SESSION —
|
|
399
|
+
* the arm coin is flipped once (P(holdout) = epsilon) and the sticky target is drawn once,
|
|
400
|
+
* uniformly over watchlist ∩ E at the session's first intersecting call.
|
|
401
|
+
*
|
|
402
|
+
* Session-level action space and behavior probabilities (they sum to 1 by construction):
|
|
403
|
+
* - full delivery (control arm observed): `1 − epsilon`;
|
|
404
|
+
* - drop candidate i of k = |watchlist ∩ E_first|: `epsilon / k` each (the draw event's logged
|
|
405
|
+
* `dropPropensity`);
|
|
406
|
+
* - sessions whose eligibility sets never intersect the watchlist: probability 1 — full
|
|
407
|
+
* delivery was certain in either arm.
|
|
408
|
+
*
|
|
409
|
+
* Per-call events are repeated observations WITHIN one session-level randomization; per-call
|
|
410
|
+
* IPW is statistically invalid for this design (per-call "propensities" imply an action
|
|
411
|
+
* distribution summing to more than 1 and bias IPS downward). Calls where the sticky target is
|
|
412
|
+
* absent from E, and adapter bypass calls, fold into the session summary (callCount /
|
|
413
|
+
* bypassCallCount) instead of generating independent propensities.
|
|
414
|
+
*
|
|
415
|
+
* Join contract: `rewards` is keyed by `sessionIdHash` (sha256(sessionId) first 16 hex — compute
|
|
416
|
+
* the same prefix over the outcome table's session ids, or log with
|
|
417
|
+
* `includePlaintextIdentifiers: true` and join on raw ids upstream). Events are grouped per
|
|
418
|
+
* (configHash, sessionIdHash) so different experiment configs are never pooled. Mixed-exposure
|
|
419
|
+
* sessions (arm or target disagreement, e.g. after registry eviction) are excluded from the
|
|
420
|
+
* trajectories and surfaced in `excluded` for the analysis to count.
|
|
421
|
+
* Pre-registration for EXP-007 stays in the research repo by design; the manifest vocabulary for
|
|
422
|
+
* it is agent-eval's `HypothesisManifest` / `signManifest` / `evaluateHypothesis` exports.
|
|
423
|
+
*/
|
|
424
|
+
declare function toOffPolicyTrajectory(events: RetrievalHoldoutEvent[], options: RetrievalHoldoutOffPolicyOptions): RetrievalHoldoutOffPolicyResult;
|
|
425
|
+
|
|
301
426
|
interface AgentMemoryImprovementSeed<TConfig> {
|
|
302
427
|
config: TConfig;
|
|
303
428
|
track: string;
|
|
@@ -420,6 +545,7 @@ interface RunAgentMemoryImprovementResult<TConfig> {
|
|
|
420
545
|
totalCostUsd: number;
|
|
421
546
|
resultJsonPath: string;
|
|
422
547
|
}
|
|
548
|
+
|
|
423
549
|
/** Searches branchable memory configurations and activates only a fresh holdout win. */
|
|
424
550
|
declare function runAgentMemoryImprovement<TConfig>(options: RunAgentMemoryImprovementOptions<TConfig>): Promise<RunAgentMemoryImprovementResult<TConfig>>;
|
|
425
551
|
|
|
@@ -580,4 +706,4 @@ declare function memoryWriteResultToSourceRecord(result: AgentMemoryWriteResult,
|
|
|
580
706
|
scope?: AgentMemoryScope;
|
|
581
707
|
}): SourceRecord;
|
|
582
708
|
|
|
583
|
-
export { AgentMemoryAcquireRunLease, type AgentMemoryActivation, type AgentMemoryActivationDriver, AgentMemoryAdapter, type AgentMemoryAttemptEvent, type AgentMemoryBranch, type AgentMemoryBranchLifetime, type AgentMemoryBranchSnapshot, AgentMemoryContext, AgentMemoryControllerMode, type AgentMemoryDimensionComparison, type AgentMemoryExperimentCandidate, type AgentMemoryExperimentRankingRow, type AgentMemoryExperimentRunLease, type AgentMemoryGovernor, AgentMemoryHit, AgentMemoryHitSchema, type AgentMemoryImprovementRunLease, type AgentMemoryImprovementSeed, type AgentMemoryJournalEntry, AgentMemoryKindSchema, AgentMemoryLifecycleTimeoutError, AgentMemoryLifecycleUnsafeError, type AgentMemoryPromotionDecision, AgentMemoryRunLease, AgentMemoryScope, AgentMemoryScopeSchema, AgentMemorySearchOptions, type AgentMemorySequence, type AgentMemorySequenceArtifact, type AgentMemorySequenceProbe, type AgentMemorySequenceProbeResult, type AgentMemorySequenceScenario, type AgentMemorySequenceStep, type AgentMemorySharingPolicy, type AgentMemoryVisibility, AgentMemoryWriteInput, AgentMemoryWriteInputSchema, AgentMemoryWriteResult, type BuildAgentMemorySequencesFromBenchmarkCasesOptions, type CreateAgentMemoryBranchOptions, DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, type ForkAgentMemoryBranchSnapshotOptions, type GraphitiMcpClientLike, type GraphitiMemoryAdapterOptions, type GraphitiToolNames, type Mem0ClientLike, type Mem0ClientMode, type Mem0MemoryAdapterOptions, type Neo4jAgentMemoryAdapterOptions, type RunAgentMemoryExperimentOptions, type RunAgentMemoryExperimentResult, type RunAgentMemoryImprovementOptions, type RunAgentMemoryImprovementResult, agentMemorySequenceJudge, buildAgentMemorySequenceScenarios, buildAgentMemorySequencesFromBenchmarkCases, createAgentMemoryBranch, createGraphitiMemoryAdapter, createMem0MemoryAdapter, createMemoryExecutionPool, createNeo4jAgentMemoryAdapter, defaultGetMemoryContext, forkAgentMemoryBranchSnapshot, graphitiMemoryAdapterIdentity, mem0MemoryAdapterIdentity, memoryHitToSourceRecord, memoryRecoveryDelayMs, memoryWriteResultToSourceRecord, renderMemoryContext, resolveMemoryCleanupTimeoutMs, runAgentMemoryExperiment, runAgentMemoryImprovement, runBoundedMemoryLifecycle, sleepForMemoryRecovery };
|
|
709
|
+
export { AgentMemoryAcquireRunLease, type AgentMemoryActivation, type AgentMemoryActivationDriver, AgentMemoryAdapter, type AgentMemoryAttemptEvent, type AgentMemoryBranch, type AgentMemoryBranchLifetime, type AgentMemoryBranchSnapshot, AgentMemoryContext, AgentMemoryControllerMode, type AgentMemoryDimensionComparison, type AgentMemoryExperimentCandidate, type AgentMemoryExperimentRankingRow, type AgentMemoryExperimentRunLease, type AgentMemoryGovernor, AgentMemoryHit, AgentMemoryHitSchema, type AgentMemoryImprovementRunLease, type AgentMemoryImprovementSeed, type AgentMemoryJournalEntry, AgentMemoryKindSchema, AgentMemoryLifecycleTimeoutError, AgentMemoryLifecycleUnsafeError, type AgentMemoryPromotionDecision, AgentMemoryRunLease, AgentMemoryScope, AgentMemoryScopeSchema, AgentMemorySearchOptions, type AgentMemorySequence, type AgentMemorySequenceArtifact, type AgentMemorySequenceProbe, type AgentMemorySequenceProbeResult, type AgentMemorySequenceScenario, type AgentMemorySequenceStep, type AgentMemorySharingPolicy, type AgentMemoryVisibility, AgentMemoryWriteInput, AgentMemoryWriteInputSchema, AgentMemoryWriteResult, type BuildAgentMemorySequencesFromBenchmarkCasesOptions, type CreateAgentMemoryBranchOptions, DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, type ForkAgentMemoryBranchSnapshotOptions, type GraphitiMcpClientLike, type GraphitiMemoryAdapterOptions, type GraphitiToolNames, type Mem0ClientLike, type Mem0ClientMode, type Mem0MemoryAdapterOptions, type Neo4jAgentMemoryAdapterOptions, RetrievalHoldoutBypassReason, RetrievalHoldoutCallContext, RetrievalHoldoutConfig, RetrievalHoldoutEvent, type RetrievalHoldoutOffPolicyOptions, type RetrievalHoldoutOffPolicyResult, RetrievalHoldoutResult, type RetrievalHoldoutSessionSummary, type RunAgentMemoryExperimentOptions, type RunAgentMemoryExperimentResult, type RunAgentMemoryImprovementOptions, type RunAgentMemoryImprovementResult, agentMemorySequenceJudge, applyRetrievalHoldout, applySessionStickyRetrievalHoldout, buildAgentMemorySequenceScenarios, buildAgentMemorySequencesFromBenchmarkCases, createAgentMemoryBranch, createGraphitiMemoryAdapter, createMem0MemoryAdapter, createMemoryExecutionPool, createNeo4jAgentMemoryAdapter, defaultGetMemoryContext, deterministicRng, emitRetrievalHoldoutBypass, forkAgentMemoryBranchSnapshot, graphitiMemoryAdapterIdentity, mem0MemoryAdapterIdentity, memoryHitToSourceRecord, memoryRecoveryDelayMs, memoryWriteResultToSourceRecord, renderMemoryContext, resetRetrievalHoldoutRegistry, resolveMemoryCleanupTimeoutMs, retrievalHoldoutConfigHash, runAgentMemoryExperiment, runAgentMemoryImprovement, runBoundedMemoryLifecycle, sleepForMemoryRecovery, toOffPolicyTrajectory };
|
package/dist/memory/index.js
CHANGED
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
runAgentMemoryExperiment,
|
|
25
25
|
runAgentMemoryImprovement,
|
|
26
26
|
toOffPolicyTrajectory
|
|
27
|
-
} from "../chunk-
|
|
27
|
+
} from "../chunk-3CAAGJCE.js";
|
|
28
28
|
import {
|
|
29
29
|
AgentMemoryLifecycleTimeoutError,
|
|
30
30
|
AgentMemoryLifecycleUnsafeError,
|
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
resolveMemoryCleanupTimeoutMs,
|
|
38
38
|
runBoundedMemoryLifecycle,
|
|
39
39
|
sleepForMemoryRecovery
|
|
40
|
-
} from "../chunk-
|
|
40
|
+
} from "../chunk-BBCIB4UL.js";
|
|
41
41
|
import "../chunk-DQ3PDMDP.js";
|
|
42
42
|
import "../chunk-YMKHCTS2.js";
|
|
43
43
|
export {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { JsonValue, Scenario, DispatchContext, RunImprovementLoopResult, ParameterCandidate, JudgeConfig, Gate, RunImprovementLoopOptions, MutableSurface, SurfaceProposer, CampaignStorage, CostLedgerHandle, CampaignResult } from '@tangle-network/agent-eval/campaign';
|
|
2
2
|
import { c as KnowledgeIndex, S as SourceRecord } from './types-6x0OpfW6.js';
|
|
3
|
-
import { OffPolicyTrajectory } from '@tangle-network/agent-eval/rl';
|
|
4
3
|
|
|
5
4
|
type RetrievalConfig = Record<string, JsonValue>;
|
|
6
5
|
type RetrievalParameterSearchSpace = Record<string, readonly JsonValue[]>;
|
|
@@ -158,30 +157,101 @@ declare function buildRetrievalParameterCandidates(searchSpace: RetrievalParamet
|
|
|
158
157
|
declare function retrievalParameterSweepProposer(options: RetrievalParameterSweepProposerOptions): SurfaceProposer;
|
|
159
158
|
declare function runRetrievalImprovementLoop(options: RunRetrievalImprovementLoopOptions): Promise<RunRetrievalImprovementLoopResult>;
|
|
160
159
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
160
|
+
type AgentMemoryKind = 'message' | 'entity' | 'fact' | 'preference' | 'observation' | 'reasoning-trace';
|
|
161
|
+
interface AgentMemoryScope {
|
|
162
|
+
tenantId?: string;
|
|
163
|
+
userId?: string;
|
|
164
|
+
agentId?: string;
|
|
165
|
+
teamId?: string;
|
|
166
|
+
runId?: string;
|
|
167
|
+
sessionId?: string;
|
|
168
|
+
namespace?: string;
|
|
169
|
+
tags?: Record<string, string>;
|
|
164
170
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
171
|
+
interface AgentMemoryHit {
|
|
172
|
+
id: string;
|
|
173
|
+
uri: string;
|
|
174
|
+
kind: AgentMemoryKind;
|
|
175
|
+
text: string;
|
|
176
|
+
title?: string;
|
|
177
|
+
score?: number;
|
|
178
|
+
normalizedScore?: number;
|
|
179
|
+
confidence?: number;
|
|
180
|
+
createdAt?: string;
|
|
181
|
+
validUntil?: string;
|
|
182
|
+
lastVerifiedAt?: string;
|
|
183
|
+
metadata?: Record<string, unknown>;
|
|
184
|
+
}
|
|
185
|
+
interface AgentMemoryContext {
|
|
186
|
+
query: string;
|
|
187
|
+
text: string;
|
|
188
|
+
hits: AgentMemoryHit[];
|
|
189
|
+
sourceRecords: SourceRecord[];
|
|
190
|
+
metadata?: Record<string, unknown>;
|
|
191
|
+
}
|
|
192
|
+
interface AgentMemorySearchOptions {
|
|
193
|
+
scope?: AgentMemoryScope;
|
|
194
|
+
limit?: number;
|
|
195
|
+
minScore?: number;
|
|
196
|
+
kinds?: AgentMemoryKind[];
|
|
197
|
+
metadata?: Record<string, unknown>;
|
|
198
|
+
/**
|
|
199
|
+
* Opt-in randomized retrieval holdout (epsilon-dropout) for treatment-effect logging.
|
|
200
|
+
* Absent by default; when absent, retrieval behavior is unchanged. See ./holdout.
|
|
201
|
+
*/
|
|
202
|
+
holdout?: RetrievalHoldoutConfig;
|
|
203
|
+
}
|
|
204
|
+
interface AgentMemoryWriteInput {
|
|
205
|
+
kind: AgentMemoryKind;
|
|
206
|
+
text: string;
|
|
207
|
+
id?: string;
|
|
208
|
+
title?: string;
|
|
209
|
+
role?: 'system' | 'user' | 'assistant' | 'tool';
|
|
210
|
+
entityName?: string;
|
|
211
|
+
entityType?: string;
|
|
212
|
+
category?: string;
|
|
213
|
+
predicate?: string;
|
|
214
|
+
subject?: string;
|
|
215
|
+
object?: string;
|
|
216
|
+
confidence?: number;
|
|
217
|
+
scope?: AgentMemoryScope;
|
|
218
|
+
metadata?: Record<string, unknown>;
|
|
219
|
+
}
|
|
220
|
+
interface AgentMemoryWriteResult {
|
|
221
|
+
accepted: boolean;
|
|
222
|
+
id: string;
|
|
223
|
+
uri: string;
|
|
224
|
+
kind: AgentMemoryKind;
|
|
225
|
+
sourceRecord?: SourceRecord;
|
|
226
|
+
metadata?: Record<string, unknown>;
|
|
227
|
+
}
|
|
228
|
+
type AgentMemoryBranchIsolation = {
|
|
229
|
+
mode: 'scoped';
|
|
230
|
+
/** False when writes may outlive the worker process that issued them. */
|
|
231
|
+
processExitSafe?: boolean;
|
|
232
|
+
/** Wait before clearing an abandoned branch so accepted asynchronous writes become visible. */
|
|
233
|
+
recoveryDelayMs?: number;
|
|
234
|
+
} | {
|
|
235
|
+
mode: 'instance';
|
|
236
|
+
branchId: string;
|
|
237
|
+
/** True only when the dedicated instance also enforces every logical scope. */
|
|
238
|
+
supportsLogicalScopes?: boolean;
|
|
239
|
+
} | {
|
|
240
|
+
mode: 'unsupported';
|
|
241
|
+
reason: string;
|
|
242
|
+
};
|
|
243
|
+
interface AgentMemoryAdapter {
|
|
244
|
+
readonly id: string;
|
|
245
|
+
/** How this adapter prevents candidate branches from reading each other's state. */
|
|
246
|
+
readonly branchIsolation?: AgentMemoryBranchIsolation;
|
|
247
|
+
search(query: string, options?: AgentMemorySearchOptions): Promise<AgentMemoryHit[]>;
|
|
248
|
+
getContext(query: string, options?: AgentMemorySearchOptions): Promise<AgentMemoryContext>;
|
|
249
|
+
write(input: AgentMemoryWriteInput): Promise<AgentMemoryWriteResult>;
|
|
250
|
+
/** Delete exactly this scope. Repeated and concurrent calls for the same scope must be safe. */
|
|
251
|
+
clear?(scope?: AgentMemoryScope): Promise<void>;
|
|
252
|
+
flush?(): Promise<void>;
|
|
253
|
+
close?(): Promise<void>;
|
|
173
254
|
}
|
|
174
|
-
declare function acquireAgentMemoryRunLease(input: {
|
|
175
|
-
experimentId: string;
|
|
176
|
-
runDir: string;
|
|
177
|
-
storage: CampaignStorage;
|
|
178
|
-
customStorage: boolean;
|
|
179
|
-
lockFileName: string;
|
|
180
|
-
label: string;
|
|
181
|
-
controllerMode?: AgentMemoryControllerMode;
|
|
182
|
-
acquireRunLease?: AgentMemoryAcquireRunLease;
|
|
183
|
-
}): Promise<OwnedAgentMemoryRunLease>;
|
|
184
|
-
|
|
185
255
|
interface RetrievalHoldoutConfig {
|
|
186
256
|
/** Per-session probability that one eligible watchlist item is suppressed. 0 logs the full schema without ever dropping. */
|
|
187
257
|
epsilon: number;
|
|
@@ -302,223 +372,30 @@ interface RetrievalHoldoutResult {
|
|
|
302
372
|
}
|
|
303
373
|
/** Adapter context paths that answer without retrieval, so no holdout draw can happen. */
|
|
304
374
|
type RetrievalHoldoutBypassReason = 'short-term-context' | 'raw-string-context';
|
|
305
|
-
/**
|
|
306
|
-
* Deterministic uniform [0,1) for assignment draws. The sha256 key derivation is ours — it makes
|
|
307
|
-
* every draw replayable from the logged keys alone (design rule D5) — while the generator core is
|
|
308
|
-
* the substrate's `mulberry32` (@tangle-network/agent-eval statistics vocabulary), seeded with the
|
|
309
|
-
* top 32 bits of the digest, so the statistical machinery is reused rather than forked.
|
|
310
|
-
*/
|
|
311
|
-
declare function deterministicRng(key: string): number;
|
|
312
|
-
/**
|
|
313
|
-
* Value-hash of the knobs that DEFINE the experiment (epsilon + watchlist, order-independent).
|
|
314
|
-
* Everything else on the config (callbacks, attribution stamps, privacy flags) does not change
|
|
315
|
-
* which assignments are drawn, so it stays out of the hash.
|
|
316
|
-
*/
|
|
317
|
-
declare function retrievalHoldoutConfigHash(config: Pick<RetrievalHoldoutConfig, 'epsilon' | 'watchlist'>): string;
|
|
318
|
-
/**
|
|
319
|
-
* Pure per-call holdout: takes post-filter hits, returns delivered hits plus the log event
|
|
320
|
-
* and the next session state. Suppression only removes items (no backfill), and it happens
|
|
321
|
-
* before rendering so a drop session's context is byte-identical to a natural smaller retrieval.
|
|
322
|
-
*/
|
|
323
|
-
declare function applyRetrievalHoldout(hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx?: RetrievalHoldoutCallContext): RetrievalHoldoutResult;
|
|
324
|
-
/**
|
|
325
|
-
* Logs a holdout event for adapter context paths that answer WITHOUT going through the
|
|
326
|
-
* search→render seam (Neo4j short-term conversation context, raw-string getContext results),
|
|
327
|
-
* so a consumer with a holdout configured still sees every call and the fraction-under-experiment
|
|
328
|
-
* denominator stays honest instead of silently losing these calls. No suppression is applied:
|
|
329
|
-
* dropping is only meaningful for retrieved memory hits, never for conversation context.
|
|
330
|
-
*/
|
|
331
|
-
declare function emitRetrievalHoldoutBypass(hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx: Omit<RetrievalHoldoutCallContext, 'session'>, bypassReason: RetrievalHoldoutBypassReason): RetrievalHoldoutEvent;
|
|
332
|
-
/**
|
|
333
|
-
* Clears all sticky-session state. For test isolation and experiment-epoch boundaries;
|
|
334
|
-
* sessions in flight afterwards re-draw deterministically (same rng keys), so with the default
|
|
335
|
-
* rng a reset is invisible in the logs unless eligibility sets changed in between.
|
|
336
|
-
*/
|
|
337
|
-
declare function resetRetrievalHoldoutRegistry(): void;
|
|
338
|
-
/**
|
|
339
|
-
* Convenience wrapper that threads session state internally, keyed by the VALUE of the
|
|
340
|
-
* experiment-defining knobs (configHash = epsilon + sorted watchlist) plus sessionId, so a fresh
|
|
341
|
-
* config object per call — the natural pattern when options are built inline — keeps full
|
|
342
|
-
* stickiness. Distinct experiments never share session state; two config objects with the same
|
|
343
|
-
* epsilon/watchlist are the same experiment by definition.
|
|
344
|
-
*/
|
|
345
|
-
declare function applySessionStickyRetrievalHoldout(hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx?: Omit<RetrievalHoldoutCallContext, 'session'>): RetrievalHoldoutResult;
|
|
346
|
-
/** Aggregated view of one session's holdout exposure — the randomization unit of this design. */
|
|
347
|
-
interface RetrievalHoldoutSessionSummary {
|
|
348
|
-
/** `${configHash}:${sessionIdHash}` — the trajectory's runId. */
|
|
349
|
-
runId: string;
|
|
350
|
-
configHash: string;
|
|
351
|
-
sessionIdHash: string;
|
|
352
|
-
sessionHoldout: boolean;
|
|
353
|
-
sessionTargetId: string | null;
|
|
354
|
-
/** Item actually suppressed in this session (equals sessionTargetId when a drop occurred). */
|
|
355
|
-
droppedId: string | null;
|
|
356
|
-
/** |watchlist ∩ E| at the session's first intersecting randomized call; 0 when never intersecting. */
|
|
357
|
-
firstCandidateCount: number;
|
|
358
|
-
/** Randomized retrieval calls observed for this session. */
|
|
359
|
-
callCount: number;
|
|
360
|
-
/** Adapter bypass calls observed for this session (logged, never randomized). */
|
|
361
|
-
bypassCallCount: number;
|
|
362
|
-
/** Session-level probability of the OBSERVED assignment under the behavior policy. */
|
|
363
|
-
behaviorProb: number;
|
|
364
|
-
/** True when the session's events disagree on arm or target (e.g. registry eviction re-draw). */
|
|
365
|
-
mixedExposure: boolean;
|
|
366
|
-
/** Present only on sessions surfaced in `excluded` rather than converted. */
|
|
367
|
-
exclusionReason?: 'mixed-exposure' | 'no-randomized-calls';
|
|
368
|
-
}
|
|
369
|
-
interface RetrievalHoldoutOffPolicyOptions {
|
|
370
|
-
/**
|
|
371
|
-
* Realized outcome PER SESSION, keyed by sessionIdHash — the session is the randomization
|
|
372
|
-
* unit, so rewards are per session, never per call. A missing entry for an included session
|
|
373
|
-
* throws: silently dropping unscored sessions would corrupt the estimator's denominator, so
|
|
374
|
-
* filter events to scored sessions upstream instead.
|
|
375
|
-
*/
|
|
376
|
-
rewards: Record<string, number>;
|
|
377
|
-
/**
|
|
378
|
-
* Target-policy probability of the session's observed assignment. Default models the
|
|
379
|
-
* always-deliver-in-full policy: 1 for full-delivery sessions, 0 for drop sessions.
|
|
380
|
-
*/
|
|
381
|
-
targetProb?: (session: RetrievalHoldoutSessionSummary) => number;
|
|
382
|
-
/** Per-session reward-model prediction enabling `doublyRobust`; when absent, qHat is null. */
|
|
383
|
-
qHat?: (session: RetrievalHoldoutSessionSummary) => number | null;
|
|
384
|
-
}
|
|
385
|
-
interface RetrievalHoldoutOffPolicyResult {
|
|
386
|
-
/** One trajectory per (configHash, sessionIdHash) — never per call. */
|
|
387
|
-
trajectories: OffPolicyTrajectory[];
|
|
388
|
-
/** 1:1 with `trajectories` (same order): the diagnostic view of each converted session. */
|
|
389
|
-
sessions: RetrievalHoldoutSessionSummary[];
|
|
390
|
-
/** Sessions surfaced but NOT converted (mixed exposure, bypass-only) — counted, never hidden. */
|
|
391
|
-
excluded: RetrievalHoldoutSessionSummary[];
|
|
392
|
-
/** Events with no sessionIdHash: outside session randomization, cannot join a reward. */
|
|
393
|
-
unattributableEvents: number;
|
|
394
|
-
}
|
|
395
|
-
/**
|
|
396
|
-
* Maps holdout logs onto agent-eval's `OffPolicyTrajectory`, ONE TRAJECTORY PER SESSION, so
|
|
397
|
-
* EXP-007's analysis consumes the substrate's `inverseProbabilityWeighting` /
|
|
398
|
-
* `selfNormalizedImportanceWeighting` / `doublyRobust` estimators directly. This matches the
|
|
399
|
-
* PREREG estimator (session-sticky self-normalized IPW): the design randomizes per SESSION —
|
|
400
|
-
* the arm coin is flipped once (P(holdout) = epsilon) and the sticky target is drawn once,
|
|
401
|
-
* uniformly over watchlist ∩ E at the session's first intersecting call.
|
|
402
|
-
*
|
|
403
|
-
* Session-level action space and behavior probabilities (they sum to 1 by construction):
|
|
404
|
-
* - full delivery (control arm observed): `1 − epsilon`;
|
|
405
|
-
* - drop candidate i of k = |watchlist ∩ E_first|: `epsilon / k` each (the draw event's logged
|
|
406
|
-
* `dropPropensity`);
|
|
407
|
-
* - sessions whose eligibility sets never intersect the watchlist: probability 1 — full
|
|
408
|
-
* delivery was certain in either arm.
|
|
409
|
-
*
|
|
410
|
-
* Per-call events are repeated observations WITHIN one session-level randomization; per-call
|
|
411
|
-
* IPW is statistically invalid for this design (per-call "propensities" imply an action
|
|
412
|
-
* distribution summing to more than 1 and bias IPS downward). Calls where the sticky target is
|
|
413
|
-
* absent from E, and adapter bypass calls, fold into the session summary (callCount /
|
|
414
|
-
* bypassCallCount) instead of generating independent propensities.
|
|
415
|
-
*
|
|
416
|
-
* Join contract: `rewards` is keyed by `sessionIdHash` (sha256(sessionId) first 16 hex — compute
|
|
417
|
-
* the same prefix over the outcome table's session ids, or log with
|
|
418
|
-
* `includePlaintextIdentifiers: true` and join on raw ids upstream). Events are grouped per
|
|
419
|
-
* (configHash, sessionIdHash) so different experiment configs are never pooled. Mixed-exposure
|
|
420
|
-
* sessions (arm or target disagreement, e.g. after registry eviction) are excluded from the
|
|
421
|
-
* trajectories and surfaced in `excluded` for the analysis to count.
|
|
422
|
-
* Pre-registration for EXP-007 stays in the research repo by design; the manifest vocabulary for
|
|
423
|
-
* it is agent-eval's `HypothesisManifest` / `signManifest` / `evaluateHypothesis` exports.
|
|
424
|
-
*/
|
|
425
|
-
declare function toOffPolicyTrajectory(events: RetrievalHoldoutEvent[], options: RetrievalHoldoutOffPolicyOptions): RetrievalHoldoutOffPolicyResult;
|
|
426
375
|
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
userId?: string;
|
|
431
|
-
agentId?: string;
|
|
432
|
-
teamId?: string;
|
|
433
|
-
runId?: string;
|
|
434
|
-
sessionId?: string;
|
|
435
|
-
namespace?: string;
|
|
436
|
-
tags?: Record<string, string>;
|
|
437
|
-
}
|
|
438
|
-
interface AgentMemoryHit {
|
|
439
|
-
id: string;
|
|
440
|
-
uri: string;
|
|
441
|
-
kind: AgentMemoryKind;
|
|
442
|
-
text: string;
|
|
443
|
-
title?: string;
|
|
444
|
-
score?: number;
|
|
445
|
-
normalizedScore?: number;
|
|
446
|
-
confidence?: number;
|
|
447
|
-
createdAt?: string;
|
|
448
|
-
validUntil?: string;
|
|
449
|
-
lastVerifiedAt?: string;
|
|
450
|
-
metadata?: Record<string, unknown>;
|
|
451
|
-
}
|
|
452
|
-
interface AgentMemoryContext {
|
|
453
|
-
query: string;
|
|
454
|
-
text: string;
|
|
455
|
-
hits: AgentMemoryHit[];
|
|
456
|
-
sourceRecords: SourceRecord[];
|
|
457
|
-
metadata?: Record<string, unknown>;
|
|
458
|
-
}
|
|
459
|
-
interface AgentMemorySearchOptions {
|
|
460
|
-
scope?: AgentMemoryScope;
|
|
461
|
-
limit?: number;
|
|
462
|
-
minScore?: number;
|
|
463
|
-
kinds?: AgentMemoryKind[];
|
|
464
|
-
metadata?: Record<string, unknown>;
|
|
465
|
-
/**
|
|
466
|
-
* Opt-in randomized retrieval holdout (epsilon-dropout) for treatment-effect logging.
|
|
467
|
-
* Absent by default; when absent, retrieval behavior is unchanged. See ./holdout.
|
|
468
|
-
*/
|
|
469
|
-
holdout?: RetrievalHoldoutConfig;
|
|
470
|
-
}
|
|
471
|
-
interface AgentMemoryWriteInput {
|
|
472
|
-
kind: AgentMemoryKind;
|
|
473
|
-
text: string;
|
|
474
|
-
id?: string;
|
|
475
|
-
title?: string;
|
|
476
|
-
role?: 'system' | 'user' | 'assistant' | 'tool';
|
|
477
|
-
entityName?: string;
|
|
478
|
-
entityType?: string;
|
|
479
|
-
category?: string;
|
|
480
|
-
predicate?: string;
|
|
481
|
-
subject?: string;
|
|
482
|
-
object?: string;
|
|
483
|
-
confidence?: number;
|
|
484
|
-
scope?: AgentMemoryScope;
|
|
485
|
-
metadata?: Record<string, unknown>;
|
|
486
|
-
}
|
|
487
|
-
interface AgentMemoryWriteResult {
|
|
488
|
-
accepted: boolean;
|
|
489
|
-
id: string;
|
|
490
|
-
uri: string;
|
|
491
|
-
kind: AgentMemoryKind;
|
|
492
|
-
sourceRecord?: SourceRecord;
|
|
493
|
-
metadata?: Record<string, unknown>;
|
|
376
|
+
interface AgentMemoryRunLease {
|
|
377
|
+
assertOwned(): Promise<void> | void;
|
|
378
|
+
release(): Promise<void> | void;
|
|
494
379
|
}
|
|
495
|
-
type
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
branchId: string;
|
|
504
|
-
/** True only when the dedicated instance also enforces every logical scope. */
|
|
505
|
-
supportsLogicalScopes?: boolean;
|
|
506
|
-
} | {
|
|
507
|
-
mode: 'unsupported';
|
|
508
|
-
reason: string;
|
|
509
|
-
};
|
|
510
|
-
interface AgentMemoryAdapter {
|
|
511
|
-
readonly id: string;
|
|
512
|
-
/** How this adapter prevents candidate branches from reading each other's state. */
|
|
513
|
-
readonly branchIsolation?: AgentMemoryBranchIsolation;
|
|
514
|
-
search(query: string, options?: AgentMemorySearchOptions): Promise<AgentMemoryHit[]>;
|
|
515
|
-
getContext(query: string, options?: AgentMemorySearchOptions): Promise<AgentMemoryContext>;
|
|
516
|
-
write(input: AgentMemoryWriteInput): Promise<AgentMemoryWriteResult>;
|
|
517
|
-
/** Delete exactly this scope. Repeated and concurrent calls for the same scope must be safe. */
|
|
518
|
-
clear?(scope?: AgentMemoryScope): Promise<void>;
|
|
519
|
-
flush?(): Promise<void>;
|
|
520
|
-
close?(): Promise<void>;
|
|
380
|
+
type AgentMemoryAcquireRunLease = (input: {
|
|
381
|
+
experimentId: string;
|
|
382
|
+
runDir: string;
|
|
383
|
+
}) => AgentMemoryRunLease | Promise<AgentMemoryRunLease>;
|
|
384
|
+
type AgentMemoryControllerMode = 'process-local';
|
|
385
|
+
interface OwnedAgentMemoryRunLease {
|
|
386
|
+
assertOwned(): Promise<void>;
|
|
387
|
+
release(): Promise<void>;
|
|
521
388
|
}
|
|
389
|
+
declare function acquireAgentMemoryRunLease(input: {
|
|
390
|
+
experimentId: string;
|
|
391
|
+
runDir: string;
|
|
392
|
+
storage: CampaignStorage;
|
|
393
|
+
customStorage: boolean;
|
|
394
|
+
lockFileName: string;
|
|
395
|
+
label: string;
|
|
396
|
+
controllerMode?: AgentMemoryControllerMode;
|
|
397
|
+
acquireRunLease?: AgentMemoryAcquireRunLease;
|
|
398
|
+
}): Promise<OwnedAgentMemoryRunLease>;
|
|
522
399
|
|
|
523
400
|
type KnowledgeBenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'memory-ingest' | 'memory-recall' | 'memory-temporal' | 'memory-update' | 'memory-forgetting' | 'memory-reasoning' | 'memory-summarization' | 'memory-recommendation' | 'memory-multiparty';
|
|
524
401
|
type KnowledgeAnswerBenchmarkTaskKind = 'rag-answer' | 'hallucination' | 'kb-improvement';
|
|
@@ -780,43 +657,5 @@ interface BuildRetrievalBenchmarkCasesFromQrelsOptions {
|
|
|
780
657
|
documentTarget?: (documentId: string, qrel: KnowledgeRetrievalBenchmarkQrel) => RetrievalGoldTarget;
|
|
781
658
|
splitOf?: (queryId: string) => KnowledgeBenchmarkSplit;
|
|
782
659
|
}
|
|
783
|
-
declare const INDUSTRY_RAG_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & {
|
|
784
|
-
taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind;
|
|
785
|
-
})[];
|
|
786
|
-
declare const INDUSTRY_MEMORY_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & {
|
|
787
|
-
taskKind: KnowledgeMemoryBenchmarkTaskKind;
|
|
788
|
-
})[];
|
|
789
|
-
declare function buildIndustryRagBenchmarkSmokeCases(specs?: readonly (KnowledgeBenchmarkSpec & {
|
|
790
|
-
taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind;
|
|
791
|
-
})[]): KnowledgeBenchmarkCase[];
|
|
792
|
-
declare function respondToIndustryRagBenchmarkSmokeCase(input: {
|
|
793
|
-
case: KnowledgeBenchmarkCase;
|
|
794
|
-
}): KnowledgeBenchmarkArtifact;
|
|
795
|
-
declare function buildIndustryMemoryBenchmarkSmokeCases(specs?: readonly (KnowledgeBenchmarkSpec & {
|
|
796
|
-
taskKind: KnowledgeMemoryBenchmarkTaskKind;
|
|
797
|
-
})[]): KnowledgeMemoryBenchmarkCase[];
|
|
798
|
-
declare function respondToIndustryMemoryBenchmarkSmokeCase(input: {
|
|
799
|
-
case: KnowledgeMemoryBenchmarkCase;
|
|
800
|
-
}): KnowledgeBenchmarkArtifact;
|
|
801
|
-
declare function buildFirstPartyMemoryLifecycleBenchmarkCases(): KnowledgeMemoryBenchmarkCase[];
|
|
802
|
-
declare function runMemoryAdapterBenchmark(options: RunMemoryAdapterBenchmarkOptions): Promise<RunMemoryAdapterBenchmarkResult>;
|
|
803
|
-
declare function createNoopMemoryBenchmarkAdapter(id?: string): AgentMemoryAdapter;
|
|
804
|
-
declare function createInMemoryBenchmarkAdapter(options?: {
|
|
805
|
-
id?: string;
|
|
806
|
-
}): AgentMemoryAdapter;
|
|
807
|
-
declare function parseKnowledgeBenchmarkJsonl<T = unknown>(text: string): T[];
|
|
808
|
-
declare function parseKnowledgeBenchmarkQrels(text: string): KnowledgeRetrievalBenchmarkQrel[];
|
|
809
|
-
declare function buildRetrievalBenchmarkCasesFromQrels(options: BuildRetrievalBenchmarkCasesFromQrelsOptions): KnowledgeRetrievalBenchmarkCase[];
|
|
810
|
-
declare function runKnowledgeBenchmarkSuite<TArtifact = KnowledgeBenchmarkArtifact>(options: RunKnowledgeBenchmarkSuiteOptions<TArtifact>): Promise<RunKnowledgeBenchmarkSuiteResult<TArtifact>>;
|
|
811
|
-
declare function renderKnowledgeBenchmarkReportMarkdown(report: KnowledgeBenchmarkReport): string;
|
|
812
|
-
declare function buildKnowledgeBenchmarkScenarios(cases: readonly KnowledgeBenchmarkCase[], splits?: readonly KnowledgeBenchmarkSplit[]): KnowledgeBenchmarkScenario[];
|
|
813
|
-
declare function knowledgeBenchmarkJudge<TArtifact = KnowledgeBenchmarkArtifact>(): JudgeConfig<TArtifact, KnowledgeBenchmarkScenario>;
|
|
814
|
-
declare function scoreKnowledgeBenchmarkArtifact<TArtifact>(testCase: KnowledgeBenchmarkCase, artifact: TArtifact): KnowledgeBenchmarkEvaluation;
|
|
815
|
-
declare function summarizeKnowledgeBenchmarkCampaign<TArtifact>(input: {
|
|
816
|
-
scenarios: readonly KnowledgeBenchmarkScenario[];
|
|
817
|
-
campaign: CampaignResult<TArtifact, KnowledgeBenchmarkScenario>;
|
|
818
|
-
}): KnowledgeBenchmarkReport;
|
|
819
|
-
declare function scoreMemoryBenchmarkArtifact<TArtifact>(testCase: KnowledgeMemoryBenchmarkCase, artifact: TArtifact): KnowledgeBenchmarkEvaluation;
|
|
820
|
-
declare function isKnowledgeMemoryBenchmarkCase(testCase: KnowledgeBenchmarkCase): testCase is KnowledgeMemoryBenchmarkCase;
|
|
821
660
|
|
|
822
|
-
export { type
|
|
661
|
+
export { type RetrievalHoldoutConfig as $, type AgentMemoryAcquireRunLease as A, type BuildRetrievalBenchmarkCasesFromQrelsOptions as B, type KnowledgeBenchmarkSpec as C, type KnowledgeBenchmarkSplit as D, type KnowledgeBenchmarkTaskKind as E, type KnowledgeClaimMatcher as F, type KnowledgeMemoryBenchmarkCase as G, type KnowledgeMemoryBenchmarkTaskKind as H, type KnowledgeMemoryEvent as I, type KnowledgeMemoryFactMatcher as J, type KnowledgeAnswerBenchmarkCase as K, type KnowledgeRetrievalBenchmarkCase as L, type KnowledgeRetrievalBenchmarkQrel as M, type KnowledgeRetrievalBenchmarkQuery as N, type MemoryAdapterBenchmarkCandidate as O, type MemoryAdapterBenchmarkRankingRow as P, type OwnedAgentMemoryRunLease as Q, type RunRetrievalImprovementLoopResult as R, type RetrievalConfig as S, type RetrievalEvalArtifact as T, type RetrievalEvalRetriever as U, type RetrievalEvalRetrieverInput as V, type RetrievalEvalRetrieverResult as W, type RetrievalEvalScenario as X, type RetrievalGoldTarget as Y, type RetrievalHoldoutBypassReason as Z, type RetrievalHoldoutCallContext as _, type RunRetrievalImprovementLoopOptions as a, type RetrievalHoldoutEligibleItem as a0, type RetrievalHoldoutEvent as a1, type RetrievalHoldoutResult as a2, type RetrievalHoldoutSessionState as a3, type RetrievalMetricSummary as a4, type RetrievalMetricWeights as a5, type RetrievalParameterSearchSpace as a6, type RetrievalParameterSweepProposerOptions as a7, type RetrievalRecallJudgeOptions as a8, type RetrievedKnowledgeHit as a9, type RetrievedSourceSpan as aa, type RunKnowledgeBenchmarkSuiteOptions as ab, type RunKnowledgeBenchmarkSuiteResult as ac, type RunMemoryAdapterBenchmarkOptions as ad, type RunMemoryAdapterBenchmarkResult as ae, acquireAgentMemoryRunLease as af, buildRetrievalEvalDispatch as ag, buildRetrievalParameterCandidates as ah, retrievalConfigFromSurface as ai, retrievalConfigSurface as aj, retrievalParameterSweepProposer as ak, retrievalRecallJudge as al, runRetrievalImprovementLoop as am, scoreRetrievalArtifact as an, type AgentMemoryAdapter as b, type AgentMemoryBranchIsolation as c, type AgentMemoryContext as d, type AgentMemoryControllerMode as e, type AgentMemoryHit as f, type AgentMemoryKind as g, type AgentMemoryRunLease as h, type AgentMemoryScope as i, type AgentMemorySearchOptions as j, type AgentMemoryWriteInput as k, type AgentMemoryWriteResult as l, type BuildRetrievalEvalDispatchOptions as m, type BuildRetrievalParameterCandidatesOptions as n, type KnowledgeAnswerBenchmarkTaskKind as o, type KnowledgeBenchmarkArtifact as p, type KnowledgeBenchmarkCase as q, type KnowledgeBenchmarkCaseBase as r, type KnowledgeBenchmarkDistribution as s, type KnowledgeBenchmarkEvaluation as t, type KnowledgeBenchmarkFamily as u, type KnowledgeBenchmarkReport as v, type KnowledgeBenchmarkResponder as w, type KnowledgeBenchmarkScenario as x, type KnowledgeBenchmarkSliceSummary as y, type KnowledgeBenchmarkSource as z };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-knowledge",
|
|
3
|
-
"version": "4.0.
|
|
3
|
+
"version": "4.0.1",
|
|
4
4
|
"description": "Source-grounded, eval-gated knowledge growth primitives for agents.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-knowledge#readme",
|
|
6
6
|
"repository": {
|