@tangle-network/agent-knowledge 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +32 -18
- package/CHANGELOG.md +32 -0
- package/README.md +31 -19
- package/dist/benchmarks/index.d.ts +51 -3
- package/dist/benchmarks/index.js +1 -1
- package/dist/{chunk-EUAXSBMH.js → chunk-AKYJG2MR.js} +224 -224
- package/dist/chunk-AKYJG2MR.js.map +1 -0
- package/dist/{chunk-DW6APRTX.js → chunk-BBCIB4UL.js} +2486 -2457
- package/dist/chunk-BBCIB4UL.js.map +1 -0
- package/dist/{chunk-UWOTQNBI.js → chunk-CPMLJYA3.js} +1886 -1842
- package/dist/chunk-CPMLJYA3.js.map +1 -0
- package/dist/{chunk-WCYW2GDA.js → chunk-MYFM6LKH.js} +2 -2
- package/dist/chunk-MYFM6LKH.js.map +1 -0
- package/dist/cli.js +1 -1
- package/dist/index.d.ts +48 -52
- package/dist/index.js +2906 -2874
- package/dist/index.js.map +1 -1
- package/dist/memory/index.d.ts +129 -6
- package/dist/memory/index.js +2 -2
- package/dist/sources/index.d.ts +17 -33
- package/dist/sources/index.js +1 -1
- package/dist/{index-Bqg1mBPt.d.ts → types-DP38encz.d.ts} +127 -284
- package/docs/eval/investment-material-facts.md +28 -29
- package/docs/eval/rag-eval-roadmap.md +6 -5
- package/docs/results/adaptive.md +13 -13
- package/docs/results/claim-grounding.md +11 -11
- package/docs/results/cost-quality.md +4 -4
- package/docs/results/investment-thesis.md +56 -56
- package/docs/results/research-driving.md +54 -54
- package/docs/two-agent-research-ab.md +94 -94
- package/package.json +4 -7
- package/dist/chunk-DW6APRTX.js.map +0 -1
- package/dist/chunk-EUAXSBMH.js.map +0 -1
- package/dist/chunk-UWOTQNBI.js.map +0 -1
- package/dist/chunk-WCYW2GDA.js.map +0 -1
package/dist/memory/index.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { b as AgentMemoryAdapter, j as AgentMemorySearchOptions, d as AgentMemoryContext, f as AgentMemoryHit, i as AgentMemoryScope, k as AgentMemoryWriteInput, l as AgentMemoryWriteResult,
|
|
2
|
-
export { c as AgentMemoryBranchIsolation, g as AgentMemoryKind,
|
|
1
|
+
import { b as AgentMemoryAdapter, j as AgentMemorySearchOptions, d as AgentMemoryContext, f as AgentMemoryHit, i as AgentMemoryScope, k as AgentMemoryWriteInput, l as AgentMemoryWriteResult, u as KnowledgeBenchmarkFamily, D as KnowledgeBenchmarkSplit, H as KnowledgeMemoryBenchmarkTaskKind, J as KnowledgeMemoryFactMatcher, h as AgentMemoryRunLease, I as KnowledgeMemoryEvent, G as KnowledgeMemoryBenchmarkCase, e as AgentMemoryControllerMode, A as AgentMemoryAcquireRunLease, $ as RetrievalHoldoutConfig, _ as RetrievalHoldoutCallContext, a2 as RetrievalHoldoutResult, Z as RetrievalHoldoutBypassReason, a1 as RetrievalHoldoutEvent } from '../types-DP38encz.js';
|
|
2
|
+
export { c as AgentMemoryBranchIsolation, g as AgentMemoryKind, Q as OwnedAgentMemoryRunLease, a0 as RetrievalHoldoutEligibleItem, a3 as RetrievalHoldoutSessionState, af as acquireAgentMemoryRunLease } from '../types-DP38encz.js';
|
|
3
3
|
import { CampaignResult, Scenario, DispatchContext, CampaignStorage, CostLedgerHandle, JudgeConfig, HeldoutSignificance, Lineage, GovernorContext, GovernorOp, SurfaceProposer, LineageStore, HeldoutSignificanceOptions } from '@tangle-network/agent-eval/campaign';
|
|
4
|
+
import { OffPolicyTrajectory } from '@tangle-network/agent-eval/rl';
|
|
4
5
|
import { z } from 'zod';
|
|
5
6
|
import { S as SourceRecord } from '../types-6x0OpfW6.js';
|
|
6
|
-
import '@tangle-network/agent-eval/rl';
|
|
7
7
|
|
|
8
8
|
declare function defaultGetMemoryContext(adapter: Pick<AgentMemoryAdapter, 'search'>, query: string, options?: AgentMemorySearchOptions): Promise<AgentMemoryContext>;
|
|
9
9
|
declare function renderMemoryContext(hits: AgentMemoryHit[]): string;
|
|
@@ -257,13 +257,16 @@ interface RunAgentMemoryExperimentResult {
|
|
|
257
257
|
attemptLogPath: string;
|
|
258
258
|
recoveryLogPath: string;
|
|
259
259
|
}
|
|
260
|
+
|
|
260
261
|
/** Converts existing ordered memory benchmark cases into executable histories. */
|
|
262
|
+
|
|
261
263
|
declare function buildAgentMemorySequencesFromBenchmarkCases(cases: readonly KnowledgeMemoryBenchmarkCase[], options?: BuildAgentMemorySequencesFromBenchmarkCasesOptions): AgentMemorySequence[];
|
|
262
|
-
/** Runs ordered, branch-isolated memory histories across candidate systems. */
|
|
263
|
-
declare function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOptions): Promise<RunAgentMemoryExperimentResult>;
|
|
264
264
|
declare function buildAgentMemorySequenceScenarios(sequences: readonly AgentMemorySequence[], candidates: readonly Pick<AgentMemoryExperimentCandidate, 'id'>[]): AgentMemorySequenceScenario[];
|
|
265
265
|
declare function agentMemorySequenceJudge(): JudgeConfig<AgentMemorySequenceArtifact, AgentMemorySequenceScenario>;
|
|
266
266
|
|
|
267
|
+
/** Runs ordered, branch-isolated memory histories across candidate systems. */
|
|
268
|
+
declare function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOptions): Promise<RunAgentMemoryExperimentResult>;
|
|
269
|
+
|
|
267
270
|
interface GraphitiMcpClientLike {
|
|
268
271
|
callTool(request: {
|
|
269
272
|
name: string;
|
|
@@ -298,6 +301,125 @@ declare function graphitiMemoryAdapterIdentity(options: Pick<GraphitiMemoryAdapt
|
|
|
298
301
|
backendRef: string;
|
|
299
302
|
}): string;
|
|
300
303
|
|
|
304
|
+
/**
|
|
305
|
+
* Deterministic uniform [0,1) for assignment draws. The sha256 key derivation is ours — it makes
|
|
306
|
+
* every draw replayable from the logged keys alone (design rule D5) — while the generator core is
|
|
307
|
+
* the substrate's `mulberry32` (@tangle-network/agent-eval statistics vocabulary), seeded with the
|
|
308
|
+
* top 32 bits of the digest, so the statistical machinery is reused rather than forked.
|
|
309
|
+
*/
|
|
310
|
+
declare function deterministicRng(key: string): number;
|
|
311
|
+
/**
|
|
312
|
+
* Value-hash of the knobs that DEFINE the experiment (epsilon + watchlist, order-independent).
|
|
313
|
+
* Everything else on the config (callbacks, attribution stamps, privacy flags) does not change
|
|
314
|
+
* which assignments are drawn, so it stays out of the hash.
|
|
315
|
+
*/
|
|
316
|
+
declare function retrievalHoldoutConfigHash(config: Pick<RetrievalHoldoutConfig, 'epsilon' | 'watchlist'>): string;
|
|
317
|
+
/**
|
|
318
|
+
* Pure per-call holdout: takes post-filter hits, returns delivered hits plus the log event
|
|
319
|
+
* and the next session state. Suppression only removes items (no backfill), and it happens
|
|
320
|
+
* before rendering so a drop session's context is byte-identical to a natural smaller retrieval.
|
|
321
|
+
*/
|
|
322
|
+
declare function applyRetrievalHoldout(hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx?: RetrievalHoldoutCallContext): RetrievalHoldoutResult;
|
|
323
|
+
/**
|
|
324
|
+
* Logs a holdout event for adapter context paths that answer WITHOUT going through the
|
|
325
|
+
* search→render seam (Neo4j short-term conversation context, raw-string getContext results),
|
|
326
|
+
* so a consumer with a holdout configured still sees every call and the fraction-under-experiment
|
|
327
|
+
* denominator stays honest instead of silently losing these calls. No suppression is applied:
|
|
328
|
+
* dropping is only meaningful for retrieved memory hits, never for conversation context.
|
|
329
|
+
*/
|
|
330
|
+
declare function emitRetrievalHoldoutBypass(hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx: Omit<RetrievalHoldoutCallContext, 'session'>, bypassReason: RetrievalHoldoutBypassReason): RetrievalHoldoutEvent;
|
|
331
|
+
/**
|
|
332
|
+
* Clears all sticky-session state. For test isolation and experiment-epoch boundaries;
|
|
333
|
+
* sessions in flight afterwards re-draw deterministically (same rng keys), so with the default
|
|
334
|
+
* rng a reset is invisible in the logs unless eligibility sets changed in between.
|
|
335
|
+
*/
|
|
336
|
+
declare function resetRetrievalHoldoutRegistry(): void;
|
|
337
|
+
/**
|
|
338
|
+
* Convenience wrapper that threads session state internally, keyed by the VALUE of the
|
|
339
|
+
* experiment-defining knobs (configHash = epsilon + sorted watchlist) plus sessionId, so a fresh
|
|
340
|
+
* config object per call — the natural pattern when options are built inline — keeps full
|
|
341
|
+
* stickiness. Distinct experiments never share session state; two config objects with the same
|
|
342
|
+
* epsilon/watchlist are the same experiment by definition.
|
|
343
|
+
*/
|
|
344
|
+
declare function applySessionStickyRetrievalHoldout(hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx?: Omit<RetrievalHoldoutCallContext, 'session'>): RetrievalHoldoutResult;
|
|
345
|
+
/** Aggregated view of one session's holdout exposure — the randomization unit of this design. */
|
|
346
|
+
interface RetrievalHoldoutSessionSummary {
|
|
347
|
+
/** `${configHash}:${sessionIdHash}` — the trajectory's runId. */
|
|
348
|
+
runId: string;
|
|
349
|
+
configHash: string;
|
|
350
|
+
sessionIdHash: string;
|
|
351
|
+
sessionHoldout: boolean;
|
|
352
|
+
sessionTargetId: string | null;
|
|
353
|
+
/** Item actually suppressed in this session (equals sessionTargetId when a drop occurred). */
|
|
354
|
+
droppedId: string | null;
|
|
355
|
+
/** |watchlist ∩ E| at the session's first intersecting randomized call; 0 when never intersecting. */
|
|
356
|
+
firstCandidateCount: number;
|
|
357
|
+
/** Randomized retrieval calls observed for this session. */
|
|
358
|
+
callCount: number;
|
|
359
|
+
/** Adapter bypass calls observed for this session (logged, never randomized). */
|
|
360
|
+
bypassCallCount: number;
|
|
361
|
+
/** Session-level probability of the OBSERVED assignment under the behavior policy. */
|
|
362
|
+
behaviorProb: number;
|
|
363
|
+
/** True when the session's events disagree on arm or target (e.g. registry eviction re-draw). */
|
|
364
|
+
mixedExposure: boolean;
|
|
365
|
+
/** Present only on sessions surfaced in `excluded` rather than converted. */
|
|
366
|
+
exclusionReason?: 'mixed-exposure' | 'no-randomized-calls';
|
|
367
|
+
}
|
|
368
|
+
interface RetrievalHoldoutOffPolicyOptions {
|
|
369
|
+
/**
|
|
370
|
+
* Realized outcome PER SESSION, keyed by sessionIdHash — the session is the randomization
|
|
371
|
+
* unit, so rewards are per session, never per call. A missing entry for an included session
|
|
372
|
+
* throws: silently dropping unscored sessions would corrupt the estimator's denominator, so
|
|
373
|
+
* filter events to scored sessions upstream instead.
|
|
374
|
+
*/
|
|
375
|
+
rewards: Record<string, number>;
|
|
376
|
+
/**
|
|
377
|
+
* Target-policy probability of the session's observed assignment. Default models the
|
|
378
|
+
* always-deliver-in-full policy: 1 for full-delivery sessions, 0 for drop sessions.
|
|
379
|
+
*/
|
|
380
|
+
targetProb?: (session: RetrievalHoldoutSessionSummary) => number;
|
|
381
|
+
/** Per-session reward-model prediction enabling `doublyRobust`; when absent, qHat is null. */
|
|
382
|
+
qHat?: (session: RetrievalHoldoutSessionSummary) => number | null;
|
|
383
|
+
}
|
|
384
|
+
interface RetrievalHoldoutOffPolicyResult {
|
|
385
|
+
/** One trajectory per (configHash, sessionIdHash), never per call. */
|
|
386
|
+
trajectories: OffPolicyTrajectory[];
|
|
387
|
+
/** 1:1 with `trajectories` (same order): the diagnostic view of each converted session. */
|
|
388
|
+
sessions: RetrievalHoldoutSessionSummary[];
|
|
389
|
+
/** Sessions not converted because exposure was mixed or retrieval was bypassed. */
|
|
390
|
+
excluded: RetrievalHoldoutSessionSummary[];
|
|
391
|
+
/** Events with no sessionIdHash: outside session randomization, cannot join a reward. */
|
|
392
|
+
unattributableEvents: number;
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* Maps retrieval-dropout events onto one `OffPolicyTrajectory` per session for use with
|
|
396
|
+
* `inverseProbabilityWeighting`, `selfNormalizedImportanceWeighting`, or `doublyRobust`.
|
|
397
|
+
* Randomization occurs once per session: the arm is selected with P(dropout) = epsilon and
|
|
398
|
+
* the sticky target is drawn once,
|
|
399
|
+
* uniformly over watchlist ∩ E at the session's first intersecting call.
|
|
400
|
+
*
|
|
401
|
+
* Session-level action space and behavior probabilities (they sum to 1 by construction):
|
|
402
|
+
* - full delivery: `1 - epsilon`;
|
|
403
|
+
* - drop candidate i of k = |watchlist ∩ E_first|: `epsilon / k` each (the draw event's logged
|
|
404
|
+
* `dropPropensity`);
|
|
405
|
+
* - sessions whose eligibility sets never intersect the watchlist: probability 1, because full
|
|
406
|
+
* delivery was certain in either arm.
|
|
407
|
+
*
|
|
408
|
+
* Per-call events are repeated observations WITHIN one session-level randomization; per-call
|
|
409
|
+
* IPW is statistically invalid for this design (per-call "propensities" imply an action
|
|
410
|
+
* distribution summing to more than 1 and bias IPS downward). Calls where the sticky target is
|
|
411
|
+
* absent from E, and adapter bypass calls, fold into the session summary (callCount /
|
|
412
|
+
* bypassCallCount) instead of generating independent propensities.
|
|
413
|
+
*
|
|
414
|
+
* Join contract: `rewards` is keyed by `sessionIdHash` (sha256(sessionId) first 16 hex; compute
|
|
415
|
+
* the same prefix over the outcome table's session ids, or log with
|
|
416
|
+
* `includePlaintextIdentifiers: true` and join on raw ids upstream). Events are grouped per
|
|
417
|
+
* (configHash, sessionIdHash) so different experiment configs are never pooled. Mixed-exposure
|
|
418
|
+
* sessions (arm or target disagreement, e.g. after registry eviction) are excluded from the
|
|
419
|
+
* trajectories and surfaced in `excluded` for analysis.
|
|
420
|
+
*/
|
|
421
|
+
declare function toOffPolicyTrajectory(events: RetrievalHoldoutEvent[], options: RetrievalHoldoutOffPolicyOptions): RetrievalHoldoutOffPolicyResult;
|
|
422
|
+
|
|
301
423
|
interface AgentMemoryImprovementSeed<TConfig> {
|
|
302
424
|
config: TConfig;
|
|
303
425
|
track: string;
|
|
@@ -420,6 +542,7 @@ interface RunAgentMemoryImprovementResult<TConfig> {
|
|
|
420
542
|
totalCostUsd: number;
|
|
421
543
|
resultJsonPath: string;
|
|
422
544
|
}
|
|
545
|
+
|
|
423
546
|
/** Searches branchable memory configurations and activates only a fresh holdout win. */
|
|
424
547
|
declare function runAgentMemoryImprovement<TConfig>(options: RunAgentMemoryImprovementOptions<TConfig>): Promise<RunAgentMemoryImprovementResult<TConfig>>;
|
|
425
548
|
|
|
@@ -580,4 +703,4 @@ declare function memoryWriteResultToSourceRecord(result: AgentMemoryWriteResult,
|
|
|
580
703
|
scope?: AgentMemoryScope;
|
|
581
704
|
}): SourceRecord;
|
|
582
705
|
|
|
583
|
-
export { AgentMemoryAcquireRunLease, type AgentMemoryActivation, type AgentMemoryActivationDriver, AgentMemoryAdapter, type AgentMemoryAttemptEvent, type AgentMemoryBranch, type AgentMemoryBranchLifetime, type AgentMemoryBranchSnapshot, AgentMemoryContext, AgentMemoryControllerMode, type AgentMemoryDimensionComparison, type AgentMemoryExperimentCandidate, type AgentMemoryExperimentRankingRow, type AgentMemoryExperimentRunLease, type AgentMemoryGovernor, AgentMemoryHit, AgentMemoryHitSchema, type AgentMemoryImprovementRunLease, type AgentMemoryImprovementSeed, type AgentMemoryJournalEntry, AgentMemoryKindSchema, AgentMemoryLifecycleTimeoutError, AgentMemoryLifecycleUnsafeError, type AgentMemoryPromotionDecision, AgentMemoryRunLease, AgentMemoryScope, AgentMemoryScopeSchema, AgentMemorySearchOptions, type AgentMemorySequence, type AgentMemorySequenceArtifact, type AgentMemorySequenceProbe, type AgentMemorySequenceProbeResult, type AgentMemorySequenceScenario, type AgentMemorySequenceStep, type AgentMemorySharingPolicy, type AgentMemoryVisibility, AgentMemoryWriteInput, AgentMemoryWriteInputSchema, AgentMemoryWriteResult, type BuildAgentMemorySequencesFromBenchmarkCasesOptions, type CreateAgentMemoryBranchOptions, DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, type ForkAgentMemoryBranchSnapshotOptions, type GraphitiMcpClientLike, type GraphitiMemoryAdapterOptions, type GraphitiToolNames, type Mem0ClientLike, type Mem0ClientMode, type Mem0MemoryAdapterOptions, type Neo4jAgentMemoryAdapterOptions, type RunAgentMemoryExperimentOptions, type RunAgentMemoryExperimentResult, type RunAgentMemoryImprovementOptions, type RunAgentMemoryImprovementResult, agentMemorySequenceJudge, buildAgentMemorySequenceScenarios, buildAgentMemorySequencesFromBenchmarkCases, createAgentMemoryBranch, createGraphitiMemoryAdapter, createMem0MemoryAdapter, createMemoryExecutionPool, createNeo4jAgentMemoryAdapter, defaultGetMemoryContext, forkAgentMemoryBranchSnapshot, graphitiMemoryAdapterIdentity, mem0MemoryAdapterIdentity, memoryHitToSourceRecord, memoryRecoveryDelayMs, memoryWriteResultToSourceRecord, renderMemoryContext, resolveMemoryCleanupTimeoutMs, runAgentMemoryExperiment, runAgentMemoryImprovement, runBoundedMemoryLifecycle, sleepForMemoryRecovery };
|
|
706
|
+
export { AgentMemoryAcquireRunLease, type AgentMemoryActivation, type AgentMemoryActivationDriver, AgentMemoryAdapter, type AgentMemoryAttemptEvent, type AgentMemoryBranch, type AgentMemoryBranchLifetime, type AgentMemoryBranchSnapshot, AgentMemoryContext, AgentMemoryControllerMode, type AgentMemoryDimensionComparison, type AgentMemoryExperimentCandidate, type AgentMemoryExperimentRankingRow, type AgentMemoryExperimentRunLease, type AgentMemoryGovernor, AgentMemoryHit, AgentMemoryHitSchema, type AgentMemoryImprovementRunLease, type AgentMemoryImprovementSeed, type AgentMemoryJournalEntry, AgentMemoryKindSchema, AgentMemoryLifecycleTimeoutError, AgentMemoryLifecycleUnsafeError, type AgentMemoryPromotionDecision, AgentMemoryRunLease, AgentMemoryScope, AgentMemoryScopeSchema, AgentMemorySearchOptions, type AgentMemorySequence, type AgentMemorySequenceArtifact, type AgentMemorySequenceProbe, type AgentMemorySequenceProbeResult, type AgentMemorySequenceScenario, type AgentMemorySequenceStep, type AgentMemorySharingPolicy, type AgentMemoryVisibility, AgentMemoryWriteInput, AgentMemoryWriteInputSchema, AgentMemoryWriteResult, type BuildAgentMemorySequencesFromBenchmarkCasesOptions, type CreateAgentMemoryBranchOptions, DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, type ForkAgentMemoryBranchSnapshotOptions, type GraphitiMcpClientLike, type GraphitiMemoryAdapterOptions, type GraphitiToolNames, type Mem0ClientLike, type Mem0ClientMode, type Mem0MemoryAdapterOptions, type Neo4jAgentMemoryAdapterOptions, RetrievalHoldoutBypassReason, RetrievalHoldoutCallContext, RetrievalHoldoutConfig, RetrievalHoldoutEvent, type RetrievalHoldoutOffPolicyOptions, type RetrievalHoldoutOffPolicyResult, RetrievalHoldoutResult, type RetrievalHoldoutSessionSummary, type RunAgentMemoryExperimentOptions, type RunAgentMemoryExperimentResult, type RunAgentMemoryImprovementOptions, type RunAgentMemoryImprovementResult, agentMemorySequenceJudge, applyRetrievalHoldout, applySessionStickyRetrievalHoldout, buildAgentMemorySequenceScenarios, buildAgentMemorySequencesFromBenchmarkCases, createAgentMemoryBranch, createGraphitiMemoryAdapter, createMem0MemoryAdapter, createMemoryExecutionPool, createNeo4jAgentMemoryAdapter, defaultGetMemoryContext, deterministicRng, emitRetrievalHoldoutBypass, forkAgentMemoryBranchSnapshot, graphitiMemoryAdapterIdentity, mem0MemoryAdapterIdentity, memoryHitToSourceRecord, memoryRecoveryDelayMs, memoryWriteResultToSourceRecord, renderMemoryContext, resetRetrievalHoldoutRegistry, resolveMemoryCleanupTimeoutMs, retrievalHoldoutConfigHash, runAgentMemoryExperiment, runAgentMemoryImprovement, runBoundedMemoryLifecycle, sleepForMemoryRecovery, toOffPolicyTrajectory };
|
package/dist/memory/index.js
CHANGED
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
runAgentMemoryExperiment,
|
|
25
25
|
runAgentMemoryImprovement,
|
|
26
26
|
toOffPolicyTrajectory
|
|
27
|
-
} from "../chunk-
|
|
27
|
+
} from "../chunk-CPMLJYA3.js";
|
|
28
28
|
import {
|
|
29
29
|
AgentMemoryLifecycleTimeoutError,
|
|
30
30
|
AgentMemoryLifecycleUnsafeError,
|
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
resolveMemoryCleanupTimeoutMs,
|
|
38
38
|
runBoundedMemoryLifecycle,
|
|
39
39
|
sleepForMemoryRecovery
|
|
40
|
-
} from "../chunk-
|
|
40
|
+
} from "../chunk-BBCIB4UL.js";
|
|
41
41
|
import "../chunk-DQ3PDMDP.js";
|
|
42
42
|
import "../chunk-YMKHCTS2.js";
|
|
43
43
|
export {
|
package/dist/sources/index.d.ts
CHANGED
|
@@ -15,9 +15,9 @@
|
|
|
15
15
|
* Sources MUST be pure with respect to local filesystem state outside the
|
|
16
16
|
* cache directory the caller hands them — they read remote authorities and
|
|
17
17
|
* return data. They MUST mark `verifiable: false` on any fragment they could
|
|
18
|
-
* not
|
|
19
|
-
* substituting empty
|
|
20
|
-
* `verifiable` to refuse promotion of
|
|
18
|
+
* not fetch and extract (block page, 4xx, parse failure) rather than silently
|
|
19
|
+
* substituting empty or partial content. The control loop downstream uses
|
|
20
|
+
* `verifiable` to refuse promotion of unusable content.
|
|
21
21
|
*
|
|
22
22
|
* @stable
|
|
23
23
|
*/
|
|
@@ -77,10 +77,11 @@ interface FragmentProvenance {
|
|
|
77
77
|
*/
|
|
78
78
|
jurisdiction?: string;
|
|
79
79
|
/**
|
|
80
|
-
* True iff the
|
|
81
|
-
* expected
|
|
82
|
-
*
|
|
83
|
-
*
|
|
80
|
+
* True iff the configured URL returned an acceptable response and the
|
|
81
|
+
* expected content was extracted. False on a block page, rate-limit
|
|
82
|
+
* response, 4xx/5xx, or selector miss. This is not publisher authentication
|
|
83
|
+
* or cryptographic content verification. Consumers MUST refuse to promote
|
|
84
|
+
* `verifiable: false` fragments into citable knowledge.
|
|
84
85
|
*/
|
|
85
86
|
verifiable: boolean;
|
|
86
87
|
/** If `verifiable === false`, the reason — surfaced to operators. */
|
|
@@ -222,33 +223,16 @@ declare function extractLinks(html: string, hrefPattern: RegExp, baseUrl: string
|
|
|
222
223
|
}[];
|
|
223
224
|
|
|
224
225
|
/**
|
|
225
|
-
* Polite HTTP fetcher
|
|
226
|
+
* Polite HTTP fetcher shared by remote sources.
|
|
226
227
|
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
* is per-host (`hostThrottle`) rather than per-source so that two
|
|
233
|
-
* independent sources targeting the same authority still cooperate.
|
|
234
|
-
*
|
|
235
|
-
* 2. On-disk content cache keyed by URL. Production sources are called
|
|
236
|
-
* from a cron loop; without a cache, every run re-hits the same
|
|
237
|
-
* pages and inflates change-detection false-positives (the authority
|
|
238
|
-
* occasionally serves slightly different boilerplate). The cache is
|
|
239
|
-
* content-addressed by URL, not by ETag — authorities like IRS.gov
|
|
240
|
-
* do not consistently send ETag/Last-Modified.
|
|
241
|
-
*
|
|
242
|
-
* 3. Block-page detection on success. A 200 with a captcha body still
|
|
243
|
-
* means "we couldn't authenticate." Sources downstream rely on
|
|
244
|
-
* `verifiable` to refuse promotion — losing that signal because the
|
|
245
|
-
* fetcher said "well, the status code was 200" is the bug class
|
|
246
|
-
* this exists to prevent.
|
|
247
|
-
*
|
|
248
|
-
* @stable
|
|
228
|
+
* Independent sources share a per-origin throttle because rate-limited sites
|
|
229
|
+
* may return block pages instead of 429 responses. Responses are cached by URL
|
|
230
|
+
* because many publishers omit reliable ETag and Last-Modified headers. Bodies
|
|
231
|
+
* are checked even after a 2xx response because captcha and block pages often
|
|
232
|
+
* use successful status codes.
|
|
249
233
|
*/
|
|
250
234
|
/** User-Agent string sent on every outbound request. */
|
|
251
|
-
declare const POLITE_USER_AGENT = "agent-knowledge
|
|
235
|
+
declare const POLITE_USER_AGENT = "agent-knowledge (+https://github.com/tangle-network/agent-knowledge)";
|
|
252
236
|
/** Minimum gap between successive requests to the same origin (ms). */
|
|
253
237
|
declare const MIN_REQUEST_GAP_MS = 1000;
|
|
254
238
|
/** Maximum response body we will buffer in memory (bytes). */
|
|
@@ -257,7 +241,7 @@ interface PoliteFetchOptions {
|
|
|
257
241
|
signal?: AbortSignal;
|
|
258
242
|
cacheDir?: string;
|
|
259
243
|
/**
|
|
260
|
-
* Cache age beyond which we re-fetch. Default 1 hour
|
|
244
|
+
* Cache age beyond which we re-fetch. Default 1 hour, long enough to
|
|
261
245
|
* batch a cron sweep across many selectors, short enough that hourly
|
|
262
246
|
* authoritative-page changes get picked up next tick.
|
|
263
247
|
*/
|
|
@@ -291,7 +275,7 @@ interface PoliteFetchResult {
|
|
|
291
275
|
}
|
|
292
276
|
/**
|
|
293
277
|
* Fetch one URL with per-host throttling, on-disk cache, and block-page
|
|
294
|
-
* detection. Never throws on network/HTTP failure
|
|
278
|
+
* detection. Never throws on network/HTTP failure. It returns a result with
|
|
295
279
|
* `verifiable: false` and `unverifiableReason` set so the caller can decide
|
|
296
280
|
* whether to skip, retry, or surface.
|
|
297
281
|
*
|