@tangle-network/agent-knowledge 4.1.0 → 5.0.1

package/dist/benchmarks/index.d.ts

@@ -1,7 +1,8 @@
-import { b as AgentMemoryAdapter, C as KnowledgeBenchmarkSpec, H as KnowledgeMemoryBenchmarkTaskKind, o as KnowledgeAnswerBenchmarkTaskKind, G as KnowledgeMemoryBenchmarkCase, q as KnowledgeBenchmarkCase, p as KnowledgeBenchmarkArtifact, ad as RunMemoryAdapterBenchmarkOptions, ae as RunMemoryAdapterBenchmarkResult, x as KnowledgeBenchmarkScenario, v as KnowledgeBenchmarkReport, B as BuildRetrievalBenchmarkCasesFromQrelsOptions, L as KnowledgeRetrievalBenchmarkCase, M as KnowledgeRetrievalBenchmarkQrel, t as KnowledgeBenchmarkEvaluation, D as KnowledgeBenchmarkSplit, ab as RunKnowledgeBenchmarkSuiteOptions, ac as RunKnowledgeBenchmarkSuiteResult } from '../types-DP38encz.js';
-export { K as KnowledgeAnswerBenchmarkCase, r as KnowledgeBenchmarkCaseBase, s as KnowledgeBenchmarkDistribution, u as KnowledgeBenchmarkFamily, w as KnowledgeBenchmarkResponder, y as KnowledgeBenchmarkSliceSummary, z as KnowledgeBenchmarkSource, E as KnowledgeBenchmarkTaskKind, F as KnowledgeClaimMatcher, I as KnowledgeMemoryEvent, J as KnowledgeMemoryFactMatcher, N as KnowledgeRetrievalBenchmarkQuery, O as MemoryAdapterBenchmarkCandidate, P as MemoryAdapterBenchmarkRankingRow } from '../types-DP38encz.js';
+import { f as AgentMemoryAdapter, F as KnowledgeBenchmarkSpec, L as KnowledgeMemoryBenchmarkTaskKind, r as KnowledgeAnswerBenchmarkTaskKind, J as KnowledgeMemoryBenchmarkCase, t as KnowledgeBenchmarkCase, s as KnowledgeBenchmarkArtifact, aa as RunMemoryAdapterBenchmarkOptions, ab as RunMemoryAdapterBenchmarkResult, C as KnowledgeBenchmarkScenario, y as KnowledgeBenchmarkReport, B as BuildRetrievalBenchmarkCasesFromQrelsOptions, O as KnowledgeRetrievalBenchmarkCase, P as KnowledgeRetrievalBenchmarkQrel, w as KnowledgeBenchmarkEvaluation, G as KnowledgeBenchmarkSplit, a8 as RunKnowledgeBenchmarkSuiteOptions, a9 as RunKnowledgeBenchmarkSuiteResult } from '../types-BY-xLVw-.js';
+export { K as KnowledgeAnswerBenchmarkCase, u as KnowledgeBenchmarkCaseBase, v as KnowledgeBenchmarkDistribution, x as KnowledgeBenchmarkFamily, z as KnowledgeBenchmarkResponder, D as KnowledgeBenchmarkSliceSummary, E as KnowledgeBenchmarkSource, H as KnowledgeBenchmarkTaskKind, I as KnowledgeClaimMatcher, M as KnowledgeMemoryEvent, N as KnowledgeMemoryFactMatcher, Q as KnowledgeRetrievalBenchmarkQuery, S as MemoryAdapterBenchmarkCandidate, T as MemoryAdapterBenchmarkRankingRow } from '../types-BY-xLVw-.js';
 import { CampaignResult, JudgeConfig } from '@tangle-network/agent-eval/campaign';
 import '../types-6x0OpfW6.js';
+import '@tangle-network/agent-interface';
 
 declare function createNoopMemoryBenchmarkAdapter(id?: string): AgentMemoryAdapter;
 declare function createInMemoryBenchmarkAdapter(options?: {

package/dist/benchmarks/index.js

@@ -20,7 +20,7 @@ import {
   scoreKnowledgeBenchmarkArtifact,
   scoreMemoryBenchmarkArtifact,
   summarizeKnowledgeBenchmarkCampaign
-} from "../chunk-BBCIB4UL.js";
+} from "../chunk-EYIA5PLQ.js";
 import "../chunk-DQ3PDMDP.js";
 import "../chunk-YMKHCTS2.js";
 export {

package/dist/{chunk-BBCIB4UL.js → chunk-EYIA5PLQ.js}

@@ -926,9 +926,6 @@ function reconcileInterruptedMemoryPaidCalls(costLedger) {
     "external memory provider"
   );
 }
-function reconcileInterruptedRunPaidCalls(costLedger, label) {
-  reconcileInterruptedPaidCallsAtMaximum(costLedger, () => true, label);
-}
 function reconcileInterruptedPaidCallsAtMaximum(costLedger, ownsActor, label) {
   if (costLedger.summary().unresolvedCalls === 0) return;
   const pending = requirePendingCostCallInspection(costLedger);
@@ -1025,7 +1022,6 @@ function reserveRecoveryAttempts(input) {
       path,
       label,
       event: {
-        schema: 1,
         attemptId,
         generation,
         recordedAt: (input.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
@@ -1102,7 +1098,7 @@ function readRecoveryAttemptGenerations(storage, path, label) {
 }
 function parseRecoveryAttemptEvent(value, path, line, label) {
   const event = value;
-  const valid = typeof event === "object" && event !== null && event.schema === 1 && typeof event.attemptId === "string" && event.attemptId.length > 0 && Number.isSafeInteger(event.generation) && event.generation > 0 && typeof event.recordedAt === "string" && !Number.isNaN(Date.parse(event.recordedAt));
+  const valid = typeof event === "object" && event !== null && typeof event.attemptId === "string" && event.attemptId.length > 0 && Number.isSafeInteger(event.generation) && event.generation > 0 && typeof event.recordedAt === "string" && !Number.isNaN(Date.parse(event.recordedAt));
   if (!valid) throw new Error(`invalid ${label} '${path}' line ${line}`);
   return value;
 }
@@ -1111,9 +1107,7 @@ function isMissingPendingCostCall(error, callId) {
 }
 function requirePendingCostCallInspection(costLedger) {
   if (!costLedger.listPending) {
-    throw new Error(
-      "interrupted memory recovery requires CostLedger.listPending() from @tangle-network/agent-eval 0.122.8 or newer"
-    );
+    throw new Error("interrupted memory recovery requires CostLedger.listPending() support");
   }
   return costLedger.listPending();
 }
@@ -1604,7 +1598,7 @@ function readActiveMemoryBenchmarkAttempts(storage, path) {
 }
 function parseMemoryBenchmarkAttemptEvent(value, path, line) {
   const event = value;
-  const valid = typeof event === "object" && event !== null && event.schema === 3 && (event.status === "started" || event.status === "cleaned") && isNonEmptyString(event.attemptId) && isNonEmptyString(event.candidateId) && isNonEmptyString(event.candidateRef) && isNonEmptyString(event.adapterId) && isNonEmptyString(event.caseId) && isNonEmptyString(event.cellId) && isAgentMemoryScope(event.scope) && typeof event.adapterCreationCostUsd === "number" && Number.isFinite(event.adapterCreationCostUsd) && event.adapterCreationCostUsd >= 0 && typeof event.costUsdPerCase === "number" && Number.isFinite(event.costUsdPerCase) && event.costUsdPerCase >= 0 && typeof event.recoveryCostUsdPerAttempt === "number" && Number.isFinite(event.recoveryCostUsdPerAttempt) && event.recoveryCostUsdPerAttempt >= 0 && typeof event.recordedAt === "string" && !Number.isNaN(Date.parse(event.recordedAt)) && typeof event.recovery === "boolean";
+  const valid = typeof event === "object" && event !== null && (event.status === "started" || event.status === "cleaned") && isNonEmptyString(event.attemptId) && isNonEmptyString(event.candidateId) && isNonEmptyString(event.candidateRef) && isNonEmptyString(event.adapterId) && isNonEmptyString(event.caseId) && isNonEmptyString(event.cellId) && isAgentMemoryScope(event.scope) && typeof event.adapterCreationCostUsd === "number" && Number.isFinite(event.adapterCreationCostUsd) && event.adapterCreationCostUsd >= 0 && typeof event.costUsdPerCase === "number" && Number.isFinite(event.costUsdPerCase) && event.costUsdPerCase >= 0 && typeof event.recoveryCostUsdPerAttempt === "number" && Number.isFinite(event.recoveryCostUsdPerAttempt) && event.recoveryCostUsdPerAttempt >= 0 && typeof event.recordedAt === "string" && !Number.isNaN(Date.parse(event.recordedAt)) && typeof event.recovery === "boolean";
   if (!valid) {
     throw new Error(`invalid memory benchmark attempt event in '${path}' line ${line}`);
   }
@@ -1702,7 +1696,6 @@ function createMemoryAdapterBenchmarkResponder(options) {
       options.scope
     );
     const attempt = {
-      schema: 3,
       status: "started",
       attemptId,
       candidateId: options.candidateId,
@@ -2002,11 +1995,7 @@ function renderSliceTable(slices) {
 }
 
 // src/retrieval-eval.ts
-import {
-  heldOutGate,
-  parameterSweepProposer,
-  runImprovementLoop
-} from "@tangle-network/agent-eval/campaign";
+import { canonicalJson as canonicalJson2 } from "@tangle-network/agent-eval";
 function retrievalConfigSurface(config) {
   return canonicalJson2(config);
 }
@@ -2129,161 +2118,36 @@ function scoreRetrievalArtifact(artifact, scenario) {
     matchedTargetIds: [...matchedTargetIds].sort()
   };
 }
-function buildRetrievalParameterCandidates(searchSpace, options = {}) {
-  const candidates = [];
-  for (const [path, values] of Object.entries(searchSpace).sort(([a], [b]) => a.localeCompare(b))) {
-    for (const value of values) {
-      if (options.baseline && canonicalJson2(getConfigPath(options.baseline, path)) === canonicalJson2(value)) {
-        continue;
-      }
-      candidates.push({
-        label: `${path}=${formatCandidateValue(value)}`,
-        rationale: `Set retrieval config ${path} to ${formatCandidateValue(value)}`,
-        changes: [{ path, value }]
-      });
-    }
-  }
-  return candidates;
-}
-function retrievalParameterSweepProposer(options) {
-  const candidates = options.candidates ?? (options.searchSpace ? buildRetrievalParameterCandidates(options.searchSpace, { baseline: options.baseline }) : []);
-  if (candidates.length === 0) {
-    throw new Error("retrievalParameterSweepProposer requires at least one candidate");
-  }
-  return parameterSweepProposer({ candidates });
-}
-async function runRetrievalImprovementLoop(options) {
-  const split = splitRetrievalScenarios(options);
-  const candidates = resolveRetrievalCandidates(options);
-  const populationSize = options.populationSize ?? Math.max(1, Math.min(4, candidates.length));
-  const maxGenerations = options.maxGenerations ?? Math.max(1, Math.ceil(candidates.length / populationSize));
-  const proposer = withTargetRecallStop(
-    retrievalParameterSweepProposer({ candidates }),
-    options.targetRecall
-  );
-  const gate = options.gate ?? heldOutGate({
-    scenarios: split.holdoutScenarios,
-    deltaThreshold: options.deltaThreshold ?? 0.02
-  });
-  const result = await runImprovementLoop({
-    baselineSurface: retrievalConfigSurface(options.baseline),
-    scenarios: split.trainScenarios,
-    holdoutScenarios: split.holdoutScenarios,
-    dispatchWithSurface: buildRetrievalEvalDispatch({
-      index: options.index,
-      defaultK: options.defaultK,
-      retrieve: options.retrieve
-    }),
-    judges: [...options.judges ?? [retrievalRecallJudge({ weights: options.metricWeights })]],
-    proposer,
-    gate,
-    autoOnPromote: "none",
-    runDir: options.runDir ?? ".agent-knowledge/retrieval-improvement",
-    seed: options.seed,
-    reps: options.reps,
-    resumable: options.resumable,
-    costCeiling: options.costCeiling,
-    maxConcurrency: options.maxConcurrency,
-    dispatchTimeoutMs: options.dispatchTimeoutMs,
-    expectUsage: options.expectUsage ?? "off",
-    tracing: options.tracing,
-    storage: options.storage,
-    populationSize,
-    maxGenerations,
-    promoteTopK: options.promoteTopK,
-    maxImprovementShots: options.maxImprovementShots,
-    report: options.report,
-    findings: options.findings,
-    now: options.now
-  });
-  return {
-    ...result,
-    baselineConfig: options.baseline,
-    winnerConfig: retrievalConfigFromSurface(result.winnerSurface),
-    trainScenarios: split.trainScenarios,
-    holdoutScenarios: split.holdoutScenarios,
-    candidates,
-    targetRecall: options.targetRecall
-  };
-}
-function splitRetrievalScenarios(options) {
-  const scenarios = [...options.scenarios];
-  if (scenarios.length === 0) {
-    throw new Error("runRetrievalImprovementLoop requires at least one training scenario");
-  }
-  if (options.holdoutScenarios) {
-    const holdoutScenarios = [...options.holdoutScenarios];
-    if (holdoutScenarios.length === 0) {
-      throw new Error("runRetrievalImprovementLoop holdoutScenarios must not be empty");
-    }
-    return { trainScenarios: scenarios, holdoutScenarios };
-  }
-  if (scenarios.length < 2) {
+function partitionRetrievalScenarios(scenarios, options = {}) {
+  if (scenarios.length < 4) {
     throw new Error(
-      "runRetrievalImprovementLoop requires at least 2 scenarios when holdoutScenarios are not provided"
+      "partitionRetrievalScenarios requires at least 4 scenarios for non-empty train and selection partitions plus 2 final scenarios"
     );
   }
-  const holdoutFraction = options.holdoutFraction ?? 0.3;
-  if (!Number.isFinite(holdoutFraction) || holdoutFraction <= 0 || holdoutFraction >= 1) {
-    throw new Error(
-      `runRetrievalImprovementLoop holdoutFraction must be > 0 and < 1, got ${String(holdoutFraction)}`
-    );
-  }
-  const shuffled = seededShuffle(scenarios, options.splitSeed ?? options.seed ?? 42);
-  const holdoutCount = Math.min(
-    shuffled.length - 1,
-    Math.max(1, Math.round(shuffled.length * holdoutFraction))
-  );
-  const splitIndex = shuffled.length - holdoutCount;
-  return {
-    trainScenarios: shuffled.slice(0, splitIndex),
-    holdoutScenarios: shuffled.slice(splitIndex)
-  };
-}
-function resolveRetrievalCandidates(options) {
-  const candidates = options.candidates ? [...options.candidates] : options.searchSpace ? buildRetrievalParameterCandidates(options.searchSpace, { baseline: options.baseline }) : [];
-  if (candidates.length === 0) {
-    throw new Error("runRetrievalImprovementLoop requires candidates or searchSpace");
-  }
-  return candidates;
-}
-function withTargetRecallStop(proposer, targetRecall) {
-  if (targetRecall === void 0) {
-    return proposer;
+  const selectionFraction = options.selectionFraction ?? 0.2;
+  const finalFraction = options.finalFraction ?? 0.2;
+  assertPartitionFraction(selectionFraction, "selectionFraction");
+  assertPartitionFraction(finalFraction, "finalFraction");
+  if (selectionFraction + finalFraction >= 1) {
+    throw new Error("selectionFraction + finalFraction must be less than 1");
   }
-  if (!Number.isFinite(targetRecall) || targetRecall < 0 || targetRecall > 1) {
-    throw new Error(`targetRecall must be between 0 and 1, got ${String(targetRecall)}`);
+  const shuffled = seededShuffle(scenarios, options.seed ?? 42);
+  const finalCount = Math.max(2, Math.round(shuffled.length * finalFraction));
+  const selectionCount = Math.max(1, Math.round(shuffled.length * selectionFraction));
+  if (finalCount + selectionCount >= shuffled.length) {
+    throw new Error("retrieval scenario fractions leave no training scenarios");
   }
+  const trainEnd = shuffled.length - selectionCount - finalCount;
   return {
-    kind: `${proposer.kind}:target-recall`,
-    propose: (context) => proposer.propose(context),
-    decide(args) {
-      const baseDecision = proposer.decide?.(args);
-      if (baseDecision?.stop) {
-        return baseDecision;
-      }
-      const bestRecall = bestObservedRecall(args.history);
-      if (bestRecall >= targetRecall) {
-        return {
-          stop: true,
-          reason: `target recall ${targetRecall} reached with train recall ${bestRecall}`
-        };
-      }
-      return { stop: false };
-    }
+    trainScenarios: shuffled.slice(0, trainEnd),
+    selectionScenarios: shuffled.slice(trainEnd, trainEnd + selectionCount),
+    finalScenarios: shuffled.slice(trainEnd + selectionCount)
   };
 }
-function bestObservedRecall(history) {
-  let best = Number.NEGATIVE_INFINITY;
-  for (const generation of history) {
-    for (const candidate of generation.candidates) {
-      const recall = candidate.dimensions.recall;
-      if (recall !== void 0 && Number.isFinite(recall) && recall > best) {
-        best = recall;
-      }
-    }
+function assertPartitionFraction(value, name) {
+  if (!Number.isFinite(value) || value <= 0 || value >= 1) {
+    throw new Error(`${name} must be greater than 0 and less than 1`);
   }
-  return best;
 }
 function retrievalK(config, scenario, defaultK) {
   const rawK = config.k ?? config.topK ?? scenario.k ?? defaultK;
@@ -2399,35 +2263,6 @@ function normalizeWeight(value) {
   }
   return value;
 }
-function getConfigPath(config, path) {
-  let current = config;
-  for (const part of path.split(".")) {
-    if (!isJsonObject(current)) {
-      return void 0;
-    }
-    current = current[part];
-  }
-  return current;
-}
-function formatCandidateValue(value) {
-  if (typeof value === "string") {
-    return value;
-  }
-  return canonicalJson2(value);
-}
-function canonicalJson2(value) {
-  if (value === void 0) {
-    return "undefined";
-  }
-  if (Array.isArray(value)) {
-    return `[${value.map(canonicalJson2).join(",")}]`;
-  }
-  if (isJsonObject(value)) {
-    const entries = Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, child]) => `${JSON.stringify(key)}:${canonicalJson2(child)}`);
-    return `{${entries.join(",")}}`;
-  }
-  return JSON.stringify(value);
-}
 function isJsonObject(value) {
   return Boolean(value) && typeof value === "object" && !Array.isArray(value);
 }
@@ -3272,9 +3107,7 @@ export {
   buildRetrievalEvalDispatch,
   retrievalRecallJudge,
   scoreRetrievalArtifact,
-  buildRetrievalParameterCandidates,
-  retrievalParameterSweepProposer,
-  runRetrievalImprovementLoop,
+  partitionRetrievalScenarios,
   memoryHitToSourceRecord,
   memoryWriteResultToSourceRecord,
   createNoopMemoryBenchmarkAdapter,
@@ -3289,7 +3122,6 @@ export {
   buildFirstPartyMemoryLifecycleBenchmarkCases,
   DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT,
   reconcileInterruptedMemoryPaidCalls,
-  reconcileInterruptedRunPaidCalls,
   assertNoInterruptedPaidCalls,
   hasSettledPaidCall,
   appendAttemptJournalEvent,
@@ -3318,4 +3150,4 @@ export {
   parseKnowledgeBenchmarkQrels,
   buildRetrievalBenchmarkCasesFromQrels
 };
-//# sourceMappingURL=chunk-BBCIB4UL.js.map
+//# sourceMappingURL=chunk-EYIA5PLQ.js.map