@tangle-network/agent-knowledge 1.7.0 → 1.9.0

package/dist/index.js

@@ -33,13 +33,14 @@ import {
   reciprocalRankFusion,
   searchKnowledge,
   sourceRegistryPath,
+  stringMetadata,
   textSourceAdapter,
   tokenizeQuery,
   validateKnowledgeIndex,
   writeJson,
   writeKnowledgeIndex,
   writeSourceRegistry
-} from "./chunk-HKYD765Q.js";
+} from "./chunk-WWY5FTKQ.js";
 import {
   AgentMemoryHitSchema,
   AgentMemoryKindSchema,
@@ -72,6 +73,9 @@ import {
   slugify,
   stableId
 } from "./chunk-YMKHCTS2.js";
+import {
+  researcherProfile
+} from "./chunk-MU5CEOGE.js";
 
 // src/changes.ts
 function detectChanges(prev, next, options = {}) {
@@ -421,10 +425,6 @@ function sourceFreshness(sources, now) {
     expiredSourceIds
   };
 }
-function stringMetadata(metadata, key) {
-  const value = metadata?.[key];
-  return typeof value === "string" ? value : void 0;
-}
 function isIsoDate(value) {
   return Boolean(value && Number.isFinite(Date.parse(value)));
 }
@@ -837,6 +837,11 @@ function knowledgeReleaseReport(input) {
     gateDecision: input.gateDecision ?? null,
     thresholds: {
       requireCorpus: false,
+      // The report gates on RunRecord-level outcomes, not on a separate scenario
+      // corpus (KnowledgeReleaseInput carries no dataset/scenarios). The scenario
+      // floor must therefore be 0 — otherwise the corpus axis fails closed on a
+      // dimension the report has no way to satisfy, masking the run-based gate.
+      minScenarioCount: 0,
       requireHoldout: input.hasHoldout ?? false,
       minHoldoutRuns: input.hasHoldout ? 1 : 0,
       minSearchRuns: 1,
@@ -859,6 +864,19 @@ import {
   blockingKnowledgeEval,
   objectiveEval
 } from "@tangle-network/agent-eval";
+
+// src/readiness-helpers.ts
+function readinessFor(options, index) {
+  if (!options.readinessSpecs?.length) return void 0;
+  return buildEvalKnowledgeBundle({
+    ...options.readiness ?? {},
+    taskId: options.readinessTaskId ?? options.goal,
+    index,
+    specs: options.readinessSpecs
+  });
+}
+
+// src/research-loop.ts
 function createKnowledgeControlLoopAdapter(options) {
   let initialized = false;
   const appliedSteps = [];
@@ -1005,15 +1023,531 @@ async function applyKnowledgeResearchDecision(options, decision, iteration = 1)
     metadata: decision.metadata
   };
 }
-function readinessFor(options, index) {
-  if (!options.readinessSpecs?.length) return void 0;
+
+// src/research-supervisor.ts
+import {
+  supervise
+} from "@tangle-network/agent-runtime/loops";
+var RESEARCH_SUPERVISOR_SYSTEM_PROMPT = [
+  "You are a research SUPERVISOR. You do not answer the research question yourself \u2014",
+  "you create and manage a team of researcher workers that grow ONE shared",
+  "knowledge base until it is ready.",
+  "",
+  "Each round:",
+  "  1. Read the goal and the gaps the readiness gate still reports.",
+  "  2. DECOMPOSE the open work into independent sub-topics. Spawn ONE researcher",
+  "     per sub-topic (spawn_agent) \u2014 split by sub-topic, never duplicate work.",
+  "     Spawn more researchers for a broad goal, fewer for a narrow one.",
+  "  3. Wait for the researchers to settle (await_event). Steer or re-spawn for",
+  "     the sub-topics that came back thin.",
+  "  4. STOP as soon as the deliverable check reports the knowledge base is ready.",
+  "",
+  "Conserve your budget: every spawn spends from one shared pool. Prefer a small",
+  "number of well-scoped researchers over a large fanout that re-covers the same",
+  "ground."
+].join("\n");
+function knowledgeReadinessDeliverable(options) {
+  return {
+    describe: `knowledge base at ${options.root} is ready for: ${options.goal}`,
+    async check() {
+      const index = await buildKnowledgeIndex(options.root);
+      const bundle = readinessFor(options, index);
+      return (bundle?.report.blockingMissingRequirements.length ?? 0) === 0;
+    }
+  };
+}
+async function runResearchSupervisor(options) {
+  await initKnowledgeBase(options.root);
+  const { profile: workerProfile } = researcherProfile({ harness: options.harness });
+  const baseInstructions = options.supervisorSystemPrompt ?? RESEARCH_SUPERVISOR_SYSTEM_PROMPT;
+  const workerContract = workerProfile.prompt?.systemPrompt;
+  const systemPrompt = workerContract ? `${baseInstructions}
+
+Each researcher worker you spawn follows this contract:
+${workerContract}` : baseInstructions;
+  const profile = {
+    name: "research-supervisor",
+    model: options.supervisorModel,
+    systemPrompt
+  };
+  return supervise(profile, options.goal, {
+    ...options.superviseOptions,
+    budget: options.budget,
+    backend: options.backend,
+    deliverable: knowledgeReadinessDeliverable(options)
+  });
+}
+
+// src/two-agent-research-loop.ts
+async function runTwoAgentResearchLoop(options) {
+  const maxRounds = Math.max(1, options.maxRounds ?? 3);
+  await initKnowledgeBase(options.root);
+  const steps = [];
+  let index = await buildKnowledgeIndex(options.root);
+  let readiness = readinessFor(options, index);
+  let ready = isReady(readiness?.report);
+  let steer;
+  for (let round2 = 1; round2 <= maxRounds && !ready; round2++) {
+    if (options.signal?.aborted) throw new Error("Two-agent research loop aborted");
+    const gaps = gapsFromReadiness(readiness);
+    const workerContribution = await options.worker({
+      root: options.root,
+      goal: options.goal,
+      round: round2,
+      index,
+      gaps,
+      steer,
+      readiness: requireReadiness(readiness, options),
+      signal: options.signal
+    });
+    const accepted = [];
+    const rejectedWorkerSources = [];
+    const existingUris = new Set(
+      index.sources.flatMap(
+        (source) => typeof source.metadata?.originalUri === "string" ? [source.metadata.originalUri] : []
+      )
+    );
+    for (const source of workerContribution.sources ?? []) {
+      if (isDuplicate(source, existingUris, accepted)) {
+        rejectedWorkerSources.push({ source, reason: "duplicate: already in the knowledge base" });
+        continue;
+      }
+      const verdict = await options.driver.verifySource(source, {
+        root: options.root,
+        goal: options.goal,
+        round: round2,
+        index,
+        gaps,
+        acceptedThisRound: accepted,
+        signal: options.signal
+      });
+      if (verdict.accept) accepted.push(source);
+      else rejectedWorkerSources.push({ source, reason: verdict.reason });
+    }
+    const acceptedWorkerSources = await registerSources(options, accepted);
+    const writtenPages = [];
+    writtenPages.push(
+      ...await applyPages(options.root, workerContribution, acceptedWorkerSources)
+    );
+    index = await buildKnowledgeIndex(options.root);
+    readiness = readinessFor(options, index);
+    let driverSources = [];
+    let driverNotes;
+    if (options.driverResearches && options.driver.research) {
+      const remainingGaps2 = gapsFromReadiness(readiness);
+      const driverContribution = await options.driver.research({
+        root: options.root,
+        goal: options.goal,
+        round: round2,
+        index,
+        remainingGaps: remainingGaps2,
+        readiness: requireReadiness(readiness, options),
+        signal: options.signal
+      });
+      driverNotes = driverContribution.notes;
+      driverSources = await registerSources(options, driverContribution.sources ?? []);
+      writtenPages.push(...await applyPages(options.root, driverContribution, driverSources));
+      index = await buildKnowledgeIndex(options.root);
+      readiness = readinessFor(options, index);
+    }
+    ready = isReady(readiness?.report);
+    const remainingGaps = gapsFromReadiness(readiness);
+    steer = ready ? void 0 : foldGaps(options.driver, remainingGaps);
+    const step = {
+      round: round2,
+      gaps,
+      acceptedWorkerSources,
+      rejectedWorkerSources,
+      driverSources,
+      writtenPages,
+      readiness,
+      ready,
+      event: createKnowledgeEvent({
+        type: "research.iteration",
+        actor: options.actor,
+        target: options.root,
+        metadata: {
+          goal: options.goal,
+          round: round2,
+          ready,
+          acceptedWorkerSourceCount: acceptedWorkerSources.length,
+          rejectedWorkerSourceCount: rejectedWorkerSources.length,
+          driverSourceCount: driverSources.length,
+          writtenPageCount: writtenPages.length,
+          remainingGapCount: remainingGaps.length
+        }
+      }),
+      notes: { worker: workerContribution.notes, driver: driverNotes }
+    };
+    steps.push(step);
+    await options.onRound?.(step);
+  }
+  return {
+    root: options.root,
+    goal: options.goal,
+    rounds: steps.length,
+    ready,
+    index,
+    readiness,
+    steps
+  };
+}
+function isReady(report) {
+  if (!report) return false;
+  return report.blockingMissingRequirements.length === 0;
+}
+function gapsFromReadiness(readiness) {
+  if (!readiness) return [];
+  const blocking = readiness.report.blockingMissingRequirements.map(
+    (requirement) => gapFor(requirement, readiness, true)
+  );
+  const nonBlocking = readiness.report.nonBlockingGaps.map(
+    (requirement) => gapFor(requirement, readiness, false)
+  );
+  return [...blocking, ...nonBlocking];
+}
+function gapFor(requirement, readiness, blocking) {
+  const spec = readiness.requirements.find((entry) => entry.id === requirement.id);
+  const query = typeof spec?.metadata?.query === "string" ? spec.metadata.query : requirement.description;
+  return { id: requirement.id, description: requirement.description, query, blocking };
+}
+function foldGaps(driver, gaps) {
+  if (gaps.length === 0) return void 0;
+  if (driver.foldGaps) return driver.foldGaps(gaps);
+  return [
+    "The knowledge base is still missing the following. Prioritise these next round:",
+    ...gaps.map(
+      (gap) => `- (${gap.blocking ? "blocking" : "soft"}) ${gap.description} [${gap.id}]`
+    )
+  ].join("\n");
+}
+function isDuplicate(source, existingUris, accepted) {
+  return existingUris.has(source.uri) || accepted.some((candidate) => candidate.uri === source.uri);
+}
+async function registerSources(options, sources) {
+  const records = [];
+  for (const source of sources) {
+    records.push(await addSourceText(options.root, source, options.sourceOptions));
+  }
+  return records;
+}
+async function applyPages(root, contribution, acceptedSources) {
+  if (acceptedSources.length === 0) return [];
+  const parts = [];
+  if (contribution.proposalText) parts.push(contribution.proposalText);
+  const built = contribution.buildPages?.(acceptedSources);
+  if (built) parts.push(built);
+  if (parts.length === 0) return [];
+  const applied = await applyKnowledgeWriteBlocks(root, parts.join("\n"));
+  return applied.written;
+}
+function requireReadiness(readiness, options) {
+  if (readiness) return readiness;
   return buildEvalKnowledgeBundle({
     ...options.readiness ?? {},
     taskId: options.readinessTaskId ?? options.goal,
-    index,
-    specs: options.readinessSpecs
+    index: emptyIndex(options.root),
+    specs: []
   });
 }
+function emptyIndex(root) {
+  return {
+    root,
+    generatedAt: (/* @__PURE__ */ new Date(0)).toISOString(),
+    sources: [],
+    pages: [],
+    graph: { nodes: [], edges: [] }
+  };
+}
+function sourceMatchesGaps(source, index, gaps) {
+  const haystack = `${source.title ?? ""}
+${source.text}`.toLowerCase();
+  const hits = [];
+  for (const gap of gaps) {
+    for (const token of gap.query.toLowerCase().split(/\s+/).filter(Boolean)) {
+      if (haystack.includes(token)) {
+        hits.push(...searchKnowledge(index, gap.query, 1));
+        break;
+      }
+    }
+  }
+  return hits;
+}
+
+// src/web-research-worker.ts
+var DEFAULT_MODEL = "glm-5.2";
+var DEFAULT_BASE_URL = "https://router.tangle.tools/v1";
+var MIN_MAX_TOKENS = 1200;
+var RouterError = class extends Error {
+  constructor(status, message) {
+    super(`router ${status}: ${message}`);
+    this.status = status;
+    this.name = "RouterError";
+  }
+  status;
+};
+function createTangleRouterClient(options = {}) {
+  const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
+  const apiKey = options.apiKey ?? process.env.TANGLE_API_KEY;
+  if (!apiKey) {
+    throw new RouterError(401, "no TANGLE_API_KEY (pass apiKey or set the env var)");
+  }
+  const model = options.model ?? DEFAULT_MODEL;
+  const headers = {
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${apiKey}`
+  };
+  return {
+    async search(query, opts) {
+      const res = await fetch(`${baseUrl}/search`, {
+        method: "POST",
+        headers,
+        signal: options.signal,
+        body: JSON.stringify({
+          query,
+          ...options.searchProvider ? { provider: options.searchProvider } : {},
+          ...opts?.maxResults != null ? { maxResults: opts.maxResults } : {}
+        })
+      });
+      if (!res.ok) {
+        throw new RouterError(res.status, await res.text().catch(() => res.statusText));
+      }
+      const body = await res.json();
+      return (body.data ?? []).filter((hit) => typeof hit?.url === "string" && hit.url.length > 0).map((hit) => ({ title: hit.title ?? hit.url, url: hit.url, snippet: hit.snippet }));
+    },
+    async chat(messages, maxTokens) {
+      const max_tokens = Math.max(MIN_MAX_TOKENS, maxTokens ?? MIN_MAX_TOKENS);
+      const res = await fetch(`${baseUrl}/chat/completions`, {
+        method: "POST",
+        headers,
+        signal: options.signal,
+        body: JSON.stringify({ model, messages, max_tokens, temperature: 0.2, stream: false })
+      });
+      if (!res.ok) {
+        throw new RouterError(res.status, await res.text().catch(() => res.statusText));
+      }
+      const body = await res.json();
+      return body.choices?.[0]?.message?.content ?? "";
+    }
+  };
+}
+function resolveRouter(opts) {
+  return opts.router ?? createTangleRouterClient(opts.router_options);
+}
+function createWebResearchWorker(options = {}) {
+  const queriesPerGap = Math.max(1, options.queriesPerGap ?? 2);
+  const resultsPerQuery = Math.max(1, options.resultsPerQuery ?? 3);
+  const maxSourcesPerRound = Math.max(1, options.maxSourcesPerRound ?? 6);
+  const minTextChars = Math.max(1, options.minTextChars ?? 200);
+  const maxTextChars = Math.max(minTextChars, options.maxTextChars ?? 4e3);
+  return async (ctx) => {
+    const router = resolveRouter(options);
+    const targetGaps = ctx.gaps.filter((gap) => gap.blocking);
+    const gaps = targetGaps.length > 0 ? targetGaps : ctx.gaps;
+    if (gaps.length === 0) {
+      return { sources: [], notes: "no open gaps to research" };
+    }
+    const queries = await formSearchQueries(router, ctx, gaps, queriesPerGap);
+    const proposals = [];
+    const seenUris = /* @__PURE__ */ new Set();
+    for (const query of queries) {
+      if (proposals.length >= maxSourcesPerRound) break;
+      if (ctx.signal?.aborted) break;
+      let hits;
+      try {
+        hits = await router.search(query, { maxResults: resultsPerQuery });
+      } catch (error) {
+        if (error.name === "AbortError") break;
+        continue;
+      }
+      for (const hit of hits) {
+        if (proposals.length >= maxSourcesPerRound) break;
+        if (seenUris.has(hit.url)) continue;
+        const fetched = await politeFetch(hit.url, {
+          signal: ctx.signal,
+          cacheDir: options.cacheDir
+        });
+        if (!fetched.verifiable) continue;
+        const text = htmlToText(fetched.body).slice(0, maxTextChars);
+        if (text.length < minTextChars) continue;
+        seenUris.add(hit.url);
+        proposals.push({
+          uri: hit.url,
+          title: hit.title || hit.url,
+          text,
+          // We just fetched + verified this page, so stamp `lastVerifiedAt` with
+          // fetch time. Do NOT set `validUntil`: a live page has no inherent
+          // future expiry, and the readiness freshness check treats any
+          // `validUntil <= now` as EXPIRED (score 0). The page's `Last-Modified`
+          // is a PAST date, so writing it here would mark every real source stale
+          // and zero out coverage. Record it as provenance metadata instead.
+          lastVerifiedAt: fetched.fetchedAt,
+          metadata: {
+            discoveredVia: query,
+            snippet: hit.snippet ?? "",
+            goal: ctx.goal,
+            sourceUpdatedAt: fetched.sourceUpdatedAt
+          }
+        });
+      }
+    }
+    return {
+      sources: proposals,
+      buildPages: buildCitingPages(proposals),
+      notes: `web-research worker: ${queries.length} queries \u2192 ${proposals.length} fetched sources`
+    };
+  };
+}
+async function formSearchQueries(router, ctx, gaps, queriesPerGap) {
+  const gapLines = gaps.map((gap, i) => `${i + 1}. ${gap.description} (readiness query: "${gap.query}")`).join("\n");
+  const want = gaps.length * queriesPerGap;
+  const system = "You are a research librarian. Turn knowledge gaps into precise web search queries that will surface authoritative primary sources (papers, docs, standards, official pages). Return ONLY a JSON array of query strings, no prose.";
+  const user = [
+    `Research goal: ${ctx.goal}`,
+    ctx.steer ? `Steer from the coordinator:
+${ctx.steer}` : "",
+    `Open knowledge gaps:
+${gapLines}`,
+    `Return up to ${want} search query strings as a JSON array (e.g. ["query one","query two"]).`
+  ].filter(Boolean).join("\n\n");
+  let raw = "";
+  try {
+    raw = await router.chat(
+      [
+        { role: "system", content: system },
+        { role: "user", content: user }
+      ],
+      MIN_MAX_TOKENS
+    );
+  } catch {
+    raw = "";
+  }
+  const parsed = parseQueryList(raw);
+  const fromLlm = parsed.slice(0, want);
+  if (fromLlm.length > 0) return dedupeStrings(fromLlm);
+  return dedupeStrings(gaps.map((gap) => gap.query || gap.description));
+}
+function parseQueryList(raw) {
+  const text = raw.trim();
+  if (!text) return [];
+  const candidates = [];
+  const arrayMatch = text.match(/\[[\s\S]*\]/);
+  if (arrayMatch) {
+    try {
+      const arr = JSON.parse(arrayMatch[0]);
+      for (const item of arr)
+        if (typeof item === "string" && item.trim()) candidates.push(item.trim());
+    } catch {
+    }
+  }
+  if (candidates.length === 0) {
+    for (const line of text.split("\n")) {
+      const cleaned = line.replace(/^\s*(?:[-*]|\d+[.)])\s*/, "").replace(/^["']|["']$/g, "").trim();
+      if (cleaned && cleaned.length > 2 && !cleaned.startsWith("{")) candidates.push(cleaned);
+    }
+  }
+  return candidates;
+}
+function dedupeStrings(values) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const value of values) {
+    const key = value.toLowerCase().trim();
+    if (key && !seen.has(key)) {
+      seen.add(key);
+      out.push(value.trim());
+    }
+  }
+  return out;
+}
+function buildCitingPages(proposals) {
+  return (acceptedSources) => {
+    if (acceptedSources.length === 0) return void 0;
+    const blocks = acceptedSources.map((record) => {
+      const proposal = proposals.find((p) => p.uri === record.metadata?.originalUri);
+      const uri = proposal?.uri ?? record.id;
+      const slug = uri.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 120) || record.id;
+      const title = proposal?.title ?? record.id;
+      const body = proposal?.text ?? "";
+      return [
+        `---FILE: knowledge/${slug}.md---`,
+        "---",
+        `title: ${escapeYaml(title)}`,
+        `sources: ["${record.id}"]`,
+        `source_url: ${uri}`,
+        "---",
+        `# ${title}`,
+        "",
+        body,
+        "",
+        `Source: ${uri}`,
+        "---END FILE---"
+      ].join("\n");
+    });
+    return blocks.join("\n");
+  };
+}
+function escapeYaml(value) {
+  return value.replace(/[\r\n]+/g, " ").replace(/"/g, "'").trim();
+}
+function createVerifyingResearchDriver(options = {}) {
+  const acceptOnParseFailure = options.acceptOnParseFailure ?? false;
+  return {
+    async verifySource(source, ctx) {
+      const router = resolveRouter(options);
+      const gapLines = ctx.gaps.map((gap) => `- ${gap.description} (query: "${gap.query}")`).join("\n");
+      const acceptedTitles = ctx.acceptedThisRound.map((accepted) => `- ${accepted.title ?? accepted.uri}`).join("\n");
+      const excerpt = source.text.slice(0, 1500);
+      const system = 'You verify whether a fetched web source belongs in a curated knowledge base. Accept a source ONLY if it is genuinely on-topic for the research goal and helps close one of the open gaps, AND it is not a near-duplicate of an already-accepted source. Reject spam, listicles, off-topic pages, marketing, and near-duplicates. Respond with ONLY a JSON object: {"accept": true|false, "reason": "<short reason>"}.';
+      const user = [
+        `Research goal: ${ctx.goal}`,
+        `Open gaps:
+${gapLines || "(none specified)"}`,
+        acceptedTitles ? `Already accepted this round:
+${acceptedTitles}` : "Nothing accepted yet this round.",
+        `Candidate source:
+URL: ${source.uri}
+Title: ${source.title ?? "(none)"}
+Excerpt:
+${excerpt}`,
+        'Verdict as JSON {"accept": boolean, "reason": string}:'
+      ].join("\n\n");
+      let raw = "";
+      try {
+        raw = await router.chat(
+          [
+            { role: "system", content: system },
+            { role: "user", content: user }
+          ],
+          MIN_MAX_TOKENS
+        );
+      } catch (error) {
+        if (error.name === "AbortError") throw error;
+        return acceptOnParseFailure ? { accept: true } : { accept: false, reason: `verifier unavailable: ${error.message}` };
+      }
+      const verdict = parseVerdict(raw);
+      if (verdict) return verdict;
+      return acceptOnParseFailure ? { accept: true } : { accept: false, reason: "verifier returned an unparseable verdict" };
+    }
+  };
+}
+function parseVerdict(raw) {
+  const text = raw.trim();
+  if (!text) return null;
+  const objMatch = text.match(/\{[\s\S]*\}/);
+  if (!objMatch) return null;
+  try {
+    const parsed = JSON.parse(objMatch[0]);
+    if (typeof parsed.accept !== "boolean") return null;
+    if (parsed.accept) return { accept: true };
+    return {
+      accept: false,
+      reason: typeof parsed.reason === "string" && parsed.reason.trim() ? parsed.reason.trim() : "rejected by verifier"
+    };
+  } catch {
+    return null;
+  }
+}
 export {
   AgentMemoryHitSchema,
   AgentMemoryKindSchema,
@@ -1033,6 +1567,8 @@ export {
   MemoryKbStore,
   POLITE_USER_AGENT,
   READINESS_SPEC_DEFAULTS,
+  RESEARCH_SUPERVISOR_SYSTEM_PROMPT,
+  RouterError,
   SCAFFOLD_PAGE_BASENAMES,
   SourceAnchorSchema,
   SourceRecordSchema,
@@ -1055,6 +1591,9 @@ export {
   createLocalDiscoveryDispatcher,
   createNeo4jAgentMemoryAdapter,
   createStateSosSource,
+  createTangleRouterClient,
+  createVerifyingResearchDriver,
+  createWebResearchWorker,
   defaultGetMemoryContext,
   defineReadinessSpec,
   detectChanges,
@@ -1069,6 +1608,7 @@ export {
   inspectKnowledgeIndex,
   isSafeKnowledgePath,
   isScaffoldPath,
+  knowledgeReadinessDeliverable,
   knowledgeReleaseReport,
   layoutFor,
   lintKnowledgeIndex,
@@ -1087,9 +1627,12 @@ export {
   reciprocalRankFusion,
   renderMemoryContext,
   runKnowledgeResearchLoop,
+  runResearchSupervisor,
+  runTwoAgentResearchLoop,
   searchKnowledge,
   sha256,
   slugify,
+  sourceMatchesGaps,
   sourceRegistryPath,
   stableId,
   stripFrontmatter,