@zeldrisho/pi-web-fetch 0.7.0 → 0.8.0

package/src/network-transport.ts

@@ -4,6 +4,7 @@ import { isIP, type LookupFunction } from "node:net";
 import { formatSize } from "@earendil-works/pi-coding-agent";
 import type { ValidatedTarget } from "./network-policy";
 
+/** Maximum size in bytes for raw HTTP response bodies. */
 export const FETCH_MAX_BYTES = 5 * 1_024 * 1_024;
 
 /**
@@ -59,12 +60,18 @@ async function requestOnce(
   address: string,
   signal: AbortSignal,
   attemptTimeoutMs: number,
+  headers: Readonly<Record<string, string>>,
 ): Promise<IncomingMessage> {
   const family = isIP(address);
   if (family !== 4 && family !== 6) throw new Error(`web_fetch could not resolve ${address}.`);
   const lookup: LookupFunction = (_hostname, options, callback) => {
-    if (options.all) callback(null, [{ address, family }]);
-    else callback(null, address, family);
+    // Match dns.lookup's asynchronous callback contract. Calling back inline
+    // lets an immediately unreachable address emit a socket error before
+    // node:http has finished installing its forwarding listeners.
+    queueMicrotask(() => {
+      if (options.all) callback(null, [{ address, family }]);
+      else callback(null, address, family);
+    });
   };
   const request = target.url.protocol === "https:" ? httpsRequest : httpRequest;
   return await new Promise((resolve, reject) => {
@@ -95,6 +102,7 @@ async function requestOnce(
         headers: {
           Accept: "text/markdown, text/html, text/plain, application/json;q=0.9, */*;q=0.1",
           "User-Agent": "Mozilla/5.0 (compatible; PiWebFetch/1.0; +https://pi.dev)",
+          ...headers,
         },
       },
       (response) => {
@@ -120,6 +128,8 @@ async function requestOnce(
 export interface RequestPinnedOptions {
   /** Per-address connect deadline used to fall back to the next address. */
   attemptTimeoutMs?: number;
+  /** Additional fixed request headers, used for conditional cache revalidation. */
+  headers?: Readonly<Record<string, string>>;
 }
 
 /**
@@ -140,7 +150,7 @@ export async function requestPinned(
   let lastError: unknown;
   for (const address of addresses) {
     try {
-      return await requestOnce(target, address, signal, attemptTimeoutMs);
+      return await requestOnce(target, address, signal, attemptTimeoutMs, options.headers ?? {});
     } catch (error) {
       lastError = error;
       if (signal.aborted) throw error;
@@ -161,6 +171,18 @@ export function responseHeader(response: IncomingMessage, name: string): string
   return Array.isArray(value) ? value[0] : value;
 }
 
+/**
+ * Read and buffer the complete response body.
+ *
+ * Enforces the maximum byte limit and respects abort signals during the read.
+ * Destroys the response stream if limits are exceeded or abort is signaled.
+ *
+ * @param response - HTTP response to read
+ * @param maxBytes - Maximum allowed response size in bytes
+ * @param signal - Optional abort signal to cancel the read
+ * @returns Buffered response body as bytes
+ * @throws When response exceeds maxBytes or abort signal fires
+ */
 export async function readResponseBytes(
   response: IncomingMessage,
   maxBytes: number,
@@ -208,6 +230,16 @@ export async function readResponseBytes(
   return output;
 }
 
+/**
+ * Decode HTTP response bytes to a string using the charset from Content-Type.
+ *
+ * Extracts the charset parameter from the Content-Type header and uses it
+ * to decode the response. Falls back to UTF-8 if charset is missing or invalid.
+ *
+ * @param bytes - Raw response bytes to decode
+ * @param contentTypeHeader - Value of the Content-Type response header
+ * @returns Decoded response string
+ */
 export function decodeResponse(bytes: Uint8Array, contentTypeHeader: string): string {
   const charset = contentTypeHeader.match(/(?:^|;)\s*charset\s*=\s*["']?([^;"'\s]+)/i)?.[1];
   try {

package/src/outline.ts

@@ -0,0 +1,240 @@
+const FENCE = /^[ \t]{0,3}(`{3,}|~{3,})/;
+const MAX_FALLBACK_HEADING_CHARACTERS = 60;
+export const MAX_OUTLINE_HEADINGS = 12;
+export const MAX_OUTLINE_HEADING_CHARACTERS = 120;
+
+/** One bounded heading exposed as untrusted document-shape metadata. */
+export interface OutlineHeading {
+  level: number;
+  text: string;
+  words: number;
+  inferred: boolean;
+}
+
+/** Bounded structural summary of a complete extracted document. */
+export interface DocumentOutline {
+  totalWords: number;
+  totalHeadings: number;
+  headings: OutlineHeading[];
+  omittedHeadings: number;
+}
+
+interface HeadingLocation {
+  level: number;
+  text: string;
+  line: number;
+  inferred: boolean;
+}
+
+interface HeadingCollection {
+  locations: HeadingLocation[];
+  total: number;
+}
+
+interface ParsedAtxHeading {
+  level: number;
+  text: string;
+}
+
+interface MarkdownFence {
+  character: string;
+  length: number;
+}
+
+function updateFence(line: string, fence: MarkdownFence | undefined): MarkdownFence | undefined {
+  const match = FENCE.exec(line);
+  if (!match) return fence;
+  const marker = match[1];
+  if (!fence) return { character: marker[0], length: marker.length };
+  const closes =
+    marker[0] === fence.character &&
+    marker.length >= fence.length &&
+    !line.slice(match[0].length).trim();
+  return closes ? undefined : fence;
+}
+
+/**
+ * Checks if a character is a space or tab.
+ *
+ * @param character - The character to check
+ * @returns `true` if the character is a space or tab
+ */
+function isSpaceOrTab(character: string | undefined): boolean {
+  return character === " " || character === "\t";
+}
+
+/** Parses the bounded ATX prefix with a linear scan to avoid backtracking on remote input. */
+function parseAtxHeading(line: string): ParsedAtxHeading | undefined {
+  let index = 0;
+  while (index < 3 && isSpaceOrTab(line[index])) index += 1;
+  if (isSpaceOrTab(line[index])) return undefined;
+
+  const markerStart = index;
+  while (line[index] === "#") index += 1;
+  const level = index - markerStart;
+  if (level < 1 || level > 6 || !isSpaceOrTab(line[index])) return undefined;
+
+  while (isSpaceOrTab(line[index])) index += 1;
+  return { level, text: line.slice(index) };
+}
+
+/**
+ * Counts the number of whitespace-separated words in a string.
+ *
+ * @param value - The string to count words in
+ * @returns The number of words, or 0 if the string is empty after trimming
+ */
+function countWords(value: string): number {
+  const trimmed = value.trim();
+  return trimmed ? trimmed.split(/\s+/).length : 0;
+}
+
+/**
+ * Counts the total words in a document while respecting fenced code blocks.
+ *
+ * @param lines - The document lines to count words in
+ * @returns The total word count across all lines, including fenced content
+ */
+function countDocumentWords(lines: string[]): number {
+  let fence: MarkdownFence | undefined;
+  return lines.reduce((total, line) => {
+    const wasInsideFence = Boolean(fence);
+    fence = updateFence(line, fence);
+    const heading = wasInsideFence || fence ? undefined : parseAtxHeading(line);
+    return total + countWords(heading ? cleanHeading(heading.text) : line);
+  }, 0);
+}
+
+/**
+ * Removes trailing whitespace and optional closing markers from heading text, then bounds the result.
+ *
+ * @param value - The raw heading text to clean
+ * @returns The cleaned and bounded heading text
+ */
+function cleanHeading(value: string): string {
+  let contentEnd = value.length;
+  while (contentEnd > 0 && isSpaceOrTab(value[contentEnd - 1])) contentEnd -= 1;
+
+  let markerStart = contentEnd;
+  while (markerStart > 0 && value[markerStart - 1] === "#") markerStart -= 1;
+  if (markerStart < contentEnd && markerStart > 0 && isSpaceOrTab(value[markerStart - 1])) {
+    contentEnd = markerStart - 1;
+    while (contentEnd > 0 && isSpaceOrTab(value[contentEnd - 1])) contentEnd -= 1;
+  }
+
+  return value.slice(0, contentEnd).trim().slice(0, MAX_OUTLINE_HEADING_CHARACTERS);
+}
+
+/**
+ * Collects ATX headings from Markdown lines while respecting fenced code blocks.
+ *
+ * @param lines - The document lines to scan for ATX headings
+ * @returns Collection of heading locations and total heading count
+ */
+function collectAtxHeadings(lines: string[]): HeadingCollection {
+  const locations: HeadingLocation[] = [];
+  let total = 0;
+  let fence: MarkdownFence | undefined;
+
+  for (const [lineNumber, line] of lines.entries()) {
+    const wasInsideFence = Boolean(fence);
+    fence = updateFence(line, fence);
+    if (wasInsideFence || fence) continue;
+    const heading = parseAtxHeading(line);
+    if (!heading) continue;
+    const text = cleanHeading(heading.text);
+    if (!text) continue;
+    total += 1;
+    if (locations.length <= MAX_OUTLINE_HEADINGS) {
+      locations.push({ level: heading.level, text, line: lineNumber, inferred: false });
+    }
+  }
+  return { locations, total };
+}
+
+/**
+ * Heuristically determines if a line looks like a heading when ATX headings are unavailable.
+ *
+ * @param line - The current line to evaluate
+ * @param nextLine - The following non-empty line for context
+ * @returns `true` if the line appears to be a heading
+ */
+function looksLikeFallbackHeading(line: string, nextLine: string): boolean {
+  if (/^\s{4,}/.test(line)) return false;
+  const candidate = line.trim();
+  if (
+    !candidate ||
+    candidate.length > MAX_FALLBACK_HEADING_CHARACTERS ||
+    !/[A-Za-z]/.test(candidate) ||
+    /[.!?…,;:]$/.test(candidate) ||
+    /[,;]/.test(candidate) ||
+    /^([-*+]|\d+\.)\s/.test(candidate) ||
+    /^(>|\||https?:\/\/)/i.test(candidate)
+  ) {
+    return false;
+  }
+  if (!/^[A-Z]/.test(candidate) && !/[`()._]/.test(candidate)) return false;
+  const following = nextLine.trim();
+  return following.length > MAX_FALLBACK_HEADING_CHARACTERS || /[.!?]$/.test(following);
+}
+
+/**
+ * Collects inferred headings using heuristics when no ATX headings are present.
+ *
+ * @param lines - The document lines to scan for heading-like content
+ * @returns Collection of inferred heading locations and total count
+ */
+function collectFallbackHeadings(lines: string[]): HeadingCollection {
+  const locations: HeadingLocation[] = [];
+  let total = 0;
+  const nextNonEmptyIndexes = new Int32Array(lines.length);
+  nextNonEmptyIndexes.fill(-1);
+  let nextNonEmptyIndex = -1;
+  for (let lineNumber = lines.length - 1; lineNumber >= 0; lineNumber -= 1) {
+    nextNonEmptyIndexes[lineNumber] = nextNonEmptyIndex;
+    if (lines[lineNumber].trim()) nextNonEmptyIndex = lineNumber;
+  }
+  let fence: MarkdownFence | undefined;
+
+  for (const [lineNumber, line] of lines.entries()) {
+    const wasInsideFence = Boolean(fence);
+    fence = updateFence(line, fence);
+    if (wasInsideFence || fence) continue;
+    const followingIndex = nextNonEmptyIndexes[lineNumber];
+    const following = followingIndex === -1 ? "" : lines[followingIndex];
+    if (looksLikeFallbackHeading(line, following)) {
+      total += 1;
+      if (locations.length <= MAX_OUTLINE_HEADINGS) {
+        locations.push({ level: 2, text: cleanHeading(line), line: lineNumber, inferred: true });
+      }
+    }
+  }
+  return { locations, total };
+}
+
+/**
+ * Builds a bounded, fence-aware outline from complete extracted Markdown.
+ *
+ * ATX headings are preferred. Only documents with no ATX headings use conservative plain-text
+ * heading inference. Heading text is remote, untrusted metadata and is bounded before exposure.
+ */
+export function createDocumentOutline(markdown: string): DocumentOutline {
+  const lines = markdown.split("\n");
+  const explicit = collectAtxHeadings(lines);
+  const collection = explicit.total > 0 ? explicit : collectFallbackHeadings(lines);
+  const headings = collection.locations.slice(0, MAX_OUTLINE_HEADINGS).map((heading, index) => {
+    const next = collection.locations[index + 1]?.line ?? lines.length;
+    return {
+      level: heading.level,
+      text: heading.text,
+      words: countWords(lines.slice(heading.line + 1, next).join("\n")),
+      inferred: heading.inferred,
+    };
+  });
+  return {
+    totalWords: countDocumentWords(lines),
+    totalHeadings: collection.total,
+    headings,
+    omittedHeadings: Math.max(0, collection.total - headings.length),
+  };
+}

package/src/service.ts

@@ -7,17 +7,25 @@ import {
 } from "@earendil-works/pi-coding-agent";
 import { ExpiringLruCache, stableKeyHash, type CachePersistence } from "./cache";
 import { sliceCompleteDocument, type CompleteDocument } from "./content";
-import { fetchCompleteDocument, type FetchRemoteDependencies } from "./fetch";
+import {
+  fetchCompleteDocument,
+  revalidateCompleteDocument,
+  type FetchRemoteDependencies,
+} from "./fetch";
+import { focusMarkdown, type FocusDetails } from "./focus";
 import { InflightCoalescer } from "./inflight";
+import { createDocumentOutline, type DocumentOutline } from "./outline";
 import {
   FETCH_DEFAULT_MAX_CHARACTERS,
   FETCH_DEFAULT_OFFSET,
   FETCH_MAX_CHARACTERS,
   FETCH_MAX_OFFSET_CHARACTERS,
+  FETCH_MAX_QUERY_CHARACTERS,
   FETCH_MIN_MAX_CHARACTERS,
 } from "./limits";
 
 const CACHE_TTL_MS = 24 * 60 * 60 * 1_000;
+const CACHE_STALE_RETENTION_MS = 7 * CACHE_TTL_MS;
 const CACHE_MAX_ENTRIES = 100;
 const CACHE_MAX_MARKDOWN_BYTES = 20 * 1_024 * 1_024;
 const MAX_INFLIGHT_REQUESTS = 100;
@@ -38,6 +46,8 @@ export type ContentKind =
 
 /** Confidence that the returned content faithfully represents the source page. */
 export type FetchConfidence = "high" | "medium" | "low";
+/** Cache evidence for this call. */
+export type CacheStatus = "hit" | "revalidated" | "miss";
 
 function safeUrl(value: string): URL | undefined {
   try {
@@ -47,6 +57,17 @@ function safeUrl(value: string): URL | undefined {
   }
 }
 
+/**
+ * Classify the kind of content returned by a fetch operation.
+ *
+ * Determines the content type based on URL patterns, extractor used,
+ * and whether an app shell was detected.
+ *
+ * @param url - The fetched URL
+ * @param extractor - Extraction method used
+ * @param shellSuspected - Whether the page appears to be an app shell
+ * @returns Content classification
+ */
 export function classifyContentKind(
   url: string,
   extractor: CompleteDocument["extractor"],
@@ -69,6 +90,17 @@ export function classifyContentKind(
   return "unknown";
 }
 
+/**
+ * Classify confidence level for fetched content quality.
+ *
+ * Determines how confident we are that the extracted content accurately
+ * represents the source page, based on extraction method and content length.
+ *
+ * @param extractor - Extraction method used
+ * @param shellSuspected - Whether the page appears to be an app shell
+ * @param markdownLength - Length of extracted markdown content
+ * @returns Confidence level (high, medium, or low)
+ */
 export function classifyConfidence(
   extractor: CompleteDocument["extractor"],
   shellSuspected: boolean,
@@ -87,10 +119,30 @@ export function classifyConfidence(
  * Returns them ordered shallow → deep, and an empty list for non-HTTP(S) targets or when
  * the page itself is already a `llms.txt` path, so a fallback never retries itself.
  */
+function isGitHubLikeHost(hostname: string): boolean {
+  return (
+    hostname === "github.com" ||
+    hostname === "raw.githubusercontent.com" ||
+    hostname === "gist.github.com" ||
+    hostname === "gist.githubusercontent.com"
+  );
+}
+
+/**
+ * Build candidate /llms.txt URLs to probe for a given page.
+ *
+ * Generates a list of potential /llms.txt index URLs by checking the site root
+ * and ancestor directories up to a maximum depth. Returns URLs ordered from
+ * shallowest to deepest.
+ *
+ * @param rawUrl - The original page URL
+ * @returns Array of candidate /llms.txt URLs to probe (empty for non-HTTP/S or GitHub)
+ */
 export function buildLlmsTxtCandidateUrls(rawUrl: string): URL[] {
   try {
     const url = new URL(rawUrl);
     if (url.protocol !== "https:" && url.protocol !== "http:") return [];
+    if (isGitHubLikeHost(url.hostname)) return [];
     if (url.pathname === "/llms.txt" || url.pathname.endsWith("/llms.txt")) return [];
     // Ancestor directories only, one level deep: for `/r2/buckets/x` probe
     // `/llms.txt` and `/r2/llms.txt`, never deeper. A trailing slash marks the final
@@ -232,6 +284,13 @@ async function fetchDocumentWithLlmsTxtSupport(
   signal: AbortSignal | undefined,
   dependencies: FetchRemoteDependencies,
 ): Promise<CompleteDocument> {
+  try {
+    if (isGitHubLikeHost(new URL(rawUrl).hostname)) {
+      return fetchCompleteDocument(rawUrl, signal, dependencies);
+    }
+  } catch {
+    // Invalid URL falls through to fetchCompleteDocument which will throw.
+  }
   const candidates = buildLlmsTxtCandidateUrls(rawUrl);
   const [primary, blindIndex] = await Promise.all([
     fetchCompleteDocument(rawUrl, signal, dependencies),
@@ -277,18 +336,23 @@ function resolveCacheDirectory(name: string): string {
   return base;
 }
 
+/** Parameters for a web fetch operation. */
 export interface WebFetchParameters {
   url: string;
+  /** Optional query used to select matching Markdown sections before continuation slicing. */
+  query?: string;
   offset?: number;
   maxCharacters?: number;
 }
 
+/** Details about content truncation and pagination strategy. */
 export interface WebFetchTruncationDetails {
   truncated: boolean;
   strategy: "continuation" | "none";
   nextOffset?: number;
 }
 
+/** Comprehensive metadata about a web fetch result. */
 export interface WebFetchDetails {
   url: string;
   requestedUrl: string;
@@ -297,15 +361,24 @@ export interface WebFetchDetails {
   title?: string;
   extractor: CompleteDocument["extractor"];
   contentKind: ContentKind;
+  /** @deprecated Use extractionDiagnostics for explicit quality evidence. */
   shellSuspected: boolean;
+  extractionDiagnostics?: CompleteDocument["extractionDiagnostics"];
+  links?: CompleteDocument["links"];
   confidence: FetchConfidence;
+  /** Bounded document-shape metadata for the complete, unfiltered document. Heading text is untrusted remote content. */
+  outline: DocumentOutline;
+  /** Deterministic focused-extraction evidence when a query was supplied. */
+  focus?: FocusDetails;
   /** True when the returned content is the site's /llms.txt served instead of the requested page. */
   llmsTxtFallback: boolean;
   /** True when the returned content is the page's advertised Markdown version served instead of a low-quality page. */
   markdownAlternateFallback: boolean;
   /** The site's /llms.txt index URL when one exists and the returned content is not that index itself. */
   llmsTxtUrl?: string;
+  /** True for a fresh cache hit or a successful 304 revalidation. */
   cached: boolean;
+  cacheStatus: CacheStatus;
   truncated: boolean;
   offset: number;
   nextOffset?: number;
@@ -333,7 +406,11 @@ const fetchCache = new ExpiringLruCache<string, CompleteDocument>(
   undefined,
   fetchCachePersistence,
 );
-const inflightFetches = new InflightCoalescer<string, CompleteDocument>(MAX_INFLIGHT_REQUESTS);
+interface FetchAcquisition {
+  document: CompleteDocument;
+  cacheStatus: Exclude<CacheStatus, "hit">;
+}
+const inflightFetches = new InflightCoalescer<string, FetchAcquisition>(MAX_INFLIGHT_REQUESTS);
 
 /**
  * Fetches a web page and returns formatted content with pagination and truncation metadata.
@@ -365,34 +442,66 @@ export async function executeWebFetch(
       `web_fetch maxCharacters must be an integer between ${FETCH_MIN_MAX_CHARACTERS} and ${FETCH_MAX_CHARACTERS}.`,
     );
   }
+  if (
+    params.query !== undefined &&
+    (params.query.trim().length === 0 || params.query.length > FETCH_MAX_QUERY_CHARACTERS)
+  ) {
+    throw new Error(
+      `web_fetch query must contain between 1 and ${FETCH_MAX_QUERY_CHARACTERS} characters.`,
+    );
+  }
   let document = fetchCache.get(params.url);
-  const cached = document !== undefined;
+  const now = Date.now();
+  const fresh = Boolean(
+    document && (document.cachedAt === undefined || now - document.cachedAt < CACHE_TTL_MS),
+  );
+  let cacheStatus: CacheStatus = fresh ? "hit" : "miss";
   onUpdate?.({
     content: [
       {
         type: "text",
-        text: cached ? `Using cached content for ${params.url}…` : `Fetching ${params.url}…`,
+        text: fresh
+          ? `Using cached content for ${params.url}…`
+          : document
+            ? `Revalidating cached content for ${params.url}…`
+            : `Fetching ${params.url}…`,
       },
     ],
     details: {},
   });
-  if (!document) {
-    document = await inflightFetches.run(
+  if (!fresh) {
+    const stale = document;
+    const acquisition = await inflightFetches.run(
       params.url,
       async (sharedSignal) => {
-        const fetched = await fetchDocumentWithLlmsTxtSupport(
-          params.url,
-          sharedSignal,
-          dependencies,
-        );
-        fetchCache.set(params.url, fetched, Date.now() + CACHE_TTL_MS);
-        return fetched;
+        let fetched: CompleteDocument;
+        let acquisitionStatus: FetchAcquisition["cacheStatus"] = "miss";
+        if (stale?.validators && !stale.llmsTxtFallback && !stale.markdownAlternateFallback) {
+          const outcome = await revalidateCompleteDocument(
+            params.url,
+            stale,
+            sharedSignal,
+            dependencies,
+          );
+          fetched = outcome.document;
+          acquisitionStatus = outcome.revalidated ? "revalidated" : "miss";
+        } else {
+          fetched = await fetchDocumentWithLlmsTxtSupport(params.url, sharedSignal, dependencies);
+        }
+        fetchCache.set(params.url, fetched, Date.now() + CACHE_STALE_RETENTION_MS);
+        return { document: fetched, cacheStatus: acquisitionStatus };
       },
       signal,
       "web_fetch was cancelled.",
     );
+    document = acquisition.document;
+    cacheStatus = acquisition.cacheStatus;
   }
-  const result = sliceCompleteDocument(document, offset, maxCharacters);
+  if (!document) throw new Error("web_fetch failed to acquire a document.");
+  const focused =
+    params.query === undefined ? undefined : focusMarkdown(document.markdown, params.query);
+  const outputDocument = focused ? { ...document, markdown: focused.markdown } : document;
+  const result = sliceCompleteDocument(outputDocument, offset, maxCharacters);
   const requestedUrl = params.url;
   const finalUrl = result.url;
   const shellSuspected = result.shellSuspected;
@@ -419,8 +528,16 @@ export async function executeWebFetch(
           "",
         ]
       : []),
+    ...(focused
+      ? [
+          focused.details.matchedSections > 0
+            ? `[Showing ${focused.details.matchedSections} of ${focused.details.totalSections} source sections selected by the focus query. Offsets apply to this focused view; omit query to read the complete document.]`
+            : "[No source sections matched the focus query. Omit query to read the complete document.]",
+          "",
+        ]
+      : []),
     `<untrusted_web_content source=${JSON.stringify(result.url)}>`,
-    result.markdown || "[The page contained no readable text.]",
+    result.markdown || (focused ? "" : "[The page contained no readable text.]"),
     "</untrusted_web_content>",
   ].join("\n");
   const outputTruncation = truncateHead(output, {
@@ -439,11 +556,16 @@ export async function executeWebFetch(
       extractor: result.extractor,
       contentKind,
       shellSuspected,
+      extractionDiagnostics: result.extractionDiagnostics,
+      links: result.links,
       confidence,
+      outline: createDocumentOutline(document.markdown),
+      focus: focused?.details,
       llmsTxtFallback: Boolean(result.llmsTxtFallback),
       markdownAlternateFallback: Boolean(result.markdownAlternateFallback),
       llmsTxtUrl: result.llmsTxtIndexUrl,
-      cached,
+      cached: cacheStatus !== "miss",
+      cacheStatus,
       truncated,
       offset: result.offset,
       nextOffset: result.nextOffset,