@zeldrisho/pi-web-fetch 0.9.2 → 0.9.3

package/src/fetch.ts

@@ -1,4 +1,13 @@
 import type { IncomingMessage } from "node:http";
+import {
+  lineFragmentOffsets,
+  markdownFragmentOffsets,
+  normalizeGitHubRawUrl,
+  parseLinkHeaderForAgentHints,
+} from "./fetch-metadata";
+
+export { normalizeGitHubRawUrl, parseLinkHeaderForAgentHints } from "./fetch-metadata";
+
 import { awaitWithAbort } from "./abort";
 import { sliceCompleteDocument, type CompleteDocument, type FetchResult } from "./content";
 import { diagnoseExtraction, extractDocumentLinks, hasExtractionWarning } from "./evidence";
@@ -14,63 +23,6 @@ import {
   responseHeaderValues,
 } from "./network-transport";
 
-/**
- * Rewrites GitHub source URLs to their raw counterparts so file contents are fetched as
- * clean plain text instead of Defuddle's noisy rendered view:
- *
- * - `github.com` `blob` URLs become their `raw.githubusercontent.com` counterpart.
- * - Bare gist pages (`gist.github.com/<user>/<id>`) get `/raw` appended; the redirect to
- *   `gist.githubusercontent.com` is followed by the normal redirect policy.
- *
- * @param rawUrl - The URL to normalize
- * @returns The rewritten raw URL, or the input unchanged for non-GitHub and non-source URLs
- */
-export function normalizeGitHubRawUrl(rawUrl: string): string {
-  try {
-    const url = new URL(rawUrl);
-
-    if (url.protocol !== "https:") return rawUrl;
-
-    if (url.hostname === "github.com") {
-      if (url.pathname.includes("/blob/")) {
-        return `https://raw.githubusercontent.com${url.pathname.replace("/blob/", "/")}${url.search}`;
-      }
-
-      // Parse pathname segments: /owner/repo/tree/ref/path
-      const segments = url.pathname.split("/").filter(Boolean);
-
-      if (segments.length >= 4 && segments[2] === "tree") {
-        const lastSegment = segments[segments.length - 1];
-
-        if (lastSegment.includes(".")) {
-          // Reconstruct path: /owner/repo/ref/path (remove "tree" segment)
-          const newPath = `/${segments[0]}/${segments[1]}/${segments.slice(3).join("/")}`;
-
-          return `https://raw.githubusercontent.com${newPath}${url.search}`;
-        }
-
-        return rawUrl;
-      }
-
-      return rawUrl;
-    }
-
-    if (url.hostname === "gist.github.com") {
-      const segments = url.pathname.split("/").filter(Boolean);
-
-      // Only rewrite bare gist pages. Subpages such as `/revisions`, `/forks`, or an
-      // explicit `/raw` path already point at a usable resource and stay untouched.
-      if (segments.length !== 2) return rawUrl;
-
-      return `https://gist.github.com/${segments[0]}/${segments[1]}/raw${url.search}`;
-    }
-
-    return rawUrl;
-  } catch {
-    return rawUrl;
-  }
-}
-
 /** @deprecated Prefer diagnoseExtraction for explicit extraction-quality signals. */
 export function detectAppShell(raw: string, markdown: string): boolean {
   return hasExtractionWarning(diagnoseExtraction(raw, markdown));
@@ -78,176 +30,12 @@ export function detectAppShell(raw: string, markdown: string): boolean {
 
 const REQUEST_TIMEOUT_MS = 20_000;
 
-/** Indexes GitHub-style Markdown heading slugs without treating fenced examples as headings. */
-function markdownFragmentOffsets(markdown: string) {
-  const offsets: Record<string, number> = {};
-  const duplicates = new Map<string, number>();
-  const lines = markdown.split("\n");
-  let offset = 0;
-  let fence: { marker: string; length: number } | undefined;
-
-  const addHeading = (text: string, headingOffset: number) => {
-    let withoutHtml = "";
-
-    for (let index = 0; index < text.length; index += 1) {
-      if (text[index] !== "<") {
-        withoutHtml += text[index];
-        continue;
-      }
-
-      const tagEnd = text.indexOf(">", index + 1);
-
-      if (tagEnd < 0) {
-        withoutHtml += text.slice(index + 1);
-        break;
-      }
-
-      index = tagEnd;
-    }
-
-    const base = withoutHtml
-      .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
-      .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
-      .replace(/[`*_~]/g, "")
-      .normalize("NFKD")
-      .replace(/[\u0300-\u036f]/g, "")
-      .toLowerCase()
-      .replace(/[^\p{Letter}\p{Number} _-]/gu, "")
-      .trim()
-      .replace(/[ _]+/g, "-");
-
-    if (!base) return;
-
-    const duplicate = duplicates.get(base) ?? 0;
-    duplicates.set(base, duplicate + 1);
-    offsets[duplicate ? `${base}-${duplicate}` : base] = headingOffset;
-  };
-
-  for (let index = 0; index < lines.length; index += 1) {
-    const line = lines[index];
-    const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
-
-    if (fenceMatch) {
-      const marker = fenceMatch[1][0];
-
-      if (!fence) fence = { marker, length: fenceMatch[1].length };
-      else if (fence.marker === marker && fenceMatch[1].length >= fence.length) fence = undefined;
-    } else if (!fence) {
-      const atxHeading = /^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$/.exec(line);
-      const setextHeading = /^\s{0,3}(=+|-+)\s*$/.test(lines[index + 1] ?? "");
-
-      if (atxHeading) addHeading(atxHeading[1], offset);
-      else if (setextHeading) addHeading(line.trim(), offset);
-    }
-
-    offset += line.length + 1;
-  }
-
-  return offsets;
-}
-
-/** Indexes GitHub line anchors such as `#L78` in plain-text source files. */
-function lineFragmentOffsets(text: string) {
-  const offsets: Record<string, number> = {};
-  let offset = 0;
-
-  for (const [index, line] of text.split("\n").entries()) {
-    offsets[`L${index + 1}`] = offset;
-    offset += line.length + 1;
-  }
-
-  return offsets;
-}
-
 /** Agent-discovery hints advertised in an HTTP `Link:` header. */
 export interface LinkHeaderAgentHints {
   describedBy?: string;
   markdownAlternate?: string;
 }
 
-/**
- * Parses an HTTP `Link:` header for the agent-discovery link relations defined by
- * llmstxt.org v2: `rel="describedby"` (the covering `llms.txt`) and
- * `rel="alternate" type="text/markdown"` (the Markdown version of the resource).
- *
- * @param header - The raw `Link:` header value
- * @param baseUrl - URL used to resolve relative references
- * @returns The first advertised target of each relation, when present
- */
-export function parseLinkHeaderForAgentHints(
-  header: string,
-  baseUrl: string | URL,
-): LinkHeaderAgentHints {
-  // Split on commas outside <...> and quoted strings, since URIs and quoted values may
-  // themselves contain commas.
-  const directives: string[] = [];
-  let current = "";
-  let inQuotes = false;
-  let inAngleBrackets = false;
-
-  for (const character of header) {
-    if (character === '"') inQuotes = !inQuotes;
-
-    if (character === "<") inAngleBrackets = true;
-
-    if (character === ">" && !inQuotes) inAngleBrackets = false;
-
-    if (character === "," && !inQuotes && !inAngleBrackets) {
-      directives.push(current);
-      current = "";
-    } else {
-      current += character;
-    }
-  }
-
-  directives.push(current);
-
-  let describedBy: string | undefined;
-  let markdownAlternate: string | undefined;
-
-  for (const directive of directives) {
-    const match = /^\s*<([^>]*)>\s*(.*)$/.exec(directive);
-
-    if (!match) continue;
-    const [, rawTarget, rawParameters] = match;
-    const parameters = new Map<string, string>();
-
-    for (const parameter of rawParameters.split(";")) {
-      const equals = parameter.indexOf("=");
-
-      if (equals === -1) continue;
-      const key = parameter.slice(0, equals).trim().toLowerCase();
-
-      const value = parameter
-        .slice(equals + 1)
-        .trim()
-        .replace(/^"|"$/g, "");
-
-      parameters.set(key, value);
-    }
-
-    const relations = (parameters.get("rel") ?? "").toLowerCase().split(/\s+/);
-    const type = (parameters.get("type") ?? "").toLowerCase();
-    let resolved: string | undefined;
-
-    try {
-      resolved = new URL(rawTarget.trim(), baseUrl).href;
-    } catch {
-      continue;
-    }
-
-    if (!resolved) continue;
-
-    if (!describedBy && relations.includes("describedby")) describedBy = resolved;
-
-    if (!markdownAlternate && relations.includes("alternate") && type === "text/markdown") {
-      markdownAlternate = resolved;
-    }
-  }
-
-  return { describedBy, markdownAlternate };
-}
-
 export interface FetchRemoteDependencies extends RedirectDependencies {
   extractHtml?: typeof extractHtmlToMarkdown;
   timeoutMs?: number;

package/src/fragment-index.ts

@@ -0,0 +1,99 @@
+function stripMarkupForFragmentSlug(value: string): string {
+  let result = "";
+
+  for (let index = 0; index < value.length; index += 1) {
+    if (value[index] !== "<") {
+      result += value[index];
+      continue;
+    }
+
+    const tagEnd = value.indexOf(">", index + 1);
+
+    if (tagEnd < 0) {
+      // Preserve the old behavior for an unterminated tag: only discard `<`.
+      result += value.slice(index + 1);
+      break;
+    }
+
+    index = tagEnd;
+  }
+
+  return result;
+}
+
+function fragmentSlug(value: string): string {
+  return stripMarkupForFragmentSlug(value)
+    .normalize("NFKD")
+    .replace(/[\u0300-\u036f]/g, "")
+    .toLowerCase()
+    .replace(/[^\p{Letter}\p{Number} _-]/gu, "")
+    .trim()
+    .replace(/[ _]+/g, "-");
+}
+
+/** Finds the extracted Markdown position corresponding to HTML fragment IDs. */
+interface FragmentOffsets {
+  [fragment: string]: number;
+}
+
+export function buildFragmentOffsets(
+  document: Document,
+  markdown: string,
+  originalIds?: ReadonlyMap<HTMLElement, string>,
+): FragmentOffsets {
+  const lines = markdown.split("\n");
+
+  const headings = lines
+    .map((line, index) => {
+      const match = /^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$/.exec(line);
+
+      return match
+        ? {
+            slug: fragmentSlug(match[1]),
+            offset: lines.slice(0, index).join("\n").length + (index ? 1 : 0),
+          }
+        : undefined;
+    })
+    .filter((heading): heading is { slug: string; offset: number } => Boolean(heading));
+
+  const offsets: Record<string, number> = {};
+  const used = new Set<number>();
+
+  for (const element of document.querySelectorAll<HTMLElement>("[id], a[name]")) {
+    const id =
+      originalIds?.get(element) ?? element.getAttribute("id") ?? element.getAttribute("name");
+
+    if (!id) continue;
+    const slug = fragmentSlug(id);
+    const textSlug = fragmentSlug(element.textContent ?? "");
+
+    const heading = headings.find(
+      (candidate) =>
+        candidate.slug === slug ||
+        (textSlug && candidate.slug === textSlug && !used.has(candidate.offset)),
+    );
+
+    if (heading) {
+      offsets[id] = heading.offset;
+      used.add(heading.offset);
+    } else {
+      const text = (element.textContent ?? "").trim();
+
+      if (text) {
+        const textOffset = markdown.toLowerCase().indexOf(text.toLowerCase());
+
+        if (textOffset >= 0) offsets[id] = textOffset;
+      }
+    }
+  }
+
+  // Most Markdown producers preserve headings but discard arbitrary HTML IDs.
+  // Resolve GitHub-style heading links even when no matching HTML ID survived.
+  for (const heading of headings) {
+    const base = heading.slug;
+
+    if (base && offsets[base] === undefined) offsets[base] = heading.offset;
+  }
+
+  return offsets;
+}

package/src/fragments.ts

@@ -0,0 +1,93 @@
+import type { CompleteDocument } from "./content";
+
+/** Removes a URL fragment before acquisition so fragment variants share cache entries. */
+export function urlWithoutFragment(rawUrl: string): string {
+  try {
+    const url = new URL(rawUrl);
+    url.hash = "";
+
+    return url.href;
+  } catch {
+    return rawUrl;
+  }
+}
+
+interface ResolvedFragment {
+  fragment?: string;
+  offset?: number;
+  endOffset?: number;
+}
+
+/** Converts a one-based source line and column into a Markdown character offset. */
+function sourceOffset(markdown: string, lineNumber: number, columnNumber = 1): number | undefined {
+  if (lineNumber < 1 || columnNumber < 1) return undefined;
+
+  const lines = markdown.split("\n");
+
+  if (lineNumber > lines.length) return undefined;
+
+  let offset = 0;
+
+  for (let index = 0; index < lineNumber - 1; index += 1) offset += lines[index].length + 1;
+
+  return offset + Math.min(columnNumber - 1, lines[lineNumber - 1].length);
+}
+
+/** Resolves heading and source-location URL fragments against a fetched document. */
+export function resolveFragmentOffset(
+  document: CompleteDocument,
+  rawUrl: string,
+): ResolvedFragment {
+  let fragment: string | undefined;
+
+  try {
+    const hash = new URL(rawUrl).hash;
+
+    if (hash) {
+      const rawFragment = hash.slice(1);
+
+      try {
+        fragment = decodeURIComponent(rawFragment);
+      } catch {
+        fragment = rawFragment;
+      }
+    }
+  } catch {
+    return {};
+  }
+
+  if (!fragment) return {};
+
+  const lineAnchor = /^L(\d+)(?:C(\d+))?(?:-L?(\d+)(?:C(\d+))?)?$/i.exec(fragment);
+
+  if (lineAnchor) {
+    const [, firstLine, firstColumn, lastLine, lastColumn] = lineAnchor;
+    const offset = sourceOffset(document.markdown, Number(firstLine), Number(firstColumn ?? 1));
+
+    if (offset === undefined) return { fragment };
+
+    if (lastLine) {
+      const endLine = Number(lastLine);
+      const endColumn = lastColumn === undefined ? undefined : Number(lastColumn);
+      const lines = document.markdown.split("\n");
+      const line = lines[endLine - 1];
+
+      if (line === undefined) return { fragment };
+
+      const endOffset =
+        endColumn === undefined
+          ? sourceOffset(document.markdown, endLine, line.length + 1)
+          : sourceOffset(document.markdown, endLine, endColumn + 1);
+
+      return endOffset !== undefined && endOffset >= offset
+        ? { fragment, offset, endOffset }
+        : { fragment };
+    }
+
+    return { fragment, offset };
+  }
+
+  const offsets = document.fragmentOffsets ?? {};
+
+  return { fragment, offset: offsets[fragment] ?? offsets[fragment.toLowerCase()] };
+}

package/src/index.ts

@@ -2,13 +2,10 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
 import { Type } from "typebox";
 import {
-  FETCH_DEFAULT_MAX_CHARACTERS,
   FETCH_DEFAULT_OFFSET,
-  FETCH_MAX_CHARACTERS,
   FETCH_MAX_OFFSET_CHARACTERS,
   FETCH_MAX_QUERY_CHARACTERS,
   FETCH_MAX_URL_CHARACTERS,
-  FETCH_MIN_MAX_CHARACTERS,
 } from "./limits";
 import { redactUrlForDisplay } from "./redact";
 import { formatCollapsibleOutput } from "./render";
@@ -54,28 +51,20 @@ export const webFetchParameters = Type.Object({
   url: Type.String({
     minLength: 1,
     maxLength: FETCH_MAX_URL_CHARACTERS,
-    description: "Public HTTP or HTTPS URL to fetch",
+    description: "HTTP or HTTPS page URL",
   }),
   query: Type.Optional(
     Type.String({
       minLength: 1,
       maxLength: FETCH_MAX_QUERY_CHARACTERS,
-      description:
-        "Optional focus query; returns matching source sections in document order (offsets apply to the focused view)",
+      description: "Focus on matching sections; offsets then apply to this focused view",
     }),
   ),
   offset: Type.Optional(
     Type.Integer({
       minimum: 0,
       maximum: FETCH_MAX_OFFSET_CHARACTERS,
-      description: `Extracted-content character offset to start reading from (default: ${FETCH_DEFAULT_OFFSET}; use nextOffset to continue)`,
-    }),
-  ),
-  maxCharacters: Type.Optional(
-    Type.Integer({
-      minimum: FETCH_MIN_MAX_CHARACTERS,
-      maximum: FETCH_MAX_CHARACTERS,
-      description: `Maximum returned content characters (default: ${FETCH_DEFAULT_MAX_CHARACTERS})`,
+      description: `Character offset in extracted content (default: ${FETCH_DEFAULT_OFFSET}; continue with nextOffset)`,
     }),
   ),
 });