@abot-ai/runtime 1.1.0 → 1.3.0

package/plugins/web/source/light/documents/response-decoding.ts

@@ -0,0 +1,120 @@
+import {
+  brotliDecompressSync,
+  gunzipSync,
+  inflateRawSync,
+  inflateSync,
+} from "node:zlib";
+
+import { WebPluginError } from "../../errors.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+
+function responseHeader(response: PublicHttpResponse, name: string): string {
+  const value = response.headers[name];
+  return (Array.isArray(value) ? value[0] : value) ?? "";
+}
+
+function responseEncoding(response: PublicHttpResponse): string {
+  const declared = responseHeader(response, "content-encoding")
+    .trim()
+    .toLowerCase();
+  if (declared && declared !== "identity") return declared;
+  const hasGzipSignature =
+    response.body[0] === 0x1f && response.body[1] === 0x8b;
+  return hasGzipSignature ? "gzip" : "identity";
+}
+
+function hasZlibWrapperHeader(body: Uint8Array): boolean {
+  if (body.byteLength < 2) return false;
+  const method = body[0]!;
+  if ((method & 0x0f) !== 8) return false;
+  if (method >> 4 > 7) return false;
+  return ((method << 8) + body[1]!) % 31 === 0;
+}
+
+function decodeBody(
+  body: Uint8Array,
+  encoding: string,
+  maxOutputLength: number,
+): Uint8Array {
+  const options = { maxOutputLength };
+  if (encoding === "gzip" || encoding === "x-gzip")
+    return gunzipSync(body, options);
+  if (encoding === "br") return brotliDecompressSync(body, options);
+  if (encoding === "deflate") {
+    // Select once: a failed wrapper decoder may already have produced output.
+    if (hasZlibWrapperHeader(body)) return inflateSync(body, options);
+    return inflateRawSync(body, options);
+  }
+  throw new WebPluginError(
+    "web_response_unsupported",
+    "The source used an unsupported content encoding.",
+  );
+}
+
+function decodedContentType(
+  response: PublicHttpResponse,
+  encoding: string,
+): string {
+  const original = responseHeader(response, "content-type");
+  const mediaType = original.split(";", 1)[0]!.trim().toLowerCase();
+  if (encoding === "identity") return original;
+  if (["application/gzip", "application/x-gzip"].includes(mediaType))
+    return "application/xml";
+  if (
+    mediaType === "application/octet-stream" &&
+    /\.xml\.gz(?:[?#]|$)/iu.test(response.finalUrl)
+  ) {
+    return "application/xml";
+  }
+  return original;
+}
+
+export function decodeLightResponse(
+  response: PublicHttpResponse,
+  maxBytes: number,
+  consumeDecodedBytes?: (count: number) => void,
+): PublicHttpResponse {
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
+    throw new RangeError("maxBytes must be a positive safe integer");
+  }
+  const limit = Math.min(maxBytes, WEB_LIMITS.responseBytes);
+  if (response.body.byteLength > WEB_LIMITS.responseBytes) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The encoded source exceeds the response limit.",
+    );
+  }
+  const encoding = responseEncoding(response);
+  let body: Uint8Array;
+  try {
+    body =
+      encoding === "identity"
+        ? response.body.subarray(0, limit)
+        : decodeBody(response.body, encoding, limit);
+  } catch (error) {
+    if (error instanceof WebPluginError) throw error;
+    // A failed synchronous decoder does not expose its partial output size.
+    // Charge its whole allowance so repeated failures cannot evade the budget.
+    consumeDecodedBytes?.(limit);
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The encoded source is invalid or exceeds the decoded response limit.",
+    );
+  }
+  consumeDecodedBytes?.(body.byteLength);
+  return Object.freeze({
+    ...response,
+    headers: Object.freeze({
+      ...response.headers,
+      "content-encoding": "identity",
+      "content-length": String(body.byteLength),
+      "content-type": decodedContentType(response, encoding),
+    }),
+    body,
+    bytesRead: body.byteLength,
+    partialContent:
+      response.partialContent ||
+      (encoding === "identity" && response.body.byteLength > limit),
+  });
+}

package/plugins/web/source/light/ranking/document-ranking.ts

@@ -0,0 +1,114 @@
+import type {
+  QueryResult,
+  SearchHit,
+  SourceFetch,
+} from "../../search-types.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type { LightCachedDocument } from "../documents/document-cache.js";
+import { queryTerms, textTerms } from "./query-normalization.js";
+
+type ScoredDocument = Readonly<{ record: LightCachedDocument; score: number }>;
+
+function isIndexablePage(record: LightCachedDocument): boolean {
+  if (record.document.noIndex) return false;
+  return record.document.kind === "page";
+}
+
+function scoreDocument(query: string, record: LightCachedDocument): number {
+  const terms = queryTerms(query);
+  if (terms.length === 0) return 0;
+  const titleTerms = queryTerms(record.document.title);
+  const title = new Set(titleTerms);
+  const body = textTerms(record.document.text);
+  const frequencies = new Map<string, number>();
+  for (const term of body)
+    frequencies.set(term, (frequencies.get(term) ?? 0) + 1);
+  let score = 0;
+  let matched = 0;
+  let titleMatches = 0;
+  for (const term of terms) {
+    const inTitle = title.has(term);
+    const frequency = frequencies.get(term) ?? 0;
+    if (!inTitle && frequency === 0) continue;
+    matched += 1;
+    if (inTitle) titleMatches += 1;
+    score +=
+      Math.log1p(Math.min(frequency, 3)) / Math.sqrt(1 + body.length / 200);
+  }
+  if (matched === 0) return 0;
+  const titleConcentration = titleMatches / Math.max(1, title.size);
+  score += 8 * titleMatches * (0.5 + titleConcentration);
+  const hasExactTopicTitle = titleTerms.join(" ") === terms.join(" ");
+  if (hasExactTopicTitle) score += 6;
+  return (score * matched) / terms.length;
+}
+
+function publishedTime(record: LightCachedDocument): number {
+  const date = record.document.publishedAt ?? record.document.updatedAt;
+  return date ? Date.parse(date) || 0 : 0;
+}
+
+function compareDocuments(left: ScoredDocument, right: ScoredDocument): number {
+  return (
+    right.score - left.score ||
+    publishedTime(right.record) - publishedTime(left.record) ||
+    left.record.document.canonicalUrl.localeCompare(
+      right.record.document.canonicalUrl,
+    )
+  );
+}
+
+function documentSnippet(query: string, text: string): string {
+  const terms = queryTerms(query);
+  const paragraphs = text
+    .split(/\n+/u)
+    .map((paragraph, index) => ({
+      paragraph,
+      index,
+      matches: terms.filter((term) => new Set(textTerms(paragraph)).has(term))
+        .length,
+    }))
+    .sort(
+      (left, right) => right.matches - left.matches || left.index - right.index,
+    );
+  return (paragraphs[0]?.paragraph ?? text).slice(0, 800);
+}
+
+export function rankLightDocuments(
+  queries: readonly string[],
+  records: readonly LightCachedDocument[],
+) {
+  const byUrl = new Map(
+    records.map((record) => [record.document.canonicalUrl, record]),
+  );
+  const results: QueryResult[] = queries.map((query) => {
+    const ranked = [...byUrl.values()]
+      .filter(isIndexablePage)
+      .map((record) => ({ record, score: scoreDocument(query, record) }))
+      .filter(({ score }) => score > 0)
+      .sort(compareDocuments)
+      .slice(0, WEB_LIMITS.searchResultsPerQuery);
+    const hits: SearchHit[] = ranked.map(({ record }, index) => ({
+      query,
+      title: record.document.title || record.document.canonicalUrl,
+      url: record.document.canonicalUrl,
+      domain: new URL(record.document.canonicalUrl).hostname,
+      snippet: documentSnippet(query, record.document.text),
+      rank: index + 1,
+    }));
+    return Object.freeze({ query, hits: Object.freeze(hits) });
+  });
+  const hits = [
+    ...new Map(
+      results.flatMap((result) => result.hits).map((hit) => [hit.url, hit]),
+    ).values(),
+  ];
+  const sourceFetches: SourceFetch[] = hits
+    .slice(0, WEB_LIMITS.searchSourceFetches)
+    .map((hit) => ({ hit, page: byUrl.get(hit.url)?.page }));
+  return Object.freeze({
+    results: Object.freeze(results),
+    hits: Object.freeze(hits),
+    sourceFetches: Object.freeze(sourceFetches),
+  });
+}

package/plugins/web/source/light/ranking/query-normalization.ts

@@ -0,0 +1,69 @@
+const QUERY_STOP_WORDS = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "for",
+  "from",
+  "how",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "the",
+  "to",
+  "was",
+  "what",
+  "when",
+  "where",
+  "which",
+  "who",
+  "with",
+  "או",
+  "איך",
+  "אילו",
+  "איזה",
+  "את",
+  "האם",
+  "הוא",
+  "היא",
+  "הם",
+  "זה",
+  "זו",
+  "מה",
+  "מי",
+  "מתי",
+  "של",
+  "על",
+  "עם",
+]);
+
+export function normalizeSearchText(value: string): string {
+  return value
+    .normalize("NFKC")
+    .toLowerCase()
+    .replace(/[\u0591-\u05BD\u05BF-\u05C2\u05C4-\u05C5\u05C7]/gu, "")
+    .replace(/[^\p{L}\p{N}]+/gu, " ")
+    .trim();
+}
+
+export function queryTerms(query: string): readonly string[] {
+  const terms = normalizeSearchText(query).split(/\s+/u).filter(Boolean);
+  const significant = terms.filter((term) => !QUERY_STOP_WORDS.has(term));
+  return Object.freeze([...new Set(significant.length ? significant : terms)]);
+}
+
+export function textTerms(text: string): readonly string[] {
+  return normalizeSearchText(text).split(/\s+/u).filter(Boolean);
+}
+
+export function countTermMatches(query: string, text: string): number {
+  const available = new Set(textTerms(text));
+  return queryTerms(query).filter((term) => available.has(term)).length;
+}

package/plugins/web/source/light/search-collection.ts

@@ -0,0 +1,136 @@
+import { isWebPluginError } from "../errors.js";
+import type { LightConfig } from "./config.js";
+import type { LightCrawlSession } from "./crawl/transport.js";
+import type {
+  LightCachedDocument,
+  LightDocumentCache,
+} from "./documents/document-cache.js";
+import { readLightDocument } from "./documents/document-reader.js";
+import {
+  createSearchFrontier,
+  type LightCrawlCandidate,
+} from "./search-frontier.js";
+import type { LightSource } from "./sources/source-definition.js";
+
+export type LightSourceFailure = Readonly<{
+  url: string;
+  errorCode: string;
+  error: string;
+}>;
+
+function isTerminalSearchError(error: unknown): boolean {
+  if (!isWebPluginError(error)) return false;
+  return error.code === "web_request_aborted";
+}
+
+export async function collectLightDocuments(params: {
+  queries: readonly string[];
+  sources: readonly LightSource[];
+  config: LightConfig;
+  session: LightCrawlSession;
+  cache: LightDocumentCache;
+}) {
+  const { session, config, cache, sources } = params;
+  const records = new Map(
+    cache
+      .forSources(new Set(config.sources.map(({ id }) => id)))
+      .map((record) => [record.document.canonicalUrl, record]),
+  );
+  const fetchedUrls = new Set<string>();
+  const consultedSources = new Set<string>();
+  const failures: LightSourceFailure[] = [];
+  const frontier = createSearchFrontier(params.queries, config);
+  let cacheHits = 0;
+  let cacheMisses = 0;
+  let successfulReads = 0;
+
+  // Round-robin entry admission gives every selected source a first opportunity.
+  const entryCount = Math.max(
+    ...sources.map(({ entryUrls }) => entryUrls.length),
+  );
+  for (let index = 0; index < entryCount; index += 1) {
+    for (const source of sources) {
+      const url = source.entryUrls[index];
+      if (url)
+        frontier.enqueue(
+          { url, title: source.title, kind: "page" },
+          source,
+          0,
+          true,
+        );
+    }
+  }
+
+  async function visit(candidate: LightCrawlCandidate): Promise<void> {
+    consultedSources.add(candidate.source.id);
+    try {
+      const { record, fromCache } = await readLightDocument({
+        url: candidate.url,
+        sourceId: candidate.source.id,
+        session,
+        config,
+        cache,
+      });
+      successfulReads += 1;
+      if (fromCache) cacheHits += 1;
+      else {
+        cacheMisses += 1;
+        fetchedUrls.add(record.document.canonicalUrl);
+      }
+      records.set(record.document.canonicalUrl, record);
+      for (const link of record.document.links) {
+        frontier.enqueue(link, candidate.source, candidate.depth + 1);
+      }
+      for (const sitemap of session.sitemapsFor(
+        new URL(candidate.url).origin,
+      )) {
+        frontier.enqueue(
+          { url: sitemap, title: "", kind: "sitemap" },
+          candidate.source,
+          0,
+        );
+      }
+    } catch (error) {
+      if (isTerminalSearchError(error)) throw error;
+      session.assertActive();
+      if (failures.length < 8)
+        failures.push(
+          Object.freeze({
+            url: candidate.url,
+            errorCode: isWebPluginError(error)
+              ? error.code
+              : "web_response_invalid",
+            error: isWebPluginError(error)
+              ? error.message.slice(0, 256)
+              : "The Light source could not be read.",
+          }),
+        );
+    }
+  }
+
+  while (frontier.hasPending()) {
+    session.assertActive();
+    if (!session.canContinue()) break;
+    const settled = await Promise.allSettled(frontier.nextBatch().map(visit));
+    const rejected = settled.find((result) => result.status === "rejected");
+    if (rejected?.status === "rejected") throw rejected.reason;
+  }
+  session.assertActive();
+  const finishedAt = Date.now();
+  const currentRecords = [...records.values()].filter(
+    (record) =>
+      fetchedUrls.has(record.document.canonicalUrl) ||
+      record.expiresAt > finishedAt,
+  );
+  return Object.freeze({
+    successfulReads,
+    records: Object.freeze(currentRecords) as readonly LightCachedDocument[],
+    fetchedUrls,
+    consultedSources: Object.freeze([...consultedSources]),
+    failures: Object.freeze(failures),
+    cacheHits,
+    cacheMisses,
+    candidatesDiscovered: frontier.discovered(),
+    candidatesOmitted: frontier.omitted(),
+  });
+}

package/plugins/web/source/light/search-frontier.ts

@@ -0,0 +1,246 @@
+import { parsePublicHttpUrl } from "../network-policy.js";
+import type { LightConfig } from "./config.js";
+import type { DiscoveryCandidate } from "./documents/document-contract.js";
+import { countTermMatches } from "./ranking/query-normalization.js";
+import type { LightSource } from "./sources/source-definition.js";
+
+export type LightCrawlCandidate = DiscoveryCandidate &
+  Readonly<{
+    source: LightSource;
+    depth: number;
+    priority: number;
+    sequence: number;
+    queryScores: readonly number[];
+    entry: boolean;
+  }>;
+
+function admittedCandidateUrl(
+  url: string,
+  source: LightSource,
+): string | undefined {
+  try {
+    const parsed = parsePublicHttpUrl(url);
+    if (!source.allowedOrigins.includes(parsed.origin)) return undefined;
+    parsed.hash = "";
+    if (parsed.toString().length > 1_024) return undefined;
+    return parsed.toString();
+  } catch {
+    return undefined;
+  }
+}
+
+export function createSearchFrontier(
+  queries: readonly string[],
+  config: LightConfig,
+) {
+  const seen = new Set<string>();
+  const pending: LightCrawlCandidate[] = [];
+  const sourcesWithEntryOpportunity = new Set<string>();
+  let sequence = 0;
+  let omitted = 0;
+  let sitemapCount = 0;
+  let nextQuery = 0;
+
+  function replacementIndex(incoming: LightCrawlCandidate): number {
+    const sourceCounts = new Map<string, number>();
+    const queryCounts = queries.map(() => 0);
+    for (const candidate of pending) {
+      sourceCounts.set(
+        candidate.source.id,
+        (sourceCounts.get(candidate.source.id) ?? 0) + 1,
+      );
+      candidate.queryScores.forEach((score, index) => {
+        if (score > 0) queryCounts[index]! += 1;
+      });
+    }
+    const candidates = pending
+      .map((candidate, index) => ({ candidate, index }))
+      .filter(({ candidate }) => !needsFirstEntryOpportunity(candidate))
+      .sort(
+        (left, right) =>
+          left.candidate.priority - right.candidate.priority ||
+          right.candidate.sequence - left.candidate.sequence,
+      );
+    const match = candidates.find(({ candidate }) =>
+      canReplaceCandidate(candidate, incoming, sourceCounts, queryCounts),
+    );
+    return match?.index ?? -1;
+  }
+
+  function enqueue(
+    candidate: DiscoveryCandidate,
+    source: LightSource,
+    depth: number,
+    entry = false,
+  ): void {
+    if (depth > config.maxDepth) {
+      omitted += 1;
+      return;
+    }
+    const url = admittedCandidateUrl(candidate.url, source);
+    if (!url) return;
+    if (seen.has(url)) return;
+    if (candidate.kind === "sitemap" && sitemapCount >= config.maxSitemaps) {
+      omitted += 1;
+      return;
+    }
+    const queryScores = queries.map((query) =>
+      countTermMatches(
+        query,
+        `${candidate.title} ${candidate.text ?? ""} ${decodeURIComponentSafe(url)}`,
+      ),
+    );
+    const relevance = Math.max(0, ...queryScores);
+    const priority = entry ? 1_000 - sequence : relevance * 100 - depth * 10;
+    const incoming = Object.freeze({
+      ...candidate,
+      url,
+      source,
+      depth,
+      priority,
+      sequence: sequence++,
+      queryScores: Object.freeze(queryScores),
+      entry,
+    });
+    if (seen.size >= config.maxCandidates) {
+      omitted += 1;
+      const index = replacementIndex(incoming);
+      if (index < 0) return;
+      seen.delete(pending[index]!.url);
+      pending.splice(index, 1);
+    }
+    seen.add(url);
+    if (candidate.kind === "sitemap") sitemapCount += 1;
+    pending.push(incoming);
+  }
+
+  function needsFirstEntryOpportunity(candidate: LightCrawlCandidate): boolean {
+    if (!candidate.entry) return false;
+    return !sourcesWithEntryOpportunity.has(candidate.source.id);
+  }
+
+  function nextCandidateIndex(origins: ReadonlySet<string>): number {
+    const available = pending
+      .map((candidate, index) => ({ candidate, index }))
+      .filter(({ candidate }) => !origins.has(new URL(candidate.url).origin));
+    const entry = available.find(({ candidate }) =>
+      needsFirstEntryOpportunity(candidate),
+    );
+    if (entry) return entry.index;
+    for (let offset = 0; offset < queries.length; offset += 1) {
+      const queryIndex = (nextQuery + offset) % queries.length;
+      const matching = available
+        .filter(({ candidate }) => candidate.queryScores[queryIndex]! > 0)
+        .sort((left, right) =>
+          compareCandidatesForQuery(
+            left.candidate,
+            right.candidate,
+            queryIndex,
+          ),
+        );
+      const match = matching[0];
+      if (!match) continue;
+      nextQuery = (queryIndex + 1) % queries.length;
+      return match.index;
+    }
+    return available[0]?.index ?? -1;
+  }
+
+  function nextBatch(): readonly LightCrawlCandidate[] {
+    pending.sort(
+      (left, right) =>
+        right.priority - left.priority || left.sequence - right.sequence,
+    );
+    const batch: LightCrawlCandidate[] = [];
+    const origins = new Set<string>();
+    while (batch.length < config.maxConcurrency) {
+      const index = nextCandidateIndex(origins);
+      if (index < 0) break;
+      const candidate = pending[index]!;
+      const origin = new URL(candidate.url).origin;
+      origins.add(origin);
+      if (candidate.entry) sourcesWithEntryOpportunity.add(candidate.source.id);
+      batch.push(candidate);
+      pending.splice(index, 1);
+    }
+    return Object.freeze(batch);
+  }
+
+  return Object.freeze({
+    enqueue,
+    nextBatch,
+    hasPending: () => pending.length > 0,
+    omitted: () => omitted,
+    discovered: () => seen.size,
+  });
+}
+
+function compareCandidatesForQuery(
+  left: LightCrawlCandidate,
+  right: LightCrawlCandidate,
+  queryIndex: number,
+): number {
+  const relevanceDifference =
+    right.queryScores[queryIndex]! - left.queryScores[queryIndex]!;
+  if (relevanceDifference !== 0) return relevanceDifference;
+  // After each source has had an entry opportunity, useful discovered content
+  // wins a relevance tie with another entry URL from an already-started source.
+  if (left.entry !== right.entry) return left.entry ? 1 : -1;
+  return left.sequence - right.sequence;
+}
+
+function decodeURIComponentSafe(value: string): string {
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return value;
+  }
+}
+
+function preservesExistingQueryCoverage(
+  existing: LightCrawlCandidate,
+  incoming: LightCrawlCandidate,
+  queryCounts: readonly number[],
+): boolean {
+  for (let index = 0; index < queryCounts.length; index += 1) {
+    if (existing.queryScores[index] === 0) continue;
+    if (incoming.queryScores[index]! > 0) continue;
+    if (queryCounts[index]! > 1) continue;
+    return false;
+  }
+  return true;
+}
+
+function canReplaceSupplementalEntry(
+  existing: LightCrawlCandidate,
+  incoming: LightCrawlCandidate,
+  queryCounts: readonly number[],
+): boolean {
+  if (!existing.entry) return false;
+  if (incoming.entry) return false;
+  if (!incoming.queryScores.some((score) => score > 0)) return false;
+  return preservesExistingQueryCoverage(existing, incoming, queryCounts);
+}
+
+function canReplaceCandidate(
+  existing: LightCrawlCandidate,
+  incoming: LightCrawlCandidate,
+  sourceCounts: ReadonlyMap<string, number>,
+  queryCounts: readonly number[],
+): boolean {
+  if (incoming.entry) return true;
+  if (canReplaceSupplementalEntry(existing, incoming, queryCounts)) return true;
+  const addsQueryCoverage = incoming.queryScores.some(
+    (score, index) => score > 0 && queryCounts[index] === 0,
+  );
+  if (
+    addsQueryCoverage &&
+    preservesExistingQueryCoverage(existing, incoming, queryCounts)
+  )
+    return true;
+  const existingCount = sourceCounts.get(existing.source.id) ?? 0;
+  const incomingCount = sourceCounts.get(incoming.source.id) ?? 0;
+  if (existingCount > incomingCount + 1) return true;
+  if (existing.source.id !== incoming.source.id) return false;
+  return incoming.priority > existing.priority;
+}