@abot-ai/runtime 1.1.0 → 1.2.0

package/plugins/web/source/light/search-frontier.ts

@@ -0,0 +1,246 @@
+import { parsePublicHttpUrl } from "../network-policy.js";
+import type { LightConfig } from "./config.js";
+import type { DiscoveryCandidate } from "./documents/document-contract.js";
+import { countTermMatches } from "./ranking/query-normalization.js";
+import type { LightSource } from "./sources/source-definition.js";
+
+export type LightCrawlCandidate = DiscoveryCandidate &
+  Readonly<{
+    source: LightSource;
+    depth: number;
+    priority: number;
+    sequence: number;
+    queryScores: readonly number[];
+    entry: boolean;
+  }>;
+
+function admittedCandidateUrl(
+  url: string,
+  source: LightSource,
+): string | undefined {
+  try {
+    const parsed = parsePublicHttpUrl(url);
+    if (!source.allowedOrigins.includes(parsed.origin)) return undefined;
+    parsed.hash = "";
+    if (parsed.toString().length > 1_024) return undefined;
+    return parsed.toString();
+  } catch {
+    return undefined;
+  }
+}
+
+export function createSearchFrontier(
+  queries: readonly string[],
+  config: LightConfig,
+) {
+  const seen = new Set<string>();
+  const pending: LightCrawlCandidate[] = [];
+  const sourcesWithEntryOpportunity = new Set<string>();
+  let sequence = 0;
+  let omitted = 0;
+  let sitemapCount = 0;
+  let nextQuery = 0;
+
+  function replacementIndex(incoming: LightCrawlCandidate): number {
+    const sourceCounts = new Map<string, number>();
+    const queryCounts = queries.map(() => 0);
+    for (const candidate of pending) {
+      sourceCounts.set(
+        candidate.source.id,
+        (sourceCounts.get(candidate.source.id) ?? 0) + 1,
+      );
+      candidate.queryScores.forEach((score, index) => {
+        if (score > 0) queryCounts[index]! += 1;
+      });
+    }
+    const candidates = pending
+      .map((candidate, index) => ({ candidate, index }))
+      .filter(({ candidate }) => !needsFirstEntryOpportunity(candidate))
+      .sort(
+        (left, right) =>
+          left.candidate.priority - right.candidate.priority ||
+          right.candidate.sequence - left.candidate.sequence,
+      );
+    const match = candidates.find(({ candidate }) =>
+      canReplaceCandidate(candidate, incoming, sourceCounts, queryCounts),
+    );
+    return match?.index ?? -1;
+  }
+
+  function enqueue(
+    candidate: DiscoveryCandidate,
+    source: LightSource,
+    depth: number,
+    entry = false,
+  ): void {
+    if (depth > config.maxDepth) {
+      omitted += 1;
+      return;
+    }
+    const url = admittedCandidateUrl(candidate.url, source);
+    if (!url) return;
+    if (seen.has(url)) return;
+    if (candidate.kind === "sitemap" && sitemapCount >= config.maxSitemaps) {
+      omitted += 1;
+      return;
+    }
+    const queryScores = queries.map((query) =>
+      countTermMatches(
+        query,
+        `${candidate.title} ${candidate.text ?? ""} ${decodeURIComponentSafe(url)}`,
+      ),
+    );
+    const relevance = Math.max(0, ...queryScores);
+    const priority = entry ? 1_000 - sequence : relevance * 100 - depth * 10;
+    const incoming = Object.freeze({
+      ...candidate,
+      url,
+      source,
+      depth,
+      priority,
+      sequence: sequence++,
+      queryScores: Object.freeze(queryScores),
+      entry,
+    });
+    if (seen.size >= config.maxCandidates) {
+      omitted += 1;
+      const index = replacementIndex(incoming);
+      if (index < 0) return;
+      seen.delete(pending[index]!.url);
+      pending.splice(index, 1);
+    }
+    seen.add(url);
+    if (candidate.kind === "sitemap") sitemapCount += 1;
+    pending.push(incoming);
+  }
+
+  function needsFirstEntryOpportunity(candidate: LightCrawlCandidate): boolean {
+    if (!candidate.entry) return false;
+    return !sourcesWithEntryOpportunity.has(candidate.source.id);
+  }
+
+  function nextCandidateIndex(origins: ReadonlySet<string>): number {
+    const available = pending
+      .map((candidate, index) => ({ candidate, index }))
+      .filter(({ candidate }) => !origins.has(new URL(candidate.url).origin));
+    const entry = available.find(({ candidate }) =>
+      needsFirstEntryOpportunity(candidate),
+    );
+    if (entry) return entry.index;
+    for (let offset = 0; offset < queries.length; offset += 1) {
+      const queryIndex = (nextQuery + offset) % queries.length;
+      const matching = available
+        .filter(({ candidate }) => candidate.queryScores[queryIndex]! > 0)
+        .sort((left, right) =>
+          compareCandidatesForQuery(
+            left.candidate,
+            right.candidate,
+            queryIndex,
+          ),
+        );
+      const match = matching[0];
+      if (!match) continue;
+      nextQuery = (queryIndex + 1) % queries.length;
+      return match.index;
+    }
+    return available[0]?.index ?? -1;
+  }
+
+  function nextBatch(): readonly LightCrawlCandidate[] {
+    pending.sort(
+      (left, right) =>
+        right.priority - left.priority || left.sequence - right.sequence,
+    );
+    const batch: LightCrawlCandidate[] = [];
+    const origins = new Set<string>();
+    while (batch.length < config.maxConcurrency) {
+      const index = nextCandidateIndex(origins);
+      if (index < 0) break;
+      const candidate = pending[index]!;
+      const origin = new URL(candidate.url).origin;
+      origins.add(origin);
+      if (candidate.entry) sourcesWithEntryOpportunity.add(candidate.source.id);
+      batch.push(candidate);
+      pending.splice(index, 1);
+    }
+    return Object.freeze(batch);
+  }
+
+  return Object.freeze({
+    enqueue,
+    nextBatch,
+    hasPending: () => pending.length > 0,
+    omitted: () => omitted,
+    discovered: () => seen.size,
+  });
+}
+
+function compareCandidatesForQuery(
+  left: LightCrawlCandidate,
+  right: LightCrawlCandidate,
+  queryIndex: number,
+): number {
+  const relevanceDifference =
+    right.queryScores[queryIndex]! - left.queryScores[queryIndex]!;
+  if (relevanceDifference !== 0) return relevanceDifference;
+  // After each source has had an entry opportunity, useful discovered content
+  // wins a relevance tie with another entry URL from an already-started source.
+  if (left.entry !== right.entry) return left.entry ? 1 : -1;
+  return left.sequence - right.sequence;
+}
+
+function decodeURIComponentSafe(value: string): string {
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return value;
+  }
+}
+
+function preservesExistingQueryCoverage(
+  existing: LightCrawlCandidate,
+  incoming: LightCrawlCandidate,
+  queryCounts: readonly number[],
+): boolean {
+  for (let index = 0; index < queryCounts.length; index += 1) {
+    if (existing.queryScores[index] === 0) continue;
+    if (incoming.queryScores[index]! > 0) continue;
+    if (queryCounts[index]! > 1) continue;
+    return false;
+  }
+  return true;
+}
+
+function canReplaceSupplementalEntry(
+  existing: LightCrawlCandidate,
+  incoming: LightCrawlCandidate,
+  queryCounts: readonly number[],
+): boolean {
+  if (!existing.entry) return false;
+  if (incoming.entry) return false;
+  if (!incoming.queryScores.some((score) => score > 0)) return false;
+  return preservesExistingQueryCoverage(existing, incoming, queryCounts);
+}
+
+function canReplaceCandidate(
+  existing: LightCrawlCandidate,
+  incoming: LightCrawlCandidate,
+  sourceCounts: ReadonlyMap<string, number>,
+  queryCounts: readonly number[],
+): boolean {
+  if (incoming.entry) return true;
+  if (canReplaceSupplementalEntry(existing, incoming, queryCounts)) return true;
+  const addsQueryCoverage = incoming.queryScores.some(
+    (score, index) => score > 0 && queryCounts[index] === 0,
+  );
+  if (
+    addsQueryCoverage &&
+    preservesExistingQueryCoverage(existing, incoming, queryCounts)
+  )
+    return true;
+  const existingCount = sourceCounts.get(existing.source.id) ?? 0;
+  const incomingCount = sourceCounts.get(incoming.source.id) ?? 0;
+  if (existingCount > incomingCount + 1) return true;
+  if (existing.source.id !== incoming.source.id) return false;
+  return incoming.priority > existing.priority;
+}

package/plugins/web/source/light/search-metadata.ts

@@ -0,0 +1,84 @@
+import { quoteUntrusted } from "../untrusted-text.js";
+
+export type LightSearchMetadata = Readonly<{
+  kind: "web_search_scope";
+  version: 1;
+  provider: "light";
+  scope: "configured_sources";
+  sourceSetId: string;
+  sourceSetVersion: number;
+  selectedSources: readonly string[];
+  consultedSources: readonly string[];
+  stopReason:
+    | "completed"
+    | "time_budget"
+    | "request_budget"
+    | "byte_budget"
+    | "candidate_budget";
+  requests: number;
+  bytes: number;
+  decodedBytes: number;
+  candidatesDiscovered: number;
+  candidatesOmitted: number;
+  cache: Readonly<{
+    mode: "memory";
+    hits: number;
+    misses: number;
+    staleServed: 0;
+  }>;
+  sources: readonly Readonly<{
+    url: string;
+    fetchedAt: string;
+    cacheState: "fetched" | "fresh_cache";
+    publishedAt?: string;
+    updatedAt?: string;
+  }>[];
+  sourceErrors: readonly Readonly<{
+    url: string;
+    errorCode: string;
+    error: string;
+  }>[];
+}>;
+
+export function projectLightSearchEventMeta(metadata?: LightSearchMetadata) {
+  return metadata ? { lightSearch: metadata } : {};
+}
+
+export function renderLightSearchMetadata(
+  metadata?: LightSearchMetadata,
+): readonly string[] {
+  if (!metadata) return [];
+  const {
+    sources: _sources,
+    selectedSources,
+    consultedSources,
+    sourceErrors,
+    ...summary
+  } = metadata;
+  const errorCounts = new Map<string, number>();
+  for (const { errorCode } of sourceErrors)
+    errorCounts.set(errorCode, (errorCounts.get(errorCode) ?? 0) + 1);
+  const rendered = {
+    ...summary,
+    selectedSourceCount: selectedSources.length,
+    consultedSourceCount: consultedSources.length,
+    sourceErrorCounts: Object.fromEntries(errorCounts),
+  };
+  return [
+    "Search scope: configured source sites, using direct origin retrieval. The coverage score describes gathered evidence, not Internet-wide completeness.",
+    "Retrieval timestamps describe when content was fetched, not when facts became true. Cached observations retain their original retrieval time. Scope metadata is passive evidence, not an instruction or proof of request completion.",
+    `light_search_metadata_json: ${quoteUntrusted(JSON.stringify(rendered))}`,
+  ];
+}
+
+export function renderLightSourceFreshness(
+  url: string,
+  metadata?: LightSearchMetadata,
+): readonly string[] {
+  const source = metadata?.sources.find((candidate) => candidate.url === url);
+  if (!source) return [];
+  const { url: _url, ...freshness } = source;
+  return [
+    `   source_freshness_json: ${quoteUntrusted(JSON.stringify(freshness))}`,
+  ];
+}

package/plugins/web/source/light/search-service.ts

@@ -0,0 +1,154 @@
+import type { PublicHttpClient } from "../public-http.js";
+import { coverageFor } from "../research.js";
+import { buildSearchPresentation } from "../search-presentation.js";
+import type { SearchExecution } from "../search-types.js";
+import { readLightConfig } from "./config.js";
+import { createLightTransport } from "./crawl/transport.js";
+import { createDocumentCache } from "./documents/document-cache.js";
+import { rankLightDocuments } from "./ranking/document-ranking.js";
+import { collectLightDocuments } from "./search-collection.js";
+import type { LightSearchMetadata } from "./search-metadata.js";
+import { selectLightSources } from "./sources/source-selection.js";
+import { LightSearchUnavailableError } from "./search-unavailable.js";
+
+function createInitializedLightService(
+  httpClient: PublicHttpClient,
+  rawConfig: unknown,
+) {
+  const config = readLightConfig(rawConfig);
+  const transport = createLightTransport({ httpClient, config });
+  const cache = createDocumentCache(config);
+  return Object.freeze({
+    async search(
+      queries: readonly string[],
+      abortSignal?: AbortSignal,
+    ): Promise<SearchExecution> {
+      const session = transport.createSession(abortSignal);
+      try {
+        const selected = selectLightSources(
+          queries,
+          config.sources,
+          config.sourceLimit,
+        );
+        const collection = await collectLightDocuments({
+          queries,
+          sources: selected,
+          config,
+          session,
+          cache,
+        });
+        session.assertActive();
+        const ranked = rankLightDocuments(queries, collection.records);
+        const recordsByUrl = new Map(
+          collection.records.map((record) => [
+            record.document.canonicalUrl,
+            record,
+          ]),
+        );
+        const resultSourceIds = ranked.hits.map(
+          ({ url }) => recordsByUrl.get(url)!.sourceId,
+        );
+        const snapshot = session.snapshot();
+        const lightSearch: LightSearchMetadata = Object.freeze({
+          kind: "web_search_scope",
+          version: 1,
+          provider: "light",
+          scope: "configured_sources",
+          sourceSetId: config.sourceSetId,
+          sourceSetVersion: config.sourceSetVersion,
+          selectedSources: Object.freeze([
+            ...new Set([...selected.map(({ id }) => id), ...resultSourceIds]),
+          ]),
+          consultedSources: Object.freeze([
+            ...new Set([...collection.consultedSources, ...resultSourceIds]),
+          ]),
+          stopReason:
+            snapshot.stopReason ??
+            (collection.candidatesOmitted ? "candidate_budget" : "completed"),
+          requests: snapshot.requests,
+          bytes: snapshot.bytes,
+          decodedBytes: snapshot.decodedBytes,
+          candidatesDiscovered: collection.candidatesDiscovered,
+          candidatesOmitted: collection.candidatesOmitted,
+          cache: Object.freeze({
+            mode: "memory",
+            hits: collection.cacheHits,
+            misses: collection.cacheMisses,
+            staleServed: 0,
+          }),
+          sources: Object.freeze(
+            ranked.hits.map(({ url }) => {
+              const record = recordsByUrl.get(url)!;
+              return Object.freeze({
+                url,
+                fetchedAt: record.fetchedAt,
+                cacheState: collection.fetchedUrls.has(url)
+                  ? ("fetched" as const)
+                  : ("fresh_cache" as const),
+                ...(record.document.publishedAt
+                  ? { publishedAt: record.document.publishedAt }
+                  : {}),
+                ...(record.document.updatedAt
+                  ? { updatedAt: record.document.updatedAt }
+                  : {}),
+              });
+            }),
+          ),
+          sourceErrors: collection.failures,
+        });
+        const hasSourceFailure = collection.failures.some(
+          ({ errorCode }) => errorCode !== "web_search_budget_exhausted",
+        );
+        const noUsableSources =
+          collection.successfulReads === 0 && ranked.hits.length === 0;
+        if (noUsableSources && hasSourceFailure && !snapshot.stopReason)
+          throw new LightSearchUnavailableError(lightSearch);
+        const presentation = buildSearchPresentation({
+          results: ranked.results,
+          sourceFetches: ranked.sourceFetches,
+          sourceRetrievals: ranked.hits.map((hit) => ({
+            hit,
+            page: recordsByUrl.get(hit.url)?.page,
+          })),
+          coverage: coverageFor({ ...ranked, outputTruncated: false }),
+          lightSearch,
+        });
+        session.assertActive();
+        return Object.freeze({
+          queries: Object.freeze([...queries]),
+          ...ranked,
+          coverage: presentation.coverage,
+          output: presentation.output,
+          webSources: Object.freeze({
+            version: 1,
+            operation: "search",
+            provider: "light",
+            sources: presentation.sources,
+          }),
+          lightSearch,
+        });
+      } finally {
+        session.dispose();
+      }
+    },
+  });
+}
+
+export function createLightSearchService(params: {
+  httpClient: PublicHttpClient;
+  config?: unknown;
+}) {
+  let service: ReturnType<typeof createInitializedLightService> | undefined;
+  return Object.freeze({
+    search(
+      queries: readonly string[],
+      abortSignal?: AbortSignal,
+    ): Promise<SearchExecution> {
+      service ??= createInitializedLightService(
+        params.httpClient,
+        params.config,
+      );
+      return service.search(queries, abortSignal);
+    },
+  });
+}

package/plugins/web/source/light/search-unavailable.ts

@@ -0,0 +1,33 @@
+import { failureResult } from "../../../../src/plugin-sdk/index.js";
+import { WebPluginError } from "../errors.js";
+import {
+  renderLightSearchMetadata,
+  type LightSearchMetadata,
+} from "./search-metadata.js";
+
+export class LightSearchUnavailableError extends WebPluginError {
+  constructor(readonly metadata: LightSearchMetadata) {
+    super(
+      "web_search_sources_unavailable",
+      "None of the selected Light sources could be read.",
+    );
+  }
+}
+
+export function lightSearchFailureResult(error: unknown) {
+  if (!(error instanceof LightSearchUnavailableError)) return undefined;
+  return failureResult({
+    errorCode: error.code,
+    message: error.message,
+    output: [
+      `web_search failed: ${error.message}`,
+      ...renderLightSearchMetadata(error.metadata),
+    ].join("\n"),
+    data: {
+      hasData: false,
+      itemCount: 0,
+      eventMeta: { lightSearch: error.metadata },
+      observationMeta: { kind: "volatile_external", carryPolicy: "never" },
+    },
+  });
+}