@abot-ai/runtime 1.1.0 → 1.3.0

package/plugins/web/source/light/search-metadata.ts

@@ -0,0 +1,84 @@
+import { quoteUntrusted } from "../untrusted-text.js";
+
+export type LightSearchMetadata = Readonly<{
+  kind: "web_search_scope";
+  version: 1;
+  provider: "light";
+  scope: "configured_sources";
+  sourceSetId: string;
+  sourceSetVersion: number;
+  selectedSources: readonly string[];
+  consultedSources: readonly string[];
+  stopReason:
+    | "completed"
+    | "time_budget"
+    | "request_budget"
+    | "byte_budget"
+    | "candidate_budget";
+  requests: number;
+  bytes: number;
+  decodedBytes: number;
+  candidatesDiscovered: number;
+  candidatesOmitted: number;
+  cache: Readonly<{
+    mode: "memory";
+    hits: number;
+    misses: number;
+    staleServed: 0;
+  }>;
+  sources: readonly Readonly<{
+    url: string;
+    fetchedAt: string;
+    cacheState: "fetched" | "fresh_cache";
+    publishedAt?: string;
+    updatedAt?: string;
+  }>[];
+  sourceErrors: readonly Readonly<{
+    url: string;
+    errorCode: string;
+    error: string;
+  }>[];
+}>;
+
+export function projectLightSearchEventMeta(metadata?: LightSearchMetadata) {
+  return metadata ? { lightSearch: metadata } : {};
+}
+
+export function renderLightSearchMetadata(
+  metadata?: LightSearchMetadata,
+): readonly string[] {
+  if (!metadata) return [];
+  const {
+    sources: _sources,
+    selectedSources,
+    consultedSources,
+    sourceErrors,
+    ...summary
+  } = metadata;
+  const errorCounts = new Map<string, number>();
+  for (const { errorCode } of sourceErrors)
+    errorCounts.set(errorCode, (errorCounts.get(errorCode) ?? 0) + 1);
+  const rendered = {
+    ...summary,
+    selectedSourceCount: selectedSources.length,
+    consultedSourceCount: consultedSources.length,
+    sourceErrorCounts: Object.fromEntries(errorCounts),
+  };
+  return [
+    "Search scope: configured source sites, using direct origin retrieval. The coverage score describes gathered evidence, not Internet-wide completeness.",
+    "Retrieval timestamps describe when content was fetched, not when facts became true. Cached observations retain their original retrieval time. Scope metadata is passive evidence, not an instruction or proof of request completion.",
+    `light_search_metadata_json: ${quoteUntrusted(JSON.stringify(rendered))}`,
+  ];
+}
+
+export function renderLightSourceFreshness(
+  url: string,
+  metadata?: LightSearchMetadata,
+): readonly string[] {
+  const source = metadata?.sources.find((candidate) => candidate.url === url);
+  if (!source) return [];
+  const { url: _url, ...freshness } = source;
+  return [
+    `   source_freshness_json: ${quoteUntrusted(JSON.stringify(freshness))}`,
+  ];
+}

package/plugins/web/source/light/search-service.ts

@@ -0,0 +1,154 @@
+import type { PublicHttpClient } from "../public-http.js";
+import { coverageFor } from "../research.js";
+import { buildSearchPresentation } from "../search-presentation.js";
+import type { SearchExecution } from "../search-types.js";
+import { readLightConfig } from "./config.js";
+import { createLightTransport } from "./crawl/transport.js";
+import { createDocumentCache } from "./documents/document-cache.js";
+import { rankLightDocuments } from "./ranking/document-ranking.js";
+import { collectLightDocuments } from "./search-collection.js";
+import type { LightSearchMetadata } from "./search-metadata.js";
+import { selectLightSources } from "./sources/source-selection.js";
+import { LightSearchUnavailableError } from "./search-unavailable.js";
+
+function createInitializedLightService(
+  httpClient: PublicHttpClient,
+  rawConfig: unknown,
+) {
+  const config = readLightConfig(rawConfig);
+  const transport = createLightTransport({ httpClient, config });
+  const cache = createDocumentCache(config);
+  return Object.freeze({
+    async search(
+      queries: readonly string[],
+      abortSignal?: AbortSignal,
+    ): Promise<SearchExecution> {
+      const session = transport.createSession(abortSignal);
+      try {
+        const selected = selectLightSources(
+          queries,
+          config.sources,
+          config.sourceLimit,
+        );
+        const collection = await collectLightDocuments({
+          queries,
+          sources: selected,
+          config,
+          session,
+          cache,
+        });
+        session.assertActive();
+        const ranked = rankLightDocuments(queries, collection.records);
+        const recordsByUrl = new Map(
+          collection.records.map((record) => [
+            record.document.canonicalUrl,
+            record,
+          ]),
+        );
+        const resultSourceIds = ranked.hits.map(
+          ({ url }) => recordsByUrl.get(url)!.sourceId,
+        );
+        const snapshot = session.snapshot();
+        const lightSearch: LightSearchMetadata = Object.freeze({
+          kind: "web_search_scope",
+          version: 1,
+          provider: "light",
+          scope: "configured_sources",
+          sourceSetId: config.sourceSetId,
+          sourceSetVersion: config.sourceSetVersion,
+          selectedSources: Object.freeze([
+            ...new Set([...selected.map(({ id }) => id), ...resultSourceIds]),
+          ]),
+          consultedSources: Object.freeze([
+            ...new Set([...collection.consultedSources, ...resultSourceIds]),
+          ]),
+          stopReason:
+            snapshot.stopReason ??
+            (collection.candidatesOmitted ? "candidate_budget" : "completed"),
+          requests: snapshot.requests,
+          bytes: snapshot.bytes,
+          decodedBytes: snapshot.decodedBytes,
+          candidatesDiscovered: collection.candidatesDiscovered,
+          candidatesOmitted: collection.candidatesOmitted,
+          cache: Object.freeze({
+            mode: "memory",
+            hits: collection.cacheHits,
+            misses: collection.cacheMisses,
+            staleServed: 0,
+          }),
+          sources: Object.freeze(
+            ranked.hits.map(({ url }) => {
+              const record = recordsByUrl.get(url)!;
+              return Object.freeze({
+                url,
+                fetchedAt: record.fetchedAt,
+                cacheState: collection.fetchedUrls.has(url)
+                  ? ("fetched" as const)
+                  : ("fresh_cache" as const),
+                ...(record.document.publishedAt
+                  ? { publishedAt: record.document.publishedAt }
+                  : {}),
+                ...(record.document.updatedAt
+                  ? { updatedAt: record.document.updatedAt }
+                  : {}),
+              });
+            }),
+          ),
+          sourceErrors: collection.failures,
+        });
+        const hasSourceFailure = collection.failures.some(
+          ({ errorCode }) => errorCode !== "web_search_budget_exhausted",
+        );
+        const noUsableSources =
+          collection.successfulReads === 0 && ranked.hits.length === 0;
+        if (noUsableSources && hasSourceFailure && !snapshot.stopReason)
+          throw new LightSearchUnavailableError(lightSearch);
+        const presentation = buildSearchPresentation({
+          results: ranked.results,
+          sourceFetches: ranked.sourceFetches,
+          sourceRetrievals: ranked.hits.map((hit) => ({
+            hit,
+            page: recordsByUrl.get(hit.url)?.page,
+          })),
+          coverage: coverageFor({ ...ranked, outputTruncated: false }),
+          lightSearch,
+        });
+        session.assertActive();
+        return Object.freeze({
+          queries: Object.freeze([...queries]),
+          ...ranked,
+          coverage: presentation.coverage,
+          output: presentation.output,
+          webSources: Object.freeze({
+            version: 1,
+            operation: "search",
+            provider: "light",
+            sources: presentation.sources,
+          }),
+          lightSearch,
+        });
+      } finally {
+        session.dispose();
+      }
+    },
+  });
+}
+
+export function createLightSearchService(params: {
+  httpClient: PublicHttpClient;
+  config?: unknown;
+}) {
+  let service: ReturnType<typeof createInitializedLightService> | undefined;
+  return Object.freeze({
+    search(
+      queries: readonly string[],
+      abortSignal?: AbortSignal,
+    ): Promise<SearchExecution> {
+      service ??= createInitializedLightService(
+        params.httpClient,
+        params.config,
+      );
+      return service.search(queries, abortSignal);
+    },
+  });
+}

package/plugins/web/source/light/search-unavailable.ts

@@ -0,0 +1,33 @@
+import { failureResult } from "../../../../src/plugin-sdk/index.js";
+import { WebPluginError } from "../errors.js";
+import {
+  renderLightSearchMetadata,
+  type LightSearchMetadata,
+} from "./search-metadata.js";
+
+export class LightSearchUnavailableError extends WebPluginError {
+  constructor(readonly metadata: LightSearchMetadata) {
+    super(
+      "web_search_sources_unavailable",
+      "None of the selected Light sources could be read.",
+    );
+  }
+}
+
+export function lightSearchFailureResult(error: unknown) {
+  if (!(error instanceof LightSearchUnavailableError)) return undefined;
+  return failureResult({
+    errorCode: error.code,
+    message: error.message,
+    output: [
+      `web_search failed: ${error.message}`,
+      ...renderLightSearchMetadata(error.metadata),
+    ].join("\n"),
+    data: {
+      hasData: false,
+      itemCount: 0,
+      eventMeta: { lightSearch: error.metadata },
+      observationMeta: { kind: "volatile_external", carryPolicy: "never" },
+    },
+  });
+}

package/plugins/web/source/light/sources/catalog.ts

@@ -0,0 +1,312 @@
+import type { LightSource } from "./source-definition.js";
+
+export const LIGHT_SOURCE_SET_ID = "abot-light-public-sources";
+export const LIGHT_SOURCE_SET_VERSION = 1;
+
+// Explicit origin pages and publisher feeds. This is a bounded source catalog,
+// not an Internet index or a promise that every source is currently reachable.
+export const DEFAULT_LIGHT_SOURCES: readonly LightSource[] = Object.freeze(
+  [
+    {
+      id: "israel-government",
+      title: "Israel Government",
+      description:
+        "Official Israeli government and Prime Minister's Office publications.",
+      keywords: [
+        "government",
+        "israel",
+        "prime minister",
+        "ממשלה",
+        "ראש ממשלה",
+        "ישראל",
+      ],
+      languages: ["he", "en"],
+      entryUrls: [
+        "https://www.gov.il/he/departments/prime_ministers_office",
+        "https://www.gov.il/en/departments/prime_ministers_office",
+      ],
+      allowedOrigins: ["https://www.gov.il"],
+    },
+    {
+      id: "uk-government",
+      title: "UK Government",
+      description:
+        "Official UK government leadership and public announcements.",
+      keywords: [
+        "government",
+        "uk",
+        "britain",
+        "prime minister",
+        "politics",
+        "ממשלה",
+        "בריטניה",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://www.gov.uk/government/ministers/prime-minister",
+        "https://www.gov.uk/government/organisations/prime-ministers-office-10-downing-street",
+      ],
+      allowedOrigins: ["https://www.gov.uk"],
+    },
+    {
+      id: "wikipedia-en",
+      title: "Wikipedia English",
+      description: "General reference articles and current events in English.",
+      keywords: [
+        "reference",
+        "encyclopedia",
+        "government",
+        "history",
+        "science",
+        "israel",
+        "prime minister",
+        "ויקיפדיה",
+        "ידע",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://en.wikipedia.org/wiki/Portal:Current_events",
+        "https://en.wikipedia.org/wiki/Prime_Minister_of_Israel",
+      ],
+      allowedOrigins: ["https://en.wikipedia.org"],
+    },
+    {
+      id: "wikipedia-he",
+      title: "Wikipedia Hebrew",
+      description: "General reference articles in Hebrew.",
+      keywords: [
+        "reference",
+        "encyclopedia",
+        "israel",
+        "ויקיפדיה",
+        "ידע",
+        "ישראל",
+        "ראש ממשלה",
+        "היסטוריה",
+        "מדע",
+      ],
+      languages: ["he"],
+      entryUrls: [
+        "https://he.wikipedia.org/wiki/ראש_ממשלת_ישראל",
+        "https://he.wikipedia.org/wiki/ישראל",
+      ],
+      allowedOrigins: ["https://he.wikipedia.org"],
+    },
+    {
+      id: "openai",
+      title: "OpenAI",
+      description: "OpenAI product, AI research, and company publications.",
+      keywords: [
+        "openai",
+        "gpt",
+        "chatgpt",
+        "ai",
+        "artificial intelligence",
+        "בינה מלאכותית",
+        "מודלים",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://openai.com/news/rss.xml",
+        "https://openai.com/news/",
+      ],
+      allowedOrigins: ["https://openai.com"],
+    },
+    {
+      id: "nodejs",
+      title: "Node.js",
+      description:
+        "Official Node.js releases, runtime news, and project announcements.",
+      keywords: [
+        "node",
+        "nodejs",
+        "node.js",
+        "javascript",
+        "runtime",
+        "release",
+        "software",
+        "תוכנה",
+        "פיתוח",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://nodejs.org/en/feed/blog.xml",
+        "https://nodejs.org/en/blog",
+      ],
+      allowedOrigins: ["https://nodejs.org"],
+    },
+    {
+      id: "nasa",
+      title: "NASA",
+      description: "Space missions, astronomy, and science news from NASA.",
+      keywords: [
+        "nasa",
+        "space",
+        "science",
+        "astronomy",
+        "research",
+        "חלל",
+        "מדע",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://www.nasa.gov/feed/",
+        "https://www.nasa.gov/news/all-news/",
+      ],
+      allowedOrigins: ["https://www.nasa.gov", "https://science.nasa.gov"],
+    },
+    {
+      id: "bbc-world",
+      title: "BBC World",
+      description: "International news from BBC News.",
+      keywords: [
+        "world",
+        "international",
+        "news",
+        "politics",
+        "government",
+        "חדשות",
+        "עולם",
+        "ממשלה",
+      ],
+      languages: ["en"],
+      entryUrls: ["https://feeds.bbci.co.uk/news/world/rss.xml"],
+      allowedOrigins: [
+        "https://feeds.bbci.co.uk",
+        "https://www.bbc.com",
+        "https://www.bbc.co.uk",
+        "https://bbc.com",
+      ],
+    },
+    {
+      id: "guardian-world",
+      title: "The Guardian World",
+      description: "International affairs and world news.",
+      keywords: ["world", "international", "news", "politics", "חדשות", "עולם"],
+      languages: ["en"],
+      entryUrls: ["https://www.theguardian.com/world/rss"],
+      allowedOrigins: ["https://www.theguardian.com"],
+    },
+    {
+      id: "cbc-world",
+      title: "CBC World",
+      description: "World news and international reporting.",
+      keywords: ["world", "international", "news", "canada", "חדשות", "עולם"],
+      languages: ["en"],
+      entryUrls: ["https://www.cbc.ca/webfeed/rss/rss-world"],
+      allowedOrigins: ["https://www.cbc.ca"],
+    },
+    {
+      id: "geektime",
+      title: "Geektime",
+      description:
+        "Technology, startups, software, and AI reporting in Hebrew.",
+      keywords: [
+        "technology",
+        "ai",
+        "gpt",
+        "software",
+        "startups",
+        "טכנולוגיה",
+        "בינה מלאכותית",
+        "כתבות",
+        "תוכנה",
+      ],
+      languages: ["he"],
+      entryUrls: ["https://www.geektime.co.il/feed/"],
+      allowedOrigins: ["https://www.geektime.co.il"],
+    },
+    {
+      id: "techcrunch",
+      title: "TechCrunch",
+      description: "Technology companies, startups, and AI news.",
+      keywords: [
+        "technology",
+        "ai",
+        "gpt",
+        "software",
+        "startups",
+        "טכנולוגיה",
+        "בינה מלאכותית",
+        "כתבות",
+      ],
+      languages: ["en"],
+      entryUrls: ["https://techcrunch.com/feed/"],
+      allowedOrigins: ["https://techcrunch.com"],
+    },
+    {
+      id: "ynet",
+      title: "Ynet",
+      description: "Israeli general news headlines in Hebrew.",
+      keywords: [
+        "israel",
+        "news",
+        "politics",
+        "ישראל",
+        "חדשות",
+        "ממשלה",
+        "ראש ממשלה",
+      ],
+      languages: ["he"],
+      entryUrls: ["https://www.ynet.co.il/Integration/StoryRss2.xml"],
+      allowedOrigins: ["https://www.ynet.co.il"],
+    },
+    {
+      id: "walla",
+      title: "Walla News",
+      description: "Israeli current affairs and general news in Hebrew.",
+      keywords: ["israel", "news", "politics", "ישראל", "חדשות", "ממשלה"],
+      languages: ["he"],
+      entryUrls: ["https://rss.walla.co.il/feed/1"],
+      allowedOrigins: [
+        "https://rss.walla.co.il",
+        "https://news.walla.co.il",
+        "https://www.walla.co.il",
+      ],
+    },
+    {
+      id: "times-of-israel",
+      title: "The Times of Israel",
+      description: "Israeli and regional news in English.",
+      keywords: [
+        "israel",
+        "news",
+        "politics",
+        "government",
+        "prime minister",
+        "ישראל",
+        "חדשות",
+        "ראש ממשלה",
+      ],
+      languages: ["en"],
+      entryUrls: ["https://www.timesofisrael.com/feed/"],
+      allowedOrigins: ["https://www.timesofisrael.com"],
+    },
+    {
+      id: "nature",
+      title: "Nature",
+      description: "Science news and research publications.",
+      keywords: [
+        "science",
+        "research",
+        "nature",
+        "ai",
+        "biology",
+        "physics",
+        "מדע",
+        "מחקר",
+      ],
+      languages: ["en"],
+      entryUrls: ["https://www.nature.com/nature.rss"],
+      allowedOrigins: ["https://www.nature.com"],
+    },
+  ].map((source) =>
+    Object.freeze({
+      ...source,
+      keywords: Object.freeze(source.keywords),
+      languages: Object.freeze(source.languages),
+      entryUrls: Object.freeze(source.entryUrls),
+      allowedOrigins: Object.freeze(source.allowedOrigins),
+    }),
+  ),
+);

package/plugins/web/source/light/sources/source-definition.ts

@@ -0,0 +1,9 @@
+export type LightSource = Readonly<{
+  id: string;
+  title: string;
+  description: string;
+  keywords: readonly string[];
+  languages: readonly string[];
+  entryUrls: readonly string[];
+  allowedOrigins: readonly string[];
+}>;

package/plugins/web/source/light/sources/source-selection.ts

@@ -0,0 +1,35 @@
+import { countTermMatches } from "../ranking/query-normalization.js";
+import type { LightSource } from "./source-definition.js";
+
+export function selectLightSources(
+  queries: readonly string[],
+  sources: readonly LightSource[],
+  limit: number,
+): readonly LightSource[] {
+  const rankings = queries.map((query) => {
+    const ranked = sources
+      .map((source, index) => ({
+        source,
+        index,
+        score: countTermMatches(
+          query,
+          `${source.title} ${source.description} ${source.keywords.join(" ")}`,
+        ),
+      }))
+      .sort(
+        (left, right) => right.score - left.score || left.index - right.index,
+      );
+    const matching = ranked.filter(({ score }) => score > 0);
+    if (matching.length > 0) return matching;
+    return ranked;
+  });
+  const selected = new Map<string, LightSource>();
+  for (let rank = 0; rank < sources.length; rank += 1) {
+    for (const ranking of rankings) {
+      if (selected.size >= limit) return Object.freeze([...selected.values()]);
+      const item = ranking[rank];
+      if (item) selected.set(item.source.id, item.source);
+    }
+  }
+  return Object.freeze([...selected.values()]);
+}

package/plugins/web/source/limits.ts

@@ -1,4 +1,7 @@
+import { PUBLIC_HTTP_LIMITS } from "../../../src/shared/public-http/limits.js";
+
 export const WEB_LIMITS = Object.freeze({
+  ...PUBLIC_HTTP_LIMITS,
   queryMaxChars: 400,
   queryMaxWords: 50,
   searchQueries: 5,
@@ -9,14 +12,9 @@ export const WEB_LIMITS = Object.freeze({
   targetReadableSources: 3,
   fetchUrls: 3,
   fetchConcurrency: 3,
-  httpConcurrency: 6,
-  httpQueueLimit: 64,
-  redirects: 3,
-  responseBytes: 512 * 1024,
   outputChars: 16_000,
   pageOutputChars: 12_000,
   sourceOutputChars: 3_000,
-  requestTimeoutMs: 10_000,
   searchTotalTimeoutMs: 30_000,
   outputBytes: 48 * 1024,
   upstreamUrlChars: 4_096,
@@ -25,5 +23,4 @@ export const WEB_LIMITS = Object.freeze({
   upstreamSnippetChars: 800,
   retryAttempts: 2,
   retryBaseMs: 750,
-  responseHeaderBytes: 32 * 1024,
 } as const);