@abot-ai/runtime 1.1.0 → 1.2.0

package/plugins/web/source/light/sources/catalog.ts

@@ -0,0 +1,312 @@
+import type { LightSource } from "./source-definition.js";
+
+export const LIGHT_SOURCE_SET_ID = "abot-light-public-sources";
+export const LIGHT_SOURCE_SET_VERSION = 1;
+
+// Explicit origin pages and publisher feeds. This is a bounded source catalog,
+// not an Internet index or a promise that every source is currently reachable.
+export const DEFAULT_LIGHT_SOURCES: readonly LightSource[] = Object.freeze(
+  [
+    {
+      id: "israel-government",
+      title: "Israel Government",
+      description:
+        "Official Israeli government and Prime Minister's Office publications.",
+      keywords: [
+        "government",
+        "israel",
+        "prime minister",
+        "ממשלה",
+        "ראש ממשלה",
+        "ישראל",
+      ],
+      languages: ["he", "en"],
+      entryUrls: [
+        "https://www.gov.il/he/departments/prime_ministers_office",
+        "https://www.gov.il/en/departments/prime_ministers_office",
+      ],
+      allowedOrigins: ["https://www.gov.il"],
+    },
+    {
+      id: "uk-government",
+      title: "UK Government",
+      description:
+        "Official UK government leadership and public announcements.",
+      keywords: [
+        "government",
+        "uk",
+        "britain",
+        "prime minister",
+        "politics",
+        "ממשלה",
+        "בריטניה",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://www.gov.uk/government/ministers/prime-minister",
+        "https://www.gov.uk/government/organisations/prime-ministers-office-10-downing-street",
+      ],
+      allowedOrigins: ["https://www.gov.uk"],
+    },
+    {
+      id: "wikipedia-en",
+      title: "Wikipedia English",
+      description: "General reference articles and current events in English.",
+      keywords: [
+        "reference",
+        "encyclopedia",
+        "government",
+        "history",
+        "science",
+        "israel",
+        "prime minister",
+        "ויקיפדיה",
+        "ידע",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://en.wikipedia.org/wiki/Portal:Current_events",
+        "https://en.wikipedia.org/wiki/Prime_Minister_of_Israel",
+      ],
+      allowedOrigins: ["https://en.wikipedia.org"],
+    },
+    {
+      id: "wikipedia-he",
+      title: "Wikipedia Hebrew",
+      description: "General reference articles in Hebrew.",
+      keywords: [
+        "reference",
+        "encyclopedia",
+        "israel",
+        "ויקיפדיה",
+        "ידע",
+        "ישראל",
+        "ראש ממשלה",
+        "היסטוריה",
+        "מדע",
+      ],
+      languages: ["he"],
+      entryUrls: [
+        "https://he.wikipedia.org/wiki/ראש_ממשלת_ישראל",
+        "https://he.wikipedia.org/wiki/ישראל",
+      ],
+      allowedOrigins: ["https://he.wikipedia.org"],
+    },
+    {
+      id: "openai",
+      title: "OpenAI",
+      description: "OpenAI product, AI research, and company publications.",
+      keywords: [
+        "openai",
+        "gpt",
+        "chatgpt",
+        "ai",
+        "artificial intelligence",
+        "בינה מלאכותית",
+        "מודלים",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://openai.com/news/rss.xml",
+        "https://openai.com/news/",
+      ],
+      allowedOrigins: ["https://openai.com"],
+    },
+    {
+      id: "nodejs",
+      title: "Node.js",
+      description:
+        "Official Node.js releases, runtime news, and project announcements.",
+      keywords: [
+        "node",
+        "nodejs",
+        "node.js",
+        "javascript",
+        "runtime",
+        "release",
+        "software",
+        "תוכנה",
+        "פיתוח",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://nodejs.org/en/feed/blog.xml",
+        "https://nodejs.org/en/blog",
+      ],
+      allowedOrigins: ["https://nodejs.org"],
+    },
+    {
+      id: "nasa",
+      title: "NASA",
+      description: "Space missions, astronomy, and science news from NASA.",
+      keywords: [
+        "nasa",
+        "space",
+        "science",
+        "astronomy",
+        "research",
+        "חלל",
+        "מדע",
+      ],
+      languages: ["en"],
+      entryUrls: [
+        "https://www.nasa.gov/feed/",
+        "https://www.nasa.gov/news/all-news/",
+      ],
+      allowedOrigins: ["https://www.nasa.gov", "https://science.nasa.gov"],
+    },
+    {
+      id: "bbc-world",
+      title: "BBC World",
+      description: "International news from BBC News.",
+      keywords: [
+        "world",
+        "international",
+        "news",
+        "politics",
+        "government",
+        "חדשות",
+        "עולם",
+        "ממשלה",
+      ],
+      languages: ["en"],
+      entryUrls: ["https://feeds.bbci.co.uk/news/world/rss.xml"],
+      allowedOrigins: [
+        "https://feeds.bbci.co.uk",
+        "https://www.bbc.com",
+        "https://www.bbc.co.uk",
+        "https://bbc.com",
+      ],
+    },
+    {
+      id: "guardian-world",
+      title: "The Guardian World",
+      description: "International affairs and world news.",
+      keywords: ["world", "international", "news", "politics", "חדשות", "עולם"],
+      languages: ["en"],
+      entryUrls: ["https://www.theguardian.com/world/rss"],
+      allowedOrigins: ["https://www.theguardian.com"],
+    },
+    {
+      id: "cbc-world",
+      title: "CBC World",
+      description: "World news and international reporting.",
+      keywords: ["world", "international", "news", "canada", "חדשות", "עולם"],
+      languages: ["en"],
+      entryUrls: ["https://www.cbc.ca/webfeed/rss/rss-world"],
+      allowedOrigins: ["https://www.cbc.ca"],
+    },
+    {
+      id: "geektime",
+      title: "Geektime",
+      description:
+        "Technology, startups, software, and AI reporting in Hebrew.",
+      keywords: [
+        "technology",
+        "ai",
+        "gpt",
+        "software",
+        "startups",
+        "טכנולוגיה",
+        "בינה מלאכותית",
+        "כתבות",
+        "תוכנה",
+      ],
+      languages: ["he"],
+      entryUrls: ["https://www.geektime.co.il/feed/"],
+      allowedOrigins: ["https://www.geektime.co.il"],
+    },
+    {
+      id: "techcrunch",
+      title: "TechCrunch",
+      description: "Technology companies, startups, and AI news.",
+      keywords: [
+        "technology",
+        "ai",
+        "gpt",
+        "software",
+        "startups",
+        "טכנולוגיה",
+        "בינה מלאכותית",
+        "כתבות",
+      ],
+      languages: ["en"],
+      entryUrls: ["https://techcrunch.com/feed/"],
+      allowedOrigins: ["https://techcrunch.com"],
+    },
+    {
+      id: "ynet",
+      title: "Ynet",
+      description: "Israeli general news headlines in Hebrew.",
+      keywords: [
+        "israel",
+        "news",
+        "politics",
+        "ישראל",
+        "חדשות",
+        "ממשלה",
+        "ראש ממשלה",
+      ],
+      languages: ["he"],
+      entryUrls: ["https://www.ynet.co.il/Integration/StoryRss2.xml"],
+      allowedOrigins: ["https://www.ynet.co.il"],
+    },
+    {
+      id: "walla",
+      title: "Walla News",
+      description: "Israeli current affairs and general news in Hebrew.",
+      keywords: ["israel", "news", "politics", "ישראל", "חדשות", "ממשלה"],
+      languages: ["he"],
+      entryUrls: ["https://rss.walla.co.il/feed/1"],
+      allowedOrigins: [
+        "https://rss.walla.co.il",
+        "https://news.walla.co.il",
+        "https://www.walla.co.il",
+      ],
+    },
+    {
+      id: "times-of-israel",
+      title: "The Times of Israel",
+      description: "Israeli and regional news in English.",
+      keywords: [
+        "israel",
+        "news",
+        "politics",
+        "government",
+        "prime minister",
+        "ישראל",
+        "חדשות",
+        "ראש ממשלה",
+      ],
+      languages: ["en"],
+      entryUrls: ["https://www.timesofisrael.com/feed/"],
+      allowedOrigins: ["https://www.timesofisrael.com"],
+    },
+    {
+      id: "nature",
+      title: "Nature",
+      description: "Science news and research publications.",
+      keywords: [
+        "science",
+        "research",
+        "nature",
+        "ai",
+        "biology",
+        "physics",
+        "מדע",
+        "מחקר",
+      ],
+      languages: ["en"],
+      entryUrls: ["https://www.nature.com/nature.rss"],
+      allowedOrigins: ["https://www.nature.com"],
+    },
+  ].map((source) =>
+    Object.freeze({
+      ...source,
+      keywords: Object.freeze(source.keywords),
+      languages: Object.freeze(source.languages),
+      entryUrls: Object.freeze(source.entryUrls),
+      allowedOrigins: Object.freeze(source.allowedOrigins),
+    }),
+  ),
+);

package/plugins/web/source/light/sources/source-definition.ts

@@ -0,0 +1,9 @@
+export type LightSource = Readonly<{
+  id: string;
+  title: string;
+  description: string;
+  keywords: readonly string[];
+  languages: readonly string[];
+  entryUrls: readonly string[];
+  allowedOrigins: readonly string[];
+}>;

package/plugins/web/source/light/sources/source-selection.ts

@@ -0,0 +1,35 @@
+import { countTermMatches } from "../ranking/query-normalization.js";
+import type { LightSource } from "./source-definition.js";
+
+export function selectLightSources(
+  queries: readonly string[],
+  sources: readonly LightSource[],
+  limit: number,
+): readonly LightSource[] {
+  const rankings = queries.map((query) => {
+    const ranked = sources
+      .map((source, index) => ({
+        source,
+        index,
+        score: countTermMatches(
+          query,
+          `${source.title} ${source.description} ${source.keywords.join(" ")}`,
+        ),
+      }))
+      .sort(
+        (left, right) => right.score - left.score || left.index - right.index,
+      );
+    const matching = ranked.filter(({ score }) => score > 0);
+    if (matching.length > 0) return matching;
+    return ranked;
+  });
+  const selected = new Map<string, LightSource>();
+  for (let rank = 0; rank < sources.length; rank += 1) {
+    for (const ranking of rankings) {
+      if (selected.size >= limit) return Object.freeze([...selected.values()]);
+      const item = ranking[rank];
+      if (item) selected.set(item.source.id, item.source);
+    }
+  }
+  return Object.freeze([...selected.values()]);
+}

package/plugins/web/source/plugin.ts

@@ -2,13 +2,13 @@ import {
   failureFromError,
   failureResult,
   readBoundedInteger,
-  successResult,
   type RuntimePluginEntrypoint,
   type RuntimePluginLoadContext,
 } from "../../../src/plugin-sdk/index.js";
 
 import { isWebPluginError } from "./errors.js";
-import { createWebFetchService, formatFetchedPages } from "./fetch-service.js";
+import { createWebFetchService } from "./fetch-service.js";
+import { buildFetchPresentation } from "./fetch-presentation.js";
 import { WEB_LIMITS } from "./limits.js";
 import {
   parseFetchParams,
@@ -20,13 +20,18 @@ import {
   createPublicHttpClient,
   type PublicHttpClient,
 } from "./public-http.js";
-import { createBraveSearchService } from "./search-service.js";
+import { selectWebSearchService } from "./search-service-selector.js";
+import { projectLightSearchEventMeta } from "./light/search-metadata.js";
+import { lightSearchFailureResult } from "./light/search-unavailable.js";
+import { successWithSourceReceipts } from "./source-receipt-budget.js";
 
 export type WebPluginDependencies = Readonly<{
   httpClient?: PublicHttpClient;
 }>;
 
 function failure(error: unknown, operation: string) {
+  const lightFailure = lightSearchFailureResult(error);
+  if (lightFailure) return lightFailure;
   if (isWebPluginError(error)) {
     return failureResult({
       errorCode: error.code,
@@ -54,11 +59,12 @@ export function createWebPlugin(
     maximum: 10_000,
     defaultValue: WEB_LIMITS.retryBaseMs,
   });
-  const searchService = createBraveSearchService({
+  const searchService = selectWebSearchService({
     apiKey: braveApiKey,
     retryBaseMs,
     httpClient,
     fetchService,
+    lightConfig: context.config?.light,
   });
 
   const handlers: RuntimePluginEntrypoint["handlers"] = {
@@ -69,25 +75,29 @@ export function createWebPlugin(
           urls,
           executionContext?.abortSignal,
         );
-        return successResult({
-          output: formatFetchedPages(pages),
-          progress: true,
-          producedNewInformation: true,
-          data: {
-            hasData: true,
-            itemCount: pages.length,
-            eventMeta: {
-              urls: pages.map(({ finalUrl }) => finalUrl),
-              partialUrls: pages
-                .filter(({ partialContent }) => partialContent)
-                .map(({ finalUrl }) => finalUrl),
-            },
-            observationMeta: {
-              kind: "volatile_external",
-              carryPolicy: "never",
+        const presentation = buildFetchPresentation(pages);
+        return successWithSourceReceipts(
+          {
+            output: presentation.output,
+            progress: true,
+            producedNewInformation: true,
+            data: {
+              hasData: true,
+              itemCount: pages.length,
+              eventMeta: {
+                urls: pages.map(({ finalUrl }) => finalUrl),
+                partialUrls: pages
+                  .filter(({ partialContent }) => partialContent)
+                  .map(({ finalUrl }) => finalUrl),
+              },
+              observationMeta: {
+                kind: "volatile_external",
+                carryPolicy: "never",
+              },
             },
           },
-        });
+          { version: 1, operation: "fetch", sources: presentation.sources },
+        );
       } catch (error) {
         return failure(error, "web_fetch");
       }
@@ -99,43 +109,47 @@ export function createWebPlugin(
           queries,
           executionContext?.abortSignal,
         );
-        return successResult({
-          output: search.output,
-          progress: search.hits.length > 0,
-          producedNewInformation: search.hits.length > 0,
-          data: {
-            hasData: search.hits.length > 0,
-            itemCount: search.hits.length,
-            eventMeta: {
-              query: queries[0],
-              queries,
-              urls: search.hits.map(({ url }) => url),
-              fetchedUrls: search.sourceFetches.flatMap(({ page }) =>
-                page ? [page.finalUrl] : [],
-              ),
-              partialFetchedUrls: search.sourceFetches.flatMap(({ page }) =>
-                page?.partialContent ? [page.finalUrl] : [],
-              ),
-              sourceFetchErrors: search.sourceFetches.flatMap(
-                ({ hit, errorCode, error }) =>
-                  error
-                    ? [
-                        {
-                          url: hit.url,
-                          errorCode,
-                          error,
-                        },
-                      ]
-                    : [],
-              ),
-              coverage: search.coverage,
-            },
-            observationMeta: {
-              kind: "volatile_external",
-              carryPolicy: "never",
+        return successWithSourceReceipts(
+          {
+            output: search.output,
+            progress: search.hits.length > 0,
+            producedNewInformation: search.hits.length > 0,
+            data: {
+              hasData: search.hits.length > 0,
+              itemCount: search.hits.length,
+              eventMeta: {
+                query: queries[0],
+                queries,
+                urls: search.hits.map(({ url }) => url),
+                fetchedUrls: search.sourceFetches.flatMap(({ page }) =>
+                  page ? [page.finalUrl] : [],
+                ),
+                partialFetchedUrls: search.sourceFetches.flatMap(({ page }) =>
+                  page?.partialContent ? [page.finalUrl] : [],
+                ),
+                sourceFetchErrors: search.sourceFetches.flatMap(
+                  ({ hit, errorCode, error }) =>
+                    error
+                      ? [
+                          {
+                            url: hit.url,
+                            errorCode,
+                            error,
+                          },
+                        ]
+                      : [],
+                ),
+                coverage: search.coverage,
+                ...projectLightSearchEventMeta(search.lightSearch),
+              },
+              observationMeta: {
+                kind: "volatile_external",
+                carryPolicy: "never",
+              },
             },
           },
-        });
+          search.webSources,
+        );
       } catch (error) {
         return failure(error, "web_search");
       }

package/plugins/web/source/public-http.ts

@@ -27,6 +27,7 @@ export type PublicHttpRequest = Readonly<{
   headers?: Readonly<Record<string, string>>;
   maxBytes?: number;
   maxRedirects?: number;
+  followRedirects?: boolean;
   timeoutMs?: number;
   abortSignal?: AbortSignal;
 }>;
@@ -37,6 +38,14 @@ export type PublicHttpClient = Readonly<{
 
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 
+function canFollowHttpRedirect(
+  request: PublicHttpRequest,
+  status: number,
+): boolean {
+  if (request.followRedirects === false) return false;
+  return REDIRECT_STATUSES.has(status);
+}
+
 function redirectLocation(response: HopResponse): string | undefined {
   const raw = response.headers.location;
   return Array.isArray(raw) ? raw[0] : raw;
@@ -107,7 +116,7 @@ export function createPublicHttpClient(
           deadline,
           request.abortSignal,
         );
-        if (!REDIRECT_STATUSES.has(response.status)) {
+        if (!canFollowHttpRedirect(request, response.status)) {
           return Object.freeze({
             requestedUrl: requested.toString(),
             finalUrl: current.toString(),