@hraness/wordcell 0.24.1 → 0.25.1

package/src/clip/metadata-search.ts

@@ -92,6 +92,10 @@ export type SearchProvider = (request: SearchProviderRequest) => Promise<SearchP
 export type RustMetadataSearchProviderOptions = {
   /** Absolute path to the reviewed binary. The file identity is pinned when the provider is created. */
   readonly binaryPath: string;
+  /** Opt in to the fixed, public Bing RSS endpoint. Disabled by default. */
+  readonly enableBingRss?: boolean;
+  /** Opt in to public OpenAlex scholarly discovery; indexed links are not fetched evidence. */
+  readonly enableOpenAlexDiscovery?: boolean;
   readonly defaultMaxResults?: number;
   readonly defaultTimeoutMs?: number;
   readonly processGraceMs?: number;
@@ -483,14 +487,19 @@ function deterministicResponse(response: MetadataSearchResponse, maximumResults:
     enginesQueried,
     enginesFailed: Object.freeze([...response.enginesFailed].sort()),
     engineStatus: response.engineStatus,
+    ...(response.engineFailures === undefined ? {} : { engineFailures: Object.freeze(
+      [...response.engineFailures].sort((a, b) => compareText(a.engine, b.engine)),
+    ) }),
   });
 }
 
 async function resolveEngineHosts(
   resolver: MetadataSearchNetworkResolver,
   timeoutMs: number,
+  enableBingRss: boolean,
+  enableOpenAlexDiscovery: boolean,
 ): Promise<readonly ResolvedEngineHost[]> {
-  const resolved = await Promise.all(METADATA_SEARCH_ENGINE_HOSTS.map(async (hostname) => {
+  const resolved = await Promise.all([...METADATA_SEARCH_ENGINE_HOSTS, ...(enableBingRss ? ["www.bing.com"] : []), ...(enableOpenAlexDiscovery ? ["api.openalex.org"] : [])].map(async (hostname) => {
     const addresses = await resolver(new URL(`https://${hostname}/`), {
       allowPrivateNetwork: false,
       timeoutMs,
@@ -548,6 +557,10 @@ export function createExactUrlSearchQuery(value: string | URL): string | null {
 
 /** Create the isolated adapter for the reviewed Rust metadata-search helper. */
 export function createRustMetadataSearchProvider(options: RustMetadataSearchProviderOptions): SearchProvider {
+  if (options.enableBingRss !== undefined && typeof options.enableBingRss !== "boolean") throw new TypeError("enableBingRss must be boolean.");
+  const enableBingRss = options.enableBingRss === true;
+  if (options.enableOpenAlexDiscovery !== undefined && typeof options.enableOpenAlexDiscovery !== "boolean") throw new TypeError("enableOpenAlexDiscovery must be boolean.");
+  const enableOpenAlexDiscovery = options.enableOpenAlexDiscovery === true;
   if (!isAbsolute(options.binaryPath)) {
     throw new TypeError("The metadata search binary path must be absolute.");
   }
@@ -609,7 +622,7 @@ export function createRustMetadataSearchProvider(options: RustMetadataSearchProv
 
     let engineHosts: readonly ResolvedEngineHost[];
     try {
-      engineHosts = await resolveEngineHosts(resolveNetworkTarget, validated.timeoutMs);
+      engineHosts = await resolveEngineHosts(resolveNetworkTarget, validated.timeoutMs, enableBingRss, enableOpenAlexDiscovery);
     } catch {
       return failure("unavailable", "Metadata search network targets are unavailable.");
     }
@@ -634,6 +647,9 @@ export function createRustMetadataSearchProvider(options: RustMetadataSearchProv
     try {
       const input = JSON.stringify({
         schema_version: REQUEST_SCHEMA_VERSION,
+        diagnostics: true,
+        ...(enableBingRss ? { bing_rss: true } : {}),
+        ...(enableOpenAlexDiscovery ? { openalex_discovery: true } : {}),
         query: validated.query,
         max_results: validated.maxResults,
         timeout_ms: validated.timeoutMs,

package/src/clip/url-intelligence.ts

@@ -35,12 +35,33 @@ export type MetadataSearchResult = {
   readonly score: number;
 };
 
+export const METADATA_SEARCH_FAILURE_CODES = [
+  "timeout", "http", "http-body", "http-status", "http-forbidden", "http-rate-limited", "http-redirect",
+  "parse", "unverified-empty", "challenge", "content-type", "content-encoding", "body-limit",
+  "body-encoding", "result-limit", "unrecognized-html", "query-mismatch",
+] as const;
+export type MetadataSearchFailureCode = (typeof METADATA_SEARCH_FAILURE_CODES)[number];
+export type MetadataSearchEngineFailure = {
+  readonly engine: string;
+  readonly code: MetadataSearchFailureCode;
+};
+
+export type MetadataSearchEngineUsage = {
+  readonly engine: "openalex";
+  readonly reportedCostUsd: number;
+  readonly billingStatus: "not-established";
+};
+
 export type MetadataSearchResponse = {
   readonly query: string;
   readonly results: readonly MetadataSearchResult[];
   readonly enginesQueried: readonly string[];
   readonly enginesFailed: readonly string[];
   readonly engineStatus: MetadataSearchEngineStatus;
+  /** Absent for older providers; present entries cover every failed engine exactly once. */
+  readonly engineFailures?: readonly MetadataSearchEngineFailure[];
+  /** Provider-reported telemetry; this does not establish a user charge. */
+  readonly engineUsage?: readonly MetadataSearchEngineUsage[];
 };
 
 export type RankMetadataSearchOptions = {
@@ -246,7 +267,7 @@ function parseMetadataResult(
 /** Strictly parse one successful metadata-search-engine-rs `GET /search` body. */
 export function parseMetadataSearchResponse(value: unknown): MetadataSearchResponse {
   const input = record(value, "Metadata search response");
-  strictKeys(input, ["query", "results", "engines_queried", "engines_failed"], "Metadata search response");
+  strictKeys(input, ["query", "results", "engines_queried", "engines_failed", "engine_failures", "engine_usage"], "Metadata search response");
   const budget: TextBudget = { bytes: 0 };
   const query = boundedString(input.query, "Metadata search response.query", MAX_METADATA_SEARCH_QUERY_UTF8_BYTES, budget);
   if (query === "" || query !== query.trim() || hasUnsafeControls(query)) {
@@ -262,13 +283,45 @@ export function parseMetadataSearchResponse(value: unknown): MetadataSearchRespo
     fail("Metadata search response.results", `must be an array with at most ${MAX_METADATA_SEARCH_RESULTS} entries.`);
   }
   const failed = new Set(enginesFailed);
+  let engineFailures: readonly MetadataSearchEngineFailure[] | undefined;
+  if (Object.hasOwn(input, "engine_failures")) {
+    if (!Array.isArray(input.engine_failures) || input.engine_failures.length !== enginesFailed.length) {
+      fail("Metadata search response.engine_failures", "must cover every failed engine exactly once.");
+    }
+    const seen = new Set<string>();
+    engineFailures = Object.freeze(input.engine_failures.map((value, index) => {
+      const label = `Metadata search response.engine_failures[${index}]`;
+      const entry = record(value, label);
+      strictKeys(entry, ["engine", "code"], label);
+      if (typeof entry.engine !== "string" || !failed.has(entry.engine) || seen.has(entry.engine)) {
+        fail(label, "must name a unique failed, queried engine.");
+      }
+      if (typeof entry.code !== "string" || !METADATA_SEARCH_FAILURE_CODES.some(code => code === entry.code)) {
+        fail(label, "must contain a known categorical failure code.");
+      }
+      seen.add(entry.engine);
+      return Object.freeze({ engine: entry.engine, code: entry.code as MetadataSearchFailureCode });
+    }));
+  }
   const results = Object.freeze(input.results.map((result, index) =>
     parseMetadataResult(result, index, queried, failed, budget)));
+  let engineUsage: readonly MetadataSearchEngineUsage[] | undefined;
+  if (Object.hasOwn(input, "engine_usage")) {
+    if (!Array.isArray(input.engine_usage) || input.engine_usage.length !== 1) fail("Metadata search response.engine_usage", "must contain one bounded OpenAlex usage record.");
+    const entry = record(input.engine_usage[0], "engine usage");
+    strictKeys(entry, ["engine", "reported_cost_usd", "billing_status"], "engine usage");
+    if (entry.engine !== "openalex" || !enginesQueried.includes("openalex") || failed.has("openalex")
+      || typeof entry.reported_cost_usd !== "number" || !Number.isFinite(entry.reported_cost_usd)
+      || entry.reported_cost_usd < 0 || entry.reported_cost_usd > 1_000_000 || entry.billing_status !== "not-established") fail("engine usage", "must preserve bounded provider-reported usage without asserting billing.");
+    engineUsage = Object.freeze([Object.freeze({ engine: "openalex" as const, reportedCostUsd: entry.reported_cost_usd, billingStatus: "not-established" as const })]);
+  }
   return Object.freeze({
     query,
     results,
     enginesQueried,
     enginesFailed,
+    ...(engineFailures === undefined ? {} : { engineFailures }),
+    ...(engineUsage === undefined ? {} : { engineUsage }),
     engineStatus: enginesFailed.length === 0
       ? "complete"
       : enginesFailed.length === enginesQueried.length ? "unavailable" : "partial",

package/src/url-intelligence.ts

@@ -86,6 +86,8 @@ export type {
   ArchiveTodayMemento,
   ArchiveTodayTimeMap,
   MetadataSearchResponse,
+  MetadataSearchEngineFailure,
+  MetadataSearchFailureCode,
   MetadataSearchResult,
   RankedMetadataSearchResult,
 } from "./clip/url-intelligence.js";