@hraness/wordcell 0.24.0 → 0.25.0

package/src/clip/url-intelligence.ts

@@ -35,12 +35,33 @@ export type MetadataSearchResult = {
   readonly score: number;
 };
 
+export const METADATA_SEARCH_FAILURE_CODES = [
+  "timeout", "http", "http-body", "http-status", "http-forbidden", "http-rate-limited", "http-redirect",
+  "parse", "unverified-empty", "challenge", "content-type", "content-encoding", "body-limit",
+  "body-encoding", "result-limit", "unrecognized-html", "query-mismatch",
+] as const;
+export type MetadataSearchFailureCode = (typeof METADATA_SEARCH_FAILURE_CODES)[number];
+export type MetadataSearchEngineFailure = {
+  readonly engine: string;
+  readonly code: MetadataSearchFailureCode;
+};
+
+export type MetadataSearchEngineUsage = {
+  readonly engine: "openalex";
+  readonly reportedCostUsd: number;
+  readonly billingStatus: "not-established";
+};
+
 export type MetadataSearchResponse = {
   readonly query: string;
   readonly results: readonly MetadataSearchResult[];
   readonly enginesQueried: readonly string[];
   readonly enginesFailed: readonly string[];
   readonly engineStatus: MetadataSearchEngineStatus;
+  /** Absent for older providers; present entries cover every failed engine exactly once. */
+  readonly engineFailures?: readonly MetadataSearchEngineFailure[];
+  /** Provider-reported telemetry; this does not establish a user charge. */
+  readonly engineUsage?: readonly MetadataSearchEngineUsage[];
 };
 
 export type RankMetadataSearchOptions = {
@@ -246,7 +267,7 @@ function parseMetadataResult(
 /** Strictly parse one successful metadata-search-engine-rs `GET /search` body. */
 export function parseMetadataSearchResponse(value: unknown): MetadataSearchResponse {
   const input = record(value, "Metadata search response");
-  strictKeys(input, ["query", "results", "engines_queried", "engines_failed"], "Metadata search response");
+  strictKeys(input, ["query", "results", "engines_queried", "engines_failed", "engine_failures", "engine_usage"], "Metadata search response");
   const budget: TextBudget = { bytes: 0 };
   const query = boundedString(input.query, "Metadata search response.query", MAX_METADATA_SEARCH_QUERY_UTF8_BYTES, budget);
   if (query === "" || query !== query.trim() || hasUnsafeControls(query)) {
@@ -262,13 +283,45 @@ export function parseMetadataSearchResponse(value: unknown): MetadataSearchRespo
     fail("Metadata search response.results", `must be an array with at most ${MAX_METADATA_SEARCH_RESULTS} entries.`);
   }
   const failed = new Set(enginesFailed);
+  let engineFailures: readonly MetadataSearchEngineFailure[] | undefined;
+  if (Object.hasOwn(input, "engine_failures")) {
+    if (!Array.isArray(input.engine_failures) || input.engine_failures.length !== enginesFailed.length) {
+      fail("Metadata search response.engine_failures", "must cover every failed engine exactly once.");
+    }
+    const seen = new Set<string>();
+    engineFailures = Object.freeze(input.engine_failures.map((value, index) => {
+      const label = `Metadata search response.engine_failures[${index}]`;
+      const entry = record(value, label);
+      strictKeys(entry, ["engine", "code"], label);
+      if (typeof entry.engine !== "string" || !failed.has(entry.engine) || seen.has(entry.engine)) {
+        fail(label, "must name a unique failed, queried engine.");
+      }
+      if (typeof entry.code !== "string" || !METADATA_SEARCH_FAILURE_CODES.some(code => code === entry.code)) {
+        fail(label, "must contain a known categorical failure code.");
+      }
+      seen.add(entry.engine);
+      return Object.freeze({ engine: entry.engine, code: entry.code as MetadataSearchFailureCode });
+    }));
+  }
   const results = Object.freeze(input.results.map((result, index) =>
     parseMetadataResult(result, index, queried, failed, budget)));
+  let engineUsage: readonly MetadataSearchEngineUsage[] | undefined;
+  if (Object.hasOwn(input, "engine_usage")) {
+    if (!Array.isArray(input.engine_usage) || input.engine_usage.length !== 1) fail("Metadata search response.engine_usage", "must contain one bounded OpenAlex usage record.");
+    const entry = record(input.engine_usage[0], "engine usage");
+    strictKeys(entry, ["engine", "reported_cost_usd", "billing_status"], "engine usage");
+    if (entry.engine !== "openalex" || !enginesQueried.includes("openalex") || failed.has("openalex")
+      || typeof entry.reported_cost_usd !== "number" || !Number.isFinite(entry.reported_cost_usd)
+      || entry.reported_cost_usd < 0 || entry.reported_cost_usd > 1_000_000 || entry.billing_status !== "not-established") fail("engine usage", "must preserve bounded provider-reported usage without asserting billing.");
+    engineUsage = Object.freeze([Object.freeze({ engine: "openalex" as const, reportedCostUsd: entry.reported_cost_usd, billingStatus: "not-established" as const })]);
+  }
   return Object.freeze({
     query,
     results,
     enginesQueried,
     enginesFailed,
+    ...(engineFailures === undefined ? {} : { engineFailures }),
+    ...(engineUsage === undefined ? {} : { engineUsage }),
     engineStatus: enginesFailed.length === 0
       ? "complete"
       : enginesFailed.length === enginesQueried.length ? "unavailable" : "partial",

package/src/clip/url-metadata.ts

@@ -1,4 +1,5 @@
 import { execFileSync } from "node:child_process";
+import { syncsDirectories } from "../directory-sync.js";
 import { randomUUID } from "node:crypto";
 import {
   closeSync,
@@ -1371,7 +1372,7 @@ export function writeUrlMetadataDocument(
       throw new Error(`URL metadata sidecar failed post-install verification: ${saved.sidecarPath}`);
     }
     assertHeldUrlMetadataWriteLockCurrent(heldLock);
-    fsyncSync(directoryDescriptor);
+    if (syncsDirectories()) fsyncSync(directoryDescriptor);
     return { changed: true, path: saved.sidecarPath };
   } catch (error) {
     operationError = error;

package/src/directory-sync.ts

@@ -0,0 +1,9 @@
+/**
+ * Whether this platform can fsync a directory handle to persist a rename or
+ * link. POSIX filesystems need it. Windows refuses (fsync on a directory
+ * handle fails with EPERM), and NTFS journals the directory metadata itself,
+ * so callers skip the directory sync there and keep every file fsync.
+ */
+export function syncsDirectories(platform: NodeJS.Platform = process.platform): boolean {
+  return platform !== "win32";
+}

package/src/graph-authority.ts

@@ -1,4 +1,5 @@
 import { createHash } from "node:crypto";
+import { syncsDirectories } from "./directory-sync.js";
 import { constants } from "node:fs";
 import { mkdir, mkdtemp, realpath, rename, rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
@@ -197,8 +198,10 @@ export async function rebuildGraph(rootInput: string, options: GraphRebuildOptio
     const handle = await fs.open(stagedPath, constants.O_RDONLY | constants.O_NOFOLLOW);
     try { await handle.sync(); } finally { await handle.close(); }
     await rename(stagedPath, path);
-    const directory = await fs.open(cache, constants.O_RDONLY);
-    try { await directory.sync(); } finally { await directory.close(); }
+    if (syncsDirectories()) {
+      const directory = await fs.open(cache, constants.O_RDONLY);
+      try { await directory.sync(); } finally { await directory.close(); }
+    }
     return result;
   } finally {
     try {

package/src/url-intelligence.ts

@@ -86,6 +86,8 @@ export type {
   ArchiveTodayMemento,
   ArchiveTodayTimeMap,
   MetadataSearchResponse,
+  MetadataSearchEngineFailure,
+  MetadataSearchFailureCode,
   MetadataSearchResult,
   RankedMetadataSearchResult,
 } from "./clip/url-intelligence.js";