@ai-agent-forge/plugin-search 0.0.0-stage → 0.88.16

package/README.md

@@ -1,3 +1,57 @@
-# Temporary Holding Version
+# Search plugin
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+First-party search capability with one URL pipeline and two model-facing entries:
+
+- `search` — merged entry, visible when the current main model has no usable
+  provider-native search. `mode:"model"` runs one isolated completion against
+  the model referenced by `defaultCapabilities.search` with native search
+  enabled; `mode:"url"` fetches the given URLs and summarizes them with the
+  current main model.
+- `search_url` — URL-only entry, visible when the main model does have usable
+  native search. It reads the URLs the model provides; discovery stays with the
+  built-in search.
+
+Only one entry is visible at a time. The host capability projection selects it
+from `getCapabilityFacts("search")`; the plugin does not branch on vendor or
+model names.
+
+## Configuration
+
+`models.json` may point the merged entry's model-search target at any registered
+model that declares both `text` and `search`:
+
+```json
+{
+  "defaultCapabilities": {
+    "search": "search-provider/search-model"
+  }
+}
+```
+
+The target's provider `search` mode determines the actual vendor protocol. If no
+target is configured, `mode:"model"` returns `search_target_unavailable`; URL
+mode still works.
+
+## Bounds
+
+- 32 URLs per call, 2048 characters per URL, 2000 characters per query.
+- 8 MiB raw response per page, 128 MiB per call, 4 MiB extracted text per
+  page.
+- 512 KiB summary input, 16 KiB summary output, 8192 output tokens.
+- 8 concurrent fetches, 20 s per URL, 120 s total fetch/model deadlines.
+- System-level disk page cache (2 GiB / 5,000 entries / 5 MiB each; 14 days
+  base, extended by hits) and a 32-entry / 10-minute in-memory summary cache.
+  Pages contain content only; no URLs, headers, cookies or validators.
+
+There are no domain, scope, or private-network restrictions in the plugin
+itself. Deployments that need them should install a policy plugin before the tool
+call. No provider credentials, cookies, or browser profiles are sent to target
+URLs; pages with no readable server-rendered text fail with `empty_content`.
+
+## Status
+
+`mechanism-verified`: deterministic unit tests cover registration, projection
+metadata, both modes, input validation, partial failures, invalid summaries,
+disk-cache persistence/retention/eviction, size/type failures, redirects, and
+cancellation. Real-task benchmarking (P4 in
+`docs/design/搜索工具设计.md`) remains a separate, authorized step.

package/agent-forge.json

@@ -0,0 +1,19 @@
+{
+	"packageFormatVersion": 1,
+	"id": "agent-forge.plugin.search",
+	"pluginVersion": "0.88.16",
+	"apiVersion": "1",
+	"minHostVersion": "0.88.14",
+	"platform": {
+		"os": [
+			"windows",
+			"macos",
+			"linux"
+		],
+		"runtime": [
+			{
+				"name": "node"
+			}
+		]
+	}
+}

package/dist/cache.d.ts

@@ -0,0 +1,103 @@
+/**
+ * System-level disk page cache (搜索缓存持久化设计).
+ *
+ * Only extracted page content is persisted: title, text and the truncation flag.
+ * URLs, response headers, cookies, validators and fetch metadata stay out of the
+ * cache. Entries are shared by every agent/profile/session of the same OS user,
+ * use a hit-count-based retention window and are evicted by LFU then LRU.
+ *
+ * All I/O is asynchronous. Writes and hit-count bookkeeping are queued so a
+ * search call never waits for the disk; `flushWrites`/`flushUsage` exist for
+ * tests and graceful shutdown.
+ */
+import type { LoggerAPI } from "@agent-forge/plugin-sdk";
+export interface CachedPageV1 {
+    readonly title?: string;
+    readonly text: string;
+    readonly truncated: boolean;
+}
+export interface PageCacheOptionsV1 {
+    readonly maxBytes?: number;
+    readonly maxEntries?: number;
+    readonly maxEntryBytes?: number;
+    readonly baseTtlMs?: number;
+    readonly ttlTiersMs?: readonly number[];
+    readonly writeQueueEntries?: number;
+    readonly writeQueueBytes?: number;
+    readonly metaFlushMs?: number;
+    readonly pruneIntervalMs?: number;
+    readonly logger?: LoggerAPI;
+}
+/** OS cache directory shared by all profiles of the current OS user. */
+export declare function defaultPageCacheDirectory(): string;
+export declare class PageCache {
+    private readonly directory;
+    private readonly maxBytes;
+    private readonly maxEntries;
+    private readonly maxEntryBytes;
+    private readonly baseTtlMs;
+    private readonly ttlTiersMs;
+    private readonly writeQueueEntries;
+    private readonly writeQueueBytes;
+    private readonly metaFlushMs;
+    private readonly pruneIntervalMs;
+    private readonly logger;
+    private readonly entries;
+    private readonly pendingWrites;
+    private readonly dirtyMeta;
+    private pendingWriteBytes;
+    private totalBytes;
+    private initPromise;
+    private unavailable;
+    private loggedUnavailable;
+    private loggedSkippedWrite;
+    private loggedWriteFailure;
+    private metaTimer;
+    private pruneTimer;
+    private pruneChain;
+    constructor(directory?: string, options?: PageCacheOptionsV1);
+    /** Fresh content only; any cache failure degrades to a miss. */
+    getFresh(rawUrl: string, now: number): Promise<CachedPageV1 | undefined>;
+    private getFreshUnsafe;
+    /** Queue a content write; any cache failure is ignored by the caller. */
+    set(rawUrl: string, page: CachedPageV1, now: number): Promise<void>;
+    private setUnsafe;
+    /** Persist queued writes and hit counts; used by tests and graceful shutdown. */
+    flushWrites(): Promise<void>;
+    flushUsage(): Promise<void>;
+    /** Remove expired entries and enforce the size/count ceiling. */
+    prune(now: number): Promise<void>;
+    private hashFor;
+    private pathsFor;
+    /** Evict low-value entries from the index before admitting a new one. */
+    private makeRoom;
+    private ttlForHits;
+    private ensureInitialized;
+    private loadIndex;
+    private loadEntryFromDisk;
+    private readMetaFile;
+    private readPage;
+    private writeEntry;
+    private writeMetaFile;
+    private atomicWrite;
+    private removeEntry;
+    private removeFiles;
+    private enqueuePrune;
+    private pruneNow;
+    private cleanupOrphanFiles;
+    private scheduleMetaFlush;
+    private schedulePrune;
+    private logUnavailable;
+    private logSkippedWrite;
+    private logDebug;
+}
+/** Small in-memory LRU for successful summaries; failures are never stored. */
+export declare class SummaryCache<T> {
+    private readonly entries;
+    private readonly maxEntries;
+    private readonly ttlMs;
+    constructor(maxEntries?: number, ttlMs?: number);
+    get(key: string, now: number): T | undefined;
+    set(key: string, value: T, now: number): void;
+}
+//# sourceMappingURL=cache.d.ts.map

package/dist/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAOH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAIzD,MAAM,WAAW,YAAY;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAkB;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC;CAC5B;AA4FD,wEAAwE;AACxE,wBAAgB,yBAAyB,IAAI,MAAM,CAWlD;AAED,qBAAa,SAAS;IACrB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAoB;IAC/C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAE/C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoC;IAClE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmC;IAC7D,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,SAAS,CAA4C;IAC7D,OAAO,CAAC,UAAU,CAA4C;IAC9D,OAAO,CAAC,UAAU,CAAoC;IAEtD,YAAY,SAAS,GAAE,MAAoC,EAAE,OAAO,GAAE,kBAAuB,EAY5F;IAED,gEAAgE;IAC1D,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAO7E;YAEa,cAAc;IAsC5B,yEAAyE;IACnE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMxE;YAEa,SAAS;IA4CvB,iFAAiF;IAC3E,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAKjC;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAkBhC;IAED,iEAAiE;IAC3D,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAItC;IAED,OAAO,CAAC,OAAO;IAWf,OAAO,CAAC,QAAQ;IAQhB,yEAAyE;IACzE,OAAO,CAAC,QAAQ;IAuBhB,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,iBAAiB;YAKX,SAAS;YAiCT,iBAAiB;YAyCjB,YAAY;YAgBZ,QAAQ;YAiBR,UAAU;YAsBV,aAAa;YAcb,WAAW;YAgBX,WAAW;YASX,WAAW;IAWzB,OAAO,CAAC,YAAY;YAMN,QAAQ;YA0BR,kBAAkB;IAoChC,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,QAAQ;CAehB;AAOD,+EAA+E;AAC/E,qBAAa,YAAY,CAAC,CAAC;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAE/B,YACC,UAAU,GAAE,MAA0C,EACtD,KAAK,GAAE,MAAwC,EAI/C;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAU3C;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAQ5C;CACD","sourcesContent":["/**\n * System-level disk page cache (搜索缓存持久化设计).\n *\n * Only extracted page content is persisted: title, text and the truncation flag.\n * URLs, response headers, cookies, validators and fetch metadata stay out of the\n * cache. Entries are shared by every agent/profile/session of the same OS user,\n * use a hit-count-based retention window and are evicted by LFU then LRU.\n *\n * All I/O is asynchronous. Writes and hit-count bookkeeping are queued so a\n * search call never waits for the disk; `flushWrites`/`flushUsage` exist for\n * tests and graceful shutdown.\n */\n\nimport { createHash, randomUUID } from \"node:crypto\";\nimport type { Dirent } from \"node:fs\";\nimport { mkdir, readdir, readFile, rename, rm, stat, unlink, writeFile } from \"node:fs/promises\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport type { LoggerAPI } from \"@agent-forge/plugin-sdk\";\nimport { SEARCH_LIMITS } from \"./limits.ts\";\nimport { byteLength, isRecord, normalizeUrl, urlHasUserInfo } from \"./text.ts\";\n\nexport interface CachedPageV1 {\n\treadonly title?: string;\n\treadonly text: string;\n\treadonly truncated: boolean;\n}\n\nexport interface PageCacheOptionsV1 {\n\treadonly maxBytes?: number;\n\treadonly maxEntries?: number;\n\treadonly maxEntryBytes?: number;\n\treadonly baseTtlMs?: number;\n\treadonly ttlTiersMs?: readonly number[];\n\treadonly writeQueueEntries?: number;\n\treadonly writeQueueBytes?: number;\n\treadonly metaFlushMs?: number;\n\treadonly pruneIntervalMs?: number;\n\treadonly logger?: LoggerAPI;\n}\n\ninterface EntryMetaV1 {\n\treadonly version: 1;\n\treadonly hash: string;\n\treadonly bytes: number;\n\treadonly storedAt: number;\n\treadonly expiresAt: number;\n\treadonly hits: number;\n\treadonly lastAccess: number;\n}\n\ninterface EntryStateV1 {\n\treadonly hash: string;\n\treadonly pagePath: string;\n\treadonly metaPath: string;\n\treadonly bytes: number;\n\treadonly storedAt: number;\n\texpiresAt: number;\n\thits: number;\n\tlastAccess: number;\n}\n\ninterface PageFileV1 {\n\treadonly version: 1;\n\treadonly title?: string;\n\treadonly text: string;\n\treadonly truncated: boolean;\n}\n\ninterface CachedPageFileV1 {\n\treadonly version: 1;\n\treadonly hash: string;\n\treadonly bytes: number;\n\treadonly storedAt: number;\n\treadonly expiresAt: number;\n\treadonly hits: number;\n\treadonly lastAccess: number;\n}\n\nconst META_SUFFIX = \".meta.json\";\nconst PAGE_SUFFIX = \".page.json\";\nconst MAX_META_BYTES = 16 * 1024;\nconst HASH_PATTERN = /^[0-9a-f]{64}$/;\nconst SHARD_PATTERN = /^[0-9a-f]{2}$/;\n\nfunction errorCode(error: unknown): string | undefined {\n\tif (typeof error !== \"object\" || error === null || !(\"code\" in error)) return undefined;\n\tconst code = (error as { code?: unknown }).code;\n\treturn typeof code === \"string\" ? code : undefined;\n}\n\nfunction isMissing(error: unknown): boolean {\n\treturn errorCode(error) === \"ENOENT\";\n}\n\nfunction finiteNonNegative(value: unknown): value is number {\n\treturn typeof value === \"number\" && Number.isFinite(value) && value >= 0;\n}\n\nfunction parseMeta(value: unknown): EntryMetaV1 | undefined {\n\tif (!isRecord(value)) return undefined;\n\tconst { version, hash, bytes, storedAt, expiresAt, hits, lastAccess } = value;\n\tif (version !== 1 || typeof hash !== \"string\" || !HASH_PATTERN.test(hash)) return undefined;\n\tif (!finiteNonNegative(bytes) || !finiteNonNegative(storedAt) || !finiteNonNegative(expiresAt)) return undefined;\n\tif (!finiteNonNegative(hits) || !Number.isInteger(hits) || !finiteNonNegative(lastAccess)) return undefined;\n\treturn { version: 1, hash, bytes, storedAt, expiresAt, hits, lastAccess };\n}\n\nfunction parsePage(value: unknown): PageFileV1 | undefined {\n\tif (!isRecord(value)) return undefined;\n\tconst { version, title, text, truncated } = value;\n\tif (version !== 1 || typeof text !== \"string\" || text === \"\" || typeof truncated !== \"boolean\") return undefined;\n\tif (title !== undefined && typeof title !== \"string\") return undefined;\n\treturn { version: 1, text, truncated, ...(title === undefined ? {} : { title }) };\n}\n\nfunction defaultTtlTiers(tiers: readonly number[] | undefined): readonly number[] {\n\tif (tiers !== undefined && tiers.length === 4) return tiers;\n\treturn SEARCH_LIMITS.cacheTtlTiersMs;\n}\n\nfunction serializePage(page: CachedPageV1): string {\n\tconst file: PageFileV1 = {\n\t\tversion: 1,\n\t\ttext: page.text,\n\t\ttruncated: page.truncated,\n\t\t...(page.title === undefined ? {} : { title: page.title }),\n\t};\n\treturn JSON.stringify(file);\n}\n\n/** OS cache directory shared by all profiles of the current OS user. */\nexport function defaultPageCacheDirectory(): string {\n\tconst home = homedir();\n\tif (process.platform === \"darwin\") {\n\t\treturn join(home, \"Library\", \"Caches\", \"agent-forge\", \"search\", \"v1\", \"pages\");\n\t}\n\tif (process.platform === \"win32\") {\n\t\tconst base = process.env.LOCALAPPDATA?.trim() || join(home, \"AppData\", \"Local\");\n\t\treturn join(base, \"agent-forge\", \"Cache\", \"search\", \"v1\", \"pages\");\n\t}\n\tconst base = process.env.XDG_CACHE_HOME?.trim() || join(home, \".cache\");\n\treturn join(base, \"agent-forge\", \"search\", \"v1\", \"pages\");\n}\n\nexport class PageCache {\n\tprivate readonly directory: string;\n\tprivate readonly maxBytes: number;\n\tprivate readonly maxEntries: number;\n\tprivate readonly maxEntryBytes: number;\n\tprivate readonly baseTtlMs: number;\n\tprivate readonly ttlTiersMs: readonly number[];\n\tprivate readonly writeQueueEntries: number;\n\tprivate readonly writeQueueBytes: number;\n\tprivate readonly metaFlushMs: number;\n\tprivate readonly pruneIntervalMs: number;\n\tprivate readonly logger: LoggerAPI | undefined;\n\n\tprivate readonly entries = new Map<string, EntryStateV1>();\n\tprivate readonly pendingWrites = new Map<string, Promise<void>>();\n\tprivate readonly dirtyMeta = new Map<string, EntryStateV1>();\n\tprivate pendingWriteBytes = 0;\n\tprivate totalBytes = 0;\n\tprivate initPromise: Promise<void> | undefined;\n\tprivate unavailable = false;\n\tprivate loggedUnavailable = false;\n\tprivate loggedSkippedWrite = false;\n\tprivate loggedWriteFailure = false;\n\tprivate metaTimer: ReturnType<typeof setTimeout> | undefined;\n\tprivate pruneTimer: ReturnType<typeof setTimeout> | undefined;\n\tprivate pruneChain: Promise<void> = Promise.resolve();\n\n\tconstructor(directory: string = defaultPageCacheDirectory(), options: PageCacheOptionsV1 = {}) {\n\t\tthis.directory = directory;\n\t\tthis.maxBytes = options.maxBytes ?? SEARCH_LIMITS.pageCacheBytes;\n\t\tthis.maxEntries = options.maxEntries ?? SEARCH_LIMITS.pageCacheEntries;\n\t\tthis.maxEntryBytes = options.maxEntryBytes ?? SEARCH_LIMITS.cacheMaxEntryBytes;\n\t\tthis.baseTtlMs = options.baseTtlMs ?? SEARCH_LIMITS.cacheBaseTtlMs;\n\t\tthis.ttlTiersMs = defaultTtlTiers(options.ttlTiersMs);\n\t\tthis.writeQueueEntries = options.writeQueueEntries ?? SEARCH_LIMITS.cacheWriteQueueEntries;\n\t\tthis.writeQueueBytes = options.writeQueueBytes ?? SEARCH_LIMITS.cacheWriteQueueBytes;\n\t\tthis.metaFlushMs = options.metaFlushMs ?? SEARCH_LIMITS.cacheMetaFlushMs;\n\t\tthis.pruneIntervalMs = options.pruneIntervalMs ?? SEARCH_LIMITS.cachePruneIntervalMs;\n\t\tthis.logger = options.logger;\n\t}\n\n\t/** Fresh content only; any cache failure degrades to a miss. */\n\tasync getFresh(rawUrl: string, now: number): Promise<CachedPageV1 | undefined> {\n\t\ttry {\n\t\t\treturn await this.getFreshUnsafe(rawUrl, now);\n\t\t} catch (error) {\n\t\t\tthis.logDebug(\"page cache read failed; treating it as a miss\", error);\n\t\t\treturn undefined;\n\t\t}\n\t}\n\n\tprivate async getFreshUnsafe(rawUrl: string, now: number): Promise<CachedPageV1 | undefined> {\n\t\tconst hash = this.hashFor(rawUrl);\n\t\tif (hash === undefined) return undefined;\n\t\tawait this.ensureInitialized(now);\n\t\tif (this.unavailable) return undefined;\n\n\t\tconst pending = this.pendingWrites.get(hash);\n\t\tif (pending !== undefined) {\n\t\t\tawait pending;\n\t\t\tif (this.unavailable) return undefined;\n\t\t}\n\n\t\tlet entry = this.entries.get(hash);\n\t\tif (entry === undefined) {\n\t\t\tentry = await this.loadEntryFromDisk(hash, now);\n\t\t\tif (entry === undefined) return undefined;\n\t\t\tthis.entries.set(hash, entry);\n\t\t\tthis.totalBytes += entry.bytes;\n\t\t}\n\t\tif (now >= entry.expiresAt) {\n\t\t\tawait this.removeEntry(entry);\n\t\t\treturn undefined;\n\t\t}\n\t\tconst content = await this.readPage(entry);\n\t\tif (content === undefined) {\n\t\t\tawait this.removeEntry(entry);\n\t\t\treturn undefined;\n\t\t}\n\t\tif (this.entries.get(hash) !== entry) return undefined;\n\n\t\tentry.hits += 1;\n\t\tentry.lastAccess = now;\n\t\tentry.expiresAt = Math.max(entry.expiresAt, now + this.ttlForHits(entry.hits));\n\t\tthis.dirtyMeta.set(hash, entry);\n\t\tthis.scheduleMetaFlush();\n\t\treturn content;\n\t}\n\n\t/** Queue a content write; any cache failure is ignored by the caller. */\n\tasync set(rawUrl: string, page: CachedPageV1, now: number): Promise<void> {\n\t\ttry {\n\t\t\tawait this.setUnsafe(rawUrl, page, now);\n\t\t} catch (error) {\n\t\t\tthis.logDebug(\"page cache write setup failed; search continues\", error);\n\t\t}\n\t}\n\n\tprivate async setUnsafe(rawUrl: string, page: CachedPageV1, now: number): Promise<void> {\n\t\tconst hash = this.hashFor(rawUrl);\n\t\tif (hash === undefined) return;\n\t\tif (page.text.trim() === \"\") return;\n\t\tconst bytes = byteLength(serializePage(page));\n\t\tif (bytes > this.maxEntryBytes) return;\n\t\tawait this.ensureInitialized(now);\n\t\tif (this.unavailable) return;\n\t\tif (this.pendingWrites.has(hash)) return;\n\t\tif (this.pendingWrites.size >= this.writeQueueEntries || this.pendingWriteBytes + bytes > this.writeQueueBytes) {\n\t\t\tthis.logSkippedWrite();\n\t\t\treturn;\n\t\t}\n\n\t\tconst paths = this.pathsFor(hash);\n\t\tconst entry: EntryStateV1 = {\n\t\t\thash,\n\t\t\tpagePath: paths.pagePath,\n\t\t\tmetaPath: paths.metaPath,\n\t\t\tbytes,\n\t\t\tstoredAt: now,\n\t\t\texpiresAt: now + this.baseTtlMs,\n\t\t\thits: 0,\n\t\t\tlastAccess: now,\n\t\t};\n\t\tconst previous = this.entries.get(hash);\n\t\tif (!this.makeRoom(hash, bytes - (previous?.bytes ?? 0), now)) {\n\t\t\tthis.schedulePrune();\n\t\t\treturn;\n\t\t}\n\t\tthis.entries.set(hash, entry);\n\t\tthis.totalBytes += bytes - (previous?.bytes ?? 0);\n\t\tthis.pendingWriteBytes += bytes;\n\t\tconst write = this.writeEntry(entry, page).finally(() => {\n\t\t\tthis.pendingWriteBytes = Math.max(0, this.pendingWriteBytes - bytes);\n\t\t\tthis.pendingWrites.delete(hash);\n\t\t});\n\t\tthis.pendingWrites.set(hash, write);\n\t\twrite.catch(() => {\n\t\t\t// writeEntry records failures and drops the index entry; never surface\n\t\t\t// an unhandled rejection from a background cache write.\n\t\t});\n\t}\n\n\t/** Persist queued writes and hit counts; used by tests and graceful shutdown. */\n\tasync flushWrites(): Promise<void> {\n\t\twhile (this.pendingWrites.size > 0) {\n\t\t\tawait Promise.allSettled([...this.pendingWrites.values()]);\n\t\t}\n\t\tawait this.flushUsage();\n\t}\n\n\tasync flushUsage(): Promise<void> {\n\t\tif (this.metaTimer !== undefined) {\n\t\t\tclearTimeout(this.metaTimer);\n\t\t\tthis.metaTimer = undefined;\n\t\t}\n\t\tconst snapshot = [...this.dirtyMeta.values()];\n\t\tthis.dirtyMeta.clear();\n\t\tawait Promise.all(\n\t\t\tsnapshot.map(async (entry) => {\n\t\t\t\tif (this.entries.get(entry.hash) !== entry) return;\n\t\t\t\ttry {\n\t\t\t\t\tawait this.writeMetaFile(entry);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tif (this.entries.get(entry.hash) === entry) this.dirtyMeta.set(entry.hash, entry);\n\t\t\t\t\tthis.logDebug(\"page cache could not flush hit bookkeeping\", error);\n\t\t\t\t}\n\t\t\t}),\n\t\t);\n\t}\n\n\t/** Remove expired entries and enforce the size/count ceiling. */\n\tasync prune(now: number): Promise<void> {\n\t\tawait this.ensureInitialized(now);\n\t\tawait this.enqueuePrune(now, false);\n\t\tawait this.flushUsage();\n\t}\n\n\tprivate hashFor(rawUrl: string): string | undefined {\n\t\tif (urlHasUserInfo(rawUrl)) return undefined;\n\t\ttry {\n\t\t\treturn createHash(\"sha256\")\n\t\t\t\t.update(`search-page-v1\\n${normalizeUrl(rawUrl)}`)\n\t\t\t\t.digest(\"hex\");\n\t\t} catch {\n\t\t\treturn undefined;\n\t\t}\n\t}\n\n\tprivate pathsFor(hash: string): { readonly pagePath: string; readonly metaPath: string } {\n\t\tconst dir = join(this.directory, hash.slice(0, 2));\n\t\treturn {\n\t\t\tpagePath: join(dir, `${hash}${PAGE_SUFFIX}`),\n\t\t\tmetaPath: join(dir, `${hash}${META_SUFFIX}`),\n\t\t};\n\t}\n\n\t/** Evict low-value entries from the index before admitting a new one. */\n\tprivate makeRoom(replacingHash: string, bytesDelta: number, now: number): boolean {\n\t\tconst within = (): boolean =>\n\t\t\tthis.entries.size + (this.entries.has(replacingHash) ? 0 : 1) <= this.maxEntries &&\n\t\t\tthis.totalBytes + bytesDelta <= this.maxBytes;\n\t\tif (within()) return true;\n\t\tconst candidates = [...this.entries.values()]\n\t\t\t.filter((entry) => entry.hash !== replacingHash && !this.pendingWrites.has(entry.hash))\n\t\t\t.sort((left, right) => {\n\t\t\t\tconst leftExpired = now >= left.expiresAt ? 1 : 0;\n\t\t\t\tconst rightExpired = now >= right.expiresAt ? 1 : 0;\n\t\t\t\tif (leftExpired !== rightExpired) return rightExpired - leftExpired;\n\t\t\t\treturn left.hits - right.hits || left.lastAccess - right.lastAccess;\n\t\t\t});\n\t\tfor (const entry of candidates) {\n\t\t\tif (within()) break;\n\t\t\tthis.entries.delete(entry.hash);\n\t\t\tthis.totalBytes = Math.max(0, this.totalBytes - entry.bytes);\n\t\t\tthis.dirtyMeta.delete(entry.hash);\n\t\t\tvoid this.removeFiles(entry.pagePath, entry.metaPath);\n\t\t}\n\t\treturn within();\n\t}\n\n\tprivate ttlForHits(hits: number): number {\n\t\tif (hits >= 50) return this.ttlTiersMs[3] ?? this.baseTtlMs;\n\t\tif (hits >= 10) return this.ttlTiersMs[2] ?? this.baseTtlMs;\n\t\tif (hits >= 3) return this.ttlTiersMs[1] ?? this.baseTtlMs;\n\t\tif (hits >= 1) return this.ttlTiersMs[0] ?? this.baseTtlMs;\n\t\treturn this.baseTtlMs;\n\t}\n\n\tprivate ensureInitialized(now: number): Promise<void> {\n\t\tthis.initPromise ??= this.loadIndex(now);\n\t\treturn this.initPromise;\n\t}\n\n\tprivate async loadIndex(now: number): Promise<void> {\n\t\tlet shards: Dirent[];\n\t\ttry {\n\t\t\tshards = await readdir(this.directory, { withFileTypes: true });\n\t\t} catch (error) {\n\t\t\tif (isMissing(error)) return;\n\t\t\tthis.unavailable = true;\n\t\t\tthis.logUnavailable(error);\n\t\t\treturn;\n\t\t}\n\t\tfor (const shard of shards) {\n\t\t\tif (!shard.isDirectory() || !SHARD_PATTERN.test(shard.name)) continue;\n\t\t\tconst shardDir = join(this.directory, shard.name);\n\t\t\tlet files: readonly string[];\n\t\t\ttry {\n\t\t\t\tfiles = await readdir(shardDir);\n\t\t\t} catch (error) {\n\t\t\t\tif (!isMissing(error)) this.logDebug(\"page cache shard is unreadable\", error);\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tfor (const file of files) {\n\t\t\t\tif (!file.endsWith(META_SUFFIX)) continue;\n\t\t\t\tconst hash = file.slice(0, -META_SUFFIX.length);\n\t\t\t\tif (!HASH_PATTERN.test(hash)) continue;\n\t\t\t\tconst entry = await this.loadEntryFromDisk(hash, now);\n\t\t\t\tif (entry === undefined) continue;\n\t\t\t\tthis.entries.set(hash, entry);\n\t\t\t\tthis.totalBytes += entry.bytes;\n\t\t\t}\n\t\t}\n\t\tawait this.enqueuePrune(now, false);\n\t}\n\n\tprivate async loadEntryFromDisk(hash: string, now: number): Promise<EntryStateV1 | undefined> {\n\t\tconst paths = this.pathsFor(hash);\n\t\tconst meta = await this.readMetaFile(paths.metaPath);\n\t\tif (meta === undefined) {\n\t\t\t// A page file can be visible for a moment before its meta file is\n\t\t\t// renamed into place by another process; never delete that fresh file.\n\t\t\treturn undefined;\n\t\t}\n\t\tif (meta.hash !== hash) {\n\t\t\tawait this.removeFiles(paths.pagePath, paths.metaPath);\n\t\t\treturn undefined;\n\t\t}\n\t\tlet pageSize: number;\n\t\ttry {\n\t\t\tconst pageStat = await stat(paths.pagePath);\n\t\t\tpageSize = pageStat.size;\n\t\t} catch (error) {\n\t\t\tif (!isMissing(error)) this.logDebug(\"page cache entry has no readable page file\", error);\n\t\t\tawait this.removeFiles(paths.pagePath, paths.metaPath);\n\t\t\treturn undefined;\n\t\t}\n\t\tif (pageSize !== meta.bytes || pageSize > this.maxEntryBytes) {\n\t\t\tawait this.removeFiles(paths.pagePath, paths.metaPath);\n\t\t\treturn undefined;\n\t\t}\n\t\tif (now >= meta.expiresAt) {\n\t\t\tawait this.removeFiles(paths.pagePath, paths.metaPath);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn {\n\t\t\thash,\n\t\t\tpagePath: paths.pagePath,\n\t\t\tmetaPath: paths.metaPath,\n\t\t\tbytes: meta.bytes,\n\t\t\tstoredAt: meta.storedAt,\n\t\t\texpiresAt: meta.expiresAt,\n\t\t\thits: meta.hits,\n\t\t\tlastAccess: meta.lastAccess,\n\t\t};\n\t}\n\n\tprivate async readMetaFile(path: string): Promise<EntryMetaV1 | undefined> {\n\t\tlet size: number;\n\t\ttry {\n\t\t\tsize = (await stat(path)).size;\n\t\t} catch {\n\t\t\treturn undefined;\n\t\t}\n\t\tif (size > MAX_META_BYTES) return undefined;\n\t\ttry {\n\t\t\tconst raw = await readFile(path, \"utf8\");\n\t\t\treturn parseMeta(JSON.parse(raw));\n\t\t} catch {\n\t\t\treturn undefined;\n\t\t}\n\t}\n\n\tprivate async readPage(entry: EntryStateV1): Promise<CachedPageV1 | undefined> {\n\t\ttry {\n\t\t\tconst info = await stat(entry.pagePath);\n\t\t\tif (info.size !== entry.bytes) return undefined;\n\t\t\tconst raw = await readFile(entry.pagePath, \"utf8\");\n\t\t\tconst parsed = parsePage(JSON.parse(raw));\n\t\t\tif (parsed === undefined) return undefined;\n\t\t\treturn {\n\t\t\t\ttext: parsed.text,\n\t\t\t\ttruncated: parsed.truncated,\n\t\t\t\t...(parsed.title === undefined ? {} : { title: parsed.title }),\n\t\t\t};\n\t\t} catch {\n\t\t\treturn undefined;\n\t\t}\n\t}\n\n\tprivate async writeEntry(entry: EntryStateV1, page: CachedPageV1): Promise<void> {\n\t\ttry {\n\t\t\tawait mkdir(join(this.directory, entry.hash.slice(0, 2)), { recursive: true, mode: 0o700 });\n\t\t\tawait this.atomicWrite(entry.pagePath, serializePage(page));\n\t\t\tawait this.writeMetaFile(entry);\n\t\t\tthis.schedulePrune();\n\t\t} catch (error) {\n\t\t\tawait this.removeFiles(entry.pagePath, entry.metaPath);\n\t\t\tif (this.entries.get(entry.hash) === entry) {\n\t\t\t\tthis.entries.delete(entry.hash);\n\t\t\t\tthis.totalBytes = Math.max(0, this.totalBytes - entry.bytes);\n\t\t\t}\n\t\t\tif (!this.loggedWriteFailure) {\n\t\t\t\tthis.loggedWriteFailure = true;\n\t\t\t\tthis.logger?.warn(\"page cache write failed; search continues without caching\", {\n\t\t\t\t\tcause: error instanceof Error ? error.message : String(error),\n\t\t\t\t});\n\t\t\t}\n\t\t\tthis.logDebug(\"page cache write failed\", error);\n\t\t}\n\t}\n\n\tprivate async writeMetaFile(entry: EntryStateV1): Promise<void> {\n\t\tconst meta: CachedPageFileV1 = {\n\t\t\tversion: 1,\n\t\t\thash: entry.hash,\n\t\t\tbytes: entry.bytes,\n\t\t\tstoredAt: entry.storedAt,\n\t\t\texpiresAt: entry.expiresAt,\n\t\t\thits: entry.hits,\n\t\t\tlastAccess: entry.lastAccess,\n\t\t};\n\t\tawait mkdir(join(this.directory, entry.hash.slice(0, 2)), { recursive: true, mode: 0o700 });\n\t\tawait this.atomicWrite(entry.metaPath, JSON.stringify(meta));\n\t}\n\n\tprivate async atomicWrite(path: string, content: string): Promise<void> {\n\t\tconst temp = `${path}.${process.pid}.${randomUUID()}.tmp`;\n\t\ttry {\n\t\t\tawait writeFile(temp, content, { encoding: \"utf8\", mode: 0o600 });\n\t\t\ttry {\n\t\t\t\tawait rename(temp, path);\n\t\t\t} catch {\n\t\t\t\t// Windows can refuse to replace an existing file; remove and retry.\n\t\t\t\tawait rm(path, { force: true });\n\t\t\t\tawait rename(temp, path);\n\t\t\t}\n\t\t} finally {\n\t\t\tawait rm(temp, { force: true }).catch(() => {});\n\t\t}\n\t}\n\n\tprivate async removeEntry(entry: EntryStateV1): Promise<void> {\n\t\tif (this.entries.get(entry.hash) === entry) {\n\t\t\tthis.entries.delete(entry.hash);\n\t\t\tthis.totalBytes = Math.max(0, this.totalBytes - entry.bytes);\n\t\t}\n\t\tthis.dirtyMeta.delete(entry.hash);\n\t\tawait this.removeFiles(entry.pagePath, entry.metaPath);\n\t}\n\n\tprivate async removeFiles(pagePath: string, metaPath: string): Promise<void> {\n\t\tawait Promise.all([\n\t\t\tunlink(pagePath).catch((error: unknown) => {\n\t\t\t\tif (!isMissing(error)) this.logDebug(\"page cache could not remove a page file\", error);\n\t\t\t}),\n\t\t\tunlink(metaPath).catch((error: unknown) => {\n\t\t\t\tif (!isMissing(error)) this.logDebug(\"page cache could not remove a meta file\", error);\n\t\t\t}),\n\t\t]);\n\t}\n\n\tprivate enqueuePrune(now: number, capacityOnly: boolean): Promise<void> {\n\t\tconst run = (): Promise<void> => this.pruneNow(now, capacityOnly);\n\t\tthis.pruneChain = this.pruneChain.then(run, run);\n\t\treturn this.pruneChain;\n\t}\n\n\tprivate async pruneNow(now: number, capacityOnly: boolean): Promise<void> {\n\t\tif (this.unavailable) return;\n\t\tconst removable = [...this.entries.values()].filter((entry) => !this.pendingWrites.has(entry.hash));\n\t\tconst victims: EntryStateV1[] = [];\n\t\tif (!capacityOnly) {\n\t\t\tfor (const entry of removable) {\n\t\t\t\tif (now >= entry.expiresAt) victims.push(entry);\n\t\t\t}\n\t\t}\n\t\tlet count = this.entries.size - victims.length;\n\t\tlet bytes = this.totalBytes - victims.reduce((sum, entry) => sum + entry.bytes, 0);\n\t\tif (count > this.maxEntries || bytes > this.maxBytes) {\n\t\t\tconst candidates = removable\n\t\t\t\t.filter((entry) => !victims.includes(entry))\n\t\t\t\t.sort((left, right) => left.hits - right.hits || left.lastAccess - right.lastAccess);\n\t\t\tfor (const entry of candidates) {\n\t\t\t\tif (count <= this.maxEntries && bytes <= this.maxBytes) break;\n\t\t\t\tvictims.push(entry);\n\t\t\t\tcount -= 1;\n\t\t\t\tbytes -= entry.bytes;\n\t\t\t}\n\t\t}\n\t\tawait Promise.all(victims.map((entry) => this.removeEntry(entry)));\n\t\tawait this.cleanupOrphanFiles(now);\n\t}\n\n\tprivate async cleanupOrphanFiles(now: number): Promise<void> {\n\t\tconst staleBefore = now - 60 * 60 * 1000;\n\t\tlet shards: Dirent[];\n\t\ttry {\n\t\t\tshards = await readdir(this.directory, { withFileTypes: true });\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tfor (const shard of shards) {\n\t\t\tif (!shard.isDirectory() || !SHARD_PATTERN.test(shard.name)) continue;\n\t\t\tconst shardDir = join(this.directory, shard.name);\n\t\t\tlet files: readonly string[];\n\t\t\ttry {\n\t\t\t\tfiles = await readdir(shardDir);\n\t\t\t} catch {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tfor (const file of files) {\n\t\t\t\tconst isPage = file.endsWith(PAGE_SUFFIX);\n\t\t\t\tconst isTemp = file.endsWith(\".tmp\");\n\t\t\t\tif (!isPage && !isTemp) continue;\n\t\t\t\tif (isPage) {\n\t\t\t\t\tconst hash = file.slice(0, -PAGE_SUFFIX.length);\n\t\t\t\t\tif (!HASH_PATTERN.test(hash) || this.entries.has(hash)) continue;\n\t\t\t\t}\n\t\t\t\tconst path = join(shardDir, file);\n\t\t\t\ttry {\n\t\t\t\t\tconst info = await stat(path);\n\t\t\t\t\tif (info.mtimeMs < staleBefore) await unlink(path);\n\t\t\t\t} catch {\n\t\t\t\t\t// A concurrent process may have removed it already.\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate scheduleMetaFlush(): void {\n\t\tif (this.metaTimer !== undefined) return;\n\t\tthis.metaTimer = setTimeout(() => {\n\t\t\tthis.metaTimer = undefined;\n\t\t\tvoid this.flushUsage();\n\t\t}, this.metaFlushMs);\n\t\tthis.metaTimer.unref?.();\n\t}\n\n\tprivate schedulePrune(): void {\n\t\tif (this.pruneTimer !== undefined) return;\n\t\tthis.pruneTimer = setTimeout(() => {\n\t\t\tthis.pruneTimer = undefined;\n\t\t\tvoid this.enqueuePrune(Date.now(), true).catch((error: unknown) => {\n\t\t\t\tthis.logDebug(\"page cache prune failed\", error);\n\t\t\t});\n\t\t}, this.pruneIntervalMs);\n\t\tthis.pruneTimer.unref?.();\n\t}\n\n\tprivate logUnavailable(error: unknown): void {\n\t\tif (this.loggedUnavailable) return;\n\t\tthis.loggedUnavailable = true;\n\t\tthis.logger?.warn(\"page cache directory is unreadable; page caching is disabled\", {\n\t\t\tcause: error instanceof Error ? error.message : String(error),\n\t\t});\n\t}\n\n\tprivate logSkippedWrite(): void {\n\t\tif (this.loggedSkippedWrite) return;\n\t\tthis.loggedSkippedWrite = true;\n\t\tthis.logger?.debug(\"page cache write queue is full; skipping background writes\");\n\t}\n\n\tprivate logDebug(message: string, error?: unknown): void {\n\t\t// A logger failure must never affect the search result.\n\t\ttry {\n\t\t\tthis.logger?.debug(\n\t\t\t\tmessage,\n\t\t\t\terror === undefined\n\t\t\t\t\t? undefined\n\t\t\t\t\t: {\n\t\t\t\t\t\t\tcause: error instanceof Error ? error.message : String(error),\n\t\t\t\t\t\t},\n\t\t\t);\n\t\t} catch {\n\t\t\t// Ignore logger failures.\n\t\t}\n\t}\n}\n\ninterface SummaryEntryV1<T> {\n\treadonly value: T;\n\treadonly expiresAt: number;\n}\n\n/** Small in-memory LRU for successful summaries; failures are never stored. */\nexport class SummaryCache<T> {\n\tprivate readonly entries = new Map<string, SummaryEntryV1<T>>();\n\tprivate readonly maxEntries: number;\n\tprivate readonly ttlMs: number;\n\n\tconstructor(\n\t\tmaxEntries: number = SEARCH_LIMITS.summaryCacheEntries,\n\t\tttlMs: number = SEARCH_LIMITS.summaryCacheTtlMs,\n\t) {\n\t\tthis.maxEntries = maxEntries;\n\t\tthis.ttlMs = ttlMs;\n\t}\n\n\tget(key: string, now: number): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (entry === undefined) return undefined;\n\t\tif (now >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\tthis.entries.delete(key);\n\t\tthis.entries.set(key, entry);\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T, now: number): void {\n\t\tthis.entries.delete(key);\n\t\tthis.entries.set(key, { value, expiresAt: now + this.ttlMs });\n\t\twhile (this.entries.size > this.maxEntries) {\n\t\t\tconst oldest = this.entries.keys().next();\n\t\t\tif (oldest.done === true) break;\n\t\t\tthis.entries.delete(oldest.value);\n\t\t}\n\t}\n}\n"]}