@ai-agent-forge/plugin-search 0.0.0-stage → 0.88.16

package/dist/cache.js

@@ -0,0 +1,666 @@
+/**
+ * System-level disk page cache (搜索缓存持久化设计).
+ *
+ * Only extracted page content is persisted: title, text and the truncation flag.
+ * URLs, response headers, cookies, validators and fetch metadata stay out of the
+ * cache. Entries are shared by every agent/profile/session of the same OS user,
+ * use a hit-count-based retention window and are evicted by LFU then LRU.
+ *
+ * All I/O is asynchronous. Writes and hit-count bookkeeping are queued so a
+ * search call never waits for the disk; `flushWrites`/`flushUsage` exist for
+ * tests and graceful shutdown.
+ */
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readdir, readFile, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { SEARCH_LIMITS } from "./limits.js";
+import { byteLength, isRecord, normalizeUrl, urlHasUserInfo } from "./text.js";
+const META_SUFFIX = ".meta.json";
+const PAGE_SUFFIX = ".page.json";
+const MAX_META_BYTES = 16 * 1024;
+const HASH_PATTERN = /^[0-9a-f]{64}$/;
+const SHARD_PATTERN = /^[0-9a-f]{2}$/;
+function errorCode(error) {
+    if (typeof error !== "object" || error === null || !("code" in error))
+        return undefined;
+    const code = error.code;
+    return typeof code === "string" ? code : undefined;
+}
+function isMissing(error) {
+    return errorCode(error) === "ENOENT";
+}
+function finiteNonNegative(value) {
+    return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+function parseMeta(value) {
+    if (!isRecord(value))
+        return undefined;
+    const { version, hash, bytes, storedAt, expiresAt, hits, lastAccess } = value;
+    if (version !== 1 || typeof hash !== "string" || !HASH_PATTERN.test(hash))
+        return undefined;
+    if (!finiteNonNegative(bytes) || !finiteNonNegative(storedAt) || !finiteNonNegative(expiresAt))
+        return undefined;
+    if (!finiteNonNegative(hits) || !Number.isInteger(hits) || !finiteNonNegative(lastAccess))
+        return undefined;
+    return { version: 1, hash, bytes, storedAt, expiresAt, hits, lastAccess };
+}
+function parsePage(value) {
+    if (!isRecord(value))
+        return undefined;
+    const { version, title, text, truncated } = value;
+    if (version !== 1 || typeof text !== "string" || text === "" || typeof truncated !== "boolean")
+        return undefined;
+    if (title !== undefined && typeof title !== "string")
+        return undefined;
+    return { version: 1, text, truncated, ...(title === undefined ? {} : { title }) };
+}
+function defaultTtlTiers(tiers) {
+    if (tiers !== undefined && tiers.length === 4)
+        return tiers;
+    return SEARCH_LIMITS.cacheTtlTiersMs;
+}
+function serializePage(page) {
+    const file = {
+        version: 1,
+        text: page.text,
+        truncated: page.truncated,
+        ...(page.title === undefined ? {} : { title: page.title }),
+    };
+    return JSON.stringify(file);
+}
+/** OS cache directory shared by all profiles of the current OS user. */
+export function defaultPageCacheDirectory() {
+    const home = homedir();
+    if (process.platform === "darwin") {
+        return join(home, "Library", "Caches", "agent-forge", "search", "v1", "pages");
+    }
+    if (process.platform === "win32") {
+        const base = process.env.LOCALAPPDATA?.trim() || join(home, "AppData", "Local");
+        return join(base, "agent-forge", "Cache", "search", "v1", "pages");
+    }
+    const base = process.env.XDG_CACHE_HOME?.trim() || join(home, ".cache");
+    return join(base, "agent-forge", "search", "v1", "pages");
+}
+export class PageCache {
+    directory;
+    maxBytes;
+    maxEntries;
+    maxEntryBytes;
+    baseTtlMs;
+    ttlTiersMs;
+    writeQueueEntries;
+    writeQueueBytes;
+    metaFlushMs;
+    pruneIntervalMs;
+    logger;
+    entries = new Map();
+    pendingWrites = new Map();
+    dirtyMeta = new Map();
+    pendingWriteBytes = 0;
+    totalBytes = 0;
+    initPromise;
+    unavailable = false;
+    loggedUnavailable = false;
+    loggedSkippedWrite = false;
+    loggedWriteFailure = false;
+    metaTimer;
+    pruneTimer;
+    pruneChain = Promise.resolve();
+    constructor(directory = defaultPageCacheDirectory(), options = {}) {
+        this.directory = directory;
+        this.maxBytes = options.maxBytes ?? SEARCH_LIMITS.pageCacheBytes;
+        this.maxEntries = options.maxEntries ?? SEARCH_LIMITS.pageCacheEntries;
+        this.maxEntryBytes = options.maxEntryBytes ?? SEARCH_LIMITS.cacheMaxEntryBytes;
+        this.baseTtlMs = options.baseTtlMs ?? SEARCH_LIMITS.cacheBaseTtlMs;
+        this.ttlTiersMs = defaultTtlTiers(options.ttlTiersMs);
+        this.writeQueueEntries = options.writeQueueEntries ?? SEARCH_LIMITS.cacheWriteQueueEntries;
+        this.writeQueueBytes = options.writeQueueBytes ?? SEARCH_LIMITS.cacheWriteQueueBytes;
+        this.metaFlushMs = options.metaFlushMs ?? SEARCH_LIMITS.cacheMetaFlushMs;
+        this.pruneIntervalMs = options.pruneIntervalMs ?? SEARCH_LIMITS.cachePruneIntervalMs;
+        this.logger = options.logger;
+    }
+    /** Fresh content only; any cache failure degrades to a miss. */
+    async getFresh(rawUrl, now) {
+        try {
+            return await this.getFreshUnsafe(rawUrl, now);
+        }
+        catch (error) {
+            this.logDebug("page cache read failed; treating it as a miss", error);
+            return undefined;
+        }
+    }
+    async getFreshUnsafe(rawUrl, now) {
+        const hash = this.hashFor(rawUrl);
+        if (hash === undefined)
+            return undefined;
+        await this.ensureInitialized(now);
+        if (this.unavailable)
+            return undefined;
+        const pending = this.pendingWrites.get(hash);
+        if (pending !== undefined) {
+            await pending;
+            if (this.unavailable)
+                return undefined;
+        }
+        let entry = this.entries.get(hash);
+        if (entry === undefined) {
+            entry = await this.loadEntryFromDisk(hash, now);
+            if (entry === undefined)
+                return undefined;
+            this.entries.set(hash, entry);
+            this.totalBytes += entry.bytes;
+        }
+        if (now >= entry.expiresAt) {
+            await this.removeEntry(entry);
+            return undefined;
+        }
+        const content = await this.readPage(entry);
+        if (content === undefined) {
+            await this.removeEntry(entry);
+            return undefined;
+        }
+        if (this.entries.get(hash) !== entry)
+            return undefined;
+        entry.hits += 1;
+        entry.lastAccess = now;
+        entry.expiresAt = Math.max(entry.expiresAt, now + this.ttlForHits(entry.hits));
+        this.dirtyMeta.set(hash, entry);
+        this.scheduleMetaFlush();
+        return content;
+    }
+    /** Queue a content write; any cache failure is ignored by the caller. */
+    async set(rawUrl, page, now) {
+        try {
+            await this.setUnsafe(rawUrl, page, now);
+        }
+        catch (error) {
+            this.logDebug("page cache write setup failed; search continues", error);
+        }
+    }
+    async setUnsafe(rawUrl, page, now) {
+        const hash = this.hashFor(rawUrl);
+        if (hash === undefined)
+            return;
+        if (page.text.trim() === "")
+            return;
+        const bytes = byteLength(serializePage(page));
+        if (bytes > this.maxEntryBytes)
+            return;
+        await this.ensureInitialized(now);
+        if (this.unavailable)
+            return;
+        if (this.pendingWrites.has(hash))
+            return;
+        if (this.pendingWrites.size >= this.writeQueueEntries || this.pendingWriteBytes + bytes > this.writeQueueBytes) {
+            this.logSkippedWrite();
+            return;
+        }
+        const paths = this.pathsFor(hash);
+        const entry = {
+            hash,
+            pagePath: paths.pagePath,
+            metaPath: paths.metaPath,
+            bytes,
+            storedAt: now,
+            expiresAt: now + this.baseTtlMs,
+            hits: 0,
+            lastAccess: now,
+        };
+        const previous = this.entries.get(hash);
+        if (!this.makeRoom(hash, bytes - (previous?.bytes ?? 0), now)) {
+            this.schedulePrune();
+            return;
+        }
+        this.entries.set(hash, entry);
+        this.totalBytes += bytes - (previous?.bytes ?? 0);
+        this.pendingWriteBytes += bytes;
+        const write = this.writeEntry(entry, page).finally(() => {
+            this.pendingWriteBytes = Math.max(0, this.pendingWriteBytes - bytes);
+            this.pendingWrites.delete(hash);
+        });
+        this.pendingWrites.set(hash, write);
+        write.catch(() => {
+            // writeEntry records failures and drops the index entry; never surface
+            // an unhandled rejection from a background cache write.
+        });
+    }
+    /** Persist queued writes and hit counts; used by tests and graceful shutdown. */
+    async flushWrites() {
+        while (this.pendingWrites.size > 0) {
+            await Promise.allSettled([...this.pendingWrites.values()]);
+        }
+        await this.flushUsage();
+    }
+    async flushUsage() {
+        if (this.metaTimer !== undefined) {
+            clearTimeout(this.metaTimer);
+            this.metaTimer = undefined;
+        }
+        const snapshot = [...this.dirtyMeta.values()];
+        this.dirtyMeta.clear();
+        await Promise.all(snapshot.map(async (entry) => {
+            if (this.entries.get(entry.hash) !== entry)
+                return;
+            try {
+                await this.writeMetaFile(entry);
+            }
+            catch (error) {
+                if (this.entries.get(entry.hash) === entry)
+                    this.dirtyMeta.set(entry.hash, entry);
+                this.logDebug("page cache could not flush hit bookkeeping", error);
+            }
+        }));
+    }
+    /** Remove expired entries and enforce the size/count ceiling. */
+    async prune(now) {
+        await this.ensureInitialized(now);
+        await this.enqueuePrune(now, false);
+        await this.flushUsage();
+    }
+    hashFor(rawUrl) {
+        if (urlHasUserInfo(rawUrl))
+            return undefined;
+        try {
+            return createHash("sha256")
+                .update(`search-page-v1\n${normalizeUrl(rawUrl)}`)
+                .digest("hex");
+        }
+        catch {
+            return undefined;
+        }
+    }
+    pathsFor(hash) {
+        const dir = join(this.directory, hash.slice(0, 2));
+        return {
+            pagePath: join(dir, `${hash}${PAGE_SUFFIX}`),
+            metaPath: join(dir, `${hash}${META_SUFFIX}`),
+        };
+    }
+    /** Evict low-value entries from the index before admitting a new one. */
+    makeRoom(replacingHash, bytesDelta, now) {
+        const within = () => this.entries.size + (this.entries.has(replacingHash) ? 0 : 1) <= this.maxEntries &&
+            this.totalBytes + bytesDelta <= this.maxBytes;
+        if (within())
+            return true;
+        const candidates = [...this.entries.values()]
+            .filter((entry) => entry.hash !== replacingHash && !this.pendingWrites.has(entry.hash))
+            .sort((left, right) => {
+            const leftExpired = now >= left.expiresAt ? 1 : 0;
+            const rightExpired = now >= right.expiresAt ? 1 : 0;
+            if (leftExpired !== rightExpired)
+                return rightExpired - leftExpired;
+            return left.hits - right.hits || left.lastAccess - right.lastAccess;
+        });
+        for (const entry of candidates) {
+            if (within())
+                break;
+            this.entries.delete(entry.hash);
+            this.totalBytes = Math.max(0, this.totalBytes - entry.bytes);
+            this.dirtyMeta.delete(entry.hash);
+            void this.removeFiles(entry.pagePath, entry.metaPath);
+        }
+        return within();
+    }
+    ttlForHits(hits) {
+        if (hits >= 50)
+            return this.ttlTiersMs[3] ?? this.baseTtlMs;
+        if (hits >= 10)
+            return this.ttlTiersMs[2] ?? this.baseTtlMs;
+        if (hits >= 3)
+            return this.ttlTiersMs[1] ?? this.baseTtlMs;
+        if (hits >= 1)
+            return this.ttlTiersMs[0] ?? this.baseTtlMs;
+        return this.baseTtlMs;
+    }
+    ensureInitialized(now) {
+        this.initPromise ??= this.loadIndex(now);
+        return this.initPromise;
+    }
+    async loadIndex(now) {
+        let shards;
+        try {
+            shards = await readdir(this.directory, { withFileTypes: true });
+        }
+        catch (error) {
+            if (isMissing(error))
+                return;
+            this.unavailable = true;
+            this.logUnavailable(error);
+            return;
+        }
+        for (const shard of shards) {
+            if (!shard.isDirectory() || !SHARD_PATTERN.test(shard.name))
+                continue;
+            const shardDir = join(this.directory, shard.name);
+            let files;
+            try {
+                files = await readdir(shardDir);
+            }
+            catch (error) {
+                if (!isMissing(error))
+                    this.logDebug("page cache shard is unreadable", error);
+                continue;
+            }
+            for (const file of files) {
+                if (!file.endsWith(META_SUFFIX))
+                    continue;
+                const hash = file.slice(0, -META_SUFFIX.length);
+                if (!HASH_PATTERN.test(hash))
+                    continue;
+                const entry = await this.loadEntryFromDisk(hash, now);
+                if (entry === undefined)
+                    continue;
+                this.entries.set(hash, entry);
+                this.totalBytes += entry.bytes;
+            }
+        }
+        await this.enqueuePrune(now, false);
+    }
+    async loadEntryFromDisk(hash, now) {
+        const paths = this.pathsFor(hash);
+        const meta = await this.readMetaFile(paths.metaPath);
+        if (meta === undefined) {
+            // A page file can be visible for a moment before its meta file is
+            // renamed into place by another process; never delete that fresh file.
+            return undefined;
+        }
+        if (meta.hash !== hash) {
+            await this.removeFiles(paths.pagePath, paths.metaPath);
+            return undefined;
+        }
+        let pageSize;
+        try {
+            const pageStat = await stat(paths.pagePath);
+            pageSize = pageStat.size;
+        }
+        catch (error) {
+            if (!isMissing(error))
+                this.logDebug("page cache entry has no readable page file", error);
+            await this.removeFiles(paths.pagePath, paths.metaPath);
+            return undefined;
+        }
+        if (pageSize !== meta.bytes || pageSize > this.maxEntryBytes) {
+            await this.removeFiles(paths.pagePath, paths.metaPath);
+            return undefined;
+        }
+        if (now >= meta.expiresAt) {
+            await this.removeFiles(paths.pagePath, paths.metaPath);
+            return undefined;
+        }
+        return {
+            hash,
+            pagePath: paths.pagePath,
+            metaPath: paths.metaPath,
+            bytes: meta.bytes,
+            storedAt: meta.storedAt,
+            expiresAt: meta.expiresAt,
+            hits: meta.hits,
+            lastAccess: meta.lastAccess,
+        };
+    }
+    async readMetaFile(path) {
+        let size;
+        try {
+            size = (await stat(path)).size;
+        }
+        catch {
+            return undefined;
+        }
+        if (size > MAX_META_BYTES)
+            return undefined;
+        try {
+            const raw = await readFile(path, "utf8");
+            return parseMeta(JSON.parse(raw));
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async readPage(entry) {
+        try {
+            const info = await stat(entry.pagePath);
+            if (info.size !== entry.bytes)
+                return undefined;
+            const raw = await readFile(entry.pagePath, "utf8");
+            const parsed = parsePage(JSON.parse(raw));
+            if (parsed === undefined)
+                return undefined;
+            return {
+                text: parsed.text,
+                truncated: parsed.truncated,
+                ...(parsed.title === undefined ? {} : { title: parsed.title }),
+            };
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async writeEntry(entry, page) {
+        try {
+            await mkdir(join(this.directory, entry.hash.slice(0, 2)), { recursive: true, mode: 0o700 });
+            await this.atomicWrite(entry.pagePath, serializePage(page));
+            await this.writeMetaFile(entry);
+            this.schedulePrune();
+        }
+        catch (error) {
+            await this.removeFiles(entry.pagePath, entry.metaPath);
+            if (this.entries.get(entry.hash) === entry) {
+                this.entries.delete(entry.hash);
+                this.totalBytes = Math.max(0, this.totalBytes - entry.bytes);
+            }
+            if (!this.loggedWriteFailure) {
+                this.loggedWriteFailure = true;
+                this.logger?.warn("page cache write failed; search continues without caching", {
+                    cause: error instanceof Error ? error.message : String(error),
+                });
+            }
+            this.logDebug("page cache write failed", error);
+        }
+    }
+    async writeMetaFile(entry) {
+        const meta = {
+            version: 1,
+            hash: entry.hash,
+            bytes: entry.bytes,
+            storedAt: entry.storedAt,
+            expiresAt: entry.expiresAt,
+            hits: entry.hits,
+            lastAccess: entry.lastAccess,
+        };
+        await mkdir(join(this.directory, entry.hash.slice(0, 2)), { recursive: true, mode: 0o700 });
+        await this.atomicWrite(entry.metaPath, JSON.stringify(meta));
+    }
+    async atomicWrite(path, content) {
+        const temp = `${path}.${process.pid}.${randomUUID()}.tmp`;
+        try {
+            await writeFile(temp, content, { encoding: "utf8", mode: 0o600 });
+            try {
+                await rename(temp, path);
+            }
+            catch {
+                // Windows can refuse to replace an existing file; remove and retry.
+                await rm(path, { force: true });
+                await rename(temp, path);
+            }
+        }
+        finally {
+            await rm(temp, { force: true }).catch(() => { });
+        }
+    }
+    async removeEntry(entry) {
+        if (this.entries.get(entry.hash) === entry) {
+            this.entries.delete(entry.hash);
+            this.totalBytes = Math.max(0, this.totalBytes - entry.bytes);
+        }
+        this.dirtyMeta.delete(entry.hash);
+        await this.removeFiles(entry.pagePath, entry.metaPath);
+    }
+    async removeFiles(pagePath, metaPath) {
+        await Promise.all([
+            unlink(pagePath).catch((error) => {
+                if (!isMissing(error))
+                    this.logDebug("page cache could not remove a page file", error);
+            }),
+            unlink(metaPath).catch((error) => {
+                if (!isMissing(error))
+                    this.logDebug("page cache could not remove a meta file", error);
+            }),
+        ]);
+    }
+    enqueuePrune(now, capacityOnly) {
+        const run = () => this.pruneNow(now, capacityOnly);
+        this.pruneChain = this.pruneChain.then(run, run);
+        return this.pruneChain;
+    }
+    async pruneNow(now, capacityOnly) {
+        if (this.unavailable)
+            return;
+        const removable = [...this.entries.values()].filter((entry) => !this.pendingWrites.has(entry.hash));
+        const victims = [];
+        if (!capacityOnly) {
+            for (const entry of removable) {
+                if (now >= entry.expiresAt)
+                    victims.push(entry);
+            }
+        }
+        let count = this.entries.size - victims.length;
+        let bytes = this.totalBytes - victims.reduce((sum, entry) => sum + entry.bytes, 0);
+        if (count > this.maxEntries || bytes > this.maxBytes) {
+            const candidates = removable
+                .filter((entry) => !victims.includes(entry))
+                .sort((left, right) => left.hits - right.hits || left.lastAccess - right.lastAccess);
+            for (const entry of candidates) {
+                if (count <= this.maxEntries && bytes <= this.maxBytes)
+                    break;
+                victims.push(entry);
+                count -= 1;
+                bytes -= entry.bytes;
+            }
+        }
+        await Promise.all(victims.map((entry) => this.removeEntry(entry)));
+        await this.cleanupOrphanFiles(now);
+    }
+    async cleanupOrphanFiles(now) {
+        const staleBefore = now - 60 * 60 * 1000;
+        let shards;
+        try {
+            shards = await readdir(this.directory, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const shard of shards) {
+            if (!shard.isDirectory() || !SHARD_PATTERN.test(shard.name))
+                continue;
+            const shardDir = join(this.directory, shard.name);
+            let files;
+            try {
+                files = await readdir(shardDir);
+            }
+            catch {
+                continue;
+            }
+            for (const file of files) {
+                const isPage = file.endsWith(PAGE_SUFFIX);
+                const isTemp = file.endsWith(".tmp");
+                if (!isPage && !isTemp)
+                    continue;
+                if (isPage) {
+                    const hash = file.slice(0, -PAGE_SUFFIX.length);
+                    if (!HASH_PATTERN.test(hash) || this.entries.has(hash))
+                        continue;
+                }
+                const path = join(shardDir, file);
+                try {
+                    const info = await stat(path);
+                    if (info.mtimeMs < staleBefore)
+                        await unlink(path);
+                }
+                catch {
+                    // A concurrent process may have removed it already.
+                }
+            }
+        }
+    }
+    scheduleMetaFlush() {
+        if (this.metaTimer !== undefined)
+            return;
+        this.metaTimer = setTimeout(() => {
+            this.metaTimer = undefined;
+            void this.flushUsage();
+        }, this.metaFlushMs);
+        this.metaTimer.unref?.();
+    }
+    schedulePrune() {
+        if (this.pruneTimer !== undefined)
+            return;
+        this.pruneTimer = setTimeout(() => {
+            this.pruneTimer = undefined;
+            void this.enqueuePrune(Date.now(), true).catch((error) => {
+                this.logDebug("page cache prune failed", error);
+            });
+        }, this.pruneIntervalMs);
+        this.pruneTimer.unref?.();
+    }
+    logUnavailable(error) {
+        if (this.loggedUnavailable)
+            return;
+        this.loggedUnavailable = true;
+        this.logger?.warn("page cache directory is unreadable; page caching is disabled", {
+            cause: error instanceof Error ? error.message : String(error),
+        });
+    }
+    logSkippedWrite() {
+        if (this.loggedSkippedWrite)
+            return;
+        this.loggedSkippedWrite = true;
+        this.logger?.debug("page cache write queue is full; skipping background writes");
+    }
+    logDebug(message, error) {
+        // A logger failure must never affect the search result.
+        try {
+            this.logger?.debug(message, error === undefined
+                ? undefined
+                : {
+                    cause: error instanceof Error ? error.message : String(error),
+                });
+        }
+        catch {
+            // Ignore logger failures.
+        }
+    }
+}
+/** Small in-memory LRU for successful summaries; failures are never stored. */
+export class SummaryCache {
+    entries = new Map();
+    maxEntries;
+    ttlMs;
+    constructor(maxEntries = SEARCH_LIMITS.summaryCacheEntries, ttlMs = SEARCH_LIMITS.summaryCacheTtlMs) {
+        this.maxEntries = maxEntries;
+        this.ttlMs = ttlMs;
+    }
+    get(key, now) {
+        const entry = this.entries.get(key);
+        if (entry === undefined)
+            return undefined;
+        if (now >= entry.expiresAt) {
+            this.entries.delete(key);
+            return undefined;
+        }
+        this.entries.delete(key);
+        this.entries.set(key, entry);
+        return entry.value;
+    }
+    set(key, value, now) {
+        this.entries.delete(key);
+        this.entries.set(key, { value, expiresAt: now + this.ttlMs });
+        while (this.entries.size > this.maxEntries) {
+            const oldest = this.entries.keys().next();
+            if (oldest.done === true)
+                break;
+            this.entries.delete(oldest.value);
+        }
+    }
+}
+//# sourceMappingURL=cache.js.map