@zhivex-ai/core 1.5.0 → 1.7.0

package/dist/middleware.js

@@ -1,7 +1,91 @@
-import { promises as fs } from "node:fs";
+import { createHash, randomUUID } from "node:crypto";
+import { constants, promises as fs } from "node:fs";
 import path from "node:path";
-const serializeInput = (input) => JSON.stringify(input);
+import { ValidationError } from "./errors.js";
+import { canonicalStoreFileStem, ensurePrivateDirectory, writePrivateFile } from "./store-security.js";
+const cacheKeySensitiveField = /(?:^|[-_])(api[-_]?key|authorization|cookie|credential|password|secret|token)(?:$|[-_])/iu;
+const cacheKeyOmittedFields = new Set([
+    "abortSignal",
+    "execute",
+    "inputGuardrails",
+    "isEnabled",
+    "onError",
+    "outputGuardrails",
+    "signal"
+]);
+const canonicalCacheInput = (value, seen = new WeakSet()) => {
+    if (value === null)
+        return "null";
+    if (value === undefined)
+        return '"[undefined]"';
+    if (typeof value === "string" || typeof value === "boolean")
+        return JSON.stringify(value);
+    if (typeof value === "number") {
+        if (Number.isNaN(value))
+            return '"[number:NaN]"';
+        if (value === Number.POSITIVE_INFINITY)
+            return '"[number:Infinity]"';
+        if (value === Number.NEGATIVE_INFINITY)
+            return '"[number:-Infinity]"';
+        return JSON.stringify(Object.is(value, -0) ? 0 : value);
+    }
+    if (typeof value === "bigint")
+        return JSON.stringify(`[bigint:${value.toString()}]`);
+    if (typeof value === "function" || typeof value === "symbol") {
+        throw new TypeError("Generate cache keys cannot canonicalize functions or symbols.");
+    }
+    if (value instanceof Date)
+        return JSON.stringify(`[date:${value.toISOString()}]`);
+    if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
+        const bytes = value instanceof ArrayBuffer
+            ? new Uint8Array(value)
+            : new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+        return JSON.stringify(`[bytes:sha256:${createHash("sha256").update(bytes).digest("hex")}]`);
+    }
+    if (typeof AbortSignal !== "undefined" && value instanceof AbortSignal) {
+        return '"[abort-signal]"';
+    }
+    if (seen.has(value)) {
+        throw new TypeError("Generate cache keys cannot canonicalize cyclic input.");
+    }
+    seen.add(value);
+    try {
+        if (Array.isArray(value)) {
+            return `[${value.map((entry) => canonicalCacheInput(entry, seen)).join(",")}]`;
+        }
+        return `{${Object.keys(value)
+            .filter((key) => !cacheKeyOmittedFields.has(key))
+            .sort()
+            .map((key) => {
+            if (cacheKeySensitiveField.test(key)) {
+                throw new TypeError("Generate cache keys cannot include sensitive fields.");
+            }
+            const entry = value[key];
+            const serialized = canonicalCacheInput(entry, seen);
+            return `${JSON.stringify(key)}:${serialized}`;
+        })
+            .join(",")}}`;
+    }
+    finally {
+        seen.delete(value);
+    }
+};
+const createDefaultGenerateCacheKey = (input) => `generate:v2:${createHash("sha256").update(canonicalCacheInput(input)).digest("hex")}`;
+const defaultGenerateCacheModelScopes = new WeakMap();
+const getDefaultGenerateCacheModelScope = (model) => {
+    const existing = defaultGenerateCacheModelScopes.get(model);
+    if (existing) {
+        return existing;
+    }
+    const created = `model-instance:${randomUUID()}`;
+    defaultGenerateCacheModelScopes.set(model, created);
+    return created;
+};
 const telemetryObserversSymbol = Symbol("zhivex-ai.telemetry-observers");
+const DEFAULT_FILE_GENERATE_CACHE_MAX_KEY_BYTES = 1024 * 1024;
+const DEFAULT_FILE_GENERATE_CACHE_MAX_ENTRY_BYTES = 16 * 1024 * 1024;
+const FILE_GENERATE_CACHE_READ_CHUNK_BYTES = 64 * 1024;
+const FILE_GENERATE_CACHE_SCHEMA_VERSION = 1;
 const getTelemetryObservers = (model) => [...(model[telemetryObserversSymbol] ?? [])];
 export const emitLanguageModelTelemetryEvent = async (model, event) => {
     const observers = getTelemetryObservers(model);
@@ -75,13 +159,26 @@ export const wrapLanguageModel = (model, middlewares) => {
     return wrappedModel;
 };
 export const createTelemetryMiddleware = (options) => {
+    let nextGenerateId = 1;
+    let nextStreamId = 1;
+    const reportTelemetry = async (event) => {
+        try {
+            await options.onEvent(event);
+        }
+        catch {
+            // Telemetry is best-effort and never owns the model operation outcome.
+        }
+    };
     const middleware = {
         name: "telemetry",
-        onTelemetryEvent: options.onEvent,
+        onTelemetryEvent: reportTelemetry,
         async wrapGenerate(context, next) {
+            const generateId = nextGenerateId;
+            nextGenerateId += 1;
             const startedAt = Date.now();
-            await options.onEvent({
+            await reportTelemetry({
                 type: "generate-start",
+                generateId,
                 model: context.model,
                 input: context.input,
                 startedAt
@@ -89,8 +186,9 @@ export const createTelemetryMiddleware = (options) => {
             try {
                 const output = await next();
                 const finishedAt = Date.now();
-                await options.onEvent({
+                await reportTelemetry({
                     type: "generate-finish",
+                    generateId,
                     model: context.model,
                     input: context.input,
                     output,
@@ -103,8 +201,9 @@ export const createTelemetryMiddleware = (options) => {
             catch (error) {
                 const finishedAt = Date.now();
                 const err = error instanceof Error ? error : new Error(String(error));
-                await options.onEvent({
+                await reportTelemetry({
                     type: "generate-error",
+                    generateId,
                     model: context.model,
                     input: context.input,
                     error: err,
@@ -116,93 +215,189 @@ export const createTelemetryMiddleware = (options) => {
             }
         },
         async wrapStream(context, next) {
-            const startedAt = Date.now();
-            await options.onEvent({
-                type: "stream-start",
-                model: context.model,
-                input: context.input,
-                startedAt
-            });
-            try {
-                const stream = await next();
-                return (async function* () {
-                    let finishReason;
-                    let providerFinishReason;
-                    let usage;
-                    try {
-                        for await (const event of stream) {
-                            if (event.type === "finish") {
-                                finishReason = event.finishReason;
-                                providerFinishReason = event.providerFinishReason;
-                                usage = event.usage;
-                            }
-                            yield event;
+            const streamId = nextStreamId;
+            nextStreamId += 1;
+            return (async function* () {
+                const operationStartedAt = Date.now();
+                let finishReason;
+                let providerFinishReason;
+                let usage;
+                let firstChunkAt;
+                let previousChunkAt;
+                let outputChunkCount = 0;
+                let terminalEventEmitted = false;
+                try {
+                    await reportTelemetry({
+                        type: "stream-start",
+                        streamId,
+                        model: context.model,
+                        input: context.input,
+                        startedAt: operationStartedAt
+                    });
+                    const stream = await next();
+                    for await (const event of stream) {
+                        if (event.type === "finish") {
+                            finishReason = event.finishReason;
+                            providerFinishReason = event.providerFinishReason;
+                            usage = event.usage;
+                            const finishedAt = Date.now();
+                            terminalEventEmitted = true;
+                            await reportTelemetry({
+                                type: "stream-finish",
+                                streamId,
+                                model: context.model,
+                                input: context.input,
+                                startedAt: operationStartedAt,
+                                finishedAt,
+                                latencyMs: finishedAt - operationStartedAt,
+                                finishReason,
+                                providerFinishReason,
+                                usage,
+                                outputChunkCount
+                            });
+                        }
+                        else if (event.type === "error") {
+                            const finishedAt = Date.now();
+                            terminalEventEmitted = true;
+                            await reportTelemetry({
+                                type: "stream-error",
+                                streamId,
+                                model: context.model,
+                                input: context.input,
+                                error: event.error,
+                                startedAt: operationStartedAt,
+                                finishedAt,
+                                latencyMs: finishedAt - operationStartedAt,
+                                outputChunkCount
+                            });
+                        }
+                        else {
+                            const chunkAt = Date.now();
+                            firstChunkAt ??= chunkAt;
+                            const timeSincePreviousChunkMs = previousChunkAt === undefined
+                                ? undefined
+                                : chunkAt - previousChunkAt;
+                            previousChunkAt = chunkAt;
+                            outputChunkCount += 1;
+                            await reportTelemetry({
+                                type: "stream-chunk",
+                                streamId,
+                                model: context.model,
+                                input: context.input,
+                                startedAt: operationStartedAt,
+                                chunkAt,
+                                chunkIndex: outputChunkCount,
+                                timeToFirstChunkMs: outputChunkCount === 1 ? chunkAt - operationStartedAt : undefined,
+                                timeSincePreviousChunkMs
+                            });
                         }
+                        yield event;
+                        if (terminalEventEmitted)
+                            return;
+                    }
+                    if (!terminalEventEmitted) {
                         const finishedAt = Date.now();
-                        await options.onEvent({
+                        terminalEventEmitted = true;
+                        await reportTelemetry({
                             type: "stream-finish",
+                            streamId,
                             model: context.model,
                             input: context.input,
-                            startedAt,
+                            startedAt: operationStartedAt,
                             finishedAt,
-                            latencyMs: finishedAt - startedAt,
+                            latencyMs: finishedAt - operationStartedAt,
                             finishReason,
                             providerFinishReason,
-                            usage
+                            usage,
+                            outputChunkCount
                         });
                     }
-                    catch (error) {
-                        const finishedAt = Date.now();
-                        const err = error instanceof Error ? error : new Error(String(error));
-                        await options.onEvent({
+                }
+                catch (error) {
+                    const finishedAt = Date.now();
+                    const err = error instanceof Error ? error : new Error(String(error));
+                    if (!terminalEventEmitted) {
+                        terminalEventEmitted = true;
+                        await reportTelemetry({
                             type: "stream-error",
+                            streamId,
                             model: context.model,
                             input: context.input,
                             error: err,
-                            startedAt,
+                            startedAt: operationStartedAt,
                             finishedAt,
-                            latencyMs: finishedAt - startedAt
+                            latencyMs: finishedAt - operationStartedAt,
+                            outputChunkCount
                         });
-                        throw error;
                     }
-                })();
-            }
-            catch (error) {
-                const finishedAt = Date.now();
-                const err = error instanceof Error ? error : new Error(String(error));
-                await options.onEvent({
-                    type: "stream-error",
-                    model: context.model,
-                    input: context.input,
-                    error: err,
-                    startedAt,
-                    finishedAt,
-                    latencyMs: finishedAt - startedAt
-                });
-                throw error;
-            }
+                    throw error;
+                }
+                finally {
+                    if (!terminalEventEmitted) {
+                        terminalEventEmitted = true;
+                        const finishedAt = Date.now();
+                        const error = Object.assign(new Error("Stream consumption ended before the provider stream completed."), { name: "AbortError" });
+                        await reportTelemetry({
+                            type: "stream-error",
+                            streamId,
+                            model: context.model,
+                            input: context.input,
+                            error,
+                            startedAt: operationStartedAt,
+                            finishedAt,
+                            latencyMs: finishedAt - operationStartedAt,
+                            outputChunkCount
+                        });
+                    }
+                }
+            })();
         }
     };
     return middleware;
 };
-export const createCachedGenerateMiddleware = (options) => ({
-    name: "cache",
-    async wrapGenerate(context, next) {
-        const key = options.getKey?.(context.input, context.model) ??
-            serializeInput({
-                provider: context.model.provider,
-                modelId: context.model.modelId,
-                input: context.input
-            });
-        const cached = await options.cache.get(key);
-        if (cached) {
-            return cached;
-        }
-        const output = await next();
-        await options.cache.set(key, output);
-        return output;
+export const createCachedGenerateMiddleware = (options) => {
+    if (typeof options.scope === "string" && options.scope.trim().length === 0) {
+        throw new ValidationError('The generate cache "scope" must be a non-empty string.');
     }
-});
+    return {
+        name: "cache",
+        async wrapGenerate(context, next) {
+            if (!options.getKey && options.scope === undefined && options.cache.scopeRequirement === "stable") {
+                return next();
+            }
+            let key;
+            if (options.getKey) {
+                key = options.getKey(context.input, context.model);
+            }
+            else {
+                const scope = typeof options.scope === "function"
+                    ? options.scope(context.input, context.model)
+                    : options.scope ?? getDefaultGenerateCacheModelScope(context.model);
+                if (typeof scope !== "string" || scope.trim().length === 0) {
+                    throw new ValidationError('The generate cache "scope" must resolve to a non-empty string.');
+                }
+                try {
+                    key = createDefaultGenerateCacheKey({
+                        scope,
+                        provider: context.model.provider,
+                        modelId: context.model.modelId,
+                        input: context.input
+                    });
+                }
+                catch {
+                    return next();
+                }
+            }
+            const cached = await options.cache.get(key);
+            if (cached) {
+                return cached;
+            }
+            const output = await next();
+            await options.cache.set(key, output);
+            return output;
+        }
+    };
+};
 export const createInMemoryGenerateCache = () => {
     const store = new Map();
     return {
@@ -214,13 +409,85 @@ export const createInMemoryGenerateCache = () => {
         }
     };
 };
+const positiveFileCacheLimit = (value, fallback, name) => {
+    const resolved = value ?? fallback;
+    if (!Number.isSafeInteger(resolved) || resolved <= 0) {
+        throw new ValidationError(`The file generate cache "${name}" limit must be a positive safe integer.`);
+    }
+    return resolved;
+};
+const readFileWithinLimit = async (filePath, maxBytes) => {
+    const handle = await fs.open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        const stat = await handle.stat();
+        if (!stat.isFile() || stat.size > maxBytes) {
+            return undefined;
+        }
+        const chunks = [];
+        let receivedBytes = 0;
+        while (receivedBytes <= maxBytes) {
+            const chunk = Buffer.allocUnsafe(Math.min(FILE_GENERATE_CACHE_READ_CHUNK_BYTES, maxBytes - receivedBytes + 1));
+            const { bytesRead } = await handle.read(chunk, 0, chunk.byteLength, null);
+            if (bytesRead === 0) {
+                return Buffer.concat(chunks, receivedBytes).toString("utf8");
+            }
+            receivedBytes += bytesRead;
+            if (receivedBytes > maxBytes) {
+                return undefined;
+            }
+            chunks.push(chunk.subarray(0, bytesRead));
+        }
+        return undefined;
+    }
+    finally {
+        await handle.close();
+    }
+};
 export const createFileGenerateCache = (options) => {
-    const getPath = (key) => path.join(options.dir, `${Buffer.from(key).toString("base64url")}.json`);
+    const maxKeyBytes = positiveFileCacheLimit(options.maxKeyBytes, DEFAULT_FILE_GENERATE_CACHE_MAX_KEY_BYTES, "maxKeyBytes");
+    const maxEntryBytes = positiveFileCacheLimit(options.maxEntryBytes, DEFAULT_FILE_GENERATE_CACHE_MAX_ENTRY_BYTES, "maxEntryBytes");
+    if (options.ttlMs !== undefined && (!Number.isSafeInteger(options.ttlMs) || options.ttlMs <= 0)) {
+        throw new ValidationError('The file generate cache "ttlMs" must be a positive safe integer.');
+    }
+    const keyIsWithinLimit = (key) => Buffer.byteLength(key, "utf8") <= maxKeyBytes;
+    const getPath = (key) => path.join(options.dir, `${canonicalStoreFileStem("generate-cache", [key])}.json`);
     return {
+        scopeRequirement: "stable",
         async get(key) {
+            if (!keyIsWithinLimit(key)) {
+                return undefined;
+            }
             try {
-                const file = await fs.readFile(getPath(key), "utf8");
-                return JSON.parse(file);
+                const file = await readFileWithinLimit(getPath(key), maxEntryBytes);
+                if (file === undefined) {
+                    return undefined;
+                }
+                try {
+                    const parsed = JSON.parse(file);
+                    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+                        return undefined;
+                    }
+                    const envelope = parsed;
+                    if (envelope.schemaVersion !== FILE_GENERATE_CACHE_SCHEMA_VERSION ||
+                        typeof envelope.createdAt !== "number" ||
+                        !Number.isSafeInteger(envelope.createdAt) ||
+                        envelope.value === null ||
+                        typeof envelope.value !== "object" ||
+                        Array.isArray(envelope.value)) {
+                        return undefined;
+                    }
+                    if (options.ttlMs !== undefined && Date.now() - envelope.createdAt >= options.ttlMs) {
+                        await fs.unlink(getPath(key)).catch(() => undefined);
+                        return undefined;
+                    }
+                    return envelope.value;
+                }
+                catch (error) {
+                    if (error instanceof SyntaxError) {
+                        return undefined;
+                    }
+                    throw error;
+                }
             }
             catch (error) {
                 const err = error;
@@ -231,57 +498,156 @@ export const createFileGenerateCache = (options) => {
             }
         },
         async set(key, value) {
-            await fs.mkdir(options.dir, { recursive: true });
-            await fs.writeFile(getPath(key), JSON.stringify(value), "utf8");
+            if (!keyIsWithinLimit(key)) {
+                return;
+            }
+            let serialized;
+            try {
+                serialized = JSON.stringify({
+                    schemaVersion: FILE_GENERATE_CACHE_SCHEMA_VERSION,
+                    createdAt: Date.now(),
+                    value
+                });
+            }
+            catch (error) {
+                if (error instanceof TypeError) {
+                    return;
+                }
+                throw error;
+            }
+            if (Buffer.byteLength(serialized, "utf8") > maxEntryBytes) {
+                return;
+            }
+            await ensurePrivateDirectory(options.dir);
+            await fs.chmod(options.dir, 0o700);
+            await writePrivateFile(getPath(key), serialized);
         }
     };
 };
 export const createCircuitBreakerMiddleware = (options) => {
-    const failureThreshold = Math.max(1, options.failureThreshold ?? 3);
-    const cooldownMs = Math.max(0, options.cooldownMs ?? 30_000);
-    const state = { failures: 0 };
+    const failureThreshold = options.failureThreshold ?? 3;
+    const cooldownMs = options.cooldownMs ?? 30_000;
+    if (!Number.isSafeInteger(failureThreshold) || failureThreshold <= 0) {
+        throw new ValidationError('The circuit breaker "failureThreshold" must be a positive safe integer.');
+    }
+    if (!Number.isSafeInteger(cooldownMs) || cooldownMs < 0) {
+        throw new ValidationError('The circuit breaker "cooldownMs" must be a non-negative safe integer.');
+    }
+    const states = new WeakMap();
+    const getState = (model) => {
+        const existing = states.get(model);
+        if (existing) {
+            return existing;
+        }
+        const created = { failures: 0, halfOpenProbeInFlight: false };
+        states.set(model, created);
+        return created;
+    };
+    const notifyStateChange = async (model, state, status) => {
+        try {
+            await options.onStateChange?.({
+                failures: state.failures,
+                openedAt: state.openedAt,
+                model,
+                status
+            });
+        }
+        catch {
+            // State observers are operational hooks and must not alter model outcomes.
+        }
+    };
+    const openError = (model) => new Error(`Circuit breaker open for model "${model.provider}/${model.modelId}".`);
+    const beginRequest = async (model) => {
+        const state = getState(model);
+        if (state.openedAt === undefined) {
+            return { state, halfOpenProbe: false };
+        }
+        const cooldownElapsed = Date.now() - state.openedAt >= cooldownMs;
+        if (!cooldownElapsed || state.halfOpenProbeInFlight) {
+            throw openError(model);
+        }
+        state.halfOpenProbeInFlight = true;
+        await notifyStateChange(model, state, "half-open");
+        return { state, halfOpenProbe: true };
+    };
+    const markSuccess = async (model, permit) => {
+        const { state } = permit;
+        if (!permit.halfOpenProbe && state.openedAt !== undefined) {
+            return;
+        }
+        const changed = state.failures > 0 || state.openedAt !== undefined || state.halfOpenProbeInFlight;
+        state.failures = 0;
+        state.openedAt = undefined;
+        state.halfOpenProbeInFlight = false;
+        if (changed) {
+            await notifyStateChange(model, state, "closed");
+        }
+    };
+    const markFailure = async (model, permit, error) => {
+        const { state } = permit;
+        const err = error instanceof Error ? error : new Error(String(error));
+        let isFailure = true;
+        try {
+            isFailure = options.isFailure?.(err) ?? true;
+        }
+        catch {
+            isFailure = true;
+        }
+        if (!isFailure) {
+            if (permit.halfOpenProbe) {
+                await markSuccess(model, permit);
+            }
+            return;
+        }
+        const wasOpen = state.openedAt !== undefined;
+        state.failures += 1;
+        if (permit.halfOpenProbe) {
+            state.halfOpenProbeInFlight = false;
+            state.openedAt = Date.now();
+            await notifyStateChange(model, state, "open");
+        }
+        else if (!wasOpen && state.failures >= failureThreshold) {
+            state.openedAt = Date.now();
+            await notifyStateChange(model, state, "open");
+        }
+    };
     return {
         name: "circuit-breaker",
         async wrapGenerate(context, next) {
-            const now = Date.now();
-            if (state.openedAt && now - state.openedAt < cooldownMs) {
-                throw new Error(`Circuit breaker open for model "${context.model.provider}/${context.model.modelId}".`);
-            }
-            if (state.openedAt && now - state.openedAt >= cooldownMs) {
-                await options.onStateChange?.({
-                    ...state,
-                    model: context.model,
-                    status: "half-open"
-                });
-            }
+            const permit = await beginRequest(context.model);
             try {
                 const result = await next();
-                state.failures = 0;
-                state.openedAt = undefined;
-                await options.onStateChange?.({
-                    ...state,
-                    model: context.model,
-                    status: "closed"
-                });
+                await markSuccess(context.model, permit);
                 return result;
             }
             catch (error) {
-                const err = error instanceof Error ? error : new Error(String(error));
-                const isFailure = options.isFailure?.(err) ?? true;
-                if (!isFailure) {
+                await markFailure(context.model, permit, error);
+                throw error;
+            }
+        },
+        async wrapStream(context, next) {
+            return (async function* () {
+                const permit = await beginRequest(context.model);
+                let settled = false;
+                try {
+                    const stream = await next();
+                    for await (const event of stream) {
+                        yield event;
+                    }
+                    await markSuccess(context.model, permit);
+                    settled = true;
+                }
+                catch (error) {
+                    await markFailure(context.model, permit, error);
+                    settled = true;
                     throw error;
                 }
-                state.failures += 1;
-                if (state.failures >= failureThreshold) {
-                    state.openedAt = Date.now();
-                    await options.onStateChange?.({
-                        ...state,
-                        model: context.model,
-                        status: "open"
-                    });
+                finally {
+                    if (!settled && permit.halfOpenProbe) {
+                        permit.state.halfOpenProbeInFlight = false;
+                    }
                 }
-                throw error;
-            }
+            })();
         }
     };
 };