@intflows/genkit-guard 0.0.11 → 0.0.12

package/README.md

@@ -208,6 +208,7 @@ By default, PII is stored in an in-memory vault scoped to a single tokenizer ins
 
 That generated namespace prevents two concurrent calls from sharing the same visible placeholder names. Vault lookups are isolated by the configured storage scope, so User A and User B can safely produce their own email tokens without cross-resolving each other's PII.
 
+For applications that need persistence, distributed workers, audits, or tenant-specific storage, provide a vault storage backend. Redis clients can be passed through the built-in helper:
 For applications that need persistence, distributed workers, audits, or tenant-specific storage, provide a vault storage backend. Redis clients can be passed through the built-in helper:
 
 ```ts
@@ -223,7 +224,8 @@ guard({
     vault: {
       storage: createRedisPiiVaultStorage(redis, {
         keyPrefix: "my-app:pii",
-        ttlSeconds: 3600
+        ttlSeconds: 3600,
+        fallbackToMemory: true
       }),
       scopeId: (req, ctx) => ctx?.auth?.sessionId ?? req?.metadata?.requestId
     }
@@ -231,6 +233,12 @@ guard({
 });
 ```
 
+`ttlSeconds` applies the configured expiry to both the scoped vault and token index. When
+`fallbackToMemory` is enabled, successful writes are also mirrored in process memory and Redis
+operation failures fall back to that mirror. The fallback is disabled by default, is local to one
+process, and is not a replacement for Redis persistence or multi-worker availability. Its in-memory
+entries observe the same TTL. Redis errors continue to propagate when fallback is disabled.
+
 For another backend, use `createPiiVaultStorage({ get, set, entries, getByToken })` with your database, cache, or secret store.
 
 Choose a `scopeId` that matches your isolation boundary, such as request ID, session ID, tenant/user ID, or a combination like `tenantId:userId:requestId`. A shared external backend should never ignore `scopeId`, because placeholders are only safe when resolved against the correct vault scope. The placeholder sent to the model uses an opaque generated namespace rather than exposing your `scopeId`.

package/dist/pii/storage.d.ts

@@ -27,7 +27,13 @@ export type RedisPiiVaultClient = {
 export type RedisPiiVaultStorageOptions = {
     keyPrefix?: string;
     tokenIndexKey?: string;
+    /** Expire Redis vault keys after this many seconds. Omit to keep them indefinitely. */
     ttlSeconds?: number;
+    /**
+     * Keep a process-local mirror and use it when a Redis operation fails.
+     * Disabled by default so Redis failures remain visible to callers.
+     */
+    fallbackToMemory?: boolean;
 };
 export declare function createRedisPiiVaultStorage(redis: RedisPiiVaultClient, options?: RedisPiiVaultStorageOptions): PiiVaultStorage;
 export declare class InMemoryPiiVaultStorage implements PiiVaultStorage {

package/dist/pii/storage.js

@@ -4,6 +4,13 @@ export function createPiiVaultStorage(adapter) {
 export function createRedisPiiVaultStorage(redis, options = {}) {
     const keyPrefix = options.keyPrefix ?? 'genkit-guard:pii';
     const tokenIndexKey = options.tokenIndexKey ?? `${keyPrefix}:tokens`;
+    const ttlSeconds = options.ttlSeconds;
+    if (ttlSeconds !== undefined && (!Number.isInteger(ttlSeconds) || ttlSeconds <= 0)) {
+        throw new Error('Redis PII vault ttlSeconds must be a positive integer.');
+    }
+    if (ttlSeconds !== undefined && !redis.expire) {
+        throw new Error('Redis PII vault ttlSeconds requires an expire method on the Redis client.');
+    }
     const hGet = redis.hGet?.bind(redis) ?? redis.hget?.bind(redis);
     const hSet = redis.hSet?.bind(redis) ?? redis.hset?.bind(redis);
     const hGetAll = redis.hGetAll?.bind(redis) ?? redis.hgetall?.bind(redis);
@@ -11,31 +18,81 @@ export function createRedisPiiVaultStorage(redis, options = {}) {
         throw new Error('Redis PII vault storage requires hGet/hSet/hGetAll or hget/hset/hgetall methods.');
     }
     const scopeKey = (scopeId) => `${keyPrefix}:scope:${scopeId}`;
+    const fallback = options.fallbackToMemory ? new ExpiringInMemoryPiiVaultStorage(ttlSeconds) : undefined;
     async function maybeExpire(key) {
-        if (options.ttlSeconds && redis.expire) {
-            await redis.expire(key, options.ttlSeconds);
+        if (ttlSeconds !== undefined) {
+            await redis.expire(key, ttlSeconds);
+        }
+    }
+    async function withFallback(redisOperation, memoryOperation) {
+        try {
+            return await redisOperation();
+        }
+        catch (error) {
+            if (!fallback)
+                throw error;
+            return memoryOperation();
         }
     }
     return createPiiVaultStorage({
         async get(scopeId, token) {
-            return (await hGet(scopeKey(scopeId), token)) ?? undefined;
+            return withFallback(async () => (await hGet(scopeKey(scopeId), token)) ?? undefined, () => fallback.get(scopeId, token));
         },
         async getByToken(token) {
-            return (await hGet(tokenIndexKey, token)) ?? undefined;
+            return withFallback(async () => (await hGet(tokenIndexKey, token)) ?? undefined, () => fallback.getByToken(token));
         },
         async set(scopeId, token, value) {
             const scopedKey = scopeKey(scopeId);
-            await hSet(scopedKey, token, value);
-            await hSet(tokenIndexKey, token, value);
-            await maybeExpire(scopedKey);
-            await maybeExpire(tokenIndexKey);
+            // Warm the opt-in fallback on every write so data written before an outage is available.
+            await fallback?.set(scopeId, token, value);
+            await withFallback(async () => {
+                await hSet(scopedKey, token, value);
+                await hSet(tokenIndexKey, token, value);
+                await maybeExpire(scopedKey);
+                await maybeExpire(tokenIndexKey);
+            }, () => undefined);
         },
         async entries(scopeId) {
-            const values = await hGetAll(scopeKey(scopeId));
-            return Object.entries(values).map(([token, value]) => ({ token, value }));
+            return withFallback(async () => {
+                const values = await hGetAll(scopeKey(scopeId));
+                return Object.entries(values).map(([token, value]) => ({ token, value }));
+            }, () => fallback.entries(scopeId));
         },
     });
 }
+class ExpiringInMemoryPiiVaultStorage {
+    ttlSeconds;
+    storage = new InMemoryPiiVaultStorage();
+    scopeExpiresAt = new Map();
+    tokenIndexExpiresAt;
+    constructor(ttlSeconds) {
+        this.ttlSeconds = ttlSeconds;
+    }
+    get(scopeId, token) {
+        return this.isScopeExpired(scopeId) ? undefined : this.storage.get(scopeId, token);
+    }
+    getByToken(token) {
+        return this.isTokenIndexExpired() ? undefined : this.storage.getByToken(token);
+    }
+    set(scopeId, token, value) {
+        this.storage.set(scopeId, token, value);
+        if (this.ttlSeconds !== undefined) {
+            const expiry = Date.now() + this.ttlSeconds * 1_000;
+            this.scopeExpiresAt.set(scopeId, expiry);
+            this.tokenIndexExpiresAt = expiry;
+        }
+    }
+    entries(scopeId) {
+        return this.isScopeExpired(scopeId) ? [] : this.storage.entries(scopeId);
+    }
+    isScopeExpired(scopeId) {
+        const expiry = this.scopeExpiresAt.get(scopeId);
+        return expiry !== undefined && expiry <= Date.now();
+    }
+    isTokenIndexExpired() {
+        return this.tokenIndexExpiresAt !== undefined && this.tokenIndexExpiresAt <= Date.now();
+    }
+}
 export class InMemoryPiiVaultStorage {
     scopes = new Map();
     tokenIndex = new Map();

package/package.json

@@ -22,7 +22,7 @@
     "huggingface"
   ],
   "license": "Apache-2.0",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "type": "module",
   "exports": "./dist/index.js",
   "types": "./dist/index.d.ts",

package/scripts/test-storage.js

@@ -1,78 +1,137 @@
-import assert from 'node:assert/strict';
-import {
-  createPiiVaultStorage,
-  createRedisPiiVaultStorage,
-  InMemoryPiiVaultStorage,
-} from '../dist/index.js';
-import { PiiTokenizer } from '../dist/pii/tokenizer.js';
-
-class FakeRedis {
-  hashes = new Map();
-  expirations = new Map();
-
-  async hGet(key, field) {
-    return this.hashes.get(key)?.[field] ?? null;
-  }
-
-  async hSet(key, field, value) {
-    const hash = this.hashes.get(key) ?? {};
-    hash[field] = value;
-    this.hashes.set(key, hash);
-  }
-
-  async hGetAll(key) {
-    return this.hashes.get(key) ?? {};
-  }
-
-  async expire(key, seconds) {
-    this.expirations.set(key, seconds);
-  }
-}
-
-const memory = new InMemoryPiiVaultStorage();
-await memory.set('scope-a', '[[EMAIL_token_0]]', 'a@example.com');
-assert.equal(await memory.get('scope-a', '[[EMAIL_token_0]]'), 'a@example.com');
-assert.equal(await memory.get('scope-b', '[[EMAIL_token_0]]'), undefined);
-assert.equal(await memory.getByToken('[[EMAIL_token_0]]'), 'a@example.com');
-
-const custom = createPiiVaultStorage({
-  async get(scopeId, token) {
-    return scopeId === 'custom' && token === '[[EMAIL_custom_0]]' ? 'custom@example.com' : undefined;
-  },
-  async set() {},
-  async entries(scopeId) {
-    return scopeId === 'custom'
-      ? [{ token: '[[EMAIL_custom_0]]', value: 'custom@example.com' }]
-      : [];
-  },
-});
-
-const customTokenizer = new PiiTokenizer({ scopeId: 'custom', storage: custom });
-assert.equal(await customTokenizer.unmask('Hi [[EMAIL_custom_0]]'), 'Hi custom@example.com');
-
-const redis = new FakeRedis();
-const redisStorage = createRedisPiiVaultStorage(redis, {
-  keyPrefix: 'test:pii',
-  ttlSeconds: 60,
-});
-
-const userATokenizer = new PiiTokenizer({ scopeId: 'tenant:userA', storage: redisStorage });
-const userBTokenizer = new PiiTokenizer({ scopeId: 'tenant:userB', storage: redisStorage });
-
-const userAMasked = await userATokenizer.mask('Email alice@example.com', [
-  { type: 'EMAIL', value: 'alice@example.com' },
-]);
-const userBMasked = await userBTokenizer.mask('Email bob@example.com', [
-  { type: 'EMAIL', value: 'bob@example.com' },
-]);
-
-assert.notEqual(userAMasked.maskedText, userBMasked.maskedText);
-assert.equal(await userATokenizer.unmask(userAMasked.maskedText), 'Email alice@example.com');
-assert.equal(await userBTokenizer.unmask(userBMasked.maskedText), 'Email bob@example.com');
-assert.equal(await userATokenizer.unmask(userBMasked.maskedText), userBMasked.maskedText);
-
-const laterPassTokenizer = new PiiTokenizer({ scopeId: 'tenant:userA:later', storage: redisStorage });
-await laterPassTokenizer.importTokens(userAMasked.maskedText);
-assert.equal(await laterPassTokenizer.unmask(userAMasked.maskedText), 'Email alice@example.com');
-
-console.log('PII vault storage tests passed.');
+import assert from 'node:assert/strict';
+import {
+  createPiiVaultStorage,
+  createRedisPiiVaultStorage,
+  InMemoryPiiVaultStorage,
+} from '../dist/index.js';
+import { PiiTokenizer } from '../dist/pii/tokenizer.js';
+
+class FakeRedis {
+  hashes = new Map();
+  expirations = new Map();
+
+  async hGet(key, field) {
+    return this.hashes.get(key)?.[field] ?? null;
+  }
+
+  async hSet(key, field, value) {
+    const hash = this.hashes.get(key) ?? {};
+    hash[field] = value;
+    this.hashes.set(key, hash);
+  }
+
+  async hGetAll(key) {
+    return this.hashes.get(key) ?? {};
+  }
+
+  async expire(key, seconds) {
+    this.expirations.set(key, seconds);
+  }
+}
+
+const memory = new InMemoryPiiVaultStorage();
+await memory.set('scope-a', '[[EMAIL_token_0]]', 'a@example.com');
+assert.equal(await memory.get('scope-a', '[[EMAIL_token_0]]'), 'a@example.com');
+assert.equal(await memory.get('scope-b', '[[EMAIL_token_0]]'), undefined);
+assert.equal(await memory.getByToken('[[EMAIL_token_0]]'), 'a@example.com');
+
+const custom = createPiiVaultStorage({
+  async get(scopeId, token) {
+    return scopeId === 'custom' && token === '[[EMAIL_custom_0]]' ? 'custom@example.com' : undefined;
+  },
+  async set() {},
+  async entries(scopeId) {
+    return scopeId === 'custom'
+      ? [{ token: '[[EMAIL_custom_0]]', value: 'custom@example.com' }]
+      : [];
+  },
+});
+
+const customTokenizer = new PiiTokenizer({ scopeId: 'custom', storage: custom });
+assert.equal(await customTokenizer.unmask('Hi [[EMAIL_custom_0]]'), 'Hi custom@example.com');
+
+const redis = new FakeRedis();
+const redisStorage = createRedisPiiVaultStorage(redis, {
+  keyPrefix: 'test:pii',
+  ttlSeconds: 60,
+});
+
+const userATokenizer = new PiiTokenizer({ scopeId: 'tenant:userA', storage: redisStorage });
+const userBTokenizer = new PiiTokenizer({ scopeId: 'tenant:userB', storage: redisStorage });
+
+const userAMasked = await userATokenizer.mask('Email alice@example.com', [
+  { type: 'EMAIL', value: 'alice@example.com' },
+]);
+const userBMasked = await userBTokenizer.mask('Email bob@example.com', [
+  { type: 'EMAIL', value: 'bob@example.com' },
+]);
+
+assert.notEqual(userAMasked.maskedText, userBMasked.maskedText);
+assert.equal(await userATokenizer.unmask(userAMasked.maskedText), 'Email alice@example.com');
+assert.equal(await userBTokenizer.unmask(userBMasked.maskedText), 'Email bob@example.com');
+assert.equal(await userATokenizer.unmask(userBMasked.maskedText), userBMasked.maskedText);
+
+const laterPassTokenizer = new PiiTokenizer({ scopeId: 'tenant:userA:later', storage: redisStorage });
+await laterPassTokenizer.importTokens(userAMasked.maskedText);
+assert.equal(await laterPassTokenizer.unmask(userAMasked.maskedText), 'Email alice@example.com');
+
+assert.equal(redis.expirations.get('test:pii:scope:tenant:userA'), 60);
+assert.equal(redis.expirations.get('test:pii:tokens'), 60);
+
+class FailingRedis extends FakeRedis {
+  failed = false;
+
+  fail() {
+    this.failed = true;
+  }
+
+  async hGet(key, field) {
+    if (this.failed) throw new Error('Redis unavailable');
+    return super.hGet(key, field);
+  }
+
+  async hSet(key, field, value) {
+    if (this.failed) throw new Error('Redis unavailable');
+    return super.hSet(key, field, value);
+  }
+
+  async hGetAll(key) {
+    if (this.failed) throw new Error('Redis unavailable');
+    return super.hGetAll(key);
+  }
+}
+
+const failingRedis = new FailingRedis();
+const resilientStorage = createRedisPiiVaultStorage(failingRedis, { fallbackToMemory: true });
+await resilientStorage.set('resilient-scope', '[[EMAIL_resilient_0]]', 'safe@example.com');
+failingRedis.fail();
+assert.equal(
+  await resilientStorage.get('resilient-scope', '[[EMAIL_resilient_0]]'),
+  'safe@example.com'
+);
+assert.equal(await resilientStorage.getByToken('[[EMAIL_resilient_0]]'), 'safe@example.com');
+assert.deepEqual(await resilientStorage.entries('resilient-scope'), [
+  { token: '[[EMAIL_resilient_0]]', value: 'safe@example.com' },
+]);
+await resilientStorage.set('resilient-scope', '[[PHONE_resilient_1]]', '0400000000');
+assert.equal(await resilientStorage.getByToken('[[PHONE_resilient_1]]'), '0400000000');
+
+const strictRedis = new FailingRedis();
+const strictStorage = createRedisPiiVaultStorage(strictRedis);
+strictRedis.fail();
+await assert.rejects(() => strictStorage.get('scope', 'token'), /Redis unavailable/);
+
+assert.throws(
+  () => createRedisPiiVaultStorage(new FakeRedis(), { ttlSeconds: 0 }),
+  /positive integer/
+);
+assert.throws(
+  () =>
+    createRedisPiiVaultStorage(
+      { hGet: async () => null, hSet: async () => undefined, hGetAll: async () => ({}) },
+      { ttlSeconds: 60 }
+    ),
+  /requires an expire method/
+);
+
+console.log('PII vault storage tests passed.');