@intflows/genkit-guard 0.0.12 → 0.0.14

package/scripts/publish-wiki.js

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readdirSync, copyFileSync, rmSync, realpathSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const args = process.argv.slice(2);
+let source = join(root, 'docs/wiki-v0.0.14');
+let repository = 'https://github.com/IntFlows/genkit-guard.wiki.git';
+let publish = false;
+for (let i = 0; i < args.length; i++) {
+  if (args[i] === '--publish') publish = true;
+  else if (['--source', '--repo'].includes(args[i])) {
+    const flag = args[i];
+    const value = args[++i];
+    if (!value || value.startsWith('-')) throw new Error(`Missing value for ${flag}`);
+    if (flag === '--source') source = resolve(value);
+    else repository = value;
+  } else if (args[i] === '--help') {
+    console.log('node scripts/publish-wiki.js [--source DIRECTORY] [--repo URL] [--publish]');
+    console.log('Default: clone wiki and preview diff. --publish commits and pushes changed numbered pages.');
+    process.exit(0);
+  } else throw new Error(`Unknown option: ${args[i]}`);
+}
+source = realpathSync(source);
+const pages = readdirSync(source, { withFileTypes: true })
+  .filter(entry => entry.isFile() && /^\d+\..+\.md$/.test(entry.name)).map(entry => entry.name).sort();
+if (!pages.length) throw new Error('No numbered Markdown wiki pages found');
+const work = mkdtempSync(join(tmpdir(), 'genkit-guard-wiki-'));
+const checkout = join(work, 'wiki');
+function git(args, cwd = work) {
+  const result = spawnSync('git', args, { cwd, encoding: 'utf8', shell: false });
+  if (result.error) throw result.error;
+  if (result.status !== 0) throw new Error(result.stderr || result.stdout || 'Git command failed');
+  return result.stdout;
+}
+try {
+  git(['clone', '--', repository, checkout]);
+  for (const page of pages) copyFileSync(join(source, page), join(checkout, page));
+  git(['add', '--', ...pages], checkout);
+  const summary = git(['diff', '--cached', '--stat'], checkout);
+  if (!summary.trim()) console.log('Wiki already matches these pages; nothing to publish.');
+  else {
+    console.log(summary);
+    console.log(git(['diff', '--cached', '--'], checkout));
+    if (!publish) console.log('Preview only. Run again with --publish to commit and push.');
+    else {
+      git(['commit', '-m', 'Update Genkit Guard wiki documentation'], checkout);
+      // Normal push: concurrent remote updates reject safely; never force-push.
+      console.log(git(['push', 'origin', 'HEAD'], checkout));
+      console.log('Wiki changes published.');
+    }
+  }
+} finally {
+  // Only remove the temporary directory created by this process.
+  rmSync(work, { recursive: true, force: true });
+}

package/scripts/test-config.js

@@ -0,0 +1,45 @@
+import assert from 'node:assert/strict';
+import { guard, initGuard, defineGuardConfig } from '../dist/index.js';
+import { ModelSingleton } from '../dist/util/singleton.js';
+import { detectPII, privacyFilterOutputToMatches } from '../dist/pii/detector.js';
+const calls = [];
+const originals = {};
+for (const method of ['getExtractor', 'getNER', 'getPIIClassifier']) {
+  originals[method] = ModelSingleton[method];
+  ModelSingleton[method] = async (name) => {
+    calls.push([method, name]);
+    return async () => method === 'getExtractor' ? { tolist: () => [[1, 0], [1, 0]] } : [{ entity: 'B-LABEL_1', word: 'Alice' }];
+  };
+}
+try {
+  await initGuard();
+  assert.deepEqual(calls.splice(0), [['getExtractor', 'Xenova/all-MiniLM-L6-v2'], ['getNER', 'Xenova/bert-base-NER']]);
+  for (const mode of ['ner', 'classifier']) {
+    const config = defineGuardConfig({
+      models: { extractor: 'custom/intent' },
+      intent: { semantic: { intents: { support: 'Customer support' } } },
+      pii: { mode, model: 'custom/pii', labelMappings: { LABEL_1: 'NAME' } },
+      logging: { enabled: false },
+    });
+    await initGuard(config);
+    const result = await guard(config).model({ prompt: 'Help Alice' }, {}, async (req) => {
+      assert.match(req.prompt, /\[\[NAME_/);
+      assert.doesNotMatch(req.prompt, /Alice/);
+      return { text: req.prompt };
+    });
+    assert.equal(result.text, 'Help Alice');
+    const expected = [['getExtractor', 'custom/intent'], [mode === 'ner' ? 'getNER' : 'getPIIClassifier', 'custom/pii']];
+    assert.deepEqual(calls.splice(0), [...expected, ...expected]);
+  }
+  await initGuard({ pii: { mode: 'classifier' } });
+  assert.equal(calls.splice(0)[1][1], 'openai/privacy-filter');
+  assert.deepEqual(privacyFilterOutputToMatches('Alice', [{ entity: 'S-private_person', word: 'Alice' }], { private_person: null }), []);
+  assert.deepEqual(privacyFilterOutputToMatches('Alice', [{ entity: 'B-label_1', word: 'Alice' }], { LABEL_1: 'CUSTOM_NAME' }), [{ type: 'CUSTOM_NAME', value: 'Alice' }]);
+  assert.deepEqual(privacyFilterOutputToMatches('Alice', [{ entity: 'toString', word: 'Alice' }]), []);
+  assert.throws(() => privacyFilterOutputToMatches('Alice', [{ entity: 'LABEL_1', word: 'Alice' }], { LABEL_1: 'bad-token' }), /uppercase/);
+  const regex = await detectPII('alice@example.com', { mode: 'classifier', labelMappings: { LABEL_1: null } });
+  assert.equal(regex.matches[0].type, 'EMAIL');
+} finally {
+  Object.assign(ModelSingleton, originals);
+}
+console.log('Shared model configuration and compatibility tests passed.');

package/scripts/test-privacy-filter.js

@@ -0,0 +1,116 @@
+import assert from 'node:assert/strict';
+import { detectPII, privacyFilterOutputToMatches } from '../dist/pii/detector.js';
+import { ModelSingleton, PRIVACY_FILTER_PIPELINE_TASK } from '../dist/util/singleton.js';
+import { InMemoryPiiVaultStorage } from '../dist/pii/storage.js';
+import { PiiTokenizer } from '../dist/pii/tokenizer.js';
+import { guard } from '../dist/index.js';
+
+assert.equal(PRIVACY_FILTER_PIPELINE_TASK, 'token-classification');
+
+const input = 'Contact Alice Smith at alice@example.com. Her key is sk-live-secret.';
+const expectedSpans = [
+  { entity_group: 'private_person', word: ' Alice Smith' },
+  { entity_group: 'private_email', word: ' alice@example.com' },
+  { entity_group: 'secret', word: ' sk-live-secret' },
+];
+
+assert.deepEqual(privacyFilterOutputToMatches(input, expectedSpans), [
+  { type: 'NAME', value: 'Alice Smith' },
+  { type: 'EMAIL', value: 'alice@example.com' },
+  { type: 'SECRET', value: 'sk-live-secret' },
+]);
+
+assert.deepEqual(
+  privacyFilterOutputToMatches('Home: 10 Green Street', [
+    { entity_group: 'private_address', start: 6, end: 21, word: 'wrong fallback' },
+    { entity_group: 'O', word: 'Home' },
+    { entity_group: 'private_date', word: 'not present' },
+    null,
+  ]),
+  [{ type: 'ADDRESS', value: '10 Green Street' }]
+);
+assert.deepEqual(privacyFilterOutputToMatches(input, { invalid: true }), []);
+
+const originalGetPIIClassifier = ModelSingleton.getPIIClassifier;
+const originalGetExtractor = ModelSingleton.getExtractor;
+let receivedOptions;
+ModelSingleton.getPIIClassifier = async () => async (_text, options) => {
+  receivedOptions = options;
+  return expectedSpans;
+};
+
+try {
+  const detection = await detectPII(input, { mode: 'classifier' });
+  assert.deepEqual(receivedOptions, { aggregation_strategy: 'simple' });
+
+  // The regex email and model email are deduplicated; model-only name and secret become matches.
+  assert.deepEqual(detection.matches, [
+    { type: 'EMAIL', value: 'alice@example.com' },
+    { type: 'NAME', value: 'Alice Smith' },
+    { type: 'SECRET', value: 'sk-live-secret' },
+  ]);
+
+  const storage = new InMemoryPiiVaultStorage();
+  const tokenizer = new PiiTokenizer({ scopeId: 'privacy-filter-test', storage });
+  const masked = await tokenizer.mask(input, detection.matches);
+
+  assert.doesNotMatch(masked.maskedText, /Alice Smith|alice@example\.com|sk-live-secret/);
+  assert.match(masked.maskedText, /\[\[NAME_/);
+  assert.match(masked.maskedText, /\[\[EMAIL_/);
+  assert.match(masked.maskedText, /\[\[SECRET_/);
+  assert.equal(await tokenizer.unmask(masked.maskedText), input);
+} finally {
+  ModelSingleton.getPIIClassifier = originalGetPIIClassifier;
+}
+
+// Reproduce a Genkit multi-turn response: an inner turn creates the token in one scope, while
+// the final response passes through middleware attached to another context and scope.
+const crossTurnStorage = new InMemoryPiiVaultStorage();
+const innerTurn = new PiiTokenizer({ scopeId: 'inner-turn', storage: crossTurnStorage });
+const crossTurnMasked = await innerTurn.mask('owner@example.com', [
+  { type: 'EMAIL', value: 'owner@example.com' },
+]);
+
+ModelSingleton.getExtractor = async () => async () => ({ tolist: () => [] });
+ModelSingleton.getPIIClassifier = async () => async () => [];
+
+try {
+  const middleware = guard({
+    intent: { semantic: { threshold: 0, intents: {} } },
+    pii: {
+      mode: 'classifier',
+      vault: { storage: crossTurnStorage, scopeId: 'outer-turn' },
+    },
+    logging: { enabled: false },
+  });
+
+  const response = await middleware.model(
+    { prompt: 'Fetch blob metadata' },
+    {},
+    async () => ({ answer: `File owner: ${crossTurnMasked.maskedText}` })
+  );
+  assert.deepEqual(response, { answer: 'File owner: owner@example.com' });
+
+  const unknownTokenResponse = await middleware.model(
+    { prompt: 'Fetch blob metadata' },
+    {},
+    async () => ({ answer: '[[EMAIL_unknownnamespace_99]]' })
+  );
+  assert.deepEqual(unknownTokenResponse, { answer: '[[EMAIL_unknownnamespace_99]]' });
+
+  let toolInput;
+  await middleware.tool(
+    { toolRequest: { name: 'sendEmail', input: { recipient: crossTurnMasked.maskedText } } },
+    {},
+    async (request) => {
+      toolInput = request.toolRequest.input;
+      return { sent: true };
+    }
+  );
+  assert.deepEqual(toolInput, { recipient: 'owner@example.com' });
+} finally {
+  ModelSingleton.getExtractor = originalGetExtractor;
+  ModelSingleton.getPIIClassifier = originalGetPIIClassifier;
+}
+
+console.log('Privacy Filter pipeline, masking and cross-turn unmasking tests passed.');

package/scripts/test-publish-wiki.js

@@ -0,0 +1,30 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+test('wiki preview never pushes; publish updates pages and preserves unrelated files', () => {
+  const work = mkdtempSync(join(tmpdir(), 'wiki-test-'));
+  const run = (cmd, args, cwd = work) => {
+    const result = spawnSync(cmd, args, { cwd, encoding: 'utf8', env: { ...process.env, GIT_AUTHOR_NAME: 'Test', GIT_AUTHOR_EMAIL: 'test@example.com', GIT_COMMITTER_NAME: 'Test', GIT_COMMITTER_EMAIL: 'test@example.com' } });
+    assert.equal(result.status, 0, result.stderr); return result.stdout;
+  };
+  try {
+    const remote = join(work, 'remote.git'); const seed = join(work, 'seed'); const source = join(work, 'pages');
+    run('git', ['init', '--bare', remote]); run('git', ['clone', remote, seed]);
+    writeFileSync(join(seed, 'Home.md'), 'Preserve me');
+    run('git', ['add', '.'], seed); run('git', ['commit', '-m', 'initial'], seed); run('git', ['push', 'origin', 'HEAD'], seed);
+    mkdirSync(source); writeFileSync(join(source, '1.-Home.md'), '# New home'); writeFileSync(join(source, 'README.md'), 'Do not publish');
+    const args = [resolve('scripts/publish-wiki.js'), '--repo', remote, '--source', source];
+    const before = run('git', ['rev-parse', 'HEAD'], remote);
+    assert.match(run(process.execPath, args), /Preview only/);
+    assert.equal(run('git', ['rev-parse', 'HEAD'], remote), before);
+    assert.match(run(process.execPath, [...args, '--publish']), /published/);
+    assert.equal(run('git', ['show', 'HEAD:1.-Home.md'], remote), '# New home');
+    assert.equal(run('git', ['show', 'HEAD:Home.md'], remote), 'Preserve me');
+    assert.doesNotMatch(run('git', ['ls-tree', '--name-only', 'HEAD'], remote), /README/);
+    assert.match(run(process.execPath, [...args, '--publish']), /nothing to publish/);
+  } finally { rmSync(work, { recursive: true, force: true }); }
+});

package/scripts/test-release1.js

@@ -0,0 +1,83 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { genkit, z } from 'genkit';
+import { guard, guardMiddleware, GuardToolError } from '../dist/index.js';
+import { ModelSingleton } from '../dist/util/singleton.js';
+ModelSingleton.getExtractor = async () => async () => ({ tolist: () => [[1, 0], [1, 0]] });
+ModelSingleton.getNER = async () => async () => [];
+const base = { intent: { semantic: { intents: { support: 'Support' } } }, logging: { enabled: false } };
+const request = () => ({ toolRequest: { name: 'sendEmail', input: { email: 'alice@example.com', nested: ['alice@example.com'] } } });
+
+test('tool policies enforce before execution and redact nested arguments', async () => {
+  for (const action of ['allow', 'block', 'redact', 'approval-required']) {
+    const decisions = [];
+    const middleware = guard({ ...base, tools: { rules: { sendEmail: action } }, logging: { enabled: false, onDecision: d => decisions.push(d) } });
+    let executed = 0;
+    const run = middleware.tool(request(), {}, async req => {
+      executed++;
+      if (action === 'redact') assert.deepEqual(req.toolRequest.input, { email: '[REDACTED]', nested: ['[REDACTED]'] });
+      else assert.equal(req.toolRequest.input.email, 'alice@example.com');
+      return { toolResponse: { name: 'sendEmail', output: 'ok' } };
+    });
+    if (action === 'block' || action === 'approval-required') await assert.rejects(run, GuardToolError);
+    else await run;
+    assert.equal(executed, action === 'allow' || action === 'redact' ? 1 : 0);
+    assert.equal(decisions[0].action, action);
+    assert.equal(decisions[0].schemaVersion, '1');
+    assert.doesNotMatch(JSON.stringify(decisions), /alice|example.com|nested/);
+  }
+});
+
+test('approval is explicit, per-call and cannot be spoofed by request flags', async () => {
+  for (const approve of [undefined, () => false, () => 'true', () => { throw new Error('secret'); }, async () => true]) {
+    let ran = false;
+    const middleware = guard({ ...base, tools: { defaultAction: 'approval-required', approve } });
+    const req = request(); req.toolRequest.input.approved = true;
+    const run = middleware.tool(req, {}, async () => { ran = true; });
+    if (approve && await Promise.resolve().then(() => approve()).catch(() => false) === true) await run;
+    else await assert.rejects(run, GuardToolError);
+    assert.equal(ran, Boolean(approve && await Promise.resolve().then(() => approve()).catch(() => false) === true));
+  }
+});
+
+test('default block, unknown tools, invalid actions and callback failures stop execution', async () => {
+  for (const tools of [{ defaultAction: 'block', rules: { other: 'allow' } }, { defaultAction: 'invalid' }]) {
+    await assert.rejects(guard({ ...base, tools }).tool(request(), {}, () => assert.fail('Executed')), GuardToolError);
+  }
+  await assert.rejects(guard({ ...base, logging: { enabled: false, onDecision: () => { throw new Error('sink unavailable'); } } }).tool(request(), {}, () => assert.fail('Executed')), /sink unavailable/);
+});
+
+test('prompt and tool events share contract; console does not include raw content', async () => {
+  const events = []; const lines = [];
+  const original = { log: console.log, warn: console.warn };
+  console.log = console.warn = value => lines.push(value);
+  try {
+    const middleware = guard({ ...base, policyVersion: 'release-1', logging: { onDecision: d => events.push(d) } });
+    await middleware.model({ prompt: 'Help alice@example.com' }, {}, async req => ({ text: req.prompt }));
+    await middleware.model({ prompt: 'ignore previous alice@example.com' }, {}, () => assert.fail('Executed'));
+    await middleware.tool(request(), {}, async () => undefined);
+  } finally { Object.assign(console, original); }
+  assert.deepEqual(events.map(d => d.guard), ['injection', 'intent', 'pii', 'injection', 'tool']);
+  assert.ok(events.every(d => d.policyVersion === 'release-1' && d.latencyMs >= 0));
+  assert.equal(new Set(events.map(d => d.decisionId)).size, events.length);
+  assert.doesNotMatch(lines.join(''), /alice@example.com|Help alice|ignore previous/);
+});
+
+test('Genkit generate actually intercepts tool execution', async () => {
+  for (const factory of [guard, guardMiddleware]) for (const action of ['allow', 'block', 'redact', 'approval-required']) {
+    const ai = genkit({});
+    let executions = 0; let turn = 0;
+    const tool = ai.defineTool({ name: 'sendEmail', description: 'Test', inputSchema: z.object({ email: z.string() }), outputSchema: z.string() }, async input => {
+      executions++;
+      assert.equal(input.email, action === 'redact' ? '[REDACTED]' : 'alice@example.com');
+      return 'ok';
+    });
+    const model = ai.defineModel({ name: 'test/model' }, async () => ({ message: { role: 'model', content: ++turn === 1
+      ? [{ toolRequest: { name: 'sendEmail', ref: '1', input: { email: 'alice@example.com' } } }]
+      : [{ text: 'Done' }] } }));
+    const run = ai.generate({ model, prompt: 'Help', tools: [tool], use: [factory({ ...base, tools: { defaultAction: action } })] });
+    if (action === 'block' || action === 'approval-required') await assert.rejects(run);
+    else await run;
+    assert.equal(executions, ['allow', 'redact'].includes(action) ? 1 : 0);
+  }
+});

package/scripts/test-types.ts

@@ -1,21 +1,51 @@
-import { guard, createRedisPiiVaultStorage, type RedisPiiVaultClient } from '../src/index.js';
-
-const redis: RedisPiiVaultClient = {
-  async hGet() {
-    return undefined;
-  },
-  async hSet() {},
-  async hGetAll() {
-    return {};
-  },
-};
-
-guard({
-  pii: {
-    reversible: true,
-    vault: {
-      storage: createRedisPiiVaultStorage(redis),
-      scopeId: (req: any, ctx: any) => ctx?.auth?.sessionId ?? req?.metadata?.requestId,
-    },
-  },
-});
+import { guard, createRedisPiiVaultStorage, type RedisPiiVaultClient } from '../src/index.js';
+
+const redis: RedisPiiVaultClient = {
+  async hGet() {
+    return undefined;
+  },
+  async hSet() {},
+  async hGetAll() {
+    return {};
+  },
+};
+
+guard({
+  pii: {
+    reversible: true,
+    vault: {
+      storage: createRedisPiiVaultStorage(redis),
+      scopeId: (req: any, ctx: any) => ctx?.auth?.sessionId ?? req?.metadata?.requestId,
+    },
+  },
+});
+
+import { defineGuardConfig, initGuard, type GuardConfig } from '../src/index.js';
+const shared = defineGuardConfig({
+  models: { extractor: 'custom/intent' },
+  pii: { model: 'custom/pii', mode: 'classifier', labelMappings: { LABEL_0: null, LABEL_1: 'NAME' } },
+});
+guard(shared);
+void initGuard(shared);
+const legacy: GuardConfig = { models: { extractor: 'legacy/intent' }, pii: { model: 'legacy/pii' } };
+guard(legacy);
+void initGuard(legacy);
+
+import type { GuardDecision } from '../src/index.js';
+const release1 = defineGuardConfig({
+  policyVersion: 'release-1',
+  tools: {
+    defaultAction: 'block',
+    rules: { lookup: 'allow', sendEmail: 'approval-required', summarize: 'redact' },
+    approve: async ({ toolName, input, context }) => {
+      const call: unknown[] = [toolName, input, context];
+      return call.length === 0;
+    },
+  },
+  logging: { onDecision: async (decision: GuardDecision) => { console.log(decision.reasonCode); } },
+});
+guard(release1);
+
+// Legacy direct invocation remains typed for configurations without tool policies.
+void guard()({ prompt: 'test' }, async (request: unknown) => request);
+void guard({ pii: { mode: 'ner' } })({ prompt: 'test' }, async (request: unknown) => request);