agent-threat-rules 0.3.1 → 1.0.0

package/dist/eval/pint-corpus.js

@@ -1,109 +0,0 @@
-/**
- * PINT Benchmark Corpus Loader
- *
- * Reads the PINT-format dataset (JSON with text/category/label/source/language)
- * built from publicly available prompt injection datasets:
- *   - deepset/prompt-injections (HuggingFace)
- *   - Lakera/gandalf_ignore_instructions (HuggingFace)
- *
- * Converts each sample into the CorpusSample interface used by the ATR eval
- * harness, allowing the PINT corpus to be evaluated alongside or instead of
- * the built-in hand-crafted corpus.
- *
- * @module agent-threat-rules/eval/pint-corpus
- */
-import { readFileSync } from 'node:fs';
-// ---------------------------------------------------------------------------
-// Difficulty assignment
-// ---------------------------------------------------------------------------
-/**
- * Assign difficulty based on language and category:
- *   - Non-English injections -> 'hard'  (multilingual evasion)
- *   - English injections     -> 'easy'  (standard patterns)
- *   - Benign samples         -> 'medium' (false-positive pressure)
- */
-function assignDifficulty(sample) {
-    if (!sample.label) {
-        return 'medium';
-    }
-    return sample.language !== 'en' ? 'hard' : 'easy';
-}
-// ---------------------------------------------------------------------------
-// Category mapping
-// ---------------------------------------------------------------------------
-/**
- * Map raw PINT categories to ATR eval categories.
- * PINT uses: prompt_injection, jailbreak, benign, hard_negatives,
- *            chat, documents, short_input, benign_input, long_input
- */
-function mapCategory(raw, isInjection) {
-    if (isInjection) {
-        if (raw === 'jailbreak')
-            return 'jailbreak';
-        return 'prompt-injection';
-    }
-    return 'benign';
-}
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-/**
- * Load the PINT benchmark corpus from a JSON file on disk.
- *
- * @param dataPath - Absolute path to pint-corpus.json
- * @returns Readonly array of CorpusSample for use with runEval()
- */
-export function loadPintCorpus(dataPath) {
-    const raw = JSON.parse(readFileSync(dataPath, 'utf-8'));
-    const seen = new Set();
-    const samples = [];
-    for (let i = 0; i < raw.length; i++) {
-        const entry = raw[i];
-        const text = entry.text.trim();
-        // Skip empty or duplicate texts
-        const key = text.toLowerCase().slice(0, 200);
-        if (!text || seen.has(key))
-            continue;
-        seen.add(key);
-        const difficulty = assignDifficulty(entry);
-        const category = mapCategory(entry.category, entry.label);
-        samples.push({
-            id: `pint-${String(i + 1).padStart(4, '0')}`,
-            text,
-            category,
-            expectedDetection: entry.label,
-            eventType: 'llm_input',
-            tier: 'any',
-            difficulty,
-            fields: {
-                source: entry.source,
-                language: entry.language,
-                originalCategory: entry.category,
-            },
-        });
-    }
-    return samples;
-}
-/**
- * Get summary statistics for the loaded PINT corpus.
- */
-export function getPintCorpusStats(corpus) {
-    const byCategory = {};
-    const byDifficulty = {};
-    const byLanguage = {};
-    for (const s of corpus) {
-        byCategory[s.category] = (byCategory[s.category] ?? 0) + 1;
-        byDifficulty[s.difficulty] = (byDifficulty[s.difficulty] ?? 0) + 1;
-        const lang = s.fields?.['language'] ?? 'unknown';
-        byLanguage[lang] = (byLanguage[lang] ?? 0) + 1;
-    }
-    return {
-        total: corpus.length,
-        attacks: corpus.filter((s) => s.expectedDetection).length,
-        benign: corpus.filter((s) => !s.expectedDetection).length,
-        byCategory,
-        byDifficulty,
-        byLanguage,
-    };
-}
-//# sourceMappingURL=pint-corpus.js.map

package/dist/eval/pint-corpus.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pint-corpus.js","sourceRoot":"","sources":["../../src/eval/pint-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAevC,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;GAKG;AACH,SAAS,gBAAgB,CACvB,MAAqB;IAErB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AACpD,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,WAAW,CAAC,GAAW,EAAE,WAAoB;IACpD,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,GAAG,KAAK,WAAW;YAAE,OAAO,WAAW,CAAC;QAC5C,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,GAAG,GAA6B,IAAI,CAAC,KAAK,CAC9C,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAChC,CAAC;IAEF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;QACtB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAE/B,gCAAgC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACrC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEd,MAAM,UAAU,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QAE1D,OAAO,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,QAAQ,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YAC5C,IAAI;YACJ,QAAQ;YACR,iBAAiB,EAAE,KAAK,CAAC,KAAK;YAC9B,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,KAAK;YACX,UAAU;YACV,MAAM,EAAE;gBACN,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,gBAAgB,EAAE,KAAK,CAAC,QAAQ;aACjC;SACF,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAA+B;IAQhE,MAAM,UAAU,GAA2B,EAAE,CAAC;IAC9C,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,MAAM,UAAU,GAA2B,EAAE,CAAC;IAE9C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3D,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC;QACjD,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC;IAED,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM;QACzD,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM;QACzD,UAAU;QACV,YAAY;QACZ,UAAU;KACX,CAAC;AACJ,CAAC"}

package/dist/eval/rule-corpus.d.ts

@@ -1,9 +0,0 @@
-/**
- * Auto-generated corpus samples extracted from ATR YAML rule test_cases.
- * Do not edit manually -- regenerate from rule YAML files.
- *
- * @module agent-threat-rules/eval/rule-corpus
- */
-import type { CorpusSample } from './corpus.js';
-export declare const RULE_CORPUS: readonly CorpusSample[];
-//# sourceMappingURL=rule-corpus.d.ts.map

package/dist/eval/rule-corpus.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"rule-corpus.d.ts","sourceRoot":"","sources":["../../src/eval/rule-corpus.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,eAAO,MAAM,WAAW,EAAE,SAAS,YAAY,EAoqJ9C,CAAC"}