agent-threat-rules 0.2.2 → 0.3.0

package/dist/eval/pint-corpus.js

@@ -0,0 +1,109 @@
+/**
+ * PINT Benchmark Corpus Loader
+ *
+ * Reads the PINT-format dataset (JSON with text/category/label/source/language)
+ * built from publicly available prompt injection datasets:
+ *   - deepset/prompt-injections (HuggingFace)
+ *   - Lakera/gandalf_ignore_instructions (HuggingFace)
+ *
+ * Converts each sample into the CorpusSample interface used by the ATR eval
+ * harness, allowing the PINT corpus to be evaluated alongside or instead of
+ * the built-in hand-crafted corpus.
+ *
+ * @module agent-threat-rules/eval/pint-corpus
+ */
+import { readFileSync } from 'node:fs';
+// ---------------------------------------------------------------------------
+// Difficulty assignment
+// ---------------------------------------------------------------------------
+/**
+ * Assign difficulty based on language and category:
+ *   - Non-English injections -> 'hard'  (multilingual evasion)
+ *   - English injections     -> 'easy'  (standard patterns)
+ *   - Benign samples         -> 'medium' (false-positive pressure)
+ */
+function assignDifficulty(sample) {
+    if (!sample.label) {
+        return 'medium';
+    }
+    return sample.language !== 'en' ? 'hard' : 'easy';
+}
+// ---------------------------------------------------------------------------
+// Category mapping
+// ---------------------------------------------------------------------------
+/**
+ * Map raw PINT categories to ATR eval categories.
+ * PINT uses: prompt_injection, jailbreak, benign, hard_negatives,
+ *            chat, documents, short_input, benign_input, long_input
+ */
+function mapCategory(raw, isInjection) {
+    if (isInjection) {
+        if (raw === 'jailbreak')
+            return 'jailbreak';
+        return 'prompt-injection';
+    }
+    return 'benign';
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Load the PINT benchmark corpus from a JSON file on disk.
+ *
+ * @param dataPath - Absolute path to pint-corpus.json
+ * @returns Readonly array of CorpusSample for use with runEval()
+ */
+export function loadPintCorpus(dataPath) {
+    const raw = JSON.parse(readFileSync(dataPath, 'utf-8'));
+    const seen = new Set();
+    const samples = [];
+    for (let i = 0; i < raw.length; i++) {
+        const entry = raw[i];
+        const text = entry.text.trim();
+        // Skip empty or duplicate texts
+        const key = text.toLowerCase().slice(0, 200);
+        if (!text || seen.has(key))
+            continue;
+        seen.add(key);
+        const difficulty = assignDifficulty(entry);
+        const category = mapCategory(entry.category, entry.label);
+        samples.push({
+            id: `pint-${String(i + 1).padStart(4, '0')}`,
+            text,
+            category,
+            expectedDetection: entry.label,
+            eventType: 'llm_input',
+            tier: 'any',
+            difficulty,
+            fields: {
+                source: entry.source,
+                language: entry.language,
+                originalCategory: entry.category,
+            },
+        });
+    }
+    return samples;
+}
+/**
+ * Get summary statistics for the loaded PINT corpus.
+ */
+export function getPintCorpusStats(corpus) {
+    const byCategory = {};
+    const byDifficulty = {};
+    const byLanguage = {};
+    for (const s of corpus) {
+        byCategory[s.category] = (byCategory[s.category] ?? 0) + 1;
+        byDifficulty[s.difficulty] = (byDifficulty[s.difficulty] ?? 0) + 1;
+        const lang = s.fields?.['language'] ?? 'unknown';
+        byLanguage[lang] = (byLanguage[lang] ?? 0) + 1;
+    }
+    return {
+        total: corpus.length,
+        attacks: corpus.filter((s) => s.expectedDetection).length,
+        benign: corpus.filter((s) => !s.expectedDetection).length,
+        byCategory,
+        byDifficulty,
+        byLanguage,
+    };
+}
+//# sourceMappingURL=pint-corpus.js.map

package/dist/eval/pint-corpus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pint-corpus.js","sourceRoot":"","sources":["../../src/eval/pint-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAevC,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;GAKG;AACH,SAAS,gBAAgB,CACvB,MAAqB;IAErB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AACpD,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,WAAW,CAAC,GAAW,EAAE,WAAoB;IACpD,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,GAAG,KAAK,WAAW;YAAE,OAAO,WAAW,CAAC;QAC5C,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,GAAG,GAA6B,IAAI,CAAC,KAAK,CAC9C,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAChC,CAAC;IAEF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;QACtB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAE/B,gCAAgC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACrC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEd,MAAM,UAAU,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QAE1D,OAAO,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,QAAQ,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YAC5C,IAAI;YACJ,QAAQ;YACR,iBAAiB,EAAE,KAAK,CAAC,KAAK;YAC9B,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,KAAK;YACX,UAAU;YACV,MAAM,EAAE;gBACN,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,gBAAgB,EAAE,KAAK,CAAC,QAAQ;aACjC;SACF,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAA+B;IAQhE,MAAM,UAAU,GAA2B,EAAE,CAAC;IAC9C,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,MAAM,UAAU,GAA2B,EAAE,CAAC;IAE9C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3D,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC;QACjD,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC;IAED,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM;QACzD,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM;QACzD,UAAU;QACV,YAAY;QACZ,UAAU;KACX,CAAC;AACJ,CAAC"}

package/dist/eval/rule-corpus.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Auto-generated corpus samples extracted from ATR YAML rule test_cases.
+ * Do not edit manually -- regenerate from rule YAML files.
+ *
+ * @module agent-threat-rules/eval/rule-corpus
+ */
+import type { CorpusSample } from './corpus.js';
+export declare const RULE_CORPUS: readonly CorpusSample[];
+//# sourceMappingURL=rule-corpus.d.ts.map

package/dist/eval/rule-corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rule-corpus.d.ts","sourceRoot":"","sources":["../../src/eval/rule-corpus.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,eAAO,MAAM,WAAW,EAAE,SAAS,YAAY,EAoqJ9C,CAAC"}