@chatwarden/core 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -1
- package/dist/src/ai/agreement.d.ts +46 -0
- package/dist/src/ai/agreement.d.ts.map +1 -0
- package/dist/src/ai/agreement.js +71 -0
- package/dist/src/ai/agreement.js.map +1 -0
- package/dist/src/ai/breaker.d.ts +31 -0
- package/dist/src/ai/breaker.d.ts.map +1 -0
- package/dist/src/ai/breaker.js +62 -0
- package/dist/src/ai/breaker.js.map +1 -0
- package/dist/src/ai/budget.d.ts +26 -0
- package/dist/src/ai/budget.d.ts.map +1 -0
- package/dist/src/ai/budget.js +57 -0
- package/dist/src/ai/budget.js.map +1 -0
- package/dist/src/ai/case.d.ts +37 -0
- package/dist/src/ai/case.d.ts.map +1 -0
- package/dist/src/ai/case.js +30 -0
- package/dist/src/ai/case.js.map +1 -0
- package/dist/src/ai/casebook.d.ts +59 -0
- package/dist/src/ai/casebook.d.ts.map +1 -0
- package/dist/src/ai/casebook.js +47 -0
- package/dist/src/ai/casebook.js.map +1 -0
- package/dist/src/ai/ceilings.d.ts +42 -0
- package/dist/src/ai/ceilings.d.ts.map +1 -0
- package/dist/src/ai/ceilings.js +59 -0
- package/dist/src/ai/ceilings.js.map +1 -0
- package/dist/src/ai/fake.d.ts +41 -0
- package/dist/src/ai/fake.d.ts.map +1 -0
- package/dist/src/ai/fake.js +75 -0
- package/dist/src/ai/fake.js.map +1 -0
- package/dist/src/ai/hash.d.ts +13 -0
- package/dist/src/ai/hash.d.ts.map +1 -0
- package/dist/src/ai/hash.js +19 -0
- package/dist/src/ai/hash.js.map +1 -0
- package/dist/src/ai/inert.d.ts +5 -0
- package/dist/src/ai/inert.d.ts.map +1 -0
- package/dist/src/ai/inert.js +22 -0
- package/dist/src/ai/inert.js.map +1 -0
- package/dist/src/ai/injection.d.ts +8 -0
- package/dist/src/ai/injection.d.ts.map +1 -0
- package/dist/src/ai/injection.js +30 -0
- package/dist/src/ai/injection.js.map +1 -0
- package/dist/src/ai/lines.d.ts +64 -0
- package/dist/src/ai/lines.d.ts.map +1 -0
- package/dist/src/ai/lines.js +172 -0
- package/dist/src/ai/lines.js.map +1 -0
- package/dist/src/ai/prompt.d.ts +44 -0
- package/dist/src/ai/prompt.d.ts.map +1 -0
- package/dist/src/ai/prompt.js +185 -0
- package/dist/src/ai/prompt.js.map +1 -0
- package/dist/src/ai/redact.d.ts +29 -0
- package/dist/src/ai/redact.d.ts.map +1 -0
- package/dist/src/ai/redact.js +160 -0
- package/dist/src/ai/redact.js.map +1 -0
- package/dist/src/ai/schema.d.ts +6 -0
- package/dist/src/ai/schema.d.ts.map +1 -0
- package/dist/src/ai/schema.js +50 -0
- package/dist/src/ai/schema.js.map +1 -0
- package/dist/src/ai/service.d.ts +328 -0
- package/dist/src/ai/service.d.ts.map +1 -0
- package/dist/src/ai/service.js +749 -0
- package/dist/src/ai/service.js.map +1 -0
- package/dist/src/ai/types.d.ts +180 -0
- package/dist/src/ai/types.d.ts.map +1 -0
- package/dist/src/ai/types.js +23 -0
- package/dist/src/ai/types.js.map +1 -0
- package/dist/src/ai/validate.d.ts +30 -0
- package/dist/src/ai/validate.d.ts.map +1 -0
- package/dist/src/ai/validate.js +160 -0
- package/dist/src/ai/validate.js.map +1 -0
- package/dist/src/ai/worker.d.ts +74 -0
- package/dist/src/ai/worker.d.ts.map +1 -0
- package/dist/src/ai/worker.js +286 -0
- package/dist/src/ai/worker.js.map +1 -0
- package/dist/src/audit.d.ts +12 -3
- package/dist/src/audit.d.ts.map +1 -1
- package/dist/src/audit.js +30 -4
- package/dist/src/audit.js.map +1 -1
- package/dist/src/chatwarden.d.ts +28 -0
- package/dist/src/chatwarden.d.ts.map +1 -1
- package/dist/src/chatwarden.js +60 -1
- package/dist/src/chatwarden.js.map +1 -1
- package/dist/src/decide.d.ts +36 -1
- package/dist/src/decide.d.ts.map +1 -1
- package/dist/src/decide.js +62 -2
- package/dist/src/decide.js.map +1 -1
- package/dist/src/documents.d.ts +1 -1
- package/dist/src/documents.d.ts.map +1 -1
- package/dist/src/documents.js +4 -1
- package/dist/src/documents.js.map +1 -1
- package/dist/src/index.d.ts +27 -5
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +17 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/profile.d.ts +30 -1
- package/dist/src/profile.d.ts.map +1 -1
- package/dist/src/profile.js +43 -4
- package/dist/src/profile.js.map +1 -1
- package/dist/src/retention.d.ts +19 -0
- package/dist/src/retention.d.ts.map +1 -1
- package/dist/src/retention.js +33 -10
- package/dist/src/retention.js.map +1 -1
- package/dist/src/types.d.ts +269 -2
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js.map +1 -1
- package/package.json +2 -2
- package/src/ai/agreement.ts +106 -0
- package/src/ai/breaker.ts +73 -0
- package/src/ai/budget.ts +65 -0
- package/src/ai/case.ts +74 -0
- package/src/ai/casebook.ts +75 -0
- package/src/ai/ceilings.ts +68 -0
- package/src/ai/fake.ts +105 -0
- package/src/ai/hash.ts +20 -0
- package/src/ai/inert.ts +26 -0
- package/src/ai/injection.ts +32 -0
- package/src/ai/lines.ts +216 -0
- package/src/ai/prompt.ts +208 -0
- package/src/ai/redact.ts +174 -0
- package/src/ai/schema.ts +51 -0
- package/src/ai/service.ts +955 -0
- package/src/ai/types.ts +165 -0
- package/src/ai/validate.ts +173 -0
- package/src/ai/worker.ts +364 -0
- package/src/audit.ts +31 -5
- package/src/chatwarden.ts +82 -1
- package/src/decide.ts +97 -4
- package/src/documents.ts +5 -2
- package/src/index.ts +52 -1
- package/src/profile.ts +68 -4
- package/src/retention.ts +47 -10
- package/src/types.ts +294 -2
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { AiOutcomeDetail, AiVerdict, SanctionKind } from '../types.js';
|
|
2
|
+
import type { AIRead } from './types.js';
|
|
3
|
+
|
|
4
|
+
// Agree or override (design section 11). Pure: it compares the stored read with what the human decided and nothing else. What a client
|
|
5
|
+
// says it did (the `accepted` echo) is kept as the one-click fact and never decides anything.
|
|
6
|
+
|
|
7
|
+
/** A step as a person would take it: a kind and a length (null for a warning, a note or a permanent one). */
|
|
8
|
+
export interface StepLike {
|
|
9
|
+
kind: SanctionKind;
|
|
10
|
+
durationMs: number | null;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** What the human decided about the reported person, as the server sees it. */
|
|
14
|
+
export interface HumanDecision {
|
|
15
|
+
verdict: 'upheld' | 'noted' | 'no_action' | 'bad_faith';
|
|
16
|
+
/** The Community Rule the human cited against the reported person, or null (a dismissal). */
|
|
17
|
+
rule: number | 'uncovered' | null;
|
|
18
|
+
/** The ids of the reported person's lines the human cited. */
|
|
19
|
+
lineIds: readonly string[];
|
|
20
|
+
/** The strongest step the human took against the reported person, or null. */
|
|
21
|
+
step: StepLike | null;
|
|
22
|
+
/** The step the ladder gives for the AI's conduct level: what the pre-fill would have been; null when there was none. */
|
|
23
|
+
prefill: StepLike | null;
|
|
24
|
+
/** How many actions the decision took on people other than the reported person (the reporter, a bystander); absent means none. */
|
|
25
|
+
others?: number;
|
|
26
|
+
/** The client says the moderator accepted the pre-fill in one click. A claim: the server counts it only when the decision is the pre-fill (see {@link agreementOf}). */
|
|
27
|
+
oneClick: boolean;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The result: the class of agreement, the AI's verdict in the casebook's three classes, and the finer facts. */
|
|
31
|
+
export interface Agreement {
|
|
32
|
+
agreement: 'agree' | 'override' | 'none';
|
|
33
|
+
/** What the AI recommended, or null when no recommendation was shown (level 1, `unclear`, or no read). */
|
|
34
|
+
aiVerdict: AiVerdict | null;
|
|
35
|
+
detail: AiOutcomeDetail;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const KIND_RANK: Record<SanctionKind, number> = { note: 1, warning: 2, chat: 3, area: 4, ban: 5 };
|
|
39
|
+
|
|
40
|
+
/** How long a step lasts for ordering: no length for a note or warning, and a permanent restriction or ban beyond any length. */
|
|
41
|
+
function lengthOf(s: StepLike): number {
|
|
42
|
+
if (s.kind === 'note' || s.kind === 'warning') return 0;
|
|
43
|
+
return s.durationMs === null ? Number.POSITIVE_INFINITY : s.durationMs;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Order two steps by how heavy they are: none, a note, a warning, a chat restriction, a ban from public rooms, a full ban; within a
|
|
48
|
+
* kind, by length (permanent last). Negative when `a` is milder than `b`, zero when equal, positive when `a` is harsher.
|
|
49
|
+
*/
|
|
50
|
+
export function compareSteps(a: StepLike | null, b: StepLike | null): number {
|
|
51
|
+
const kindA = a === null ? 0 : (KIND_RANK[a.kind] ?? 0);
|
|
52
|
+
const kindB = b === null ? 0 : (KIND_RANK[b.kind] ?? 0);
|
|
53
|
+
if (kindA !== kindB) return kindA < kindB ? -1 : 1;
|
|
54
|
+
if (a === null || b === null) return 0;
|
|
55
|
+
const x = lengthOf(a);
|
|
56
|
+
const y = lengthOf(b);
|
|
57
|
+
return x < y ? -1 : x > y ? 1 : 0;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const NO_DETAIL: AiOutcomeDetail = { verdict: false, rule: false, lines: 0, outcome: 'none', oneClick: false };
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The decision is the pre-fill as it stood: the same kind of step, no longer than the pre-fill (a moderator may trim a length and it is
|
|
64
|
+
* still their one click), or both without a step (a dismissal accepted as one).
|
|
65
|
+
*/
|
|
66
|
+
function stepAccepted(step: StepLike | null, prefill: StepLike | null): boolean {
|
|
67
|
+
if (step === null || prefill === null) return step === null && prefill === null;
|
|
68
|
+
if (step.kind !== prefill.kind) return false;
|
|
69
|
+
if (step.durationMs === prefill.durationMs) return true;
|
|
70
|
+
return step.durationMs !== null && prefill.durationMs !== null && step.durationMs <= prefill.durationMs;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Compare the AI's recommendation in `read` with `decision`. `level` is the AI level in force: below 2 the recommendation was not
|
|
75
|
+
* shown, so there is nothing to agree with (`none`). `upheld` and `noted` are one class (the AI wanted to act, or to record);
|
|
76
|
+
* `no_action` is the other; a `bad_faith` verdict is always an override of a recommendation (the AI can never call a report bad faith).
|
|
77
|
+
* A rule that differs is not a disagreement: it is `rule: false` in the detail (and for a `no_action` the rule means nothing: `true`).
|
|
78
|
+
* `detail.oneClick` is the server's verdict, not the client's claim: true only when the client said so AND the decision is an
|
|
79
|
+
* agreement that equals the pre-fill (the same verdict, the same rule, the same kind of step and no longer) and the decision takes no
|
|
80
|
+
* action on anyone else: a decision that also sanctions the reporter or a bystander is the moderator's own, not the AI's one key.
|
|
81
|
+
*/
|
|
82
|
+
export function agreementOf(read: AIRead | null, decision: HumanDecision, level: 0 | 1 | 2 = 2): Agreement {
|
|
83
|
+
const rec = read?.recommendation ?? null;
|
|
84
|
+
if (level < 2 || rec === null || rec.verdict === 'unclear') {
|
|
85
|
+
return { agreement: 'none', aiVerdict: null, detail: { ...NO_DETAIL } };
|
|
86
|
+
}
|
|
87
|
+
const aiVerdict: AiVerdict = rec.verdict;
|
|
88
|
+
const aiActs = aiVerdict === 'upheld' || aiVerdict === 'noted';
|
|
89
|
+
const humanActs = decision.verdict === 'upheld' || decision.verdict === 'noted';
|
|
90
|
+
const same = decision.verdict !== 'bad_faith' && aiActs === humanActs;
|
|
91
|
+
const cited = new Set(rec.evidence);
|
|
92
|
+
const lines = new Set(decision.lineIds.filter((id) => cited.has(id))).size;
|
|
93
|
+
const cmp = compareSteps(decision.step, decision.prefill);
|
|
94
|
+
const ruleMatches = aiVerdict === 'no_action' || rec.rule === decision.rule;
|
|
95
|
+
return {
|
|
96
|
+
agreement: same ? 'agree' : 'override',
|
|
97
|
+
aiVerdict,
|
|
98
|
+
detail: {
|
|
99
|
+
verdict: same,
|
|
100
|
+
rule: ruleMatches,
|
|
101
|
+
lines,
|
|
102
|
+
outcome: cmp === 0 ? 'same' : cmp < 0 ? 'milder' : 'harsher',
|
|
103
|
+
oneClick: decision.oneClick && same && rec.verdict === decision.verdict && ruleMatches && (decision.others ?? 0) === 0 && stepAccepted(decision.step, decision.prefill),
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// The breaker (design section 4, "A breaker"). Five consecutive failures, or any `fatal`, stop new calls for five minutes; then
|
|
2
|
+
// ONE probe call decides: its success closes the breaker, its failure opens it for another five minutes. While it is open nothing is
|
|
3
|
+
// called. A refusal (`declined`) or an unusable answer (`invalid`) is an answer, so it counts as a success here: the provider is up.
|
|
4
|
+
|
|
5
|
+
/** Consecutive failures that open the breaker. */
|
|
6
|
+
export const BREAKER_THRESHOLD = 5;
|
|
7
|
+
/** How long an open breaker refuses calls before one probe is let through. */
|
|
8
|
+
export const BREAKER_OPEN_MS = 5 * 60_000;
|
|
9
|
+
|
|
10
|
+
/** Whether a call may be made now: freely, as the one probe, or not at all. */
|
|
11
|
+
export type BreakerGate = 'go' | 'probe' | 'no';
|
|
12
|
+
|
|
13
|
+
/** What the hub shows: `closed` is working, `open` and `half_open` are "unavailable". */
|
|
14
|
+
export type BreakerState = 'closed' | 'open' | 'half_open';
|
|
15
|
+
|
|
16
|
+
export class Breaker {
|
|
17
|
+
private failures = 0;
|
|
18
|
+
private openUntil: number | null = null;
|
|
19
|
+
private probing = false;
|
|
20
|
+
|
|
21
|
+
private readonly clock: { now(): number };
|
|
22
|
+
|
|
23
|
+
constructor(clock: { now(): number }) {
|
|
24
|
+
this.clock = clock;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Ask to make a call. A `probe` is handed out once; report its end with {@link success}, {@link failure} or {@link release}. */
|
|
28
|
+
gate(): BreakerGate {
|
|
29
|
+
if (this.openUntil === null) return 'go';
|
|
30
|
+
if (this.clock.now() < this.openUntil) return 'no';
|
|
31
|
+
if (this.probing) return 'no';
|
|
32
|
+
this.probing = true;
|
|
33
|
+
return 'probe';
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Whether a call could go out now: closed, or half-open with the probe free. Takes nothing (see {@link gate}). */
|
|
37
|
+
available(): boolean {
|
|
38
|
+
if (this.openUntil === null) return true;
|
|
39
|
+
return this.clock.now() >= this.openUntil && !this.probing;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
state(): BreakerState {
|
|
43
|
+
if (this.openUntil === null) return 'closed';
|
|
44
|
+
return this.clock.now() < this.openUntil ? 'open' : 'half_open';
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** When an open breaker lets the probe through (epoch ms), or null when closed. */
|
|
48
|
+
reopensAt(): number | null {
|
|
49
|
+
return this.openUntil;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** A call produced an answer (even a refusal or an unusable one). Closes the breaker. */
|
|
53
|
+
success(): void {
|
|
54
|
+
this.failures = 0;
|
|
55
|
+
this.openUntil = null;
|
|
56
|
+
this.probing = false;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** A call failed. `fatal` opens at once; so does a failed probe and the fifth failure in a row. */
|
|
60
|
+
failure(fatal = false): void {
|
|
61
|
+
this.failures += 1;
|
|
62
|
+
if (fatal || this.probing || this.failures >= BREAKER_THRESHOLD) {
|
|
63
|
+
this.openUntil = this.clock.now() + BREAKER_OPEN_MS;
|
|
64
|
+
this.failures = 0;
|
|
65
|
+
}
|
|
66
|
+
this.probing = false;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** A probe was handed out but no call came of it (for example the budget stopped it): let the next ask be the probe. */
|
|
70
|
+
release(): void {
|
|
71
|
+
this.probing = false;
|
|
72
|
+
}
|
|
73
|
+
}
|
package/src/ai/budget.ts
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// The spend cap and the call-rate cap (design section 4). The cap is over the UTC day and computed from the stored spend, so it
|
|
2
|
+
// survives a restart and is shared by every process on the store; the helpers here only name the day.
|
|
3
|
+
|
|
4
|
+
const DAY_MS = 24 * 3_600_000;
|
|
5
|
+
|
|
6
|
+
/** Epoch ms of 00:00 UTC on the day that holds `ms`. */
|
|
7
|
+
export function startOfUtcDay(ms: number): number {
|
|
8
|
+
return Math.floor(ms / DAY_MS) * DAY_MS;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** Epoch ms of the next 00:00 UTC after `ms`: when a day's cap resets. */
|
|
12
|
+
export function nextUtcMidnight(ms: number): number {
|
|
13
|
+
return startOfUtcDay(ms) + DAY_MS;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** The UTC day as `YYYY-MM-DD`. */
|
|
17
|
+
export function utcDayLabel(ms: number): string {
|
|
18
|
+
return new Date(startOfUtcDay(ms)).toISOString().slice(0, 10);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* At most `limit` calls in any `windowMs` (the per-minute cap). In-memory: it guards a burst in this process; the daily spend cap is
|
|
23
|
+
* what the store shares.
|
|
24
|
+
*/
|
|
25
|
+
export class CallWindow {
|
|
26
|
+
private readonly at: number[] = [];
|
|
27
|
+
|
|
28
|
+
private readonly limit: number;
|
|
29
|
+
private readonly windowMs: number;
|
|
30
|
+
|
|
31
|
+
constructor(limit: number, windowMs = 60_000) {
|
|
32
|
+
this.limit = limit;
|
|
33
|
+
this.windowMs = windowMs;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Take a slot at `now`; false (nothing taken) when the window is full. */
|
|
37
|
+
tryTake(now: number): boolean {
|
|
38
|
+
this.trim(now);
|
|
39
|
+
if (this.at.length >= this.limit) return false;
|
|
40
|
+
this.at.push(now);
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Whether a slot is free at `now`, taking none. */
|
|
45
|
+
peek(now: number): boolean {
|
|
46
|
+
this.trim(now);
|
|
47
|
+
return this.at.length < this.limit;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** When a slot is free: `now` if one is, else the moment the oldest call leaves the window. */
|
|
51
|
+
nextFree(now: number): number {
|
|
52
|
+
this.trim(now);
|
|
53
|
+
return this.at.length < this.limit ? now : (this.at[0] as number) + this.windowMs;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Give back the slot taken at `now` (the call was not made after all). */
|
|
57
|
+
giveBack(now: number): void {
|
|
58
|
+
const i = this.at.lastIndexOf(now);
|
|
59
|
+
if (i >= 0) this.at.splice(i, 1);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
private trim(now: number): void {
|
|
63
|
+
while (this.at.length > 0 && (this.at[0] as number) <= now - this.windowMs) this.at.shift();
|
|
64
|
+
}
|
|
65
|
+
}
|
package/src/ai/case.ts
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { DocumentPolicy } from '../documents.js';
|
|
2
|
+
import { reasonLabel } from '../reporting.js';
|
|
3
|
+
import type { ReportReason } from '../reporting.js';
|
|
4
|
+
import type { ReportRecord } from '../types.js';
|
|
5
|
+
import { inputHash } from './hash.js';
|
|
6
|
+
import { looksLikeInjection } from './injection.js';
|
|
7
|
+
import { chooseLines } from './lines.js';
|
|
8
|
+
import type { ChosenLines } from './lines.js';
|
|
9
|
+
import { buildAIInput } from './prompt.js';
|
|
10
|
+
import type { AIExampleInput, AIReadInput } from './types.js';
|
|
11
|
+
|
|
12
|
+
// One report, made ready for a read: the lines chosen and redacted, the prompt rendered, the input hash for this adapter, and the
|
|
13
|
+
// facts the answer is checked against. Pure. The hash leaves the casebook examples out (ruling A3), so the same case always has the
|
|
14
|
+
// same hash whatever the casebook holds; `build` renders the input again with examples for the call itself.
|
|
15
|
+
|
|
16
|
+
/** A report ready to be read (or to be compared with a stored read). */
|
|
17
|
+
export interface Prepared {
|
|
18
|
+
/** The input without casebook examples: what the hash is of. */
|
|
19
|
+
input: AIReadInput;
|
|
20
|
+
chosen: ChosenLines;
|
|
21
|
+
/** `inputHash(input, adapterId)`. */
|
|
22
|
+
hash: string;
|
|
23
|
+
/** A line or the note looks like an instruction to the AI (the tripwire). */
|
|
24
|
+
injection: boolean;
|
|
25
|
+
/** Every line ref that is in the prompt. */
|
|
26
|
+
refs: ReadonlySet<string>;
|
|
27
|
+
/** The numbers of the rules that were active when the prompt was built. */
|
|
28
|
+
activeRules: ReadonlySet<number>;
|
|
29
|
+
/** The refs of the lines the reported person wrote. */
|
|
30
|
+
ownRefs: ReadonlySet<string>;
|
|
31
|
+
/** The real line id behind each ref. Never handed to an adapter. */
|
|
32
|
+
idOf: ReadonlyMap<string, string>;
|
|
33
|
+
/** The real ids of the lines the reported person wrote (among the chosen). */
|
|
34
|
+
ownLineIds: ReadonlySet<string>;
|
|
35
|
+
/** The same input with casebook examples in it (at most two are used). The hash does not change. */
|
|
36
|
+
build(examples: readonly AIExampleInput[]): AIReadInput;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface PrepareArgs {
|
|
40
|
+
report: ReportRecord;
|
|
41
|
+
policy: DocumentPolicy;
|
|
42
|
+
reasons: readonly ReportReason[];
|
|
43
|
+
adapterId: string;
|
|
44
|
+
/** `blockedLines: 'mask'`: send a line the filter touched masked, not raw. */
|
|
45
|
+
mask: boolean;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function prepareCase(args: PrepareArgs): Prepared {
|
|
49
|
+
const { report, policy } = args;
|
|
50
|
+
const chosen = chooseLines(report, { mask: args.mask });
|
|
51
|
+
const reasonId = report.reportedReason ?? null;
|
|
52
|
+
const reason = reasonId === null ? null : { id: reasonId, label: reasonLabel(args.reasons, reasonId) ?? reasonId };
|
|
53
|
+
const build = (examples: readonly AIExampleInput[]): AIReadInput => buildAIInput({ policy, reason, chosen, examples });
|
|
54
|
+
const input = build([]);
|
|
55
|
+
|
|
56
|
+
// The tripwire reads what the players typed (the raw text), whatever is sent.
|
|
57
|
+
const rawOf = new Map(report.excerpt.map((l) => [l.lineId, l.raw]));
|
|
58
|
+
const injection =
|
|
59
|
+
(report.note !== null && looksLikeInjection(report.note)) ||
|
|
60
|
+
chosen.lines.some((l) => looksLikeInjection(rawOf.get(l.lineId) ?? l.text));
|
|
61
|
+
|
|
62
|
+
return {
|
|
63
|
+
input,
|
|
64
|
+
chosen,
|
|
65
|
+
hash: inputHash(input, args.adapterId),
|
|
66
|
+
injection,
|
|
67
|
+
refs: new Set(chosen.lines.map((l) => l.ref)),
|
|
68
|
+
activeRules: new Set(input.rules.map((r) => r.number)),
|
|
69
|
+
ownRefs: new Set(chosen.lines.filter((l) => l.byReported).map((l) => l.ref)),
|
|
70
|
+
idOf: new Map(chosen.lines.map((l) => [l.ref, l.lineId])),
|
|
71
|
+
ownLineIds: new Set(chosen.lines.filter((l) => l.byReported).map((l) => l.lineId)),
|
|
72
|
+
build,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { RuleRef } from '../rules.js';
|
|
2
|
+
import type { AiOutcomeInsert, ConductLevel, ReportRecord, SanctionKind, SubjectId } from '../types.js';
|
|
3
|
+
import { compareSteps } from './agreement.js';
|
|
4
|
+
import type { StepLike } from './agreement.js';
|
|
5
|
+
import type { ChosenLines } from './lines.js';
|
|
6
|
+
import { redactText } from './redact.js';
|
|
7
|
+
|
|
8
|
+
// The casebook's pure parts (design section 11): which lines of a decided case become a pseudonymised example, and what the human
|
|
9
|
+
// did to the reported person. The service puts them together with the store; nothing here reads a store or acts on anyone.
|
|
10
|
+
|
|
11
|
+
/** How many teachable decisions must exist before any example is shown to the model. */
|
|
12
|
+
export const AI_EXAMPLES_MIN = 30;
|
|
13
|
+
/** The most lines in one example. */
|
|
14
|
+
export const EXAMPLE_LINES_MAX = 6;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Up to six lines of the case, as aliases and redacted text: first the lines the human or the AI cited, then the rest of what the
|
|
18
|
+
* reported person wrote, then the other lines, the newest first within each group; shown in time order. The text goes through the
|
|
19
|
+
* same redaction as a read (it is idempotent on text that already went through it). No handle, seat, id or line id is returned.
|
|
20
|
+
*/
|
|
21
|
+
export function pickExampleLines(chosen: ChosenLines, cited: ReadonlySet<string>): Array<{ who: string; text: string }> {
|
|
22
|
+
const lines = chosen.lines;
|
|
23
|
+
const ranked = lines
|
|
24
|
+
.map((l, i) => ({ i, rank: (cited.has(l.lineId) ? 0 : l.byReported ? 1 : 2) * 1000 + (lines.length - i) }))
|
|
25
|
+
.sort((a, b) => a.rank - b.rank)
|
|
26
|
+
.slice(0, EXAMPLE_LINES_MAX)
|
|
27
|
+
.map((e) => e.i)
|
|
28
|
+
.sort((a, b) => a - b);
|
|
29
|
+
return ranked.map((i) => {
|
|
30
|
+
const line = lines[i] as ChosenLines['lines'][number];
|
|
31
|
+
return { who: line.who, text: redactText(line.text, [], () => null) };
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** One action the decision took, as the casebook needs it. */
|
|
36
|
+
export interface TakenAction {
|
|
37
|
+
subject: string;
|
|
38
|
+
kind: SanctionKind;
|
|
39
|
+
durationMs: number | null;
|
|
40
|
+
rule: number | 'uncovered';
|
|
41
|
+
lineIds: readonly string[];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* What the human did to the reported person: their strongest step, the rule it cited and the lines they showed. A decision that
|
|
46
|
+
* acted on nobody but others (the reporter, a bystander) did nothing to the reported person; `others` counts the actions on those others.
|
|
47
|
+
*/
|
|
48
|
+
export function actionsOnTarget(actions: readonly TakenAction[], target: string): { step: StepLike | null; rule: number | 'uncovered' | null; lineIds: string[]; others: number } {
|
|
49
|
+
const own = actions.filter((a) => a.subject === target);
|
|
50
|
+
const others = actions.length - own.length;
|
|
51
|
+
if (own.length === 0) return { step: null, rule: null, lineIds: [], others };
|
|
52
|
+
let strongest = own[0] as TakenAction;
|
|
53
|
+
for (const a of own) if (compareSteps(a, strongest) > 0) strongest = a;
|
|
54
|
+
return {
|
|
55
|
+
step: { kind: strongest.kind, durationMs: strongest.durationMs },
|
|
56
|
+
rule: strongest.rule,
|
|
57
|
+
lineIds: [...new Set(own.flatMap((a) => a.lineIds))],
|
|
58
|
+
others,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** What the casebook is told about a decision that is about to commit (see `DecisionServiceDeps.aiOutcome`). */
|
|
63
|
+
export interface AiOutcomeArgs {
|
|
64
|
+
report: ReportRecord;
|
|
65
|
+
/** The canonical subject the report is about. */
|
|
66
|
+
target: SubjectId;
|
|
67
|
+
verdict: 'upheld' | 'noted' | 'no_action' | 'bad_faith';
|
|
68
|
+
actions: ReadonlyArray<{ subject: SubjectId; kind: SanctionKind; durationMs: number | null; rule: RuleRef; lineIds: readonly string[] }>;
|
|
69
|
+
/** The echo of the decision, or all-empty (`readId` null, not accepted, no severity) when it carried none. */
|
|
70
|
+
ai: { readId: string | null; accepted: boolean; severity: ConductLevel | null };
|
|
71
|
+
now: number;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Computes the casebook row for a decision, from the stored read and the decision. Returns null for none. */
|
|
75
|
+
export type AiOutcomeHook = (args: AiOutcomeArgs) => AiOutcomeInsert | null;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { SanctionKind } from '../types.js';
|
|
2
|
+
|
|
3
|
+
// The ceilings (design section 8). They hold in code, not by convention: the AI module cannot write a sanction at all (an ESLint
|
|
4
|
+
// rule forbids importing the sanction modules), and anything that pre-fills a step for a human, or one day acts, passes through
|
|
5
|
+
// `assertWithinCeilings`.
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The longest step the one-key accept may pre-fill, and the longest a later level 3 may take: 24 hours.
|
|
9
|
+
* @experimental
|
|
10
|
+
*/
|
|
11
|
+
export const AI_MAX_PREFILL_MS = 24 * 3_600_000;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The kinds an AI step may ever be: a note, a warning or a chat restriction. Never a ban and never an area ban. A frozen list.
|
|
15
|
+
* @experimental
|
|
16
|
+
*/
|
|
17
|
+
export const AI_MAX_KIND: readonly SanctionKind[] = Object.freeze(['note', 'warning', 'chat'] as const);
|
|
18
|
+
|
|
19
|
+
/** True for a kind in {@link AI_MAX_KIND}. Anything else (a ban, an area ban, a prototype key, a non-string) is false. */
|
|
20
|
+
export function isAllowedKind(kind: unknown): kind is SanctionKind {
|
|
21
|
+
return typeof kind === 'string' && AI_MAX_KIND.includes(kind as SanctionKind);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Thrown by `assertWithinCeilings`. `code` is a fixed string, never echoed text.
|
|
26
|
+
* @experimental
|
|
27
|
+
*/
|
|
28
|
+
export class AICeilingError extends Error {
|
|
29
|
+
readonly code: 'ceiling_kind' | 'ceiling_duration' | 'ceiling_evidence';
|
|
30
|
+
constructor(code: AICeilingError['code'], message: string) {
|
|
31
|
+
super(message);
|
|
32
|
+
this.name = 'AICeilingError';
|
|
33
|
+
this.code = code;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Throws when a step an AI proposes would break a ceiling: a kind outside {@link AI_MAX_KIND} (a ban, an area ban), a chat
|
|
39
|
+
* restriction with no length, any length (any kind, a warning or note included) that is not a finite number above 0 and at most
|
|
40
|
+
* {@link AI_MAX_PREFILL_MS}, or evidence with no line by the reported person.
|
|
41
|
+
* `evidence.cited` is the refs the AI cited; `evidence.ownRefs` the refs of lines the reported person wrote.
|
|
42
|
+
* @experimental
|
|
43
|
+
*/
|
|
44
|
+
export function assertWithinCeilings(
|
|
45
|
+
step: { kind: SanctionKind; durationMs: number | null },
|
|
46
|
+
evidence: { cited: readonly string[]; ownRefs: ReadonlySet<string> },
|
|
47
|
+
): void {
|
|
48
|
+
if (!isAllowedKind(step.kind)) throw new AICeilingError('ceiling_kind', 'this kind is beyond what the AI may do');
|
|
49
|
+
// No kind may carry more than 24 hours, a warning or note included; a chat restriction must carry a length.
|
|
50
|
+
const d: unknown = step.durationMs;
|
|
51
|
+
const lengthOk = typeof d === 'number' && Number.isFinite(d) && d > 0 && d <= AI_MAX_PREFILL_MS;
|
|
52
|
+
if (!(d === null && step.kind !== 'chat') && !lengthOk) {
|
|
53
|
+
throw new AICeilingError('ceiling_duration', 'duration is beyond what the AI may do (24 hours at most)');
|
|
54
|
+
}
|
|
55
|
+
if (!evidence.cited.some((ref) => evidence.ownRefs.has(ref))) {
|
|
56
|
+
throw new AICeilingError('ceiling_evidence', 'no cited evidence line is by the reported person');
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Whether the AI may act on this person. Always true now; level 3 will back it with a "do not automate" mark. Called on every
|
|
62
|
+
* path that could ever act. Reads and drafts are advice, so they are produced either way.
|
|
63
|
+
* @experimental
|
|
64
|
+
*/
|
|
65
|
+
export function automationAllowed(person: string): boolean {
|
|
66
|
+
void person;
|
|
67
|
+
return true;
|
|
68
|
+
}
|
package/src/ai/fake.ts
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { AIError } from './types.js';
|
|
2
|
+
import type { AbortSignalLike, AIRead, AIReadInput, AIReadResult, AIUsage, ModerationAI } from './types.js';
|
|
3
|
+
|
|
4
|
+
// A scripted ModerationAI for tests and the demo. No network, no key. A script is a function, or a queue of steps used one per call;
|
|
5
|
+
// when the queue runs dry the fake reads the case by a simple rule (see `fakeRead`), so a test only scripts what it cares about.
|
|
6
|
+
|
|
7
|
+
/** One scripted call: an answer (with optional usage), a thrown error, a call that never returns, or the default answer with this usage. */
|
|
8
|
+
export type FakeStep =
|
|
9
|
+
| { output: unknown; usage?: Partial<AIUsage> }
|
|
10
|
+
| { error: AIError }
|
|
11
|
+
| { hang: true }
|
|
12
|
+
| { usage: Partial<AIUsage> };
|
|
13
|
+
|
|
14
|
+
/** What a function script is given. `call` counts from 1. */
|
|
15
|
+
export interface FakeCallContext {
|
|
16
|
+
call: number;
|
|
17
|
+
signal: AbortSignalLike;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export type FakeScript = readonly FakeStep[] | ((input: AIReadInput, ctx: FakeCallContext) => FakeStep | Promise<FakeStep>);
|
|
21
|
+
|
|
22
|
+
/** The fake adapter, with what it was asked kept for the test to look at. */
|
|
23
|
+
export interface FakeModerationAI extends ModerationAI {
|
|
24
|
+
/** Every input it was called with, oldest first. */
|
|
25
|
+
readonly calls: AIReadInput[];
|
|
26
|
+
/** The abort signal of each call. */
|
|
27
|
+
readonly signals: AbortSignalLike[];
|
|
28
|
+
readonly callCount: number;
|
|
29
|
+
/** Add a step to the end of the queue (a queue script only). */
|
|
30
|
+
push(...steps: FakeStep[]): void;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Usage of a fake call unless a step says otherwise: 1 000 micro-dollars. */
|
|
34
|
+
export const FAKE_USAGE: AIUsage = Object.freeze({
|
|
35
|
+
model: 'fake-1',
|
|
36
|
+
inputTokens: 1000,
|
|
37
|
+
outputTokens: 100,
|
|
38
|
+
cacheReadTokens: 0,
|
|
39
|
+
cacheWriteTokens: 0,
|
|
40
|
+
costMicros: 1000,
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* A valid answer for `input`, by a simple rule: nothing by P1 means banter and no action; otherwise P1 was rude, citing the first
|
|
45
|
+
* line P1 wrote, with a recommendation to uphold under the first active rule. Overrides replace fields.
|
|
46
|
+
*/
|
|
47
|
+
export function fakeRead(input: AIReadInput, over: Partial<AIRead> = {}): AIRead {
|
|
48
|
+
const own = input.lines.filter((l) => l.who === 'P1').map((l) => l.ref);
|
|
49
|
+
const rule: number | 'uncovered' = input.rules[0]?.number ?? 'uncovered';
|
|
50
|
+
const base: AIRead =
|
|
51
|
+
own.length === 0
|
|
52
|
+
? {
|
|
53
|
+
severity: 'banter',
|
|
54
|
+
rule: null,
|
|
55
|
+
summary: 'Nothing in the lines breaks a rule.',
|
|
56
|
+
evidence: [],
|
|
57
|
+
confidence: 'medium',
|
|
58
|
+
injectionSuspected: false,
|
|
59
|
+
recommendation: { verdict: 'no_action', severity: 'banter', rule: null, evidence: [] },
|
|
60
|
+
}
|
|
61
|
+
: {
|
|
62
|
+
severity: 'rude',
|
|
63
|
+
rule,
|
|
64
|
+
summary: 'The reported player was rude in chat.',
|
|
65
|
+
evidence: [own[0] as string],
|
|
66
|
+
confidence: 'medium',
|
|
67
|
+
injectionSuspected: false,
|
|
68
|
+
recommendation: { verdict: 'upheld', severity: 'rude', rule, evidence: [own[0] as string] },
|
|
69
|
+
};
|
|
70
|
+
return { ...base, ...over };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** The adapter id is what the audit trail shows, so it keeps to the audit actor's characters. */
|
|
74
|
+
export function createFakeModerationAI(script: FakeScript = [], opts: { id?: string } = {}): FakeModerationAI {
|
|
75
|
+
const queue: FakeStep[] = typeof script === 'function' ? [] : [...script];
|
|
76
|
+
const calls: AIReadInput[] = [];
|
|
77
|
+
const signals: AbortSignalLike[] = [];
|
|
78
|
+
return {
|
|
79
|
+
id: opts.id ?? 'fake:model',
|
|
80
|
+
calls,
|
|
81
|
+
signals,
|
|
82
|
+
get callCount() {
|
|
83
|
+
return calls.length;
|
|
84
|
+
},
|
|
85
|
+
push: (...steps) => void queue.push(...steps),
|
|
86
|
+
async read(input, options): Promise<AIReadResult> {
|
|
87
|
+
calls.push(input);
|
|
88
|
+
signals.push(options.signal);
|
|
89
|
+
const call = calls.length;
|
|
90
|
+
const step: FakeStep | undefined = typeof script === 'function' ? await script(input, { call, signal: options.signal }) : queue.shift();
|
|
91
|
+
if (step !== undefined && 'error' in step) throw step.error;
|
|
92
|
+
if (step !== undefined && 'hang' in step) {
|
|
93
|
+
// Never answers; settles only when the caller gives up, as a real client does on abort.
|
|
94
|
+
return await new Promise<AIReadResult>((_, reject) => {
|
|
95
|
+
const stop = (): void => reject(new AIError('timeout', 'aborted'));
|
|
96
|
+
if (options.signal.aborted) stop();
|
|
97
|
+
else options.signal.addEventListener('abort', stop, { once: true });
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
const usage: AIUsage = { ...FAKE_USAGE, ...(step !== undefined && 'usage' in step ? step.usage : {}) };
|
|
101
|
+
const output = step !== undefined && 'output' in step ? step.output : fakeRead(input);
|
|
102
|
+
return { output, usage };
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
package/src/ai/hash.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { sha256Hex } from '../crypto.js';
|
|
2
|
+
import type { AIReadInput } from './types.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The input hash a stored read is valid for (design section 6): sha256 over the prompt version, the rendered stable prefix (the
|
|
6
|
+
* policy text that changes meaning), the case's own rendered inputs and the adapter id, in that order. The casebook examples are
|
|
7
|
+
* NOT part of it: a growing casebook must not make every stored read look stale, and the read records the ids of the examples it
|
|
8
|
+
* used instead (`AIStoredRead.exampleIds`). The `example:` rows are therefore left out of the variable part before hashing; no
|
|
9
|
+
* other row of the data block can begin that way (a chat line starts `L<n> `, the rest are fixed keys). The fields are framed as a
|
|
10
|
+
* JSON array, so no part can slide into the next. Object key order in `input` cannot matter: only those values are read. A hex
|
|
11
|
+
* string of 64 characters.
|
|
12
|
+
* @experimental
|
|
13
|
+
*/
|
|
14
|
+
export function inputHash(input: AIReadInput, adapterId: string): string {
|
|
15
|
+
const caseInputs = input.prompt.variable
|
|
16
|
+
.split('\n')
|
|
17
|
+
.filter((row) => !row.startsWith('example: '))
|
|
18
|
+
.join('\n');
|
|
19
|
+
return sha256Hex(JSON.stringify([input.promptVersion, input.prompt.stable, caseInputs, adapterId]));
|
|
20
|
+
}
|
package/src/ai/inert.ts
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { AiFacade, AiStatsView, AiStatus } from './service.js';
|
|
2
|
+
|
|
3
|
+
// The facade when no AI is configured. Apart from the service so that a build with no AI in it never loads the service (the kit imports the
|
|
4
|
+
// service only when `createChatwarden` is given an adapter): a browser bundle of the kit carries these few lines and not the AI module.
|
|
5
|
+
|
|
6
|
+
/** The facade when no AI is configured: always off, never touches the store. */
|
|
7
|
+
export function createInertAi(): AiFacade {
|
|
8
|
+
const status: AiStatus = { available: false, level: 0, maxLevel: 0, state: 'off', spentMicros: 0, capMicros: 0, capCalls: 0, day: '', reads: 0, cacheHitRate: null, invalid: {} };
|
|
9
|
+
return {
|
|
10
|
+
level: () => 0,
|
|
11
|
+
status: () => ({ ...status, invalid: {} }),
|
|
12
|
+
read: () => null,
|
|
13
|
+
expected: () => false,
|
|
14
|
+
readAgain: () => Promise.resolve('off'),
|
|
15
|
+
briefs: () => new Map(),
|
|
16
|
+
stats: (q) => emptyStats(q?.since ?? 0),
|
|
17
|
+
markReversed: () => false,
|
|
18
|
+
dontTeach: () => false,
|
|
19
|
+
idle: () => Promise.resolve(),
|
|
20
|
+
sweep: () => Promise.resolve(),
|
|
21
|
+
close: () => undefined,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const emptyStats = (since: number): AiStatsView => ({ since, decided: 0, agree: 0, override: 0, none: 0, falsePositives: 0, oneClick: 0, agreementRate: null, falsePositiveRate: null, oneClickRate: null, rules: [] });
|
|
26
|
+
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { cleanText } from './redact.js';
|
|
2
|
+
|
|
3
|
+
// A tripwire, not a defence (design section 8, layer 5). A line that looks like it is instructing the AI sets `injectionSuspected`:
|
|
4
|
+
// the case is shown with a plain note, its confidence is forced to low and the AI may not lower its queue rank. The patterns are
|
|
5
|
+
// phrase-shaped on purpose, so ordinary banter ("ignore me lol", "you are now in my way") does not trip them. A false positive costs
|
|
6
|
+
// one visible mark; a miss costs nothing the other layers (no authority, validated output, a human reads it) do not already cover.
|
|
7
|
+
|
|
8
|
+
const IGNORE = String.raw`\b(?:ignore|disregard|forget|override)\s+(?:(?:all|any|every|of|the|your|my|these|those|previous|prior|earlier|above|preceding|everything|anything|what)\s+)*(?:above|previous|prior|earlier|preceding|instructions?|rules?|prompts?|guidelines?|directions?|commands?|polic(?:y|ies)|messages?|context|system)\b`;
|
|
9
|
+
// "you are now" followed by a role, a mode or nothing: not "you are now in my way" or "you are now at 3 kills".
|
|
10
|
+
const NOW = String.raw`\byou(?:'re|\s+are|\s+r)\s+now\s*(?:$|[.,:;!?-]|(?:an?|the)\b|(?:dan|free|unfiltered|unrestricted|jailbroken|acting|playing)\b|my\s+(?:assistant|servant|ai)\b|in\s+(?:developer|debug|admin|god|dan|unrestricted|jailbreak)\b|going\s+to\s+(?:act|ignore|pretend)\b)`;
|
|
11
|
+
// A role marker at the start of the text or after punctuation or a quote ("system: ...", "hey! assistant: ..."), not mid-sentence.
|
|
12
|
+
const ROLE = String.raw`(?:^|[.!?;"'\[(<>|\x60])\s*(?:system|assistant|developer|admin)\s*[:>\]]`;
|
|
13
|
+
const TAG = String.raw`<\/?\s*(?:system|assistant|instructions?|data|prompt)\s*>|<\|[a-z_]+\|>|\[\/?inst\]|<<\/?sys>>`;
|
|
14
|
+
const AS_AI = String.raw`\bas an? (?:ai|a\.i\.|language model|large language model|llm|assistant|artificial intelligence|chatbot)\b`;
|
|
15
|
+
const NEW = String.raw`\b(?:new|updated|revised|additional)\s+(?:instructions?|rules|prompt|system prompt)\s*:|\bfrom now on\b[^.]{0,30}\b(?:you|ignore|respond|answer|act|only|always)\b|\bdo anything now\b`;
|
|
16
|
+
const TELL = String.raw`\btell\s+(?:the\s+|your\s+)?(?:moderators?|mods?|admins?|ai|assistant|reviewer|bot|model)\b[^.]{0,60}\b(?:fine|ok|okay|nothing|no action|safe|innocent|dismiss|banter|false|lying|harmless)\b`;
|
|
17
|
+
const PRETEND = String.raw`\bpretend\s+(?:that\s+)?you(?:'re|\s+are|\s+were)\b`;
|
|
18
|
+
const REVEAL = String.raw`\b(?:reveal|show|print|repeat|output|leak)\s+(?:me\s+)?(?:your|the)\s+(?:system\s+|initial\s+|hidden\s+)?(?:prompt|instructions?)\b`;
|
|
19
|
+
|
|
20
|
+
const PATTERNS: readonly RegExp[] = [IGNORE, NOW, ROLE, TAG, AS_AI, NEW, TELL, PRETEND, REVEAL].map((p) => new RegExp(p, 'u'));
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* True when a chat line or note reads like an attempt to instruct the AI ("ignore the above", "you are now ...", "system: ...",
|
|
24
|
+
* "as an AI", "disregard your rules", "tell the moderator this is fine"). Case, width, zero-width characters and extra whitespace
|
|
25
|
+
* are ignored. Never throws.
|
|
26
|
+
* @experimental
|
|
27
|
+
*/
|
|
28
|
+
export function looksLikeInjection(text: string): boolean {
|
|
29
|
+
if (typeof text !== 'string' || text === '') return false;
|
|
30
|
+
const t = cleanText(text.normalize('NFKC')).toLowerCase().replace(/[‘’]/gu, "'");
|
|
31
|
+
return PATTERNS.some((re) => re.test(t));
|
|
32
|
+
}
|