@consensus-tools/universal 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +451 -0
- package/dist/__tests__/defaults.test.d.ts +2 -0
- package/dist/__tests__/defaults.test.d.ts.map +1 -0
- package/dist/__tests__/defaults.test.js +55 -0
- package/dist/__tests__/defaults.test.js.map +1 -0
- package/dist/__tests__/fail-policy.test.d.ts +2 -0
- package/dist/__tests__/fail-policy.test.d.ts.map +1 -0
- package/dist/__tests__/fail-policy.test.js +80 -0
- package/dist/__tests__/fail-policy.test.js.map +1 -0
- package/dist/__tests__/frameworks.test.d.ts +2 -0
- package/dist/__tests__/frameworks.test.d.ts.map +1 -0
- package/dist/__tests__/frameworks.test.js +86 -0
- package/dist/__tests__/frameworks.test.js.map +1 -0
- package/dist/__tests__/logger.test.d.ts +2 -0
- package/dist/__tests__/logger.test.d.ts.map +1 -0
- package/dist/__tests__/logger.test.js +77 -0
- package/dist/__tests__/logger.test.js.map +1 -0
- package/dist/__tests__/resolve.test.d.ts +2 -0
- package/dist/__tests__/resolve.test.d.ts.map +1 -0
- package/dist/__tests__/resolve.test.js +71 -0
- package/dist/__tests__/resolve.test.js.map +1 -0
- package/dist/__tests__/wrap.test.d.ts +2 -0
- package/dist/__tests__/wrap.test.d.ts.map +1 -0
- package/dist/__tests__/wrap.test.js +90 -0
- package/dist/__tests__/wrap.test.js.map +1 -0
- package/dist/defaults.d.ts +20 -0
- package/dist/defaults.d.ts.map +1 -0
- package/dist/defaults.js +48 -0
- package/dist/defaults.js.map +1 -0
- package/dist/errors.d.ts +23 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +31 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +38 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +239 -0
- package/dist/index.js.map +1 -0
- package/dist/logger.d.ts +12 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +55 -0
- package/dist/logger.js.map +1 -0
- package/dist/resolve.d.ts +9 -0
- package/dist/resolve.d.ts.map +1 -0
- package/dist/resolve.js +25 -0
- package/dist/resolve.js.map +1 -0
- package/dist/types.d.ts +35 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +82 -0
- package/src/__tests__/defaults.test.ts +71 -0
- package/src/__tests__/fail-policy.test.ts +107 -0
- package/src/__tests__/frameworks.test.ts +106 -0
- package/src/__tests__/logger.test.ts +93 -0
- package/src/__tests__/resolve.test.ts +80 -0
- package/src/__tests__/wrap.test.ts +110 -0
- package/src/consensus-llm.test.ts +260 -0
- package/src/defaults.ts +124 -0
- package/src/errors.ts +35 -0
- package/src/index.ts +386 -0
- package/src/logger.ts +65 -0
- package/src/persona-reviewer-factory.ts +387 -0
- package/src/reputation-manager.test.ts +131 -0
- package/src/reputation-manager.ts +168 -0
- package/src/resolve.ts +30 -0
- package/src/risk-tiers.test.ts +36 -0
- package/src/risk-tiers.ts +49 -0
- package/src/types.ts +127 -0
package/src/resolve.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { ToolExecutor, Wrappable } from "./types.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Resolves a Wrappable into a plain ToolExecutor function.
|
|
5
|
+
*
|
|
6
|
+
* Resolution order: direct function > .execute > .invoke > .call
|
|
7
|
+
* First match wins.
|
|
8
|
+
*/
|
|
9
|
+
export function resolveWrappable(wrappable: Wrappable): ToolExecutor {
|
|
10
|
+
if (typeof wrappable === "function") {
|
|
11
|
+
return wrappable;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
if (typeof wrappable === "object" && wrappable !== null) {
|
|
15
|
+
if ("execute" in wrappable && typeof wrappable.execute === "function") {
|
|
16
|
+
return wrappable.execute.bind(wrappable);
|
|
17
|
+
}
|
|
18
|
+
if ("invoke" in wrappable && typeof wrappable.invoke === "function") {
|
|
19
|
+
return wrappable.invoke.bind(wrappable);
|
|
20
|
+
}
|
|
21
|
+
if ("call" in wrappable && typeof wrappable.call === "function") {
|
|
22
|
+
return wrappable.call.bind(wrappable);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
throw new TypeError(
|
|
27
|
+
"Expected a Wrappable: a function, or an object with .execute(), .invoke(), or .call() method. " +
|
|
28
|
+
`Received: ${typeof wrappable}`,
|
|
29
|
+
);
|
|
30
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { classifyTool } from "./risk-tiers.js";
|
|
3
|
+
|
|
4
|
+
describe("classifyTool", () => {
|
|
5
|
+
it("classifies read-only tools as low risk", () => {
|
|
6
|
+
expect(classifyTool("get_weather")).toBe("low");
|
|
7
|
+
expect(classifyTool("list_files")).toBe("low");
|
|
8
|
+
expect(classifyTool("search_documents")).toBe("low");
|
|
9
|
+
expect(classifyTool("fetch_user")).toBe("low");
|
|
10
|
+
expect(classifyTool("read_config")).toBe("low");
|
|
11
|
+
expect(classifyTool("query_database")).toBe("low");
|
|
12
|
+
expect(classifyTool("check_status")).toBe("low");
|
|
13
|
+
expect(classifyTool("view_dashboard")).toBe("low");
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it("classifies write/send/delete tools as high risk", () => {
|
|
17
|
+
expect(classifyTool("send_email")).toBe("high");
|
|
18
|
+
expect(classifyTool("delete_user")).toBe("high");
|
|
19
|
+
expect(classifyTool("write_file")).toBe("high");
|
|
20
|
+
expect(classifyTool("deploy_service")).toBe("high");
|
|
21
|
+
expect(classifyTool("merge_branch")).toBe("high");
|
|
22
|
+
expect(classifyTool("grant_permission")).toBe("high");
|
|
23
|
+
expect(classifyTool("execute_command")).toBe("high");
|
|
24
|
+
expect(classifyTool("transfer_funds")).toBe("high");
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it("defaults unknown tools to high risk", () => {
|
|
28
|
+
expect(classifyTool("some_unknown_tool")).toBe("high");
|
|
29
|
+
expect(classifyTool("foobar")).toBe("high");
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("respects user overrides", () => {
|
|
33
|
+
expect(classifyTool("send_email", { send_email: "low" })).toBe("low");
|
|
34
|
+
expect(classifyTool("get_weather", { get_weather: "high" })).toBe("high");
|
|
35
|
+
});
|
|
36
|
+
});
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { RiskTier, RiskTierMap } from "./types.js";
|
|
2
|
+
|
|
3
|
+
// ── Default Risk Classification ──────────────────────────────────────
|
|
4
|
+
// Tools are classified by name prefix/pattern. Write/send/delete/mutate
|
|
5
|
+
// operations get full LLM deliberation. Read-only operations fast-path
|
|
6
|
+
// through regex only.
|
|
7
|
+
//
|
|
8
|
+
// Users override via config.riskTiers: { "my_tool": "low" }
|
|
9
|
+
|
|
10
|
+
const HIGH_RISK_PATTERNS = [
|
|
11
|
+
/^(send|email|mail)/i,
|
|
12
|
+
/^(delete|remove|drop|destroy)/i,
|
|
13
|
+
/^(write|update|patch|put|post|create|insert)/i,
|
|
14
|
+
/^(deploy|release|publish|push)/i,
|
|
15
|
+
/^(merge|commit|approve)/i,
|
|
16
|
+
/^(grant|revoke|escalate|permission)/i,
|
|
17
|
+
/^(execute|run|eval|exec)/i,
|
|
18
|
+
/^(transfer|pay|charge|refund)/i,
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
const LOW_RISK_PATTERNS = [
|
|
22
|
+
/^(get|fetch|read|list|search|query|find|lookup)/i,
|
|
23
|
+
/^(check|verify|validate|inspect|describe)/i,
|
|
24
|
+
/^(count|sum|aggregate|stats)/i,
|
|
25
|
+
/^(view|show|display|render|preview)/i,
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Classify a tool name into a risk tier.
|
|
30
|
+
*
|
|
31
|
+
* Priority: user overrides > low-risk patterns > high-risk patterns > default high.
|
|
32
|
+
* Unknown tools default to high-risk (safe by default).
|
|
33
|
+
*/
|
|
34
|
+
export function classifyTool(toolName: string, overrides?: RiskTierMap): RiskTier {
|
|
35
|
+
if (overrides?.[toolName]) {
|
|
36
|
+
return overrides[toolName];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
for (const pattern of LOW_RISK_PATTERNS) {
|
|
40
|
+
if (pattern.test(toolName)) return "low";
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
for (const pattern of HIGH_RISK_PATTERNS) {
|
|
44
|
+
if (pattern.test(toolName)) return "high";
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Unknown tools default to high-risk (safe by default)
|
|
48
|
+
return "high";
|
|
49
|
+
}
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import type { IStorage } from "@consensus-tools/storage";
|
|
2
|
+
import type { DecisionResult } from "@consensus-tools/wrapper";
|
|
3
|
+
import type { PersonaConfig } from "@consensus-tools/personas";
|
|
4
|
+
|
|
5
|
+
// ── Wrappable ────────────────────────────────────────────────────────
|
|
6
|
+
// Any tool executor that consensus can wrap.
|
|
7
|
+
// Resolution order: execute > invoke > call (first match wins).
|
|
8
|
+
|
|
9
|
+
export type ToolExecutor = (toolName: string, args: Record<string, unknown>) => Promise<unknown>;
|
|
10
|
+
|
|
11
|
+
export type Wrappable =
|
|
12
|
+
| ToolExecutor
|
|
13
|
+
| { execute: ToolExecutor }
|
|
14
|
+
| { invoke: ToolExecutor }
|
|
15
|
+
| { call: ToolExecutor };
|
|
16
|
+
|
|
17
|
+
// ── Fail Policy ──────────────────────────────────────────────────────
|
|
18
|
+
|
|
19
|
+
export type FailPolicy = "closed" | "open";
|
|
20
|
+
|
|
21
|
+
// ── Execution Mode ───────────────────────────────────────────────────
|
|
22
|
+
|
|
23
|
+
export type ExecutionMode = "enforce" | "shadow";
|
|
24
|
+
|
|
25
|
+
// ── Model Adapter ────────────────────────────────────────────────────
|
|
26
|
+
// Provider-agnostic LLM interface. Accepts structured messages,
|
|
27
|
+
// returns raw text. Users wrap their SDK call:
|
|
28
|
+
//
|
|
29
|
+
// model: (msgs) => openai.chat({ messages: msgs }).then(r => r.choices[0].message.content)
|
|
30
|
+
// model: (msgs) => anthropic.messages.create({ messages: msgs }).then(r => r.content[0].text)
|
|
31
|
+
|
|
32
|
+
export interface ModelMessage {
|
|
33
|
+
role: "system" | "user";
|
|
34
|
+
content: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export type ModelAdapter = (messages: ModelMessage[]) => Promise<string>;
|
|
38
|
+
|
|
39
|
+
// ── Risk Tiers ───────────────────────────────────────────────────────
|
|
40
|
+
|
|
41
|
+
export type RiskTier = "low" | "high";
|
|
42
|
+
|
|
43
|
+
export type RiskTierMap = Record<string, RiskTier>;
|
|
44
|
+
|
|
45
|
+
// ── Feedback ─────────────────────────────────────────────────────────
|
|
46
|
+
// Human feedback signal for reputation ground truth.
|
|
47
|
+
|
|
48
|
+
export interface FeedbackSignal {
|
|
49
|
+
/** The decision ID this feedback relates to. */
|
|
50
|
+
decisionId: string;
|
|
51
|
+
/** "override_block" = human unblocked a blocked call. "flag_miss" = human flagged a missed risk. */
|
|
52
|
+
type: "override_block" | "flag_miss";
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// ── Logger ───────────────────────────────────────────────────────────
|
|
56
|
+
|
|
57
|
+
export interface LogEvent {
|
|
58
|
+
event: string;
|
|
59
|
+
data: Record<string, unknown>;
|
|
60
|
+
timestamp: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// ── LLM Decision Result ─────────────────────────────────────────────
|
|
64
|
+
// Result from the LLM persona deliberation path.
|
|
65
|
+
|
|
66
|
+
export interface LlmDecisionResult {
|
|
67
|
+
/** Unique decision ID for feedback reference. */
|
|
68
|
+
decisionId: string;
|
|
69
|
+
/** Final action: allow, block, or escalate. */
|
|
70
|
+
action: "allow" | "block" | "escalate";
|
|
71
|
+
/** Per-persona vote breakdown. */
|
|
72
|
+
votes: Array<{
|
|
73
|
+
personaId: string;
|
|
74
|
+
personaName: string;
|
|
75
|
+
vote: "YES" | "NO" | "REWRITE";
|
|
76
|
+
confidence: number;
|
|
77
|
+
rationale: string;
|
|
78
|
+
source: "llm" | "regex_fallback";
|
|
79
|
+
}>;
|
|
80
|
+
/** Policy used for resolution. */
|
|
81
|
+
policy: string;
|
|
82
|
+
/** Consensus trace from resolveConsensus. */
|
|
83
|
+
consensusTrace: Record<string, unknown>;
|
|
84
|
+
/** Aggregate score (0-1). */
|
|
85
|
+
aggregateScore: number;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// ── Config ───────────────────────────────────────────────────────────
|
|
89
|
+
|
|
90
|
+
export interface UniversalConfig {
|
|
91
|
+
/** Consensus policy name. For regex mode: 'majority', 'supermajority', 'unanimous', 'threshold:X'.
|
|
92
|
+
* For LLM mode: also supports all 9 core policies (WEIGHTED_REPUTATION, MAJORITY_VOTE, etc.). */
|
|
93
|
+
policy?: string;
|
|
94
|
+
/** Guard domain names to use as reviewers (regex mode). */
|
|
95
|
+
guards?: string[];
|
|
96
|
+
/** Behavior on error: 'closed' blocks, 'open' allows. Default: 'closed'. */
|
|
97
|
+
failPolicy?: FailPolicy;
|
|
98
|
+
/** Storage backend: 'memory' for in-memory, or an IStorage instance. Default: 'memory'. */
|
|
99
|
+
storage?: "memory" | IStorage;
|
|
100
|
+
/** Logging: true for console.debug, false to disable, or a custom function. Default: true. */
|
|
101
|
+
logger?: boolean | ((event: LogEvent) => void);
|
|
102
|
+
/** Called after every consensus decision (both regex and LLM modes). */
|
|
103
|
+
onDecision?: (decision: DecisionResult<unknown> | LlmDecisionResult) => void | Promise<void>;
|
|
104
|
+
/** Called when an error occurs during deliberation. */
|
|
105
|
+
onError?: (err: Error, action: unknown) => void;
|
|
106
|
+
|
|
107
|
+
// ── LLM Persona Mode (activated when `model` is provided) ─────────
|
|
108
|
+
|
|
109
|
+
/** LLM model adapter. When provided, enables LLM persona mode. */
|
|
110
|
+
model?: ModelAdapter;
|
|
111
|
+
/** Persona pack name. Default: 'default' (security, compliance, operations). */
|
|
112
|
+
pack?: string;
|
|
113
|
+
/** Custom personas (overrides pack). */
|
|
114
|
+
personas?: PersonaConfig[];
|
|
115
|
+
/** Execution mode: 'enforce' blocks on consensus rejection, 'shadow' logs but never blocks. Default: 'enforce'. */
|
|
116
|
+
mode?: ExecutionMode;
|
|
117
|
+
/** Tool risk tier overrides. Keys are tool names, values are 'low' or 'high'. */
|
|
118
|
+
riskTiers?: RiskTierMap;
|
|
119
|
+
/** Human feedback callback for reputation ground truth. */
|
|
120
|
+
onFeedback?: (signal: FeedbackSignal) => void;
|
|
121
|
+
/** Storage for persisting reputation across restarts. Default: in-memory (lost on restart). */
|
|
122
|
+
reputationStore?: IStorage;
|
|
123
|
+
/** Reputation threshold below which persona respawn triggers. Default: 0.15. */
|
|
124
|
+
respawnThreshold?: number;
|
|
125
|
+
/** Per-persona LLM call timeout in milliseconds. Default: 3000. */
|
|
126
|
+
personaTimeout?: number;
|
|
127
|
+
}
|