@effect-uai/core 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/decision-model/Decision.d.mts +155 -0
- package/dist/decision-model/Decision.d.mts.map +1 -0
- package/dist/decision-model/Decision.mjs +118 -0
- package/dist/decision-model/Decision.mjs.map +1 -0
- package/dist/decision-model/Decision.test.d.mts +1 -0
- package/dist/decision-model/Decision.test.mjs +164 -0
- package/dist/decision-model/Decision.test.mjs.map +1 -0
- package/dist/decision-model/DecisionModel.d.mts +45 -0
- package/dist/decision-model/DecisionModel.d.mts.map +1 -0
- package/dist/decision-model/DecisionModel.mjs +25 -0
- package/dist/decision-model/DecisionModel.mjs.map +1 -0
- package/dist/index.d.mts +4 -1
- package/dist/index.mjs +4 -1
- package/dist/math/Probability.d.mts +21 -0
- package/dist/math/Probability.d.mts.map +1 -0
- package/dist/math/Probability.mjs +31 -0
- package/dist/math/Probability.mjs.map +1 -0
- package/dist/math/Probability.test.d.mts +1 -0
- package/dist/math/Probability.test.mjs +95 -0
- package/dist/math/Probability.test.mjs.map +1 -0
- package/package.json +13 -1
- package/src/decision-model/Decision.test.ts +146 -0
- package/src/decision-model/Decision.ts +273 -0
- package/src/decision-model/DecisionModel.ts +53 -0
- package/src/index.ts +3 -0
- package/src/math/Probability.test.ts +61 -0
- package/src/math/Probability.ts +44 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
declare namespace Probability_d_exports {
|
|
2
|
+
export { confidence, margin };
|
|
3
|
+
}
|
|
4
|
+
/**
|
|
5
|
+
* 1 minus normalized Shannon entropy: 1 when all mass sits on one outcome, 0
|
|
6
|
+
* when spread evenly. Fewer than two outcomes yields 1.
|
|
7
|
+
*
|
|
8
|
+
* Conservative on two outcomes, where `[0.94, 0.06]` scores ~0.67, so a
|
|
9
|
+
* threshold tuned on a long vector does not transfer. {@link margin} is the
|
|
10
|
+
* more stable gate across vectors of different length.
|
|
11
|
+
*/
|
|
12
|
+
declare const confidence: (probabilities: ReadonlyArray<number>) => number;
|
|
13
|
+
/**
|
|
14
|
+
* Top mass minus runner-up mass, which unlike {@link confidence} separates a
|
|
15
|
+
* close second from a flat spread. A single outcome yields its own
|
|
16
|
+
* probability; empty yields 0.
|
|
17
|
+
*/
|
|
18
|
+
declare const margin: (probabilities: ReadonlyArray<number>) => number;
|
|
19
|
+
//#endregion
|
|
20
|
+
export { confidence, margin, Probability_d_exports as t };
|
|
21
|
+
//# sourceMappingURL=Probability.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Probability.d.mts","names":[],"sources":["../../src/math/Probability.ts"],"mappings":";;;;;;;;;;;cAaa,aAAU,eAAmB;;;;;;cAiB7B,SAAM,eAAmB"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { n as __exportAll } from "../rolldown-runtime-EAsCxrDo.mjs";
|
|
2
|
+
import { Array, Number, Option, Order, pipe } from "effect";
|
|
3
|
+
//#region src/math/Probability.ts
|
|
4
|
+
/** Certainty measures over a probability vector. */
|
|
5
|
+
var Probability_exports = /* @__PURE__ */ __exportAll({
|
|
6
|
+
confidence: () => confidence,
|
|
7
|
+
margin: () => margin
|
|
8
|
+
});
|
|
9
|
+
const descending = Order.flip(Order.Number);
|
|
10
|
+
/**
|
|
11
|
+
* 1 minus normalized Shannon entropy: 1 when all mass sits on one outcome, 0
|
|
12
|
+
* when spread evenly. Fewer than two outcomes yields 1.
|
|
13
|
+
*
|
|
14
|
+
* Conservative on two outcomes, where `[0.94, 0.06]` scores ~0.67, so a
|
|
15
|
+
* threshold tuned on a long vector does not transfer. {@link margin} is the
|
|
16
|
+
* more stable gate across vectors of different length.
|
|
17
|
+
*/
|
|
18
|
+
const confidence = (probabilities) => probabilities.length < 2 ? 1 : 1 - pipe(probabilities, Array.filter((p) => p > 0), Array.map((p) => -p * Math.log(p)), Number.sumAll) / Math.log(probabilities.length);
|
|
19
|
+
/**
|
|
20
|
+
* Top mass minus runner-up mass, which unlike {@link confidence} separates a
|
|
21
|
+
* close second from a flat spread. A single outcome yields its own
|
|
22
|
+
* probability; empty yields 0.
|
|
23
|
+
*/
|
|
24
|
+
const margin = (probabilities) => {
|
|
25
|
+
const top = pipe(probabilities, Array.sort(descending), Array.take(2));
|
|
26
|
+
return pipe(Array.get(top, 0), Option.map((first) => pipe(Array.get(top, 1), Option.getOrElse(() => 0), (second) => first - second)), Option.getOrElse(() => 0));
|
|
27
|
+
};
|
|
28
|
+
//#endregion
|
|
29
|
+
export { confidence, margin, Probability_exports as t };
|
|
30
|
+
|
|
31
|
+
//# sourceMappingURL=Probability.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Probability.mjs","names":["A","N"],"sources":["../../src/math/Probability.ts"],"sourcesContent":["/** Certainty measures over a probability vector. */\nimport { Array as A, Number as N, Option, Order, pipe } from \"effect\"\n\nconst descending = Order.flip(Order.Number)\n\n/**\n * 1 minus normalized Shannon entropy: 1 when all mass sits on one outcome, 0\n * when spread evenly. Fewer than two outcomes yields 1.\n *\n * Conservative on two outcomes, where `[0.94, 0.06]` scores ~0.67, so a\n * threshold tuned on a long vector does not transfer. {@link margin} is the\n * more stable gate across vectors of different length.\n */\nexport const confidence = (probabilities: ReadonlyArray<number>): number =>\n probabilities.length < 2\n ? 1\n : 1 -\n pipe(\n probabilities,\n A.filter((p) => p > 0),\n A.map((p) => -p * Math.log(p)),\n N.sumAll,\n ) /\n Math.log(probabilities.length)\n\n/**\n * Top mass minus runner-up mass, which unlike {@link confidence} separates a\n * close second from a flat spread. A single outcome yields its own\n * probability; empty yields 0.\n */\nexport const margin = (probabilities: ReadonlyArray<number>): number => {\n const top = pipe(probabilities, A.sort(descending), A.take(2))\n return pipe(\n A.get(top, 0),\n Option.map((first) =>\n pipe(\n A.get(top, 1),\n Option.getOrElse(() => 0),\n (second) => first - second,\n ),\n ),\n Option.getOrElse(() => 0),\n )\n}\n"],"mappings":";;;;;;;;AAGA,MAAM,aAAa,MAAM,KAAK,MAAM,MAAM;;;;;;;;;AAU1C,MAAa,cAAc,kBACzB,cAAc,SAAS,IACnB,IACA,IACA,KACE,eACAA,MAAE,QAAQ,MAAM,IAAI,CAAC,GACrBA,MAAE,KAAK,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,GAC7BC,OAAE,MACJ,IACE,KAAK,IAAI,cAAc,MAAM;;;;;;AAOrC,MAAa,UAAU,kBAAiD;CACtE,MAAM,MAAM,KAAK,eAAeD,MAAE,KAAK,UAAU,GAAGA,MAAE,KAAK,CAAC,CAAC;CAC7D,OAAO,KACLA,MAAE,IAAI,KAAK,CAAC,GACZ,OAAO,KAAK,UACV,KACEA,MAAE,IAAI,KAAK,CAAC,GACZ,OAAO,gBAAgB,CAAC,IACvB,WAAW,QAAQ,MACtB,CACF,GACA,OAAO,gBAAgB,CAAC,CAC1B;AACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { confidence, margin } from "./Probability.mjs";
|
|
2
|
+
import { m as it, o as describe } from "../utils-CiH_DjFW.mjs";
|
|
3
|
+
import { n as globalExpect } from "../test.DNmyFkvJ-YPzDR0Co.mjs";
|
|
4
|
+
//#region src/math/Probability.test.ts
|
|
5
|
+
describe("confidence", () => {
|
|
6
|
+
it("is 1 when all mass sits on one outcome", () => {
|
|
7
|
+
globalExpect(confidence([
|
|
8
|
+
1,
|
|
9
|
+
0,
|
|
10
|
+
0
|
|
11
|
+
])).toBeCloseTo(1, 10);
|
|
12
|
+
});
|
|
13
|
+
it("is 0 when mass is spread evenly, at any length", () => {
|
|
14
|
+
globalExpect(confidence([.5, .5])).toBeCloseTo(0, 10);
|
|
15
|
+
globalExpect(confidence([
|
|
16
|
+
.25,
|
|
17
|
+
.25,
|
|
18
|
+
.25,
|
|
19
|
+
.25
|
|
20
|
+
])).toBeCloseTo(0, 10);
|
|
21
|
+
});
|
|
22
|
+
it("is 1 for fewer than two outcomes, which have nothing to be uncertain between", () => {
|
|
23
|
+
globalExpect(confidence([])).toBe(1);
|
|
24
|
+
globalExpect(confidence([1])).toBe(1);
|
|
25
|
+
});
|
|
26
|
+
it("rises as mass concentrates", () => {
|
|
27
|
+
const spread = confidence([
|
|
28
|
+
.4,
|
|
29
|
+
.35,
|
|
30
|
+
.25
|
|
31
|
+
]);
|
|
32
|
+
const leaning = confidence([
|
|
33
|
+
.6,
|
|
34
|
+
.25,
|
|
35
|
+
.15
|
|
36
|
+
]);
|
|
37
|
+
const decided = confidence([
|
|
38
|
+
.9,
|
|
39
|
+
.07,
|
|
40
|
+
.03
|
|
41
|
+
]);
|
|
42
|
+
globalExpect(spread).toBeLessThan(leaning);
|
|
43
|
+
globalExpect(leaning).toBeLessThan(decided);
|
|
44
|
+
});
|
|
45
|
+
it("is conservative on two outcomes, so a threshold does not transfer across lengths", () => {
|
|
46
|
+
globalExpect(confidence([.94, .06])).toBeCloseTo(.67, 2);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
describe("margin", () => {
|
|
50
|
+
it("is the gap between the top two, whatever order they arrive in", () => {
|
|
51
|
+
globalExpect(margin([
|
|
52
|
+
.6,
|
|
53
|
+
.3,
|
|
54
|
+
.1
|
|
55
|
+
])).toBeCloseTo(.3, 10);
|
|
56
|
+
globalExpect(margin([
|
|
57
|
+
.1,
|
|
58
|
+
.6,
|
|
59
|
+
.3
|
|
60
|
+
])).toBeCloseTo(.3, 10);
|
|
61
|
+
});
|
|
62
|
+
it("is 0 on a tie and 0 when empty", () => {
|
|
63
|
+
globalExpect(margin([.5, .5])).toBeCloseTo(0, 10);
|
|
64
|
+
globalExpect(margin([])).toBe(0);
|
|
65
|
+
});
|
|
66
|
+
it("yields the lone probability for a single outcome", () => {
|
|
67
|
+
globalExpect(margin([.7])).toBeCloseTo(.7, 10);
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
describe("confidence and margin disagree, which is why both exist", () => {
|
|
71
|
+
const closeSecond = [
|
|
72
|
+
.45,
|
|
73
|
+
.42,
|
|
74
|
+
.13
|
|
75
|
+
];
|
|
76
|
+
const flatSpread = [
|
|
77
|
+
.45,
|
|
78
|
+
.11,
|
|
79
|
+
.11,
|
|
80
|
+
.11,
|
|
81
|
+
.11,
|
|
82
|
+
.11
|
|
83
|
+
];
|
|
84
|
+
it("confidence cannot separate them", () => {
|
|
85
|
+
globalExpect(Math.abs(confidence(closeSecond) - confidence(flatSpread))).toBeLessThan(.05);
|
|
86
|
+
});
|
|
87
|
+
it("margin separates them sharply", () => {
|
|
88
|
+
globalExpect(margin(closeSecond)).toBeCloseTo(.03, 10);
|
|
89
|
+
globalExpect(margin(flatSpread)).toBeCloseTo(.34, 10);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
//#endregion
|
|
93
|
+
export {};
|
|
94
|
+
|
|
95
|
+
//# sourceMappingURL=Probability.test.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Probability.test.mjs","names":[],"sources":["../../src/math/Probability.test.ts"],"sourcesContent":["import { describe, expect, it } from \"vitest\"\nimport { confidence, margin } from \"./Probability.js\"\n\ndescribe(\"confidence\", () => {\n it(\"is 1 when all mass sits on one outcome\", () => {\n expect(confidence([1, 0, 0])).toBeCloseTo(1, 10)\n })\n\n it(\"is 0 when mass is spread evenly, at any length\", () => {\n expect(confidence([0.5, 0.5])).toBeCloseTo(0, 10)\n expect(confidence([0.25, 0.25, 0.25, 0.25])).toBeCloseTo(0, 10)\n })\n\n it(\"is 1 for fewer than two outcomes, which have nothing to be uncertain between\", () => {\n expect(confidence([])).toBe(1)\n expect(confidence([1])).toBe(1)\n })\n\n it(\"rises as mass concentrates\", () => {\n const spread = confidence([0.4, 0.35, 0.25])\n const leaning = confidence([0.6, 0.25, 0.15])\n const decided = confidence([0.9, 0.07, 0.03])\n expect(spread).toBeLessThan(leaning)\n expect(leaning).toBeLessThan(decided)\n })\n\n it(\"is conservative on two outcomes, so a threshold does not transfer across lengths\", () => {\n expect(confidence([0.94, 0.06])).toBeCloseTo(0.67, 2)\n })\n})\n\ndescribe(\"margin\", () => {\n it(\"is the gap between the top two, whatever order they arrive in\", () => {\n expect(margin([0.6, 0.3, 0.1])).toBeCloseTo(0.3, 10)\n expect(margin([0.1, 0.6, 0.3])).toBeCloseTo(0.3, 10)\n })\n\n it(\"is 0 on a tie and 0 when empty\", () => {\n expect(margin([0.5, 0.5])).toBeCloseTo(0, 10)\n expect(margin([])).toBe(0)\n })\n\n it(\"yields the lone probability for a single outcome\", () => {\n expect(margin([0.7])).toBeCloseTo(0.7, 10)\n })\n})\n\ndescribe(\"confidence and margin disagree, which is why both exist\", () => {\n // Two plausible answers versus no idea. Same winner, same top mass.\n const closeSecond = [0.45, 0.42, 0.13]\n const flatSpread = [0.45, 0.11, 0.11, 0.11, 0.11, 0.11]\n\n it(\"confidence cannot separate them\", () => {\n expect(Math.abs(confidence(closeSecond) - confidence(flatSpread))).toBeLessThan(0.05)\n })\n\n it(\"margin separates them sharply\", () => {\n expect(margin(closeSecond)).toBeCloseTo(0.03, 10)\n expect(margin(flatSpread)).toBeCloseTo(0.34, 10)\n })\n})\n"],"mappings":";;;;AAGA,SAAS,oBAAoB;CAC3B,GAAG,gDAAgD;EACjD,aAAO,WAAW;GAAC;GAAG;GAAG;EAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,EAAE;CACjD,CAAC;CAED,GAAG,wDAAwD;EACzD,aAAO,WAAW,CAAC,IAAK,EAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,EAAE;EAChD,aAAO,WAAW;GAAC;GAAM;GAAM;GAAM;EAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,EAAE;CAChE,CAAC;CAED,GAAG,sFAAsF;EACvF,aAAO,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;EAC7B,aAAO,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;CAChC,CAAC;CAED,GAAG,oCAAoC;EACrC,MAAM,SAAS,WAAW;GAAC;GAAK;GAAM;EAAI,CAAC;EAC3C,MAAM,UAAU,WAAW;GAAC;GAAK;GAAM;EAAI,CAAC;EAC5C,MAAM,UAAU,WAAW;GAAC;GAAK;GAAM;EAAI,CAAC;EAC5C,aAAO,MAAM,CAAC,CAAC,aAAa,OAAO;EACnC,aAAO,OAAO,CAAC,CAAC,aAAa,OAAO;CACtC,CAAC;CAED,GAAG,0FAA0F;EAC3F,aAAO,WAAW,CAAC,KAAM,GAAI,CAAC,CAAC,CAAC,CAAC,YAAY,KAAM,CAAC;CACtD,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;CACvB,GAAG,uEAAuE;EACxE,aAAO,OAAO;GAAC;GAAK;GAAK;EAAG,CAAC,CAAC,CAAC,CAAC,YAAY,IAAK,EAAE;EACnD,aAAO,OAAO;GAAC;GAAK;GAAK;EAAG,CAAC,CAAC,CAAC,CAAC,YAAY,IAAK,EAAE;CACrD,CAAC;CAED,GAAG,wCAAwC;EACzC,aAAO,OAAO,CAAC,IAAK,EAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,EAAE;EAC5C,aAAO,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;CAC3B,CAAC;CAED,GAAG,0DAA0D;EAC3D,aAAO,OAAO,CAAC,EAAG,CAAC,CAAC,CAAC,CAAC,YAAY,IAAK,EAAE;CAC3C,CAAC;AACH,CAAC;AAED,SAAS,iEAAiE;CAExE,MAAM,cAAc;EAAC;EAAM;EAAM;CAAI;CACrC,MAAM,aAAa;EAAC;EAAM;EAAM;EAAM;EAAM;EAAM;CAAI;CAEtD,GAAG,yCAAyC;EAC1C,aAAO,KAAK,IAAI,WAAW,WAAW,IAAI,WAAW,UAAU,CAAC,CAAC,CAAC,CAAC,aAAa,GAAI;CACtF,CAAC;CAED,GAAG,uCAAuC;EACxC,aAAO,OAAO,WAAW,CAAC,CAAC,CAAC,YAAY,KAAM,EAAE;EAChD,aAAO,OAAO,UAAU,CAAC,CAAC,CAAC,YAAY,KAAM,EAAE;CACjD,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@effect-uai/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "Low-level primitives (loop, conversation, items, tools, streaming codecs) for building AI agents with Effect.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agents",
|
|
@@ -106,6 +106,14 @@
|
|
|
106
106
|
"types": "./dist/domain/Turn.d.mts",
|
|
107
107
|
"import": "./dist/domain/Turn.mjs"
|
|
108
108
|
},
|
|
109
|
+
"./Decision": {
|
|
110
|
+
"types": "./dist/decision-model/Decision.d.mts",
|
|
111
|
+
"import": "./dist/decision-model/Decision.mjs"
|
|
112
|
+
},
|
|
113
|
+
"./DecisionModel": {
|
|
114
|
+
"types": "./dist/decision-model/DecisionModel.d.mts",
|
|
115
|
+
"import": "./dist/decision-model/DecisionModel.mjs"
|
|
116
|
+
},
|
|
109
117
|
"./Embedding": {
|
|
110
118
|
"types": "./dist/embedding-model/Embedding.d.mts",
|
|
111
119
|
"import": "./dist/embedding-model/Embedding.mjs"
|
|
@@ -202,6 +210,10 @@
|
|
|
202
210
|
"types": "./dist/retry/Retry.d.mts",
|
|
203
211
|
"import": "./dist/retry/Retry.mjs"
|
|
204
212
|
},
|
|
213
|
+
"./Probability": {
|
|
214
|
+
"types": "./dist/math/Probability.d.mts",
|
|
215
|
+
"import": "./dist/math/Probability.mjs"
|
|
216
|
+
},
|
|
205
217
|
"./Vector": {
|
|
206
218
|
"types": "./dist/math/Vector.d.mts",
|
|
207
219
|
"import": "./dist/math/Vector.mjs"
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { describe, it } from "@effect/vitest"
|
|
2
|
+
import { Effect, Layer, Option, Ref, Schema } from "effect"
|
|
3
|
+
import { expect, expectTypeOf } from "vitest"
|
|
4
|
+
import * as AiError from "../domain/AiError.js"
|
|
5
|
+
import * as Decision from "./Decision.js"
|
|
6
|
+
import { decide, DecisionModel, type DecisionModelService } from "./DecisionModel.js"
|
|
7
|
+
|
|
8
|
+
const triage = Decision.make({
|
|
9
|
+
inputSchema: Schema.Struct({ message: Schema.String }),
|
|
10
|
+
decisions: {
|
|
11
|
+
department: Decision.classify({
|
|
12
|
+
instructions: "Which team should handle this?",
|
|
13
|
+
criteria: { billing: "Payments, invoicing", technical: "Bugs, outages" },
|
|
14
|
+
}),
|
|
15
|
+
severity: Decision.rate({
|
|
16
|
+
instructions: "How severe is it?",
|
|
17
|
+
criteria: ["Cosmetic", "Degraded", "Blocking"],
|
|
18
|
+
}),
|
|
19
|
+
urgent: Decision.probability({ instructions: "Does this convey urgency?" }),
|
|
20
|
+
},
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
const fixture = {
|
|
24
|
+
department: { _tag: "Classify", labels: { billing: 0.62, technical: 0.38 } },
|
|
25
|
+
severity: { _tag: "Rate", levels: [0, 0.7, 0.3], legend: ["Cosmetic", "Degraded", "Blocking"] },
|
|
26
|
+
urgent: { _tag: "Probability", probability: 0.94 },
|
|
27
|
+
} as const
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* A fixed fixture cannot prove it satisfies the caller's `Answers<D>`. This is
|
|
31
|
+
* the boundary a provider casts at too, once it has decoded the wire.
|
|
32
|
+
*/
|
|
33
|
+
const fixed = <D extends Decision.Decisions>(
|
|
34
|
+
answers: Readonly<Record<string, Decision.Answer>>,
|
|
35
|
+
): Decision.Answers<D> => answers as Decision.Answers<D>
|
|
36
|
+
|
|
37
|
+
type Call = { readonly input: unknown; readonly model: string }
|
|
38
|
+
|
|
39
|
+
/** A `DecisionModel` layer that records what reached it. */
|
|
40
|
+
const recording = Effect.gen(function* () {
|
|
41
|
+
const calls = yield* Ref.make<ReadonlyArray<Call>>([])
|
|
42
|
+
const service: DecisionModelService = {
|
|
43
|
+
decide: (_definition, request) =>
|
|
44
|
+
Effect.as(
|
|
45
|
+
Ref.update(calls, (seen) => [...seen, { input: request.input, model: request.model }]),
|
|
46
|
+
{ answers: fixed(fixture), usage: {} },
|
|
47
|
+
),
|
|
48
|
+
}
|
|
49
|
+
return { calls, layer: Layer.succeed(DecisionModel, service) }
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
describe("the answer map is derived from the definition", () => {
|
|
53
|
+
it.effect("keeps each decision's label keys and rejects labels it never had", () =>
|
|
54
|
+
Effect.gen(function* () {
|
|
55
|
+
const { calls, layer } = yield* recording
|
|
56
|
+
const result = yield* decide(triage, {
|
|
57
|
+
model: "jev-latest",
|
|
58
|
+
input: { message: "hi" },
|
|
59
|
+
}).pipe(Effect.provide(layer))
|
|
60
|
+
|
|
61
|
+
expectTypeOf(result.answers.department.labels).toEqualTypeOf<{
|
|
62
|
+
readonly billing: number
|
|
63
|
+
readonly technical: number
|
|
64
|
+
}>()
|
|
65
|
+
// @ts-expect-error shipping is not a label of this decision
|
|
66
|
+
result.answers.department.labels.shipping
|
|
67
|
+
// @ts-expect-error a Probability answer has no labels
|
|
68
|
+
result.answers.urgent.labels
|
|
69
|
+
|
|
70
|
+
// `decide` forwards both arguments rather than dropping either.
|
|
71
|
+
expect(yield* Ref.get(calls)).toEqual([{ input: { message: "hi" }, model: "jev-latest" }])
|
|
72
|
+
}),
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
it("keeps the rubric as a tuple, so levels are positional rather than parsed", () => {
|
|
76
|
+
type Severity = (typeof triage)["decisions"]["severity"]
|
|
77
|
+
expectTypeOf<Decision.AnswerFor<Severity>["levels"]>().toEqualTypeOf<
|
|
78
|
+
readonly [number, number, number]
|
|
79
|
+
>()
|
|
80
|
+
expectTypeOf<Decision.AnswerFor<Severity>["legend"]>().toEqualTypeOf<
|
|
81
|
+
readonly ["Cosmetic", "Degraded", "Blocking"]
|
|
82
|
+
>()
|
|
83
|
+
})
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
describe("reading an answer", () => {
|
|
87
|
+
it("derives a Probability answer's distribution as [p, 1 - p]", () => {
|
|
88
|
+
const [yes, no] = Decision.probabilitiesOf(fixture.urgent)
|
|
89
|
+
expect(yes).toBeCloseTo(0.94, 10)
|
|
90
|
+
expect(no).toBeCloseTo(0.06, 10)
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
it("ranks labels best first and names the winner", () => {
|
|
94
|
+
expect(Decision.ranked(fixture.department)).toEqual([
|
|
95
|
+
{ label: "billing", probability: 0.62 },
|
|
96
|
+
{ label: "technical", probability: 0.38 },
|
|
97
|
+
])
|
|
98
|
+
expect(Decision.winner(fixture.department)).toStrictEqual(Option.some("billing"))
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
it("has no winner for an empty label set", () => {
|
|
102
|
+
expect(Decision.winner({ _tag: "Classify", labels: {} })).toStrictEqual(Option.none())
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
it("reads a rubric by index and by expected value", () => {
|
|
106
|
+
expect(Decision.topLevel(fixture.severity)).toBe(1)
|
|
107
|
+
expect(Decision.expectedLevel(fixture.severity)).toBeCloseTo(1.3, 10)
|
|
108
|
+
expect(fixture.severity.legend[Decision.topLevel(fixture.severity)]).toBe("Degraded")
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
it("resolves a level tie to the lowest index", () => {
|
|
112
|
+
expect(Decision.topLevel({ _tag: "Rate", levels: [0.5, 0.5], legend: ["a", "b"] })).toBe(0)
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
it("computes certainty for every kind, including Probability", () => {
|
|
116
|
+
expect(Decision.confidence(fixture.urgent)).toBeCloseTo(0.67, 2)
|
|
117
|
+
expect(Decision.margin(fixture.urgent)).toBeCloseTo(0.88, 10)
|
|
118
|
+
expect(Decision.margin(fixture.department)).toBeCloseTo(0.24, 10)
|
|
119
|
+
})
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
describe("assertKinds", () => {
|
|
123
|
+
const supported = ["Classify", "Probability"] as const
|
|
124
|
+
|
|
125
|
+
it.effect("passes when every kind is supported", () => {
|
|
126
|
+
const { department, urgent } = triage.decisions
|
|
127
|
+
return Decision.assertKinds({ department, urgent }, supported, "nli")
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
it.effect("fails Unsupported and names every offender at once", () =>
|
|
131
|
+
Effect.gen(function* () {
|
|
132
|
+
const decisions = {
|
|
133
|
+
...triage.decisions,
|
|
134
|
+
impact: Decision.rate({ instructions: "How bad?", criteria: ["low", "high"] }),
|
|
135
|
+
}
|
|
136
|
+
const error = yield* Effect.flip(Decision.assertKinds(decisions, supported, "nli"))
|
|
137
|
+
|
|
138
|
+
expect(error).toBeInstanceOf(AiError.Unsupported)
|
|
139
|
+
if (error._tag !== "Unsupported") return
|
|
140
|
+
|
|
141
|
+
expect(error.reason).toContain('"severity" is a Rate')
|
|
142
|
+
expect(error.reason).toContain('"impact" is a Rate')
|
|
143
|
+
expect(error.reason).not.toContain("department")
|
|
144
|
+
}),
|
|
145
|
+
)
|
|
146
|
+
})
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed questions about one input, answered with probabilities.
|
|
3
|
+
*
|
|
4
|
+
* A {@link Definition} pairs the shape being read with the questions asked of
|
|
5
|
+
* it. Build one at module scope and apply it per call with
|
|
6
|
+
* `DecisionModel.decide`.
|
|
7
|
+
*
|
|
8
|
+
* @experimental The model class this abstracts is new and the shape may change.
|
|
9
|
+
*/
|
|
10
|
+
import {
|
|
11
|
+
Array as A,
|
|
12
|
+
Effect,
|
|
13
|
+
Match,
|
|
14
|
+
Number as N,
|
|
15
|
+
Option,
|
|
16
|
+
Order,
|
|
17
|
+
Record,
|
|
18
|
+
type Schema,
|
|
19
|
+
pipe,
|
|
20
|
+
} from "effect"
|
|
21
|
+
import * as AiError from "../domain/AiError.js"
|
|
22
|
+
import * as Probability from "../math/Probability.js"
|
|
23
|
+
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
// Criteria
|
|
26
|
+
// ---------------------------------------------------------------------------
|
|
27
|
+
|
|
28
|
+
/** Labels mapped to what each one means. */
|
|
29
|
+
export type Labels = Readonly<Record<string, string>>
|
|
30
|
+
|
|
31
|
+
/** An ordered rubric, lowest level first. */
|
|
32
|
+
export type Levels = readonly [string, string, ...ReadonlyArray<string>]
|
|
33
|
+
|
|
34
|
+
/** What a true and a false answer mean, when the boundary is subtle. */
|
|
35
|
+
export type Boundary = { readonly true?: string; readonly false?: string }
|
|
36
|
+
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// Decisions
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
|
|
41
|
+
export type ClassifyDecision<L extends Labels = Labels> = {
|
|
42
|
+
readonly _tag: "Classify"
|
|
43
|
+
readonly instructions: string
|
|
44
|
+
readonly criteria: L
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export type RateDecision<V extends Levels = Levels> = {
|
|
48
|
+
readonly _tag: "Rate"
|
|
49
|
+
readonly instructions: string
|
|
50
|
+
readonly criteria: V
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export type ProbabilityDecision = {
|
|
54
|
+
readonly _tag: "Probability"
|
|
55
|
+
readonly instructions: string
|
|
56
|
+
readonly criteria?: Boundary
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export type Decision = ClassifyDecision | RateDecision | ProbabilityDecision
|
|
60
|
+
|
|
61
|
+
/** Pick one label. Use {@link rate} when the options form a spectrum. */
|
|
62
|
+
export const classify = <const L extends Labels>(decision: {
|
|
63
|
+
readonly instructions: string
|
|
64
|
+
readonly criteria: L
|
|
65
|
+
}): ClassifyDecision<L> => ({ _tag: "Classify", ...decision })
|
|
66
|
+
|
|
67
|
+
/** Place the input on an ordered rubric. */
|
|
68
|
+
export const rate = <const V extends Levels>(decision: {
|
|
69
|
+
readonly instructions: string
|
|
70
|
+
readonly criteria: V
|
|
71
|
+
}): RateDecision<V> => ({ _tag: "Rate", ...decision })
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* How likely a statement is to hold. Named for its answer rather than for an
|
|
75
|
+
* act, unlike its siblings, so nobody reads the result as a boolean.
|
|
76
|
+
*/
|
|
77
|
+
export const probability = (decision: {
|
|
78
|
+
readonly instructions: string
|
|
79
|
+
readonly criteria?: Boundary
|
|
80
|
+
}): ProbabilityDecision => ({ _tag: "Probability", ...decision })
|
|
81
|
+
|
|
82
|
+
/** Decisions keyed by the name their answer comes back under. */
|
|
83
|
+
export type Decisions = { readonly [name: string]: Decision }
|
|
84
|
+
|
|
85
|
+
// ---------------------------------------------------------------------------
|
|
86
|
+
// Definition
|
|
87
|
+
// ---------------------------------------------------------------------------
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* A reusable decision set bound to the shape it reads.
|
|
91
|
+
*
|
|
92
|
+
* `inputSchema` encodes the input on the way out, so a schema that omits a
|
|
93
|
+
* field keeps it from the model. That projection is the supported way to hold
|
|
94
|
+
* irrelevant or sensitive context out of a request.
|
|
95
|
+
*/
|
|
96
|
+
export type Definition<A_, D extends Decisions = Decisions> = {
|
|
97
|
+
readonly inputSchema: Schema.Codec<A_, unknown, any, never>
|
|
98
|
+
readonly decisions: D
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Build a decision set. Declare it once at module scope: the questions and
|
|
103
|
+
* their rubrics are the reviewed artifact, the input changes per call.
|
|
104
|
+
*
|
|
105
|
+
* @example
|
|
106
|
+
* ```ts
|
|
107
|
+
* const triage = Decision.make({
|
|
108
|
+
* inputSchema: Schema.Struct({ message: Schema.String }),
|
|
109
|
+
* decisions: {
|
|
110
|
+
* department: Decision.classify({
|
|
111
|
+
* instructions: "Which team should handle this?",
|
|
112
|
+
* criteria: { billing: "Payments, invoicing", technical: "Bugs, outages" },
|
|
113
|
+
* }),
|
|
114
|
+
* },
|
|
115
|
+
* })
|
|
116
|
+
* ```
|
|
117
|
+
*/
|
|
118
|
+
export const make = <
|
|
119
|
+
S extends Schema.Codec<any, any, any, never>,
|
|
120
|
+
const D extends Decisions,
|
|
121
|
+
>(definition: {
|
|
122
|
+
readonly inputSchema: S
|
|
123
|
+
readonly decisions: D
|
|
124
|
+
}): Definition<S["Type"], D> => definition
|
|
125
|
+
|
|
126
|
+
// ---------------------------------------------------------------------------
|
|
127
|
+
// Answers
|
|
128
|
+
// ---------------------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Score contract for {@link ClassifyAnswer.labels} and
|
|
132
|
+
* {@link RateAnswer.levels}: probabilities sum to 1 within floating-point
|
|
133
|
+
* error and are comparable within one answer. Whether they are *calibrated*,
|
|
134
|
+
* so a stated 0.8 means right about 80% of the time, is a property of the
|
|
135
|
+
* Layer and is documented per provider. No implementor promises calibration
|
|
136
|
+
* across requests, across decision kinds, or between a statement and its
|
|
137
|
+
* negation: `P(claim) != 1 - P(not claim)`. A threshold fitted against one
|
|
138
|
+
* model does not transfer to another.
|
|
139
|
+
*/
|
|
140
|
+
export type ClassifyAnswer<L extends Labels = Labels> = {
|
|
141
|
+
readonly _tag: "Classify"
|
|
142
|
+
readonly labels: Readonly<Record<keyof L & string, number>>
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export type RateAnswer<V extends Levels = Levels> = {
|
|
146
|
+
readonly _tag: "Rate"
|
|
147
|
+
/** Index-aligned with {@link RateAnswer.legend}. */
|
|
148
|
+
readonly levels: { readonly [K in keyof V]: number }
|
|
149
|
+
/** The rubric, echoed so an index reads back as prose. */
|
|
150
|
+
readonly legend: Readonly<V>
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export type ProbabilityAnswer = {
|
|
154
|
+
readonly _tag: "Probability"
|
|
155
|
+
readonly probability: number
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
export type Answer = ClassifyAnswer | RateAnswer | ProbabilityAnswer
|
|
159
|
+
|
|
160
|
+
/** The answer a decision determines, keeping its labels or rubric. */
|
|
161
|
+
export type AnswerFor<D> = D extends {
|
|
162
|
+
readonly _tag: "Classify"
|
|
163
|
+
readonly criteria: infer L extends Labels
|
|
164
|
+
}
|
|
165
|
+
? ClassifyAnswer<L>
|
|
166
|
+
: D extends { readonly _tag: "Rate"; readonly criteria: infer V extends Levels }
|
|
167
|
+
? RateAnswer<V>
|
|
168
|
+
: ProbabilityAnswer
|
|
169
|
+
|
|
170
|
+
/** Answers keyed by their decision's name. */
|
|
171
|
+
export type Answers<D extends Decisions> = { readonly [K in keyof D]: AnswerFor<D[K]> }
|
|
172
|
+
|
|
173
|
+
// ---------------------------------------------------------------------------
|
|
174
|
+
// Guards
|
|
175
|
+
// ---------------------------------------------------------------------------
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Fail `AiError.Unsupported` naming every decision whose kind this provider
|
|
179
|
+
* cannot express. Bucket 1: omitting an answer would leave a hole where
|
|
180
|
+
* {@link Answers} promises a key.
|
|
181
|
+
*
|
|
182
|
+
* Reports all offenders at once, since the definition is known up front.
|
|
183
|
+
*/
|
|
184
|
+
export const assertKinds = (
|
|
185
|
+
decisions: Decisions,
|
|
186
|
+
supported: ReadonlyArray<Decision["_tag"]>,
|
|
187
|
+
provider: string,
|
|
188
|
+
): Effect.Effect<void, AiError.AiError> => {
|
|
189
|
+
const offenders = pipe(
|
|
190
|
+
Record.toEntries(decisions),
|
|
191
|
+
A.filter(([, decision]) => !supported.includes(decision._tag)),
|
|
192
|
+
A.map(([name, decision]) => `"${name}" is a ${decision._tag}`),
|
|
193
|
+
)
|
|
194
|
+
return offenders.length === 0
|
|
195
|
+
? Effect.void
|
|
196
|
+
: Effect.fail(
|
|
197
|
+
new AiError.Unsupported({
|
|
198
|
+
provider,
|
|
199
|
+
capability: "decision",
|
|
200
|
+
reason: `${provider} answers ${supported.join(" / ")} only; ${offenders.join(", ")}.`,
|
|
201
|
+
}),
|
|
202
|
+
)
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// ---------------------------------------------------------------------------
|
|
206
|
+
// Reading an answer
|
|
207
|
+
// ---------------------------------------------------------------------------
|
|
208
|
+
|
|
209
|
+
/** A `Probability` answer's implied distribution is `[p, 1 - p]`. */
|
|
210
|
+
export const probabilitiesOf: (answer: Answer) => ReadonlyArray<number> = Match.type<Answer>().pipe(
|
|
211
|
+
Match.discriminatorsExhaustive("_tag")({
|
|
212
|
+
Classify: (answer) => Record.values(answer.labels),
|
|
213
|
+
Rate: (answer) => answer.levels,
|
|
214
|
+
Probability: (answer) => [answer.probability, 1 - answer.probability],
|
|
215
|
+
}),
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
/** See {@link Probability.confidence}. Defined for every kind. */
|
|
219
|
+
export const confidence = (answer: Answer): number =>
|
|
220
|
+
Probability.confidence(probabilitiesOf(answer))
|
|
221
|
+
|
|
222
|
+
/** See {@link Probability.margin}. Defined for every kind. */
|
|
223
|
+
export const margin = (answer: Answer): number => Probability.margin(probabilitiesOf(answer))
|
|
224
|
+
|
|
225
|
+
const byProbability = Order.mapInput(
|
|
226
|
+
Order.flip(Order.Number),
|
|
227
|
+
(outcome: { readonly probability: number }) => outcome.probability,
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Labels best first. Ties keep first-seen order. Also the bridge to rank
|
|
232
|
+
* fusion: map to `label` and pass alongside another retriever's ordering.
|
|
233
|
+
*/
|
|
234
|
+
export const ranked = <L extends Labels>(
|
|
235
|
+
answer: ClassifyAnswer<L>,
|
|
236
|
+
): ReadonlyArray<{ readonly label: keyof L & string; readonly probability: number }> =>
|
|
237
|
+
pipe(
|
|
238
|
+
Record.toEntries(answer.labels),
|
|
239
|
+
A.map(([label, probability]) => ({ label, probability })),
|
|
240
|
+
A.sort(byProbability),
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
/** Highest-probability label. `None` only when the label set is empty. */
|
|
244
|
+
export const winner = <L extends Labels>(
|
|
245
|
+
answer: ClassifyAnswer<L>,
|
|
246
|
+
): Option.Option<keyof L & string> =>
|
|
247
|
+
pipe(
|
|
248
|
+
ranked(answer),
|
|
249
|
+
A.head,
|
|
250
|
+
Option.map((outcome) => outcome.label),
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
/** Index of the highest-probability level. Ties resolve to the lowest. */
|
|
254
|
+
export const topLevel = (answer: RateAnswer): number =>
|
|
255
|
+
pipe(
|
|
256
|
+
answer.levels as ReadonlyArray<number>,
|
|
257
|
+
A.reduce({ index: 0, probability: -Infinity }, (best, probability, index) =>
|
|
258
|
+
probability > best.probability ? { index, probability } : best,
|
|
259
|
+
),
|
|
260
|
+
(best) => best.index,
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Probability-weighted mean of the level indices, so it can land between
|
|
265
|
+
* them. Weakly calibrated in practice: a summary, not a magnitude to
|
|
266
|
+
* interpolate. Use {@link topLevel} with `legend` to name a level instead.
|
|
267
|
+
*/
|
|
268
|
+
export const expectedLevel = (answer: RateAnswer): number =>
|
|
269
|
+
pipe(
|
|
270
|
+
answer.levels as ReadonlyArray<number>,
|
|
271
|
+
A.map((probability, index) => index * probability),
|
|
272
|
+
N.sumAll,
|
|
273
|
+
)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { Context, Effect } from "effect"
|
|
2
|
+
import type * as AiError from "../domain/AiError.js"
|
|
3
|
+
import type * as Decision from "./Decision.js"
|
|
4
|
+
|
|
5
|
+
/** Optional throughout: not every backing reports token counts. */
|
|
6
|
+
export type DecideUsage = {
|
|
7
|
+
readonly inputTokens?: number
|
|
8
|
+
readonly outputTokens?: number
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export type DecideResponse<D extends Decision.Decisions = Decision.Decisions> = {
|
|
12
|
+
readonly answers: Decision.Answers<D>
|
|
13
|
+
readonly usage: DecideUsage
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Cross-provider decide request. The input is billed once for the whole
|
|
18
|
+
* definition, so batching decisions is cheaper than repeating the input.
|
|
19
|
+
*/
|
|
20
|
+
export type CommonDecideRequest<A> = {
|
|
21
|
+
readonly input: A
|
|
22
|
+
/** Each provider narrows this to its typed literal union. */
|
|
23
|
+
readonly model: string
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type DecisionModelService = {
|
|
27
|
+
readonly decide: <A, D extends Decision.Decisions>(
|
|
28
|
+
definition: Decision.Definition<A, D>,
|
|
29
|
+
request: CommonDecideRequest<A>,
|
|
30
|
+
) => Effect.Effect<DecideResponse<D>, AiError.AiError>
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Answering typed questions about an input with probabilities instead of
|
|
35
|
+
* generating text. The model reports beliefs; the decision stays in the
|
|
36
|
+
* caller's code.
|
|
37
|
+
*
|
|
38
|
+
* Implementor contract: answer every decision or fail. Omitting one leaves a
|
|
39
|
+
* hole where {@link DecideResponse} promises a key, so a kind the backing
|
|
40
|
+
* cannot express is rejected up front with `Decision.assertKinds`.
|
|
41
|
+
*
|
|
42
|
+
* @experimental The model class this abstracts is new and the shape may change.
|
|
43
|
+
*/
|
|
44
|
+
export class DecisionModel extends Context.Service<DecisionModel, DecisionModelService>()(
|
|
45
|
+
"@betalyra/effect-uai/DecisionModel",
|
|
46
|
+
) {}
|
|
47
|
+
|
|
48
|
+
/** Answer a definition's decisions about one input in one call. */
|
|
49
|
+
export const decide = <A, D extends Decision.Decisions>(
|
|
50
|
+
definition: Decision.Definition<A, D>,
|
|
51
|
+
request: CommonDecideRequest<A>,
|
|
52
|
+
): Effect.Effect<DecideResponse<D>, AiError.AiError, DecisionModel> =>
|
|
53
|
+
Effect.flatMap(DecisionModel, (m) => m.decide(definition, request))
|