@postalsys/email-content 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -17
- package/dist/.tsbuildinfo +1 -1
- package/dist/analyze.d.ts +3 -0
- package/dist/analyze.d.ts.map +1 -1
- package/dist/analyze.js +10 -4
- package/dist/analyze.js.map +1 -1
- package/dist/annotate/agreement.d.ts +99 -0
- package/dist/annotate/agreement.d.ts.map +1 -0
- package/dist/annotate/agreement.js +126 -0
- package/dist/annotate/agreement.js.map +1 -0
- package/dist/annotate/client.d.ts +7 -19
- package/dist/annotate/client.d.ts.map +1 -1
- package/dist/annotate/client.js +28 -82
- package/dist/annotate/client.js.map +1 -1
- package/dist/annotate/http.d.ts +57 -0
- package/dist/annotate/http.d.ts.map +1 -0
- package/dist/annotate/http.js +99 -0
- package/dist/annotate/http.js.map +1 -0
- package/dist/annotate/llm.d.ts +17 -0
- package/dist/annotate/llm.d.ts.map +1 -1
- package/dist/annotate/llm.js +68 -36
- package/dist/annotate/llm.js.map +1 -1
- package/dist/annotate/probabilities.d.ts +48 -0
- package/dist/annotate/probabilities.d.ts.map +1 -0
- package/dist/annotate/probabilities.js +119 -0
- package/dist/annotate/probabilities.js.map +1 -0
- package/dist/annotate/providers.d.ts +26 -0
- package/dist/annotate/providers.d.ts.map +1 -0
- package/dist/annotate/providers.js +43 -0
- package/dist/annotate/providers.js.map +1 -0
- package/dist/annotate/store.d.ts +3 -0
- package/dist/annotate/store.d.ts.map +1 -1
- package/dist/annotate/store.js +3 -2
- package/dist/annotate/store.js.map +1 -1
- package/dist/annotate/typesafe-client.d.ts +64 -0
- package/dist/annotate/typesafe-client.d.ts.map +1 -0
- package/dist/annotate/typesafe-client.js +76 -0
- package/dist/annotate/typesafe-client.js.map +1 -0
- package/dist/annotate/typesafe-prompt.d.ts +64 -0
- package/dist/annotate/typesafe-prompt.d.ts.map +1 -0
- package/dist/annotate/typesafe-prompt.js +170 -0
- package/dist/annotate/typesafe-prompt.js.map +1 -0
- package/dist/classify/rules.d.ts +7 -0
- package/dist/classify/rules.d.ts.map +1 -1
- package/dist/classify/rules.js +117 -18
- package/dist/classify/rules.js.map +1 -1
- package/dist/cli/agreement.d.ts +27 -0
- package/dist/cli/agreement.d.ts.map +1 -0
- package/dist/cli/agreement.js +153 -0
- package/dist/cli/agreement.js.map +1 -0
- package/dist/cli/annotate.d.ts.map +1 -1
- package/dist/cli/annotate.js +28 -16
- package/dist/cli/annotate.js.map +1 -1
- package/dist/cli/audit.d.ts +12 -0
- package/dist/cli/audit.d.ts.map +1 -0
- package/dist/cli/audit.js +142 -0
- package/dist/cli/audit.js.map +1 -0
- package/dist/cli/disagreements.d.ts.map +1 -1
- package/dist/cli/disagreements.js +20 -3
- package/dist/cli/disagreements.js.map +1 -1
- package/dist/cli/evaluate.d.ts.map +1 -1
- package/dist/cli/evaluate.js +31 -72
- package/dist/cli/evaluate.js.map +1 -1
- package/dist/cli/main.js +28 -2
- package/dist/cli/main.js.map +1 -1
- package/dist/corpus/analyze.d.ts.map +1 -1
- package/dist/corpus/analyze.js +4 -5
- package/dist/corpus/analyze.js.map +1 -1
- package/dist/corpus/migrations/index.d.ts.map +1 -1
- package/dist/corpus/migrations/index.js +14 -0
- package/dist/corpus/migrations/index.js.map +1 -1
- package/dist/corpus/stats.d.ts +1 -0
- package/dist/corpus/stats.d.ts.map +1 -1
- package/dist/corpus/stats.js +1 -0
- package/dist/corpus/stats.js.map +1 -1
- package/dist/detect/patterns/forward-headers.json +24 -0
- package/dist/detect/patterns/html-clients.json +27 -0
- package/dist/detect/patterns/quoted-headers.json +7 -0
- package/dist/detect/patterns/reply-headers.json +410 -0
- package/dist/detect/patterns.d.ts +2 -0
- package/dist/detect/patterns.d.ts.map +1 -1
- package/dist/detect/patterns.js +3 -0
- package/dist/detect/patterns.js.map +1 -1
- package/dist/detect/reply-header.d.ts +7 -0
- package/dist/detect/reply-header.d.ts.map +1 -1
- package/dist/detect/reply-header.js +51 -4
- package/dist/detect/reply-header.js.map +1 -1
- package/dist/detect/subject.d.ts +16 -0
- package/dist/detect/subject.d.ts.map +1 -0
- package/dist/detect/subject.js +22 -0
- package/dist/detect/subject.js.map +1 -0
- package/dist/eval/metrics.d.ts +7 -0
- package/dist/eval/metrics.d.ts.map +1 -1
- package/dist/eval/metrics.js +1 -1
- package/dist/eval/metrics.js.map +1 -1
- package/dist/eval/pattern-audit.d.ts +56 -0
- package/dist/eval/pattern-audit.d.ts.map +1 -0
- package/dist/eval/pattern-audit.js +111 -0
- package/dist/eval/pattern-audit.js.map +1 -0
- package/dist/eval/reference.d.ts +51 -0
- package/dist/eval/reference.d.ts.map +1 -0
- package/dist/eval/reference.js +96 -0
- package/dist/eval/reference.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/mime/flowed.d.ts +14 -0
- package/dist/mime/flowed.d.ts.map +1 -0
- package/dist/mime/flowed.js +71 -0
- package/dist/mime/flowed.js.map +1 -0
- package/dist/mime/parse.js +2 -5
- package/dist/mime/parse.js.map +1 -1
- package/dist/segment/features.d.ts.map +1 -1
- package/dist/segment/features.js +2 -1
- package/dist/segment/features.js.map +1 -1
- package/dist/segment/html-blocks.d.ts.map +1 -1
- package/dist/segment/html-blocks.js +33 -9
- package/dist/segment/html-blocks.js.map +1 -1
- package/dist/segment/text-blocks.d.ts.map +1 -1
- package/dist/segment/text-blocks.js +18 -3
- package/dist/segment/text-blocks.js.map +1 -1
- package/dist/segment/types.d.ts +7 -0
- package/dist/segment/types.d.ts.map +1 -1
- package/dist/types.d.ts +3 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +2 -2
- package/dist/types.js.map +1 -1
- package/package.json +5 -5
package/dist/annotate/llm.d.ts
CHANGED
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
import { type Block } from '../segment/types';
|
|
14
14
|
import { type Label } from '../types';
|
|
15
15
|
import { type CompletionClient } from './client';
|
|
16
|
+
import { type LabelProbabilities } from './probabilities';
|
|
17
|
+
import { type SystemOneClient } from './typesafe-client';
|
|
16
18
|
/** Characters of each block sent to the model. Enough for a decision, bounded for cost. */
|
|
17
19
|
export declare const MAX_BLOCK_CHARS = 700;
|
|
18
20
|
/** Blocks per message. Longer messages are truncated with a recorded flag. */
|
|
@@ -21,6 +23,8 @@ export interface LlmLabel {
|
|
|
21
23
|
index: number;
|
|
22
24
|
label: Label;
|
|
23
25
|
confidence: number;
|
|
26
|
+
/** Full distribution, from annotators that return one. The label is derived from it. */
|
|
27
|
+
probabilities?: LabelProbabilities;
|
|
24
28
|
}
|
|
25
29
|
export interface AnnotationResult {
|
|
26
30
|
labels: LlmLabel[];
|
|
@@ -43,4 +47,17 @@ export declare function annotatableBlocks(blocks: readonly Block[]): Block[];
|
|
|
43
47
|
* @param isReply whether the message carries threading headers
|
|
44
48
|
*/
|
|
45
49
|
export declare function annotateBlocks(client: CompletionClient, blocks: readonly Block[], isReply: boolean): Promise<AnnotationResult>;
|
|
50
|
+
/**
|
|
51
|
+
* Annotate one message's blocks with TypeSafe: one Choice question per block.
|
|
52
|
+
*
|
|
53
|
+
* The stored label is the cost rule applied to the returned distribution at DEFAULT_COST_RATIO, not
|
|
54
|
+
* the model's top choice, and the raw distribution is kept alongside so other ratios can be applied
|
|
55
|
+
* later without re-running inference. An answer without a usable distribution is treated as
|
|
56
|
+
* missing: the cost rule cannot be checked against it, so it must not be allowed to hide anything.
|
|
57
|
+
*
|
|
58
|
+
* @param client configured TypeSafe client
|
|
59
|
+
* @param blocks blocks from the primary representation
|
|
60
|
+
* @param isReply whether the message carries threading headers
|
|
61
|
+
*/
|
|
62
|
+
export declare function annotateBlocksWithTypeSafe(client: SystemOneClient, blocks: readonly Block[], isReply: boolean): Promise<AnnotationResult>;
|
|
46
63
|
//# sourceMappingURL=llm.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,EAAwC,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAEhG,OAAO,EAAE,KAAK,eAAe,EAAyB,MAAM,mBAAmB,CAAC;AAGhF,2FAA2F;AAC3F,eAAO,MAAM,eAAe,MAAM,CAAC;AAEnC,8EAA8E;AAC9E,eAAO,MAAM,UAAU,KAAK,CAAC;AAE7B,MAAM,WAAW,QAAQ;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,wFAAwF;IACxF,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACtC;AAED,MAAM,WAAW,gBAAgB;IAC7B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,wEAAwE;AACxE,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,KAAK,EAAE,CAEnE;AA8CD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA+BpI;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,0BAA0B,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA4B/I"}
|
package/dist/annotate/llm.js
CHANGED
|
@@ -15,7 +15,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
15
15
|
exports.MAX_BLOCKS = exports.MAX_BLOCK_CHARS = void 0;
|
|
16
16
|
exports.annotatableBlocks = annotatableBlocks;
|
|
17
17
|
exports.annotateBlocks = annotateBlocks;
|
|
18
|
+
exports.annotateBlocksWithTypeSafe = annotateBlocksWithTypeSafe;
|
|
19
|
+
const probabilities_1 = require("./probabilities");
|
|
18
20
|
const prompt_1 = require("./prompt");
|
|
21
|
+
const typesafe_prompt_1 = require("./typesafe-prompt");
|
|
19
22
|
/** Characters of each block sent to the model. Enough for a decision, bounded for cost. */
|
|
20
23
|
exports.MAX_BLOCK_CHARS = 700;
|
|
21
24
|
/** Blocks per message. Longer messages are truncated with a recorded flag. */
|
|
@@ -39,6 +42,27 @@ function toPromptBlocks(blocks) {
|
|
|
39
42
|
};
|
|
40
43
|
});
|
|
41
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Close the loop on what was sent: every sent block comes back exactly once, in index order.
|
|
47
|
+
*
|
|
48
|
+
* A block the model declined to label, or answered unusably, is UNKNOWN with confidence zero, which
|
|
49
|
+
* stays visible. Silently dropping it would let an omission look like agreement later.
|
|
50
|
+
*/
|
|
51
|
+
function finish(sent, accepted, spurious, meta) {
|
|
52
|
+
const labels = [...accepted.values()];
|
|
53
|
+
const missing = [];
|
|
54
|
+
for (const block of sent) {
|
|
55
|
+
if (!accepted.has(block.index)) {
|
|
56
|
+
missing.push(block.index);
|
|
57
|
+
labels.push({ index: block.index, label: 'UNKNOWN', confidence: 0 });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
labels.sort((a, b) => a.index - b.index);
|
|
61
|
+
return { ...meta, labels, missing, spurious, blocksSent: sent.length };
|
|
62
|
+
}
|
|
63
|
+
function clampConfidence(value) {
|
|
64
|
+
return typeof value === 'number' && Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0.5;
|
|
65
|
+
}
|
|
42
66
|
/**
|
|
43
67
|
* Annotate one message's blocks.
|
|
44
68
|
*
|
|
@@ -49,16 +73,9 @@ function toPromptBlocks(blocks) {
|
|
|
49
73
|
async function annotateBlocks(client, blocks, isReply) {
|
|
50
74
|
const candidates = annotatableBlocks(blocks);
|
|
51
75
|
const sent = toPromptBlocks(candidates);
|
|
76
|
+
const meta = { model: client.model, promptVersion: prompt_1.PROMPT_VERSION, blocksTotal: candidates.length };
|
|
52
77
|
if (!sent.length) {
|
|
53
|
-
return
|
|
54
|
-
labels: [],
|
|
55
|
-
model: client.model,
|
|
56
|
-
promptVersion: prompt_1.PROMPT_VERSION,
|
|
57
|
-
missing: [],
|
|
58
|
-
spurious: [],
|
|
59
|
-
blocksSent: 0,
|
|
60
|
-
blocksTotal: candidates.length
|
|
61
|
-
};
|
|
78
|
+
return finish(sent, new Map(), [], meta);
|
|
62
79
|
}
|
|
63
80
|
const response = await client.complete({
|
|
64
81
|
system: prompt_1.SYSTEM_PROMPT,
|
|
@@ -69,39 +86,54 @@ async function annotateBlocks(client, blocks, isReply) {
|
|
|
69
86
|
// Validate against the blocks we actually sent. The schema constrains shape and label values,
|
|
70
87
|
// but nothing stops the model returning an index we never asked about, or omitting one.
|
|
71
88
|
const expected = new Set(sent.map(block => block.index));
|
|
72
|
-
const
|
|
73
|
-
const labels = [];
|
|
89
|
+
const accepted = new Map();
|
|
74
90
|
const spurious = [];
|
|
75
91
|
for (const item of response.labels ?? []) {
|
|
76
|
-
if (!expected.has(item.index) ||
|
|
92
|
+
if (!expected.has(item.index) || accepted.has(item.index)) {
|
|
77
93
|
spurious.push(item.index);
|
|
78
94
|
continue;
|
|
79
95
|
}
|
|
80
|
-
|
|
81
|
-
labels.push({
|
|
82
|
-
index: item.index,
|
|
83
|
-
label: item.label,
|
|
84
|
-
confidence: Number.isFinite(item.confidence) ? Math.min(1, Math.max(0, item.confidence)) : 0.5
|
|
85
|
-
});
|
|
96
|
+
accepted.set(item.index, { index: item.index, label: item.label, confidence: clampConfidence(item.confidence) });
|
|
86
97
|
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
98
|
+
return finish(sent, accepted, spurious, meta);
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Annotate one message's blocks with TypeSafe: one Choice question per block.
|
|
102
|
+
*
|
|
103
|
+
* The stored label is the cost rule applied to the returned distribution at DEFAULT_COST_RATIO, not
|
|
104
|
+
* the model's top choice, and the raw distribution is kept alongside so other ratios can be applied
|
|
105
|
+
* later without re-running inference. An answer without a usable distribution is treated as
|
|
106
|
+
* missing: the cost rule cannot be checked against it, so it must not be allowed to hide anything.
|
|
107
|
+
*
|
|
108
|
+
* @param client configured TypeSafe client
|
|
109
|
+
* @param blocks blocks from the primary representation
|
|
110
|
+
* @param isReply whether the message carries threading headers
|
|
111
|
+
*/
|
|
112
|
+
async function annotateBlocksWithTypeSafe(client, blocks, isReply) {
|
|
113
|
+
const candidates = annotatableBlocks(blocks);
|
|
114
|
+
const sent = toPromptBlocks(candidates);
|
|
115
|
+
const requests = (0, typesafe_prompt_1.planRequests)(sent, isReply);
|
|
116
|
+
// Independent: each request carries the full state and its own share of the questions.
|
|
117
|
+
const responses = await Promise.all(requests.map(request => client.ask(request)));
|
|
118
|
+
const accepted = new Map();
|
|
119
|
+
const spurious = [];
|
|
120
|
+
responses.forEach((response, i) => {
|
|
121
|
+
const asked = requests[i].questions;
|
|
122
|
+
for (const [id, answer] of Object.entries(response.answers)) {
|
|
123
|
+
// Matched against what this request asked. Anything else is counted and discarded,
|
|
124
|
+
// exactly like an invented index on the OpenAI path.
|
|
125
|
+
const index = (0, typesafe_prompt_1.parseQuestionId)(id);
|
|
126
|
+
if (index === null || !Object.hasOwn(asked, id) || accepted.has(index)) {
|
|
127
|
+
spurious.push(index ?? -1);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
const probabilities = (0, probabilities_1.sanitizeProbabilities)(answer?.probabilities);
|
|
131
|
+
if (!probabilities) {
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
accepted.set(index, { index, label: (0, probabilities_1.applyCostRule)(probabilities), confidence: clampConfidence(answer.confidence), probabilities });
|
|
94
135
|
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
return {
|
|
98
|
-
labels,
|
|
99
|
-
model: client.model,
|
|
100
|
-
promptVersion: prompt_1.PROMPT_VERSION,
|
|
101
|
-
missing,
|
|
102
|
-
spurious,
|
|
103
|
-
blocksSent: sent.length,
|
|
104
|
-
blocksTotal: candidates.length
|
|
105
|
-
};
|
|
136
|
+
});
|
|
137
|
+
return finish(sent, accepted, spurious, { model: client.model, promptVersion: typesafe_prompt_1.TYPESAFE_PROMPT_VERSION, blocksTotal: candidates.length });
|
|
106
138
|
}
|
|
107
139
|
//# sourceMappingURL=llm.js.map
|
package/dist/annotate/llm.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;;
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;;AAqCH,8CAEC;AAqDD,wCA+BC;AAcD,gEA4BC;AAhKD,mDAAgG;AAChG,qCAA8G;AAE9G,uDAA2F;AAE3F,2FAA2F;AAC9E,QAAA,eAAe,GAAG,GAAG,CAAC;AAEnC,8EAA8E;AACjE,QAAA,UAAU,GAAG,EAAE,CAAC;AAsB7B,wEAAwE;AACxE,SAAgB,iBAAiB,CAAC,MAAwB;IACtD,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,cAAc,CAAC,MAAwB;IAC5C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAU,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,uBAAe,CAAC;QAChD,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAe,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;YAC/D,SAAS;YACT,8EAA8E;YAC9E,qFAAqF;YACrF,8BAA8B;YAC9B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC;SAClF,CAAC;IACN,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;GAKG;AACH,SAAS,MAAM,CACX,IAA4B,EAC5B,QAA+B,EAC/B,QAAkB,EAClB,IAAmE;IAEnE,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACtC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AAC3E,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACvG,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc,CAAC,MAAwB,EAAE,MAAwB,EAAE,OAAgB;IACrG,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,uBAAc,EAAE,WAAW,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;IAEpG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO,MAAM,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAoE;QACtG,MAAM,EAAE,sBAAa;QACrB,IAAI,EAAE,IAAA,yBAAgB,EAAC,IAAI,EAAE,OAAO,CAAC;QACrC,UAAU,EAAE,cAAc;QAC1B,MAAM,EAAE,wBAAe;KAC1B,CAAC,CAAC;IAEH,8FAA8F;IAC9F,wFAAwF;IACxF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC7C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,SAAS;QACb,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;GAWG;AACI,KAAK,UAAU,0BAA0B,CAAC,MAAuB,EAAE,MAAwB,EAAE,OAAgB;IAChH,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAA,8BAAY,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7C,uFAAuF;IACvF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAElF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC7C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAI,QAAQ,CAAC,CAAC,CAAsB,CAAC,SAAS,CAAC;QAC1D,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,mFAAmF;YACnF,qDAAqD;YACrD,MAAM,KAAK,GAAG,IAAA,iCAAe,EAAC,EAAE,CAAC,CAAC;YAClC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrE,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3B,SAAS;YACb,CAAC;YACD,MAAM,aAAa,GAAG,IAAA,qCAAqB,EAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YACnE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACjB,SAAS;YACb,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAA,6BAAa,EAAC,aAAa,CAAC,EAAE,UAAU,EAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC;QACvI,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,yCAAuB,EAAE,WAAW,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;AAC7I,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Label probability distributions from a probabilistic annotator, and the decision rule applied to
|
|
3
|
+
* them.
|
|
4
|
+
*
|
|
5
|
+
* The OpenAI annotator is asked, in its prompt, to treat hiding authored text as ten times worse
|
|
6
|
+
* than leaving history visible. An annotator that returns a full distribution does not need to be
|
|
7
|
+
* asked to be biased: the raw distribution is stored unbiased and the asymmetry is applied here, in
|
|
8
|
+
* code, where it can be tuned and re-applied without re-running inference.
|
|
9
|
+
*/
|
|
10
|
+
import { type Label } from '../types';
|
|
11
|
+
export type LabelProbabilities = Partial<Record<Label, number>>;
|
|
12
|
+
/**
|
|
13
|
+
* Hiding authored text is treated as this many times worse than leaving history visible. Part of
|
|
14
|
+
* the `annotate-ts-v1` convention: stored labels always use it, so changing it means a new prompt
|
|
15
|
+
* version. Other ratios are applied at read time, from the stored distribution.
|
|
16
|
+
*/
|
|
17
|
+
export declare const DEFAULT_COST_RATIO = 10;
|
|
18
|
+
/**
|
|
19
|
+
* Labels whose content was written for this message and must stay visible. Hiding one of these is
|
|
20
|
+
* the expensive error every cost rule, gate and disagreement filter is built around.
|
|
21
|
+
*/
|
|
22
|
+
export declare const AUTHORED_LABELS: ReadonlySet<Label>;
|
|
23
|
+
/**
|
|
24
|
+
* Keep only known labels with finite values, clamped into [0, 1]. Returns null when nothing usable
|
|
25
|
+
* is left, so a malformed answer is recorded as missing rather than as a confident zero.
|
|
26
|
+
*/
|
|
27
|
+
export declare function sanitizeProbabilities(raw: unknown): LabelProbabilities | null;
|
|
28
|
+
/** Most probable label. Ties go to the earlier label in LABELS, which puts NEW_CONTENT first. */
|
|
29
|
+
export declare function topLabel(probabilities: LabelProbabilities): Label;
|
|
30
|
+
/**
|
|
31
|
+
* Apply the cost asymmetry to a distribution.
|
|
32
|
+
*
|
|
33
|
+
* Hiding a block wrongly costs `ratio` times the probability that it was authored, summed over
|
|
34
|
+
* AUTHORED_LABELS since hiding is wrong if it is any of them. Leaving it visible wrongly costs the
|
|
35
|
+
* probability that it was history. So REPLY_HISTORY survives only when
|
|
36
|
+
* P(REPLY_HISTORY) > ratio * (P(NEW_CONTENT) + P(INLINE_QUOTE)); otherwise the block falls back to
|
|
37
|
+
* whichever authored label is more likely. Every other label is taken as is, since none of them
|
|
38
|
+
* hides anything.
|
|
39
|
+
*/
|
|
40
|
+
export declare function applyCostRule(probabilities: LabelProbabilities, ratio?: number): Label;
|
|
41
|
+
/**
|
|
42
|
+
* Gap between the two most probable labels. Small means genuinely ambiguous, which is what review
|
|
43
|
+
* triage should sort by instead of a model's self reported confidence.
|
|
44
|
+
*/
|
|
45
|
+
export declare function probabilityMargin(probabilities: LabelProbabilities): number;
|
|
46
|
+
/** Parse the JSON stored in `annotations.probabilities`. Null for rows written without one. */
|
|
47
|
+
export declare function parseStoredProbabilities(json: string | null | undefined): LabelProbabilities | null;
|
|
48
|
+
//# sourceMappingURL=probabilities.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probabilities.d.ts","sourceRoot":"","sources":["../../src/annotate/probabilities.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAU,KAAK,KAAK,EAAE,MAAM,UAAU,CAAC;AAE9C,MAAM,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEhE;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,KAAK,CAAC;AAErC;;;GAGG;AACH,eAAO,MAAM,eAAe,EAAE,WAAW,CAAC,KAAK,CAAmD,CAAC;AAEnG;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAc7E;AAED,iGAAiG;AACjG,wBAAgB,QAAQ,CAAC,aAAa,EAAE,kBAAkB,GAAG,KAAK,CAWjE;AAED;;;;;;;;;GASG;AACH,wBAAgB,aAAa,CAAC,aAAa,EAAE,kBAAkB,EAAE,KAAK,GAAE,MAA2B,GAAG,KAAK,CAe1G;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,aAAa,EAAE,kBAAkB,GAAG,MAAM,CAa3E;AAED,+FAA+F;AAC/F,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,kBAAkB,GAAG,IAAI,CASnG"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Label probability distributions from a probabilistic annotator, and the decision rule applied to
|
|
4
|
+
* them.
|
|
5
|
+
*
|
|
6
|
+
* The OpenAI annotator is asked, in its prompt, to treat hiding authored text as ten times worse
|
|
7
|
+
* than leaving history visible. An annotator that returns a full distribution does not need to be
|
|
8
|
+
* asked to be biased: the raw distribution is stored unbiased and the asymmetry is applied here, in
|
|
9
|
+
* code, where it can be tuned and re-applied without re-running inference.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.AUTHORED_LABELS = exports.DEFAULT_COST_RATIO = void 0;
|
|
13
|
+
exports.sanitizeProbabilities = sanitizeProbabilities;
|
|
14
|
+
exports.topLabel = topLabel;
|
|
15
|
+
exports.applyCostRule = applyCostRule;
|
|
16
|
+
exports.probabilityMargin = probabilityMargin;
|
|
17
|
+
exports.parseStoredProbabilities = parseStoredProbabilities;
|
|
18
|
+
const types_1 = require("../types");
|
|
19
|
+
/**
|
|
20
|
+
* Hiding authored text is treated as this many times worse than leaving history visible. Part of
|
|
21
|
+
* the `annotate-ts-v1` convention: stored labels always use it, so changing it means a new prompt
|
|
22
|
+
* version. Other ratios are applied at read time, from the stored distribution.
|
|
23
|
+
*/
|
|
24
|
+
exports.DEFAULT_COST_RATIO = 10;
|
|
25
|
+
/**
|
|
26
|
+
* Labels whose content was written for this message and must stay visible. Hiding one of these is
|
|
27
|
+
* the expensive error every cost rule, gate and disagreement filter is built around.
|
|
28
|
+
*/
|
|
29
|
+
exports.AUTHORED_LABELS = new Set(['NEW_CONTENT', 'INLINE_QUOTE']);
|
|
30
|
+
/**
|
|
31
|
+
* Keep only known labels with finite values, clamped into [0, 1]. Returns null when nothing usable
|
|
32
|
+
* is left, so a malformed answer is recorded as missing rather than as a confident zero.
|
|
33
|
+
*/
|
|
34
|
+
function sanitizeProbabilities(raw) {
|
|
35
|
+
if (!raw || typeof raw !== 'object') {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
const out = {};
|
|
39
|
+
let any = false;
|
|
40
|
+
for (const label of types_1.LABELS) {
|
|
41
|
+
const value = raw[label];
|
|
42
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
43
|
+
out[label] = Math.min(1, Math.max(0, value));
|
|
44
|
+
any = true;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return any ? out : null;
|
|
48
|
+
}
|
|
49
|
+
/** Most probable label. Ties go to the earlier label in LABELS, which puts NEW_CONTENT first. */
|
|
50
|
+
function topLabel(probabilities) {
|
|
51
|
+
let best = 'UNKNOWN';
|
|
52
|
+
let bestValue = -1;
|
|
53
|
+
for (const label of types_1.LABELS) {
|
|
54
|
+
const value = probabilities[label] ?? 0;
|
|
55
|
+
if (value > bestValue) {
|
|
56
|
+
best = label;
|
|
57
|
+
bestValue = value;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return best;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Apply the cost asymmetry to a distribution.
|
|
64
|
+
*
|
|
65
|
+
* Hiding a block wrongly costs `ratio` times the probability that it was authored, summed over
|
|
66
|
+
* AUTHORED_LABELS since hiding is wrong if it is any of them. Leaving it visible wrongly costs the
|
|
67
|
+
* probability that it was history. So REPLY_HISTORY survives only when
|
|
68
|
+
* P(REPLY_HISTORY) > ratio * (P(NEW_CONTENT) + P(INLINE_QUOTE)); otherwise the block falls back to
|
|
69
|
+
* whichever authored label is more likely. Every other label is taken as is, since none of them
|
|
70
|
+
* hides anything.
|
|
71
|
+
*/
|
|
72
|
+
function applyCostRule(probabilities, ratio = exports.DEFAULT_COST_RATIO) {
|
|
73
|
+
const top = topLabel(probabilities);
|
|
74
|
+
if (top !== 'REPLY_HISTORY') {
|
|
75
|
+
return top;
|
|
76
|
+
}
|
|
77
|
+
let authoredMass = 0;
|
|
78
|
+
let fallback = 'NEW_CONTENT';
|
|
79
|
+
for (const label of exports.AUTHORED_LABELS) {
|
|
80
|
+
const value = probabilities[label] ?? 0;
|
|
81
|
+
authoredMass += value;
|
|
82
|
+
if (value > (probabilities[fallback] ?? 0)) {
|
|
83
|
+
fallback = label;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return (probabilities.REPLY_HISTORY ?? 0) > ratio * authoredMass ? 'REPLY_HISTORY' : fallback;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Gap between the two most probable labels. Small means genuinely ambiguous, which is what review
|
|
90
|
+
* triage should sort by instead of a model's self reported confidence.
|
|
91
|
+
*/
|
|
92
|
+
function probabilityMargin(probabilities) {
|
|
93
|
+
let first = 0;
|
|
94
|
+
let second = 0;
|
|
95
|
+
for (const label of types_1.LABELS) {
|
|
96
|
+
const value = probabilities[label] ?? 0;
|
|
97
|
+
if (value > first) {
|
|
98
|
+
second = first;
|
|
99
|
+
first = value;
|
|
100
|
+
}
|
|
101
|
+
else if (value > second) {
|
|
102
|
+
second = value;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return first - second;
|
|
106
|
+
}
|
|
107
|
+
/** Parse the JSON stored in `annotations.probabilities`. Null for rows written without one. */
|
|
108
|
+
function parseStoredProbabilities(json) {
|
|
109
|
+
if (!json) {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
try {
|
|
113
|
+
return sanitizeProbabilities(JSON.parse(json));
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=probabilities.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probabilities.js","sourceRoot":"","sources":["../../src/annotate/probabilities.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;;AAuBH,sDAcC;AAGD,4BAWC;AAYD,sCAeC;AAMD,8CAaC;AAGD,4DASC;AA3GD,oCAA8C;AAI9C;;;;GAIG;AACU,QAAA,kBAAkB,GAAG,EAAE,CAAC;AAErC;;;GAGG;AACU,QAAA,eAAe,GAAuB,IAAI,GAAG,CAAQ,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC;AAEnG;;;GAGG;AACH,SAAgB,qBAAqB,CAAC,GAAY;IAC9C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,GAAG,GAAuB,EAAE,CAAC;IACnC,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,KAAK,MAAM,KAAK,IAAI,cAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAI,GAA+B,CAAC,KAAK,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7C,GAAG,GAAG,IAAI,CAAC;QACf,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5B,CAAC;AAED,iGAAiG;AACjG,SAAgB,QAAQ,CAAC,aAAiC;IACtD,IAAI,IAAI,GAAU,SAAS,CAAC;IAC5B,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;IACnB,KAAK,MAAM,KAAK,IAAI,cAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACpB,IAAI,GAAG,KAAK,CAAC;YACb,SAAS,GAAG,KAAK,CAAC;QACtB,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;;;;;GASG;AACH,SAAgB,aAAa,CAAC,aAAiC,EAAE,QAAgB,0BAAkB;IAC/F,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC;IACpC,IAAI,GAAG,KAAK,eAAe,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IACD,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAU,aAAa,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,uBAAe,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,YAAY,IAAI,KAAK,CAAC;QACtB,IAAI,KAAK,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACzC,QAAQ,GAAG,KAAK,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,CAAC,aAAa,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClG,CAAC;AAED;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,aAAiC;IAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,cAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,KAAK,EAAE,CAAC;YAChB,MAAM,GAAG,KAAK,CAAC;YACf,KAAK,GAAG,KAAK,CAAC;QAClB,CAAC;aAAM,IAAI,KAAK,GAAG,MAAM,EAAE,CAAC;YACxB,MAAM,GAAG,KAAK,CAAC;QACnB,CAAC;IACL,CAAC;IACD,OAAO,KAAK,GAAG,MAAM,CAAC;AAC1B,CAAC;AAED,+FAA+F;AAC/F,SAAgB,wBAAwB,CAAC,IAA+B;IACpE,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,CAAC;QACD,OAAO,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Annotation providers behind one interface.
|
|
3
|
+
*
|
|
4
|
+
* Each provider owns its client, default model, prompt version and response validation. Commands
|
|
5
|
+
* only pick one by name, so adding a provider never means editing a command's control flow.
|
|
6
|
+
*/
|
|
7
|
+
import { type Usage } from './http';
|
|
8
|
+
import { type AnnotationResult } from './llm';
|
|
9
|
+
import { type Block } from '../segment/types';
|
|
10
|
+
export interface Annotator {
|
|
11
|
+
readonly model: string;
|
|
12
|
+
readonly promptVersion: string;
|
|
13
|
+
readonly usage: Usage;
|
|
14
|
+
annotate(blocks: readonly Block[], isReply: boolean): Promise<AnnotationResult>;
|
|
15
|
+
}
|
|
16
|
+
interface ProviderSpec {
|
|
17
|
+
defaultModel: string;
|
|
18
|
+
promptVersion: string;
|
|
19
|
+
/** Throws, before anything is sent, when the provider's key is not configured. */
|
|
20
|
+
create(model: string): Annotator;
|
|
21
|
+
}
|
|
22
|
+
export declare const PROVIDERS: Readonly<Record<string, ProviderSpec>>;
|
|
23
|
+
/** The provider registered under `name`, or null. Own properties only, so `constructor` is not one. */
|
|
24
|
+
export declare function provider(name: string): ProviderSpec | null;
|
|
25
|
+
export {};
|
|
26
|
+
//# sourceMappingURL=providers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../../src/annotate/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAA8C,KAAK,gBAAgB,EAAE,MAAM,OAAO,CAAC;AAI1F,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACnF;AAED,UAAU,YAAY;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,kFAAkF;IAClF,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC;AAED,eAAO,MAAM,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAsB5D,CAAC;AAEF,uGAAuG;AACvG,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAE1D"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Annotation providers behind one interface.
|
|
4
|
+
*
|
|
5
|
+
* Each provider owns its client, default model, prompt version and response validation. Commands
|
|
6
|
+
* only pick one by name, so adding a provider never means editing a command's control flow.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.PROVIDERS = void 0;
|
|
10
|
+
exports.provider = provider;
|
|
11
|
+
const client_1 = require("./client");
|
|
12
|
+
const llm_1 = require("./llm");
|
|
13
|
+
const prompt_1 = require("./prompt");
|
|
14
|
+
const typesafe_client_1 = require("./typesafe-client");
|
|
15
|
+
const typesafe_prompt_1 = require("./typesafe-prompt");
|
|
16
|
+
exports.PROVIDERS = {
|
|
17
|
+
openai: {
|
|
18
|
+
defaultModel: 'gpt-5.6-sol',
|
|
19
|
+
promptVersion: prompt_1.PROMPT_VERSION,
|
|
20
|
+
create(model) {
|
|
21
|
+
const client = new client_1.OpenAIClient((0, client_1.configFromEnv)(model));
|
|
22
|
+
return { model, promptVersion: prompt_1.PROMPT_VERSION, usage: client.usage, annotate: (blocks, isReply) => (0, llm_1.annotateBlocks)(client, blocks, isReply) };
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
typesafe: {
|
|
26
|
+
defaultModel: typesafe_client_1.TYPESAFE_DEFAULT_MODEL,
|
|
27
|
+
promptVersion: typesafe_prompt_1.TYPESAFE_PROMPT_VERSION,
|
|
28
|
+
create(model) {
|
|
29
|
+
const client = new typesafe_client_1.TypeSafeClient((0, typesafe_client_1.typeSafeConfigFromEnv)(model));
|
|
30
|
+
return {
|
|
31
|
+
model,
|
|
32
|
+
promptVersion: typesafe_prompt_1.TYPESAFE_PROMPT_VERSION,
|
|
33
|
+
usage: client.usage,
|
|
34
|
+
annotate: (blocks, isReply) => (0, llm_1.annotateBlocksWithTypeSafe)(client, blocks, isReply)
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
/** The provider registered under `name`, or null. Own properties only, so `constructor` is not one. */
|
|
40
|
+
function provider(name) {
|
|
41
|
+
return Object.hasOwn(exports.PROVIDERS, name) ? exports.PROVIDERS[name] : null;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=providers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/annotate/providers.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAiDH,4BAEC;AAjDD,qCAAuD;AAEvD,+BAA0F;AAC1F,qCAA0C;AAC1C,uDAAkG;AAClG,uDAA4D;AAiB/C,QAAA,SAAS,GAA2C;IAC7D,MAAM,EAAE;QACJ,YAAY,EAAE,aAAa;QAC3B,aAAa,EAAE,uBAAc;QAC7B,MAAM,CAAC,KAAK;YACR,MAAM,MAAM,GAAG,IAAI,qBAAY,CAAC,IAAA,sBAAa,EAAC,KAAK,CAAC,CAAC,CAAC;YACtD,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,uBAAc,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAA,oBAAc,EAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;QACjJ,CAAC;KACJ;IACD,QAAQ,EAAE;QACN,YAAY,EAAE,wCAAsB;QACpC,aAAa,EAAE,yCAAuB;QACtC,MAAM,CAAC,KAAK;YACR,MAAM,MAAM,GAAG,IAAI,gCAAc,CAAC,IAAA,uCAAqB,EAAC,KAAK,CAAC,CAAC,CAAC;YAChE,OAAO;gBACH,KAAK;gBACL,aAAa,EAAE,yCAAuB;gBACtC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAA,gCAA0B,EAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;aACrF,CAAC;QACN,CAAC;KACJ;CACJ,CAAC;AAEF,uGAAuG;AACvG,SAAgB,QAAQ,CAAC,IAAY;IACjC,OAAO,MAAM,CAAC,MAAM,CAAC,iBAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAE,iBAAS,CAAC,IAAI,CAAkB,CAAC,CAAC,CAAC,IAAI,CAAC;AACrF,CAAC"}
|
package/dist/annotate/store.d.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { type CorpusStore } from '../corpus/store';
|
|
10
10
|
import { type Label, type Reason } from '../types';
|
|
11
|
+
import { type LabelProbabilities } from './probabilities';
|
|
11
12
|
export interface AnnotationRow {
|
|
12
13
|
path: string;
|
|
13
14
|
representation: 'text' | 'html';
|
|
@@ -22,6 +23,8 @@ export interface AnnotationRow {
|
|
|
22
23
|
baselineLabel: Label | null;
|
|
23
24
|
baselineConfidence: number | null;
|
|
24
25
|
baselineReasons: readonly Reason[];
|
|
26
|
+
/** Full label distribution, when the annotator returns one. */
|
|
27
|
+
probabilities?: LabelProbabilities | null;
|
|
25
28
|
}
|
|
26
29
|
export interface AnnotationRunInfo {
|
|
27
30
|
source: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAqB,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAqB,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EAAqB,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAE7E,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,GAAG,MAAM,CAAC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;IAC7C,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,KAAK,GAAG,IAAI,CAAC;IAC5B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,+DAA+D;IAC/D,aAAa,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,iBAAiB;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM,CAYtF;AAED,wBAAgB,mBAAmB,CAC/B,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,GAC/G,IAAI,CAaN;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,aAAa,EAAE,GAAG,IAAI,CA+BxG;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,EAAE,aAAa,GAAE,MAAM,GAAG,IAAW,GAAG,GAAG,CAAC,MAAM,CAAC,CAgBzI"}
|
package/dist/annotate/store.js
CHANGED
|
@@ -13,6 +13,7 @@ exports.finishAnnotationRun = finishAnnotationRun;
|
|
|
13
13
|
exports.writeAnnotations = writeAnnotations;
|
|
14
14
|
exports.annotatedPaths = annotatedPaths;
|
|
15
15
|
const types_1 = require("../types");
|
|
16
|
+
const probabilities_1 = require("./probabilities");
|
|
16
17
|
function startAnnotationRun(store, info) {
|
|
17
18
|
const result = store.run(`INSERT INTO annotation_runs (source, model, prompt_version, extractor_version, started_at, notes)
|
|
18
19
|
VALUES (?, ?, ?, ?, ?, ?)`, info.source, info.model, info.promptVersion, types_1.EXTRACTOR_VERSION, Date.now(), info.notes ?? null);
|
|
@@ -31,8 +32,8 @@ function writeAnnotations(store, runId, rows) {
|
|
|
31
32
|
for (const row of rows) {
|
|
32
33
|
store.run(`INSERT INTO annotations (
|
|
33
34
|
run_id, path, representation, block_index, mime_part, block_start, block_end, char_count,
|
|
34
|
-
source, label, confidence, baseline_label, baseline_conf, baseline_reasons, agrees
|
|
35
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, runId, row.path, row.representation, row.blockIndex, row.mimePart, row.blockStart, row.blockEnd, row.charCount, row.source, row.label, row.confidence, row.baselineLabel, row.baselineConfidence, row.baselineReasons.join(','), row.baselineLabel === row.label ? 1 : 0);
|
|
35
|
+
source, label, confidence, baseline_label, baseline_conf, baseline_reasons, agrees, probabilities, margin
|
|
36
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, runId, row.path, row.representation, row.blockIndex, row.mimePart, row.blockStart, row.blockEnd, row.charCount, row.source, row.label, row.confidence, row.baselineLabel, row.baselineConfidence, row.baselineReasons.join(','), row.baselineLabel === row.label ? 1 : 0, row.probabilities ? JSON.stringify(row.probabilities) : null, row.probabilities ? (0, probabilities_1.probabilityMargin)(row.probabilities) : null);
|
|
36
37
|
}
|
|
37
38
|
});
|
|
38
39
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;AA+BH,gDAYC;AAED,kDAiBC;AAED,4CA+BC;AASD,wCAgBC;AArHD,oCAAsE;AACtE,mDAA6E;AA2B7E,SAAgB,kBAAkB,CAAC,KAAkB,EAAE,IAAuB;IAC1E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CACpB;mCAC2B,EAC3B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,aAAa,EAClB,yBAAiB,EACjB,IAAI,CAAC,GAAG,EAAE,EACV,IAAI,CAAC,KAAK,IAAI,IAAI,CACrB,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,SAAgB,mBAAmB,CAC/B,KAAkB,EAClB,KAAa,EACb,MAA8G;IAE9G,KAAK,CAAC,GAAG,CACL;;sBAEc,EACd,IAAI,CAAC,GAAG,EAAE,EACV,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,gBAAgB,EACvB,KAAK,CACR,CAAC;AACN,CAAC;AAED,SAAgB,gBAAgB,CAAC,KAAkB,EAAE,KAAa,EAAE,IAA8B;IAC9F,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO;IACX,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,KAAK,CAAC,GAAG,CACL;;;8EAG8D,EAC9D,KAAK,EACL,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,cAAc,EAClB,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,QAAQ,EACZ,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,QAAQ,EACZ,GAAG,CAAC,SAAS,EACb,GAAG,CAAC,MAAM,EACV,GAAG,CAAC,KAAK,EACT,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,aAAa,EACjB,GAAG,CAAC,kBAAkB,EACtB,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAC7B,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,EAC5D,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAA,iCAAiB,EAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAClE,CAAC;QACN,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,KAAkB,EAAE,MAAc,EAAE,KAAoB,EAAE,gBAA+B,IAAI;IACxH,MAAM,OAAO,GAAG,CAAC,cAAc,CAAC,CAAC;IACjC,MAAM,MAAM,GAAa,CAAC,MAAM,CAAC,CAAC;IAClC,IAAI,KAAK,EAAE,CAAC;QACR,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CACpB,qGAAqG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAC5H,GAAG,MAAM,CACZ,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9C,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal TypeSafe System One client.
|
|
3
|
+
*
|
|
4
|
+
* No SDK dependency, for the same reason as the OpenAI client: the package ships three runtime
|
|
5
|
+
* dependencies and none of them opens a socket. `@typesafe-ai/sdk` could only be a dev dependency,
|
|
6
|
+
* and `dist/cli` is part of the published tarball, so an installed `email-content annotate` would
|
|
7
|
+
* fail to load it. The surface used here is one POST.
|
|
8
|
+
*
|
|
9
|
+
* This is the second place in the codebase that sends message content off the machine, under the
|
|
10
|
+
* same terms as the first: only reachable from `email-content annotate`, opt in, fails closed
|
|
11
|
+
* without a key. See docs/threat-model.md, "Configured exception: LLM annotation".
|
|
12
|
+
*/
|
|
13
|
+
import { ApiError, type ApiConfig, type Usage } from './http';
|
|
14
|
+
/**
|
|
15
|
+
* Pinned rather than `jev-latest`: labels are stored with the model that produced them, and an
|
|
16
|
+
* alias that silently moves would put two models' output under one name.
|
|
17
|
+
*/
|
|
18
|
+
export declare const TYPESAFE_DEFAULT_MODEL = "jev-1.13.0";
|
|
19
|
+
export type TypeSafeConfig = ApiConfig;
|
|
20
|
+
export interface ChoiceQuestion {
|
|
21
|
+
type: 'choice';
|
|
22
|
+
instructions: string | Record<string, unknown>;
|
|
23
|
+
criteria: Record<string, string | Record<string, unknown> | null>;
|
|
24
|
+
}
|
|
25
|
+
export interface SystemOneRequest {
|
|
26
|
+
state: unknown;
|
|
27
|
+
questions: Record<string, ChoiceQuestion>;
|
|
28
|
+
}
|
|
29
|
+
/** One answer as returned. Fields are unvalidated until the annotator checks them. */
|
|
30
|
+
export interface ChoiceAnswer {
|
|
31
|
+
choice?: unknown;
|
|
32
|
+
probabilities?: unknown;
|
|
33
|
+
confidence?: unknown;
|
|
34
|
+
}
|
|
35
|
+
export interface SystemOneResponse {
|
|
36
|
+
answers: Record<string, ChoiceAnswer>;
|
|
37
|
+
}
|
|
38
|
+
/** The narrow surface the annotator depends on, so its validation can be tested without a network. */
|
|
39
|
+
export interface SystemOneClient {
|
|
40
|
+
readonly model: string;
|
|
41
|
+
ask(request: SystemOneRequest): Promise<SystemOneResponse>;
|
|
42
|
+
}
|
|
43
|
+
export declare class TypeSafeError extends ApiError {
|
|
44
|
+
constructor(message: string, status: number, retryable: boolean);
|
|
45
|
+
}
|
|
46
|
+
/** Load configuration from the environment, after process.loadEnvFile(). */
|
|
47
|
+
export declare function typeSafeConfigFromEnv(model: string, overrides?: Partial<TypeSafeConfig>): TypeSafeConfig;
|
|
48
|
+
export declare class TypeSafeClient implements SystemOneClient {
|
|
49
|
+
private readonly config;
|
|
50
|
+
/** Input tokens are reported as prompt tokens, output tokens as completion tokens. */
|
|
51
|
+
readonly usage: Usage;
|
|
52
|
+
constructor(config: TypeSafeConfig);
|
|
53
|
+
get model(): string;
|
|
54
|
+
/**
|
|
55
|
+
* Ask a set of questions about one state.
|
|
56
|
+
*
|
|
57
|
+
* Retries on rate limits (429), overload (529) and other server errors. Anything else, a 422
|
|
58
|
+
* validation failure in particular, is a bug in the request and is not retried. Error messages
|
|
59
|
+
* carry the status only, never the response body: a validation error may echo the state back,
|
|
60
|
+
* and the state is private mail.
|
|
61
|
+
*/
|
|
62
|
+
ask(request: SystemOneRequest): Promise<SystemOneResponse>;
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=typesafe-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"typesafe-client.d.ts","sourceRoot":"","sources":["../../src/annotate/typesafe-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,QAAQ,EAA8B,KAAK,SAAS,EAAE,KAAK,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE1F;;;GAGG;AACH,eAAO,MAAM,sBAAsB,eAAe,CAAC;AAEnD,MAAM,MAAM,cAAc,GAAG,SAAS,CAAC;AAEvC,MAAM,WAAW,cAAc;IAC3B,IAAI,EAAE,QAAQ,CAAC;IACf,YAAY,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/C,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;CACrE;AAED,MAAM,WAAW,gBAAgB;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;CAC7C;AAED,sFAAsF;AACtF,MAAM,WAAW,YAAY;IACzB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACzC;AAED,sGAAsG;AACtG,MAAM,WAAW,eAAe;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAC9D;AAED,qBAAa,aAAc,SAAQ,QAAQ;gBAC3B,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO;CAIlE;AAED,4EAA4E;AAC5E,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,OAAO,CAAC,cAAc,CAAM,GAAG,cAAc,CAE5G;AAED,qBAAa,cAAe,YAAW,eAAe;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAyE;gBAElF,MAAM,EAAE,cAAc;IAIlC,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;;;;;;OAOG;IACG,GAAG,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC;CAwBnE"}
|