@postalsys/email-content 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -12
- package/dist/.tsbuildinfo +1 -1
- package/dist/analyze.d.ts +6 -0
- package/dist/analyze.d.ts.map +1 -1
- package/dist/analyze.js +27 -6
- package/dist/analyze.js.map +1 -1
- package/dist/annotate/agreement.d.ts +99 -0
- package/dist/annotate/agreement.d.ts.map +1 -0
- package/dist/annotate/agreement.js +126 -0
- package/dist/annotate/agreement.js.map +1 -0
- package/dist/annotate/client.d.ts +7 -19
- package/dist/annotate/client.d.ts.map +1 -1
- package/dist/annotate/client.js +28 -82
- package/dist/annotate/client.js.map +1 -1
- package/dist/annotate/http.d.ts +57 -0
- package/dist/annotate/http.d.ts.map +1 -0
- package/dist/annotate/http.js +99 -0
- package/dist/annotate/http.js.map +1 -0
- package/dist/annotate/llm.d.ts +17 -0
- package/dist/annotate/llm.d.ts.map +1 -1
- package/dist/annotate/llm.js +68 -36
- package/dist/annotate/llm.js.map +1 -1
- package/dist/annotate/probabilities.d.ts +48 -0
- package/dist/annotate/probabilities.d.ts.map +1 -0
- package/dist/annotate/probabilities.js +119 -0
- package/dist/annotate/probabilities.js.map +1 -0
- package/dist/annotate/providers.d.ts +26 -0
- package/dist/annotate/providers.d.ts.map +1 -0
- package/dist/annotate/providers.js +43 -0
- package/dist/annotate/providers.js.map +1 -0
- package/dist/annotate/store.d.ts +3 -0
- package/dist/annotate/store.d.ts.map +1 -1
- package/dist/annotate/store.js +3 -2
- package/dist/annotate/store.js.map +1 -1
- package/dist/annotate/typesafe-client.d.ts +64 -0
- package/dist/annotate/typesafe-client.d.ts.map +1 -0
- package/dist/annotate/typesafe-client.js +76 -0
- package/dist/annotate/typesafe-client.js.map +1 -0
- package/dist/annotate/typesafe-prompt.d.ts +64 -0
- package/dist/annotate/typesafe-prompt.d.ts.map +1 -0
- package/dist/annotate/typesafe-prompt.js +170 -0
- package/dist/annotate/typesafe-prompt.js.map +1 -0
- package/dist/canonical/text.d.ts +15 -3
- package/dist/canonical/text.d.ts.map +1 -1
- package/dist/canonical/text.js +20 -4
- package/dist/canonical/text.js.map +1 -1
- package/dist/classify/rules.d.ts +7 -0
- package/dist/classify/rules.d.ts.map +1 -1
- package/dist/classify/rules.js +67 -14
- package/dist/classify/rules.js.map +1 -1
- package/dist/cli/agreement.d.ts +27 -0
- package/dist/cli/agreement.d.ts.map +1 -0
- package/dist/cli/agreement.js +153 -0
- package/dist/cli/agreement.js.map +1 -0
- package/dist/cli/annotate.d.ts.map +1 -1
- package/dist/cli/annotate.js +28 -16
- package/dist/cli/annotate.js.map +1 -1
- package/dist/cli/audit.d.ts +12 -0
- package/dist/cli/audit.d.ts.map +1 -0
- package/dist/cli/audit.js +134 -0
- package/dist/cli/audit.js.map +1 -0
- package/dist/cli/disagreements.d.ts.map +1 -1
- package/dist/cli/disagreements.js +20 -3
- package/dist/cli/disagreements.js.map +1 -1
- package/dist/cli/evaluate.d.ts.map +1 -1
- package/dist/cli/evaluate.js +5 -70
- package/dist/cli/evaluate.js.map +1 -1
- package/dist/cli/main.js +28 -2
- package/dist/cli/main.js.map +1 -1
- package/dist/corpus/analyze.d.ts.map +1 -1
- package/dist/corpus/analyze.js +4 -5
- package/dist/corpus/analyze.js.map +1 -1
- package/dist/corpus/migrations/index.d.ts.map +1 -1
- package/dist/corpus/migrations/index.js +14 -0
- package/dist/corpus/migrations/index.js.map +1 -1
- package/dist/corpus/stats.d.ts +1 -0
- package/dist/corpus/stats.d.ts.map +1 -1
- package/dist/corpus/stats.js +1 -0
- package/dist/corpus/stats.js.map +1 -1
- package/dist/detect/patterns/forward-headers.json +24 -0
- package/dist/detect/patterns/html-clients.json +27 -0
- package/dist/detect/patterns/reply-headers.json +368 -0
- package/dist/detect/patterns.d.ts +19 -0
- package/dist/detect/patterns.d.ts.map +1 -1
- package/dist/detect/patterns.js +53 -25
- package/dist/detect/patterns.js.map +1 -1
- package/dist/detect/reply-header.d.ts.map +1 -1
- package/dist/detect/reply-header.js +12 -1
- package/dist/detect/reply-header.js.map +1 -1
- package/dist/detect/subject.d.ts +16 -0
- package/dist/detect/subject.d.ts.map +1 -0
- package/dist/detect/subject.js +22 -0
- package/dist/detect/subject.js.map +1 -0
- package/dist/eval/pattern-audit.d.ts +56 -0
- package/dist/eval/pattern-audit.d.ts.map +1 -0
- package/dist/eval/pattern-audit.js +111 -0
- package/dist/eval/pattern-audit.js.map +1 -0
- package/dist/eval/reference.d.ts +44 -0
- package/dist/eval/reference.d.ts.map +1 -0
- package/dist/eval/reference.js +87 -0
- package/dist/eval/reference.js.map +1 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/mime/flowed.d.ts +14 -0
- package/dist/mime/flowed.d.ts.map +1 -0
- package/dist/mime/flowed.js +71 -0
- package/dist/mime/flowed.js.map +1 -0
- package/dist/mime/parse.js +2 -5
- package/dist/mime/parse.js.map +1 -1
- package/dist/segment/html-blocks.d.ts.map +1 -1
- package/dist/segment/html-blocks.js +11 -7
- package/dist/segment/html-blocks.js.map +1 -1
- package/dist/segment/text-blocks.d.ts.map +1 -1
- package/dist/segment/text-blocks.js +14 -3
- package/dist/segment/text-blocks.js.map +1 -1
- package/dist/types.d.ts +18 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +2 -2
- package/package.json +8 -8
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Retrying JSON POST shared by the annotator clients.
|
|
4
|
+
*
|
|
5
|
+
* Both providers need the same thing: one authenticated POST, exponential backoff on rate limits
|
|
6
|
+
* and overload, and no retry once the provider has said the request itself is wrong. Keeping that
|
|
7
|
+
* loop in one place means a fix to it cannot land in one client and not the other.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.ApiError = void 0;
|
|
11
|
+
exports.apiConfigFromEnv = apiConfigFromEnv;
|
|
12
|
+
exports.postJson = postJson;
|
|
13
|
+
/**
|
|
14
|
+
* Load a provider's configuration from the environment, after process.loadEnvFile().
|
|
15
|
+
*
|
|
16
|
+
* Throws when the key is missing. That is the fail closed guarantee in docs/threat-model.md: a
|
|
17
|
+
* provider without a configured key never gets as far as reading a message.
|
|
18
|
+
*/
|
|
19
|
+
function apiConfigFromEnv(env, model, overrides = {}) {
|
|
20
|
+
const apiKey = process.env[env.keyVar];
|
|
21
|
+
if (!apiKey) {
|
|
22
|
+
throw new Error(`${env.keyVar} is not set. Copy .env.example to .env and fill it in.`);
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
apiKey,
|
|
26
|
+
baseUrl: process.env[env.urlVar] || env.defaultUrl,
|
|
27
|
+
model,
|
|
28
|
+
timeoutMs: 120000,
|
|
29
|
+
maxRetries: 5,
|
|
30
|
+
...overrides
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/** An HTTP or protocol failure from an annotation provider. */
|
|
34
|
+
class ApiError extends Error {
|
|
35
|
+
status;
|
|
36
|
+
retryable;
|
|
37
|
+
constructor(message, status, retryable) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.name = 'ApiError';
|
|
40
|
+
this.status = status;
|
|
41
|
+
this.retryable = retryable;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
exports.ApiError = ApiError;
|
|
45
|
+
function sleep(ms) {
|
|
46
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
47
|
+
}
|
|
48
|
+
async function postJson(options) {
|
|
49
|
+
let lastError = null;
|
|
50
|
+
// Payloads run to hundreds of kilobytes, and nothing about them changes between attempts.
|
|
51
|
+
const body = JSON.stringify(options.payload);
|
|
52
|
+
for (let attempt = 0; attempt <= options.policy.maxRetries; attempt++) {
|
|
53
|
+
if (attempt > 0) {
|
|
54
|
+
// Full jitter, capped, so a burst of workers does not retry in lockstep.
|
|
55
|
+
const backoff = Math.min(30000, 500 * 2 ** (attempt - 1));
|
|
56
|
+
await sleep(Math.random() * backoff);
|
|
57
|
+
}
|
|
58
|
+
const controller = new AbortController();
|
|
59
|
+
const timer = setTimeout(() => controller.abort(), options.policy.timeoutMs);
|
|
60
|
+
try {
|
|
61
|
+
const response = await fetch(options.url, {
|
|
62
|
+
method: 'POST',
|
|
63
|
+
signal: controller.signal,
|
|
64
|
+
headers: {
|
|
65
|
+
Authorization: `Bearer ${options.apiKey}`,
|
|
66
|
+
'Content-Type': 'application/json'
|
|
67
|
+
},
|
|
68
|
+
body
|
|
69
|
+
});
|
|
70
|
+
if (!response.ok) {
|
|
71
|
+
let detail = null;
|
|
72
|
+
try {
|
|
73
|
+
detail = await response.json();
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// Body was not JSON. The status alone is enough.
|
|
77
|
+
}
|
|
78
|
+
const error = options.httpError(response.status, detail);
|
|
79
|
+
if (!error.retryable) {
|
|
80
|
+
throw error;
|
|
81
|
+
}
|
|
82
|
+
lastError = error;
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
return options.parse(await response.json());
|
|
86
|
+
}
|
|
87
|
+
catch (err) {
|
|
88
|
+
if (err instanceof ApiError && !err.retryable) {
|
|
89
|
+
throw err;
|
|
90
|
+
}
|
|
91
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
92
|
+
}
|
|
93
|
+
finally {
|
|
94
|
+
clearTimeout(timer);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
throw lastError ?? new Error('request failed');
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=http.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../../src/annotate/http.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;AAqBH,4CAaC;AAyCD,4BAqDC;AAjHD;;;;;GAKG;AACH,SAAgB,gBAAgB,CAAC,GAA2D,EAAE,KAAa,EAAE,YAAgC,EAAE;IAC3I,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,wDAAwD,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO;QACH,MAAM;QACN,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU;QAClD,KAAK;QACL,SAAS,EAAE,MAAM;QACjB,UAAU,EAAE,CAAC;QACb,GAAG,SAAS;KACf,CAAC;AACN,CAAC;AAUD,+DAA+D;AAC/D,MAAa,QAAS,SAAQ,KAAK;IACtB,MAAM,CAAS;IACf,SAAS,CAAU;IAE5B,YAAY,OAAe,EAAE,MAAc,EAAE,SAAkB;QAC3D,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;CACJ;AAVD,4BAUC;AAgBD,SAAS,KAAK,CAAC,EAAU;IACrB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAEM,KAAK,UAAU,QAAQ,CAAI,OAA2B;IACzD,IAAI,SAAS,GAAiB,IAAI,CAAC;IACnC,0FAA0F;IAC1F,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACpE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACd,yEAAyE;YACzE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAE7E,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE;gBACtC,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE;oBACL,aAAa,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE;oBACzC,cAAc,EAAE,kBAAkB;iBACrC;gBACD,IAAI;aACP,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,IAAI,MAAM,GAAY,IAAI,CAAC;gBAC3B,IAAI,CAAC;oBACD,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,CAAC;gBAAC,MAAM,CAAC;oBACL,iDAAiD;gBACrD,CAAC;gBACD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACzD,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;oBACnB,MAAM,KAAK,CAAC;gBAChB,CAAC;gBACD,SAAS,GAAG,KAAK,CAAC;gBAClB,SAAS;YACb,CAAC;YAED,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,YAAY,QAAQ,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;gBAC5C,MAAM,GAAG,CAAC;YACd,CAAC;YACD,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,CAAC;gBAAS,CAAC;YACP,YAAY,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACL,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;AACnD,CAAC"}
|
package/dist/annotate/llm.d.ts
CHANGED
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
import { type Block } from '../segment/types';
|
|
14
14
|
import { type Label } from '../types';
|
|
15
15
|
import { type CompletionClient } from './client';
|
|
16
|
+
import { type LabelProbabilities } from './probabilities';
|
|
17
|
+
import { type SystemOneClient } from './typesafe-client';
|
|
16
18
|
/** Characters of each block sent to the model. Enough for a decision, bounded for cost. */
|
|
17
19
|
export declare const MAX_BLOCK_CHARS = 700;
|
|
18
20
|
/** Blocks per message. Longer messages are truncated with a recorded flag. */
|
|
@@ -21,6 +23,8 @@ export interface LlmLabel {
|
|
|
21
23
|
index: number;
|
|
22
24
|
label: Label;
|
|
23
25
|
confidence: number;
|
|
26
|
+
/** Full distribution, from annotators that return one. The label is derived from it. */
|
|
27
|
+
probabilities?: LabelProbabilities;
|
|
24
28
|
}
|
|
25
29
|
export interface AnnotationResult {
|
|
26
30
|
labels: LlmLabel[];
|
|
@@ -43,4 +47,17 @@ export declare function annotatableBlocks(blocks: readonly Block[]): Block[];
|
|
|
43
47
|
* @param isReply whether the message carries threading headers
|
|
44
48
|
*/
|
|
45
49
|
export declare function annotateBlocks(client: CompletionClient, blocks: readonly Block[], isReply: boolean): Promise<AnnotationResult>;
|
|
50
|
+
/**
|
|
51
|
+
* Annotate one message's blocks with TypeSafe: one Choice question per block.
|
|
52
|
+
*
|
|
53
|
+
* The stored label is the cost rule applied to the returned distribution at DEFAULT_COST_RATIO, not
|
|
54
|
+
* the model's top choice, and the raw distribution is kept alongside so other ratios can be applied
|
|
55
|
+
* later without re-running inference. An answer without a usable distribution is treated as
|
|
56
|
+
* missing: the cost rule cannot be checked against it, so it must not be allowed to hide anything.
|
|
57
|
+
*
|
|
58
|
+
* @param client configured TypeSafe client
|
|
59
|
+
* @param blocks blocks from the primary representation
|
|
60
|
+
* @param isReply whether the message carries threading headers
|
|
61
|
+
*/
|
|
62
|
+
export declare function annotateBlocksWithTypeSafe(client: SystemOneClient, blocks: readonly Block[], isReply: boolean): Promise<AnnotationResult>;
|
|
46
63
|
//# sourceMappingURL=llm.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,EAAwC,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAEhG,OAAO,EAAE,KAAK,eAAe,EAAyB,MAAM,mBAAmB,CAAC;AAGhF,2FAA2F;AAC3F,eAAO,MAAM,eAAe,MAAM,CAAC;AAEnC,8EAA8E;AAC9E,eAAO,MAAM,UAAU,KAAK,CAAC;AAE7B,MAAM,WAAW,QAAQ;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,wFAAwF;IACxF,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACtC;AAED,MAAM,WAAW,gBAAgB;IAC7B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,wEAAwE;AACxE,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,KAAK,EAAE,CAEnE;AA8CD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA+BpI;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,0BAA0B,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA4B/I"}
|
package/dist/annotate/llm.js
CHANGED
|
@@ -15,7 +15,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
15
15
|
exports.MAX_BLOCKS = exports.MAX_BLOCK_CHARS = void 0;
|
|
16
16
|
exports.annotatableBlocks = annotatableBlocks;
|
|
17
17
|
exports.annotateBlocks = annotateBlocks;
|
|
18
|
+
exports.annotateBlocksWithTypeSafe = annotateBlocksWithTypeSafe;
|
|
19
|
+
const probabilities_1 = require("./probabilities");
|
|
18
20
|
const prompt_1 = require("./prompt");
|
|
21
|
+
const typesafe_prompt_1 = require("./typesafe-prompt");
|
|
19
22
|
/** Characters of each block sent to the model. Enough for a decision, bounded for cost. */
|
|
20
23
|
exports.MAX_BLOCK_CHARS = 700;
|
|
21
24
|
/** Blocks per message. Longer messages are truncated with a recorded flag. */
|
|
@@ -39,6 +42,27 @@ function toPromptBlocks(blocks) {
|
|
|
39
42
|
};
|
|
40
43
|
});
|
|
41
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Close the loop on what was sent: every sent block comes back exactly once, in index order.
|
|
47
|
+
*
|
|
48
|
+
* A block the model declined to label, or answered unusably, is UNKNOWN with confidence zero, which
|
|
49
|
+
* stays visible. Silently dropping it would let an omission look like agreement later.
|
|
50
|
+
*/
|
|
51
|
+
function finish(sent, accepted, spurious, meta) {
|
|
52
|
+
const labels = [...accepted.values()];
|
|
53
|
+
const missing = [];
|
|
54
|
+
for (const block of sent) {
|
|
55
|
+
if (!accepted.has(block.index)) {
|
|
56
|
+
missing.push(block.index);
|
|
57
|
+
labels.push({ index: block.index, label: 'UNKNOWN', confidence: 0 });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
labels.sort((a, b) => a.index - b.index);
|
|
61
|
+
return { ...meta, labels, missing, spurious, blocksSent: sent.length };
|
|
62
|
+
}
|
|
63
|
+
function clampConfidence(value) {
|
|
64
|
+
return typeof value === 'number' && Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0.5;
|
|
65
|
+
}
|
|
42
66
|
/**
|
|
43
67
|
* Annotate one message's blocks.
|
|
44
68
|
*
|
|
@@ -49,16 +73,9 @@ function toPromptBlocks(blocks) {
|
|
|
49
73
|
async function annotateBlocks(client, blocks, isReply) {
|
|
50
74
|
const candidates = annotatableBlocks(blocks);
|
|
51
75
|
const sent = toPromptBlocks(candidates);
|
|
76
|
+
const meta = { model: client.model, promptVersion: prompt_1.PROMPT_VERSION, blocksTotal: candidates.length };
|
|
52
77
|
if (!sent.length) {
|
|
53
|
-
return
|
|
54
|
-
labels: [],
|
|
55
|
-
model: client.model,
|
|
56
|
-
promptVersion: prompt_1.PROMPT_VERSION,
|
|
57
|
-
missing: [],
|
|
58
|
-
spurious: [],
|
|
59
|
-
blocksSent: 0,
|
|
60
|
-
blocksTotal: candidates.length
|
|
61
|
-
};
|
|
78
|
+
return finish(sent, new Map(), [], meta);
|
|
62
79
|
}
|
|
63
80
|
const response = await client.complete({
|
|
64
81
|
system: prompt_1.SYSTEM_PROMPT,
|
|
@@ -69,39 +86,54 @@ async function annotateBlocks(client, blocks, isReply) {
|
|
|
69
86
|
// Validate against the blocks we actually sent. The schema constrains shape and label values,
|
|
70
87
|
// but nothing stops the model returning an index we never asked about, or omitting one.
|
|
71
88
|
const expected = new Set(sent.map(block => block.index));
|
|
72
|
-
const
|
|
73
|
-
const labels = [];
|
|
89
|
+
const accepted = new Map();
|
|
74
90
|
const spurious = [];
|
|
75
91
|
for (const item of response.labels ?? []) {
|
|
76
|
-
if (!expected.has(item.index) ||
|
|
92
|
+
if (!expected.has(item.index) || accepted.has(item.index)) {
|
|
77
93
|
spurious.push(item.index);
|
|
78
94
|
continue;
|
|
79
95
|
}
|
|
80
|
-
|
|
81
|
-
labels.push({
|
|
82
|
-
index: item.index,
|
|
83
|
-
label: item.label,
|
|
84
|
-
confidence: Number.isFinite(item.confidence) ? Math.min(1, Math.max(0, item.confidence)) : 0.5
|
|
85
|
-
});
|
|
96
|
+
accepted.set(item.index, { index: item.index, label: item.label, confidence: clampConfidence(item.confidence) });
|
|
86
97
|
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
98
|
+
return finish(sent, accepted, spurious, meta);
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Annotate one message's blocks with TypeSafe: one Choice question per block.
|
|
102
|
+
*
|
|
103
|
+
* The stored label is the cost rule applied to the returned distribution at DEFAULT_COST_RATIO, not
|
|
104
|
+
* the model's top choice, and the raw distribution is kept alongside so other ratios can be applied
|
|
105
|
+
* later without re-running inference. An answer without a usable distribution is treated as
|
|
106
|
+
* missing: the cost rule cannot be checked against it, so it must not be allowed to hide anything.
|
|
107
|
+
*
|
|
108
|
+
* @param client configured TypeSafe client
|
|
109
|
+
* @param blocks blocks from the primary representation
|
|
110
|
+
* @param isReply whether the message carries threading headers
|
|
111
|
+
*/
|
|
112
|
+
async function annotateBlocksWithTypeSafe(client, blocks, isReply) {
|
|
113
|
+
const candidates = annotatableBlocks(blocks);
|
|
114
|
+
const sent = toPromptBlocks(candidates);
|
|
115
|
+
const requests = (0, typesafe_prompt_1.planRequests)(sent, isReply);
|
|
116
|
+
// Independent: each request carries the full state and its own share of the questions.
|
|
117
|
+
const responses = await Promise.all(requests.map(request => client.ask(request)));
|
|
118
|
+
const accepted = new Map();
|
|
119
|
+
const spurious = [];
|
|
120
|
+
responses.forEach((response, i) => {
|
|
121
|
+
const asked = requests[i].questions;
|
|
122
|
+
for (const [id, answer] of Object.entries(response.answers)) {
|
|
123
|
+
// Matched against what this request asked. Anything else is counted and discarded,
|
|
124
|
+
// exactly like an invented index on the OpenAI path.
|
|
125
|
+
const index = (0, typesafe_prompt_1.parseQuestionId)(id);
|
|
126
|
+
if (index === null || !Object.hasOwn(asked, id) || accepted.has(index)) {
|
|
127
|
+
spurious.push(index ?? -1);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
const probabilities = (0, probabilities_1.sanitizeProbabilities)(answer?.probabilities);
|
|
131
|
+
if (!probabilities) {
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
accepted.set(index, { index, label: (0, probabilities_1.applyCostRule)(probabilities), confidence: clampConfidence(answer.confidence), probabilities });
|
|
94
135
|
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
return {
|
|
98
|
-
labels,
|
|
99
|
-
model: client.model,
|
|
100
|
-
promptVersion: prompt_1.PROMPT_VERSION,
|
|
101
|
-
missing,
|
|
102
|
-
spurious,
|
|
103
|
-
blocksSent: sent.length,
|
|
104
|
-
blocksTotal: candidates.length
|
|
105
|
-
};
|
|
136
|
+
});
|
|
137
|
+
return finish(sent, accepted, spurious, { model: client.model, promptVersion: typesafe_prompt_1.TYPESAFE_PROMPT_VERSION, blocksTotal: candidates.length });
|
|
106
138
|
}
|
|
107
139
|
//# sourceMappingURL=llm.js.map
|
package/dist/annotate/llm.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;;
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;;AAqCH,8CAEC;AAqDD,wCA+BC;AAcD,gEA4BC;AAhKD,mDAAgG;AAChG,qCAA8G;AAE9G,uDAA2F;AAE3F,2FAA2F;AAC9E,QAAA,eAAe,GAAG,GAAG,CAAC;AAEnC,8EAA8E;AACjE,QAAA,UAAU,GAAG,EAAE,CAAC;AAsB7B,wEAAwE;AACxE,SAAgB,iBAAiB,CAAC,MAAwB;IACtD,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,cAAc,CAAC,MAAwB;IAC5C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAU,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,uBAAe,CAAC;QAChD,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAe,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;YAC/D,SAAS;YACT,8EAA8E;YAC9E,qFAAqF;YACrF,8BAA8B;YAC9B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC;SAClF,CAAC;IACN,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;GAKG;AACH,SAAS,MAAM,CACX,IAA4B,EAC5B,QAA+B,EAC/B,QAAkB,EAClB,IAAmE;IAEnE,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACtC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AAC3E,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACvG,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc,CAAC,MAAwB,EAAE,MAAwB,EAAE,OAAgB;IACrG,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,uBAAc,EAAE,WAAW,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;IAEpG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO,MAAM,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAoE;QACtG,MAAM,EAAE,sBAAa;QACrB,IAAI,EAAE,IAAA,yBAAgB,EAAC,IAAI,EAAE,OAAO,CAAC;QACrC,UAAU,EAAE,cAAc;QAC1B,MAAM,EAAE,wBAAe;KAC1B,CAAC,CAAC;IAEH,8FAA8F;IAC9F,wFAAwF;IACxF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC7C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,SAAS;QACb,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;GAWG;AACI,KAAK,UAAU,0BAA0B,CAAC,MAAuB,EAAE,MAAwB,EAAE,OAAgB;IAChH,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAA,8BAAY,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7C,uFAAuF;IACvF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAElF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC7C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAI,QAAQ,CAAC,CAAC,CAAsB,CAAC,SAAS,CAAC;QAC1D,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,mFAAmF;YACnF,qDAAqD;YACrD,MAAM,KAAK,GAAG,IAAA,iCAAe,EAAC,EAAE,CAAC,CAAC;YAClC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrE,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3B,SAAS;YACb,CAAC;YACD,MAAM,aAAa,GAAG,IAAA,qCAAqB,EAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YACnE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACjB,SAAS;YACb,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAA,6BAAa,EAAC,aAAa,CAAC,EAAE,UAAU,EAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC;QACvI,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,yCAAuB,EAAE,WAAW,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;AAC7I,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Label probability distributions from a probabilistic annotator, and the decision rule applied to
|
|
3
|
+
* them.
|
|
4
|
+
*
|
|
5
|
+
* The OpenAI annotator is asked, in its prompt, to treat hiding authored text as ten times worse
|
|
6
|
+
* than leaving history visible. An annotator that returns a full distribution does not need to be
|
|
7
|
+
* asked to be biased: the raw distribution is stored unbiased and the asymmetry is applied here, in
|
|
8
|
+
* code, where it can be tuned and re-applied without re-running inference.
|
|
9
|
+
*/
|
|
10
|
+
import { type Label } from '../types';
|
|
11
|
+
export type LabelProbabilities = Partial<Record<Label, number>>;
|
|
12
|
+
/**
|
|
13
|
+
* Hiding authored text is treated as this many times worse than leaving history visible. Part of
|
|
14
|
+
* the `annotate-ts-v1` convention: stored labels always use it, so changing it means a new prompt
|
|
15
|
+
* version. Other ratios are applied at read time, from the stored distribution.
|
|
16
|
+
*/
|
|
17
|
+
export declare const DEFAULT_COST_RATIO = 10;
|
|
18
|
+
/**
|
|
19
|
+
* Labels whose content was written for this message and must stay visible. Hiding one of these is
|
|
20
|
+
* the expensive error every cost rule, gate and disagreement filter is built around.
|
|
21
|
+
*/
|
|
22
|
+
export declare const AUTHORED_LABELS: ReadonlySet<Label>;
|
|
23
|
+
/**
|
|
24
|
+
* Keep only known labels with finite values, clamped into [0, 1]. Returns null when nothing usable
|
|
25
|
+
* is left, so a malformed answer is recorded as missing rather than as a confident zero.
|
|
26
|
+
*/
|
|
27
|
+
export declare function sanitizeProbabilities(raw: unknown): LabelProbabilities | null;
|
|
28
|
+
/** Most probable label. Ties go to the earlier label in LABELS, which puts NEW_CONTENT first. */
|
|
29
|
+
export declare function topLabel(probabilities: LabelProbabilities): Label;
|
|
30
|
+
/**
|
|
31
|
+
* Apply the cost asymmetry to a distribution.
|
|
32
|
+
*
|
|
33
|
+
* Hiding a block wrongly costs `ratio` times the probability that it was authored, summed over
|
|
34
|
+
* AUTHORED_LABELS since hiding is wrong if it is any of them. Leaving it visible wrongly costs the
|
|
35
|
+
* probability that it was history. So REPLY_HISTORY survives only when
|
|
36
|
+
* P(REPLY_HISTORY) > ratio * (P(NEW_CONTENT) + P(INLINE_QUOTE)); otherwise the block falls back to
|
|
37
|
+
* whichever authored label is more likely. Every other label is taken as is, since none of them
|
|
38
|
+
* hides anything.
|
|
39
|
+
*/
|
|
40
|
+
export declare function applyCostRule(probabilities: LabelProbabilities, ratio?: number): Label;
|
|
41
|
+
/**
|
|
42
|
+
* Gap between the two most probable labels. Small means genuinely ambiguous, which is what review
|
|
43
|
+
* triage should sort by instead of a model's self reported confidence.
|
|
44
|
+
*/
|
|
45
|
+
export declare function probabilityMargin(probabilities: LabelProbabilities): number;
|
|
46
|
+
/** Parse the JSON stored in `annotations.probabilities`. Null for rows written without one. */
|
|
47
|
+
export declare function parseStoredProbabilities(json: string | null | undefined): LabelProbabilities | null;
|
|
48
|
+
//# sourceMappingURL=probabilities.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probabilities.d.ts","sourceRoot":"","sources":["../../src/annotate/probabilities.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAU,KAAK,KAAK,EAAE,MAAM,UAAU,CAAC;AAE9C,MAAM,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEhE;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,KAAK,CAAC;AAErC;;;GAGG;AACH,eAAO,MAAM,eAAe,EAAE,WAAW,CAAC,KAAK,CAAmD,CAAC;AAEnG;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAc7E;AAED,iGAAiG;AACjG,wBAAgB,QAAQ,CAAC,aAAa,EAAE,kBAAkB,GAAG,KAAK,CAWjE;AAED;;;;;;;;;GASG;AACH,wBAAgB,aAAa,CAAC,aAAa,EAAE,kBAAkB,EAAE,KAAK,GAAE,MAA2B,GAAG,KAAK,CAe1G;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,aAAa,EAAE,kBAAkB,GAAG,MAAM,CAa3E;AAED,+FAA+F;AAC/F,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,kBAAkB,GAAG,IAAI,CASnG"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Label probability distributions from a probabilistic annotator, and the decision rule applied to
|
|
4
|
+
* them.
|
|
5
|
+
*
|
|
6
|
+
* The OpenAI annotator is asked, in its prompt, to treat hiding authored text as ten times worse
|
|
7
|
+
* than leaving history visible. An annotator that returns a full distribution does not need to be
|
|
8
|
+
* asked to be biased: the raw distribution is stored unbiased and the asymmetry is applied here, in
|
|
9
|
+
* code, where it can be tuned and re-applied without re-running inference.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.AUTHORED_LABELS = exports.DEFAULT_COST_RATIO = void 0;
|
|
13
|
+
exports.sanitizeProbabilities = sanitizeProbabilities;
|
|
14
|
+
exports.topLabel = topLabel;
|
|
15
|
+
exports.applyCostRule = applyCostRule;
|
|
16
|
+
exports.probabilityMargin = probabilityMargin;
|
|
17
|
+
exports.parseStoredProbabilities = parseStoredProbabilities;
|
|
18
|
+
const types_1 = require("../types");
|
|
19
|
+
/**
|
|
20
|
+
* Hiding authored text is treated as this many times worse than leaving history visible. Part of
|
|
21
|
+
* the `annotate-ts-v1` convention: stored labels always use it, so changing it means a new prompt
|
|
22
|
+
* version. Other ratios are applied at read time, from the stored distribution.
|
|
23
|
+
*/
|
|
24
|
+
exports.DEFAULT_COST_RATIO = 10;
|
|
25
|
+
/**
|
|
26
|
+
* Labels whose content was written for this message and must stay visible. Hiding one of these is
|
|
27
|
+
* the expensive error every cost rule, gate and disagreement filter is built around.
|
|
28
|
+
*/
|
|
29
|
+
exports.AUTHORED_LABELS = new Set(['NEW_CONTENT', 'INLINE_QUOTE']);
|
|
30
|
+
/**
|
|
31
|
+
* Keep only known labels with finite values, clamped into [0, 1]. Returns null when nothing usable
|
|
32
|
+
* is left, so a malformed answer is recorded as missing rather than as a confident zero.
|
|
33
|
+
*/
|
|
34
|
+
function sanitizeProbabilities(raw) {
|
|
35
|
+
if (!raw || typeof raw !== 'object') {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
const out = {};
|
|
39
|
+
let any = false;
|
|
40
|
+
for (const label of types_1.LABELS) {
|
|
41
|
+
const value = raw[label];
|
|
42
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
43
|
+
out[label] = Math.min(1, Math.max(0, value));
|
|
44
|
+
any = true;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return any ? out : null;
|
|
48
|
+
}
|
|
49
|
+
/** Most probable label. Ties go to the earlier label in LABELS, which puts NEW_CONTENT first. */
|
|
50
|
+
function topLabel(probabilities) {
|
|
51
|
+
let best = 'UNKNOWN';
|
|
52
|
+
let bestValue = -1;
|
|
53
|
+
for (const label of types_1.LABELS) {
|
|
54
|
+
const value = probabilities[label] ?? 0;
|
|
55
|
+
if (value > bestValue) {
|
|
56
|
+
best = label;
|
|
57
|
+
bestValue = value;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return best;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Apply the cost asymmetry to a distribution.
|
|
64
|
+
*
|
|
65
|
+
* Hiding a block wrongly costs `ratio` times the probability that it was authored, summed over
|
|
66
|
+
* AUTHORED_LABELS since hiding is wrong if it is any of them. Leaving it visible wrongly costs the
|
|
67
|
+
* probability that it was history. So REPLY_HISTORY survives only when
|
|
68
|
+
* P(REPLY_HISTORY) > ratio * (P(NEW_CONTENT) + P(INLINE_QUOTE)); otherwise the block falls back to
|
|
69
|
+
* whichever authored label is more likely. Every other label is taken as is, since none of them
|
|
70
|
+
* hides anything.
|
|
71
|
+
*/
|
|
72
|
+
function applyCostRule(probabilities, ratio = exports.DEFAULT_COST_RATIO) {
|
|
73
|
+
const top = topLabel(probabilities);
|
|
74
|
+
if (top !== 'REPLY_HISTORY') {
|
|
75
|
+
return top;
|
|
76
|
+
}
|
|
77
|
+
let authoredMass = 0;
|
|
78
|
+
let fallback = 'NEW_CONTENT';
|
|
79
|
+
for (const label of exports.AUTHORED_LABELS) {
|
|
80
|
+
const value = probabilities[label] ?? 0;
|
|
81
|
+
authoredMass += value;
|
|
82
|
+
if (value > (probabilities[fallback] ?? 0)) {
|
|
83
|
+
fallback = label;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return (probabilities.REPLY_HISTORY ?? 0) > ratio * authoredMass ? 'REPLY_HISTORY' : fallback;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Gap between the two most probable labels. Small means genuinely ambiguous, which is what review
|
|
90
|
+
* triage should sort by instead of a model's self reported confidence.
|
|
91
|
+
*/
|
|
92
|
+
function probabilityMargin(probabilities) {
|
|
93
|
+
let first = 0;
|
|
94
|
+
let second = 0;
|
|
95
|
+
for (const label of types_1.LABELS) {
|
|
96
|
+
const value = probabilities[label] ?? 0;
|
|
97
|
+
if (value > first) {
|
|
98
|
+
second = first;
|
|
99
|
+
first = value;
|
|
100
|
+
}
|
|
101
|
+
else if (value > second) {
|
|
102
|
+
second = value;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return first - second;
|
|
106
|
+
}
|
|
107
|
+
/** Parse the JSON stored in `annotations.probabilities`. Null for rows written without one. */
|
|
108
|
+
function parseStoredProbabilities(json) {
|
|
109
|
+
if (!json) {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
try {
|
|
113
|
+
return sanitizeProbabilities(JSON.parse(json));
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=probabilities.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probabilities.js","sourceRoot":"","sources":["../../src/annotate/probabilities.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;;AAuBH,sDAcC;AAGD,4BAWC;AAYD,sCAeC;AAMD,8CAaC;AAGD,4DASC;AA3GD,oCAA8C;AAI9C;;;;GAIG;AACU,QAAA,kBAAkB,GAAG,EAAE,CAAC;AAErC;;;GAGG;AACU,QAAA,eAAe,GAAuB,IAAI,GAAG,CAAQ,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC;AAEnG;;;GAGG;AACH,SAAgB,qBAAqB,CAAC,GAAY;IAC9C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,GAAG,GAAuB,EAAE,CAAC;IACnC,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,KAAK,MAAM,KAAK,IAAI,cAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAI,GAA+B,CAAC,KAAK,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7C,GAAG,GAAG,IAAI,CAAC;QACf,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5B,CAAC;AAED,iGAAiG;AACjG,SAAgB,QAAQ,CAAC,aAAiC;IACtD,IAAI,IAAI,GAAU,SAAS,CAAC;IAC5B,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;IACnB,KAAK,MAAM,KAAK,IAAI,cAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACpB,IAAI,GAAG,KAAK,CAAC;YACb,SAAS,GAAG,KAAK,CAAC;QACtB,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;;;;;GASG;AACH,SAAgB,aAAa,CAAC,aAAiC,EAAE,QAAgB,0BAAkB;IAC/F,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC;IACpC,IAAI,GAAG,KAAK,eAAe,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IACD,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAU,aAAa,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,uBAAe,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,YAAY,IAAI,KAAK,CAAC;QACtB,IAAI,KAAK,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACzC,QAAQ,GAAG,KAAK,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,CAAC,aAAa,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClG,CAAC;AAED;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,aAAiC;IAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,cAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,KAAK,EAAE,CAAC;YAChB,MAAM,GAAG,KAAK,CAAC;YACf,KAAK,GAAG,KAAK,CAAC;QAClB,CAAC;aAAM,IAAI,KAAK,GAAG,MAAM,EAAE,CAAC;YACxB,MAAM,GAAG,KAAK,CAAC;QACnB,CAAC;IACL,CAAC;IACD,OAAO,KAAK,GAAG,MAAM,CAAC;AAC1B,CAAC;AAED,+FAA+F;AAC/F,SAAgB,wBAAwB,CAAC,IAA+B;IACpE,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,CAAC;QACD,OAAO,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Annotation providers behind one interface.
|
|
3
|
+
*
|
|
4
|
+
* Each provider owns its client, default model, prompt version and response validation. Commands
|
|
5
|
+
* only pick one by name, so adding a provider never means editing a command's control flow.
|
|
6
|
+
*/
|
|
7
|
+
import { type Usage } from './http';
|
|
8
|
+
import { type AnnotationResult } from './llm';
|
|
9
|
+
import { type Block } from '../segment/types';
|
|
10
|
+
export interface Annotator {
|
|
11
|
+
readonly model: string;
|
|
12
|
+
readonly promptVersion: string;
|
|
13
|
+
readonly usage: Usage;
|
|
14
|
+
annotate(blocks: readonly Block[], isReply: boolean): Promise<AnnotationResult>;
|
|
15
|
+
}
|
|
16
|
+
interface ProviderSpec {
|
|
17
|
+
defaultModel: string;
|
|
18
|
+
promptVersion: string;
|
|
19
|
+
/** Throws, before anything is sent, when the provider's key is not configured. */
|
|
20
|
+
create(model: string): Annotator;
|
|
21
|
+
}
|
|
22
|
+
export declare const PROVIDERS: Readonly<Record<string, ProviderSpec>>;
|
|
23
|
+
/** The provider registered under `name`, or null. Own properties only, so `constructor` is not one. */
|
|
24
|
+
export declare function provider(name: string): ProviderSpec | null;
|
|
25
|
+
export {};
|
|
26
|
+
//# sourceMappingURL=providers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../../src/annotate/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAA8C,KAAK,gBAAgB,EAAE,MAAM,OAAO,CAAC;AAI1F,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACnF;AAED,UAAU,YAAY;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,kFAAkF;IAClF,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC;AAED,eAAO,MAAM,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAsB5D,CAAC;AAEF,uGAAuG;AACvG,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAE1D"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Annotation providers behind one interface.
|
|
4
|
+
*
|
|
5
|
+
* Each provider owns its client, default model, prompt version and response validation. Commands
|
|
6
|
+
* only pick one by name, so adding a provider never means editing a command's control flow.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.PROVIDERS = void 0;
|
|
10
|
+
exports.provider = provider;
|
|
11
|
+
const client_1 = require("./client");
|
|
12
|
+
const llm_1 = require("./llm");
|
|
13
|
+
const prompt_1 = require("./prompt");
|
|
14
|
+
const typesafe_client_1 = require("./typesafe-client");
|
|
15
|
+
const typesafe_prompt_1 = require("./typesafe-prompt");
|
|
16
|
+
exports.PROVIDERS = {
|
|
17
|
+
openai: {
|
|
18
|
+
defaultModel: 'gpt-5.6-sol',
|
|
19
|
+
promptVersion: prompt_1.PROMPT_VERSION,
|
|
20
|
+
create(model) {
|
|
21
|
+
const client = new client_1.OpenAIClient((0, client_1.configFromEnv)(model));
|
|
22
|
+
return { model, promptVersion: prompt_1.PROMPT_VERSION, usage: client.usage, annotate: (blocks, isReply) => (0, llm_1.annotateBlocks)(client, blocks, isReply) };
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
typesafe: {
|
|
26
|
+
defaultModel: typesafe_client_1.TYPESAFE_DEFAULT_MODEL,
|
|
27
|
+
promptVersion: typesafe_prompt_1.TYPESAFE_PROMPT_VERSION,
|
|
28
|
+
create(model) {
|
|
29
|
+
const client = new typesafe_client_1.TypeSafeClient((0, typesafe_client_1.typeSafeConfigFromEnv)(model));
|
|
30
|
+
return {
|
|
31
|
+
model,
|
|
32
|
+
promptVersion: typesafe_prompt_1.TYPESAFE_PROMPT_VERSION,
|
|
33
|
+
usage: client.usage,
|
|
34
|
+
annotate: (blocks, isReply) => (0, llm_1.annotateBlocksWithTypeSafe)(client, blocks, isReply)
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
/** The provider registered under `name`, or null. Own properties only, so `constructor` is not one. */
|
|
40
|
+
function provider(name) {
|
|
41
|
+
return Object.hasOwn(exports.PROVIDERS, name) ? exports.PROVIDERS[name] : null;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=providers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/annotate/providers.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAiDH,4BAEC;AAjDD,qCAAuD;AAEvD,+BAA0F;AAC1F,qCAA0C;AAC1C,uDAAkG;AAClG,uDAA4D;AAiB/C,QAAA,SAAS,GAA2C;IAC7D,MAAM,EAAE;QACJ,YAAY,EAAE,aAAa;QAC3B,aAAa,EAAE,uBAAc;QAC7B,MAAM,CAAC,KAAK;YACR,MAAM,MAAM,GAAG,IAAI,qBAAY,CAAC,IAAA,sBAAa,EAAC,KAAK,CAAC,CAAC,CAAC;YACtD,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,uBAAc,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAA,oBAAc,EAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;QACjJ,CAAC;KACJ;IACD,QAAQ,EAAE;QACN,YAAY,EAAE,wCAAsB;QACpC,aAAa,EAAE,yCAAuB;QACtC,MAAM,CAAC,KAAK;YACR,MAAM,MAAM,GAAG,IAAI,gCAAc,CAAC,IAAA,uCAAqB,EAAC,KAAK,CAAC,CAAC,CAAC;YAChE,OAAO;gBACH,KAAK;gBACL,aAAa,EAAE,yCAAuB;gBACtC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAA,gCAA0B,EAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;aACrF,CAAC;QACN,CAAC;KACJ;CACJ,CAAC;AAEF,uGAAuG;AACvG,SAAgB,QAAQ,CAAC,IAAY;IACjC,OAAO,MAAM,CAAC,MAAM,CAAC,iBAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAE,iBAAS,CAAC,IAAI,CAAkB,CAAC,CAAC,CAAC,IAAI,CAAC;AACrF,CAAC"}
|
package/dist/annotate/store.d.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { type CorpusStore } from '../corpus/store';
|
|
10
10
|
import { type Label, type Reason } from '../types';
|
|
11
|
+
import { type LabelProbabilities } from './probabilities';
|
|
11
12
|
export interface AnnotationRow {
|
|
12
13
|
path: string;
|
|
13
14
|
representation: 'text' | 'html';
|
|
@@ -22,6 +23,8 @@ export interface AnnotationRow {
|
|
|
22
23
|
baselineLabel: Label | null;
|
|
23
24
|
baselineConfidence: number | null;
|
|
24
25
|
baselineReasons: readonly Reason[];
|
|
26
|
+
/** Full label distribution, when the annotator returns one. */
|
|
27
|
+
probabilities?: LabelProbabilities | null;
|
|
25
28
|
}
|
|
26
29
|
export interface AnnotationRunInfo {
|
|
27
30
|
source: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAqB,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAqB,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EAAqB,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAE7E,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,GAAG,MAAM,CAAC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;IAC7C,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,KAAK,GAAG,IAAI,CAAC;IAC5B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,+DAA+D;IAC/D,aAAa,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,iBAAiB;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM,CAYtF;AAED,wBAAgB,mBAAmB,CAC/B,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,GAC/G,IAAI,CAaN;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,aAAa,EAAE,GAAG,IAAI,CA+BxG;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,EAAE,aAAa,GAAE,MAAM,GAAG,IAAW,GAAG,GAAG,CAAC,MAAM,CAAC,CAgBzI"}
|
package/dist/annotate/store.js
CHANGED
|
@@ -13,6 +13,7 @@ exports.finishAnnotationRun = finishAnnotationRun;
|
|
|
13
13
|
exports.writeAnnotations = writeAnnotations;
|
|
14
14
|
exports.annotatedPaths = annotatedPaths;
|
|
15
15
|
const types_1 = require("../types");
|
|
16
|
+
const probabilities_1 = require("./probabilities");
|
|
16
17
|
function startAnnotationRun(store, info) {
|
|
17
18
|
const result = store.run(`INSERT INTO annotation_runs (source, model, prompt_version, extractor_version, started_at, notes)
|
|
18
19
|
VALUES (?, ?, ?, ?, ?, ?)`, info.source, info.model, info.promptVersion, types_1.EXTRACTOR_VERSION, Date.now(), info.notes ?? null);
|
|
@@ -31,8 +32,8 @@ function writeAnnotations(store, runId, rows) {
|
|
|
31
32
|
for (const row of rows) {
|
|
32
33
|
store.run(`INSERT INTO annotations (
|
|
33
34
|
run_id, path, representation, block_index, mime_part, block_start, block_end, char_count,
|
|
34
|
-
source, label, confidence, baseline_label, baseline_conf, baseline_reasons, agrees
|
|
35
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, runId, row.path, row.representation, row.blockIndex, row.mimePart, row.blockStart, row.blockEnd, row.charCount, row.source, row.label, row.confidence, row.baselineLabel, row.baselineConfidence, row.baselineReasons.join(','), row.baselineLabel === row.label ? 1 : 0);
|
|
35
|
+
source, label, confidence, baseline_label, baseline_conf, baseline_reasons, agrees, probabilities, margin
|
|
36
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, runId, row.path, row.representation, row.blockIndex, row.mimePart, row.blockStart, row.blockEnd, row.charCount, row.source, row.label, row.confidence, row.baselineLabel, row.baselineConfidence, row.baselineReasons.join(','), row.baselineLabel === row.label ? 1 : 0, row.probabilities ? JSON.stringify(row.probabilities) : null, row.probabilities ? (0, probabilities_1.probabilityMargin)(row.probabilities) : null);
|
|
36
37
|
}
|
|
37
38
|
});
|
|
38
39
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;AA+BH,gDAYC;AAED,kDAiBC;AAED,4CA+BC;AASD,wCAgBC;AArHD,oCAAsE;AACtE,mDAA6E;AA2B7E,SAAgB,kBAAkB,CAAC,KAAkB,EAAE,IAAuB;IAC1E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CACpB;mCAC2B,EAC3B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,aAAa,EAClB,yBAAiB,EACjB,IAAI,CAAC,GAAG,EAAE,EACV,IAAI,CAAC,KAAK,IAAI,IAAI,CACrB,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,SAAgB,mBAAmB,CAC/B,KAAkB,EAClB,KAAa,EACb,MAA8G;IAE9G,KAAK,CAAC,GAAG,CACL;;sBAEc,EACd,IAAI,CAAC,GAAG,EAAE,EACV,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,gBAAgB,EACvB,KAAK,CACR,CAAC;AACN,CAAC;AAED,SAAgB,gBAAgB,CAAC,KAAkB,EAAE,KAAa,EAAE,IAA8B;IAC9F,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO;IACX,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,KAAK,CAAC,GAAG,CACL;;;8EAG8D,EAC9D,KAAK,EACL,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,cAAc,EAClB,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,QAAQ,EACZ,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,QAAQ,EACZ,GAAG,CAAC,SAAS,EACb,GAAG,CAAC,MAAM,EACV,GAAG,CAAC,KAAK,EACT,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,aAAa,EACjB,GAAG,CAAC,kBAAkB,EACtB,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAC7B,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,EAC5D,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAA,iCAAiB,EAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAClE,CAAC;QACN,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,KAAkB,EAAE,MAAc,EAAE,KAAoB,EAAE,gBAA+B,IAAI;IACxH,MAAM,OAAO,GAAG,CAAC,cAAc,CAAC,CAAC;IACjC,MAAM,MAAM,GAAa,CAAC,MAAM,CAAC,CAAC;IAClC,IAAI,KAAK,EAAE,CAAC;QACR,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CACpB,qGAAqG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAC5H,GAAG,MAAM,CACZ,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9C,CAAC"}
|