@kontourai/survey 6.1.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/calibration.d.ts +15 -0
- package/dist/src/calibration.js +11 -0
- package/dist/src/canonical-reviewed-trust-input.d.ts +10 -0
- package/dist/src/canonical-reviewed-trust-input.js +45 -1
- package/dist/src/console/review-console-server.js +22 -4
- package/dist/src/decision-carry-forward.d.ts +157 -0
- package/dist/src/decision-carry-forward.js +295 -0
- package/dist/src/extraction-envelope.d.ts +4 -3
- package/dist/src/extraction-envelope.js +3 -3
- package/dist/src/field-states.d.ts +100 -0
- package/dist/src/field-states.js +169 -0
- package/dist/src/index.d.ts +6 -4
- package/dist/src/index.js +3 -1
- package/dist/src/mcp/review-mcp.js +173 -45
- package/dist/src/review-resource.d.ts +36 -0
- package/dist/src/review-session-file.d.ts +7 -0
- package/dist/src/review-workbench/audit-rows.d.ts +1 -1
- package/dist/src/review-workbench/audit-rows.js +1 -0
- package/dist/src/review-workbench/audit-sample.d.ts +32 -0
- package/dist/src/review-workbench/audit-sample.js +53 -0
- package/dist/src/review-workbench/extraction-inspector.d.ts +1 -1
- package/dist/src/review-workbench/extraction-inspector.js +1 -1
- package/dist/src/review-workbench/field-state-view.d.ts +15 -0
- package/dist/src/review-workbench/field-state-view.js +84 -0
- package/dist/src/review-workbench/queue-binding.d.ts +45 -2
- package/dist/src/review-workbench/queue-binding.js +45 -0
- package/dist/src/review-workbench/review-presentation.d.ts +23 -0
- package/dist/src/review-workbench/review-presentation.js +51 -10
- package/dist/src/review-workbench/review-queue-session.d.ts +93 -5
- package/dist/src/review-workbench/review-queue-session.js +147 -11
- package/dist/src/review-workbench/review-session-replay.d.ts +11 -1
- package/dist/src/review-workbench/review-session-replay.js +49 -1
- package/dist/src/review-workbench/review-workbench-css.generated.js +126 -0
- package/dist/src/review-workbench/review-workbench-element.d.ts +10 -0
- package/dist/src/review-workbench/review-workbench-element.js +21 -3
- package/dist/src/review-workbench/review-workbench.css +126 -0
- package/dist/src/review-workbench/review-workbench.d.ts +40 -8
- package/dist/src/review-workbench/review-workbench.js +192 -71
- package/dist/src/review-workbench/review-workbench.standalone.css +126 -0
- package/dist/src/review-workbench/server-review-session.d.ts +18 -0
- package/dist/src/review-workbench/server-review-session.js +20 -2
- package/dist/src/surface-reviewed-extraction.d.ts +23 -0
- package/dist/src/surface-reviewed-extraction.js +64 -1
- package/dist/src/to-surface.js +32 -3
- package/package.json +1 -1
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { ReviewItem } from "../review-resource.js";
|
|
2
|
+
import { type ReviewQueueSessionState } from "./review-queue-session.js";
|
|
3
|
+
export interface RandomAuditSampleOptions {
|
|
4
|
+
/** Inclusion probability for each item, greater than 0 and at most 1. */
|
|
5
|
+
readonly rate: number;
|
|
6
|
+
/** Any non-empty string. The same seed and items always draw the same sample. */
|
|
7
|
+
readonly seed: string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Draws a seeded random audit sample from `items`, for example the items an
|
|
11
|
+
* auto-accept threshold would have accepted without review.
|
|
12
|
+
*
|
|
13
|
+
* Each item is included on its own when a SHA-256 of `(seed, item name)`,
|
|
14
|
+
* read as a number in [0, 1), is below `rate`. The draw is deterministic, does
|
|
15
|
+
* not depend on the order of `items`, and an item's inclusion does not change
|
|
16
|
+
* when other items are added or removed. Items keep their input order. Item
|
|
17
|
+
* names must be unique.
|
|
18
|
+
*/
|
|
19
|
+
export declare function drawRandomAuditSample(items: readonly ReviewItem[], options: RandomAuditSampleOptions): ReviewItem[];
|
|
20
|
+
/**
|
|
21
|
+
* Draws a random audit sample (see {@link drawRandomAuditSample}) and opens it
|
|
22
|
+
* as a score-blind review session: the session hides every confidence and
|
|
23
|
+
* verifier result, and each decision made in it records
|
|
24
|
+
* `presentation: { scoreBlind: true }` and
|
|
25
|
+
* `sampling: { kind: "random-audit", rate, seed }`.
|
|
26
|
+
*
|
|
27
|
+
* Throws when the draw selects no item, since a session needs one.
|
|
28
|
+
*/
|
|
29
|
+
export declare function openRandomAuditSession(items: readonly ReviewItem[], options: RandomAuditSampleOptions & {
|
|
30
|
+
readonly reviewedAt?: string;
|
|
31
|
+
readonly actorId?: string;
|
|
32
|
+
}): ReviewQueueSessionState;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { sha256Hex } from "../sha256.js";
|
|
2
|
+
import { canonicalJson } from "./canonical.js";
|
|
3
|
+
import { assertAuditRate, assertAuditSeed, initialReviewQueueSessionState, } from "./review-queue-session.js";
|
|
4
|
+
/**
|
|
5
|
+
* Draws a seeded random audit sample from `items`, for example the items an
|
|
6
|
+
* auto-accept threshold would have accepted without review.
|
|
7
|
+
*
|
|
8
|
+
* Each item is included on its own when a SHA-256 of `(seed, item name)`,
|
|
9
|
+
* read as a number in [0, 1), is below `rate`. The draw is deterministic, does
|
|
10
|
+
* not depend on the order of `items`, and an item's inclusion does not change
|
|
11
|
+
* when other items are added or removed. Items keep their input order. Item
|
|
12
|
+
* names must be unique.
|
|
13
|
+
*/
|
|
14
|
+
export function drawRandomAuditSample(items, options) {
|
|
15
|
+
assertAuditRate(options.rate);
|
|
16
|
+
assertAuditSeed(options.seed);
|
|
17
|
+
const names = new Set();
|
|
18
|
+
for (const item of items) {
|
|
19
|
+
if (names.has(item.metadata.name)) {
|
|
20
|
+
throw new Error(`Audit sample items must have unique names; ${item.metadata.name} appears twice.`);
|
|
21
|
+
}
|
|
22
|
+
names.add(item.metadata.name);
|
|
23
|
+
}
|
|
24
|
+
return items.filter((item) => auditDraw(options.seed, item.metadata.name) < options.rate);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Draws a random audit sample (see {@link drawRandomAuditSample}) and opens it
|
|
28
|
+
* as a score-blind review session: the session hides every confidence and
|
|
29
|
+
* verifier result, and each decision made in it records
|
|
30
|
+
* `presentation: { scoreBlind: true }` and
|
|
31
|
+
* `sampling: { kind: "random-audit", rate, seed }`.
|
|
32
|
+
*
|
|
33
|
+
* Throws when the draw selects no item, since a session needs one.
|
|
34
|
+
*/
|
|
35
|
+
export function openRandomAuditSession(items, options) {
|
|
36
|
+
const sample = drawRandomAuditSample(items, options);
|
|
37
|
+
if (sample.length === 0) {
|
|
38
|
+
throw new Error(`The random audit sample at rate ${options.rate} drew no items from ${items.length}; use a higher rate or more items.`);
|
|
39
|
+
}
|
|
40
|
+
const base = initialReviewQueueSessionState(sample);
|
|
41
|
+
return {
|
|
42
|
+
...base,
|
|
43
|
+
...(options.reviewedAt !== undefined ? { reviewedAt: options.reviewedAt } : {}),
|
|
44
|
+
...(options.actorId !== undefined ? { actorId: options.actorId } : {}),
|
|
45
|
+
presentation: { scoreBlind: true },
|
|
46
|
+
sampling: { kind: "random-audit", rate: options.rate, seed: options.seed },
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
/** A deterministic number in [0, 1) from the first 52 bits of a SHA-256. */
|
|
50
|
+
function auditDraw(seed, itemName) {
|
|
51
|
+
const hex = sha256Hex(canonicalJson(["survey.random-audit.v1", seed, itemName])).slice(0, 13);
|
|
52
|
+
return Number.parseInt(hex, 16) / 2 ** 52;
|
|
53
|
+
}
|
|
@@ -88,7 +88,7 @@ export interface ExtractionInspectorSource {
|
|
|
88
88
|
}>;
|
|
89
89
|
/**
|
|
90
90
|
* Whether the import checked excerpts against the prepared artifact
|
|
91
|
-
* (`status.provenance
|
|
91
|
+
* (`status.provenance`, required on every import record). Always set by
|
|
92
92
|
* {@link buildExtractionInspectorModel}; optional for hand-authored models.
|
|
93
93
|
*/
|
|
94
94
|
importProvenance?: "verified" | "unverified";
|
|
@@ -25,7 +25,7 @@ export function buildExtractionInspectorModel(input) {
|
|
|
25
25
|
if (sourceKeys.has(sourceKey))
|
|
26
26
|
throw new Error("Extraction inspector source identity collision.");
|
|
27
27
|
sourceKeys.add(sourceKey);
|
|
28
|
-
const source = sourceModel(sourceKey, record.metadata.name, prepared, record.status.diagnostics, entry.artifact, envelope.result.ocrDerived, envelope.result.proposals, record.status.provenance
|
|
28
|
+
const source = sourceModel(sourceKey, record.metadata.name, prepared, record.status.diagnostics, entry.artifact, envelope.result.ocrDerived, envelope.result.proposals, record.status.provenance);
|
|
29
29
|
sources.push(source);
|
|
30
30
|
const candidateStart = candidates.length;
|
|
31
31
|
const itemByProposalIndex = new Map();
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { FieldState } from "../field-states.js";
|
|
2
|
+
import { type ReviewQueueSessionState } from "./review-queue-session.js";
|
|
3
|
+
/**
|
|
4
|
+
* The field-state panel: one row per field the extraction was asked for or
|
|
5
|
+
* proposed, with its derived content state, its lifecycle, and its signals.
|
|
6
|
+
* Fields with no review card (not read, only excluded proposals, no proposal)
|
|
7
|
+
* are listed too, so a run that lost content never reads as complete.
|
|
8
|
+
*
|
|
9
|
+
* The lifecycle of a field in the queue is read from the live session, so it
|
|
10
|
+
* follows the reviewer's decisions. Only what the session cannot know comes
|
|
11
|
+
* from the derived states: a decision carried forward from an earlier round,
|
|
12
|
+
* and a superseded one. In a score-blind session a content state derived from
|
|
13
|
+
* verifier records is shown as the state its candidates alone give.
|
|
14
|
+
*/
|
|
15
|
+
export declare function renderFieldStatesHtml(states: readonly FieldState[], session: ReviewQueueSessionState): string;
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { isScoreBlind } from "./review-queue-session.js";
|
|
2
|
+
const CONTENT_TEXT = {
|
|
3
|
+
value: "Value found",
|
|
4
|
+
conflicting: "Conflicting values",
|
|
5
|
+
unsupported: "Not supported by verifier",
|
|
6
|
+
excluded: "Proposals excluded",
|
|
7
|
+
not_covered: "Not read",
|
|
8
|
+
};
|
|
9
|
+
const LIFECYCLE_TEXT = {
|
|
10
|
+
pending: "Needs review",
|
|
11
|
+
accepted: "Accepted",
|
|
12
|
+
rejected: "Rejected",
|
|
13
|
+
could_not_confirm: "Could not confirm",
|
|
14
|
+
superseded: "Superseded",
|
|
15
|
+
};
|
|
16
|
+
const WORKBENCH_LIFECYCLE = {
|
|
17
|
+
"accept-proposed": "accepted",
|
|
18
|
+
"select-proposed": "accepted",
|
|
19
|
+
"keep-current": "accepted",
|
|
20
|
+
"reject-proposed": "rejected",
|
|
21
|
+
"could-not-confirm": "could_not_confirm",
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* The field-state panel: one row per field the extraction was asked for or
|
|
25
|
+
* proposed, with its derived content state, its lifecycle, and its signals.
|
|
26
|
+
* Fields with no review card (not read, only excluded proposals, no proposal)
|
|
27
|
+
* are listed too, so a run that lost content never reads as complete.
|
|
28
|
+
*
|
|
29
|
+
* The lifecycle of a field in the queue is read from the live session, so it
|
|
30
|
+
* follows the reviewer's decisions. Only what the session cannot know comes
|
|
31
|
+
* from the derived states: a decision carried forward from an earlier round,
|
|
32
|
+
* and a superseded one. In a score-blind session a content state derived from
|
|
33
|
+
* verifier records is shown as the state its candidates alone give.
|
|
34
|
+
*/
|
|
35
|
+
export function renderFieldStatesHtml(states, session) {
|
|
36
|
+
if (states.length === 0)
|
|
37
|
+
return "";
|
|
38
|
+
const itemsByName = new Map(session.items.map((item) => [item.metadata.name, item]));
|
|
39
|
+
const rows = states.map((state) => {
|
|
40
|
+
const item = state.reviewItemName !== undefined ? itemsByName.get(state.reviewItemName) : undefined;
|
|
41
|
+
const content = state.content === "unsupported" && isScoreBlind(session)
|
|
42
|
+
? (item && item.spec.candidates.filter((candidate) => candidate.role === "proposed").length > 1 ? "conflicting" : "value")
|
|
43
|
+
: state.content;
|
|
44
|
+
const live = item ? session.decisionsByItemName[item.metadata.name] : undefined;
|
|
45
|
+
const lifecycle = live !== undefined
|
|
46
|
+
? WORKBENCH_LIFECYCLE[live]
|
|
47
|
+
: state.lifecycle === "superseded" || state.decisionBasis === "carried-forward"
|
|
48
|
+
? state.lifecycle
|
|
49
|
+
: state.reviewItemName !== undefined ? "pending" : undefined;
|
|
50
|
+
const carried = live === undefined && state.decisionBasis === "carried-forward";
|
|
51
|
+
const signals = [
|
|
52
|
+
...(state.signals.incompleteRun ? ["The extraction stopped before reading all of the source."] : []),
|
|
53
|
+
...(state.signals.excludedProposals ? [`${state.signals.excludedProposals} proposal${state.signals.excludedProposals === 1 ? " was" : "s were"} excluded: the excerpt is not at the cited span.`] : []),
|
|
54
|
+
...(state.signals.unresolvedImport ? ["The import could not ground any value."] : []),
|
|
55
|
+
];
|
|
56
|
+
const label = `${state.slot.fieldOrBehavior}${state.slot.pathIndices?.length ? `[${state.slot.pathIndices.join("][")}]` : ""}`;
|
|
57
|
+
return `
|
|
58
|
+
<li class="field-state-row" data-testid="field-state" data-field="${escapeHtml(label)}" data-content="${content ?? "none"}" data-lifecycle="${lifecycle ?? "none"}"${carried ? ' data-basis="carried-forward"' : ""}>
|
|
59
|
+
<span class="field-state-name">${escapeHtml(label)}</span>
|
|
60
|
+
<span class="field-state-chips">
|
|
61
|
+
<span class="chip state-content ${content ?? "none"}" data-testid="field-state-content">${escapeHtml(content === undefined ? "No proposal" : CONTENT_TEXT[content])}</span>
|
|
62
|
+
${lifecycle === undefined ? "" : `<span class="chip state-lifecycle ${lifecycle}" data-testid="field-state-lifecycle">${escapeHtml(LIFECYCLE_TEXT[lifecycle])}${carried ? " · carried forward" : ""}</span>`}
|
|
63
|
+
</span>
|
|
64
|
+
${content === undefined ? `<span class="field-state-note">No value was proposed. That is not the same as the source not stating one.</span>` : ""}
|
|
65
|
+
${signals.map((signal) => `<span class="field-state-note signal" data-testid="field-state-signal">${escapeHtml(signal)}</span>`).join("")}
|
|
66
|
+
</li>`;
|
|
67
|
+
}).join("");
|
|
68
|
+
const unread = states.filter((state) => state.content === "not_covered").length;
|
|
69
|
+
return `
|
|
70
|
+
<section class="field-states" data-testid="field-states" aria-label="Field states">
|
|
71
|
+
<h2>Field states</h2>
|
|
72
|
+
${unread > 0 ? `<p class="field-states-lead" data-testid="field-states-incomplete">${unread} field${unread === 1 ? " was" : "s were"} not read: the extraction stopped short. ${unread === 1 ? "It has" : "They have"} no value to review, and that does not mean the source lacks ${unread === 1 ? "it" : "them"}.</p>` : ""}
|
|
73
|
+
<ul>${rows}</ul>
|
|
74
|
+
</section>
|
|
75
|
+
`;
|
|
76
|
+
}
|
|
77
|
+
function escapeHtml(value) {
|
|
78
|
+
return String(value)
|
|
79
|
+
.replaceAll("&", "&")
|
|
80
|
+
.replaceAll("<", "<")
|
|
81
|
+
.replaceAll(">", ">")
|
|
82
|
+
.replaceAll('"', """)
|
|
83
|
+
.replaceAll("'", "'");
|
|
84
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ReviewQueueSessionState } from "./review-queue-session.js";
|
|
2
2
|
import { reviewResourceApiVersion, type ReviewItem } from "../review-resource.js";
|
|
3
|
-
import { type ExtractionEnvelopeImportResult } from "../extraction-envelope.js";
|
|
3
|
+
import { type ExtractionEnvelopeImport, type ExtractionEnvelopeImportResult } from "../extraction-envelope.js";
|
|
4
4
|
/**
|
|
5
5
|
* The durable attestation record. Serializable; a consumer stores it beside the
|
|
6
6
|
* queue when the round opens and presents it, unchanged, at every later
|
|
@@ -78,7 +78,7 @@ export declare function bindReviewQueue(snapshot: ReviewQueueSessionState, optio
|
|
|
78
78
|
*/
|
|
79
79
|
export declare function validateReviewQueueBinding(binding: ReviewQueueBinding, snapshot: ReviewQueueSessionState, options?: ValidateReviewQueueBindingOptions): ReviewQueueBindingIssue[];
|
|
80
80
|
export declare function assertReviewQueueBinding(binding: ReviewQueueBinding, snapshot: ReviewQueueSessionState, options?: ValidateReviewQueueBindingOptions): void;
|
|
81
|
-
export type ReviewQueueExtractionIssueCode = "import-not-grounded" | "empty-queue" | "item-missing-from-queue" | "item-not-in-extraction" | "item-diverges-from-extraction";
|
|
81
|
+
export type ReviewQueueExtractionIssueCode = "import-invalid" | "import-not-grounded" | "empty-queue" | "item-missing-from-queue" | "item-not-in-extraction" | "item-diverges-from-extraction";
|
|
82
82
|
export interface ReviewQueueExtractionIssue {
|
|
83
83
|
readonly code: ReviewQueueExtractionIssueCode;
|
|
84
84
|
readonly message: string;
|
|
@@ -124,4 +124,47 @@ export declare class UnattestedExtractionQueueError extends Error {
|
|
|
124
124
|
* cross-check the label.
|
|
125
125
|
*/
|
|
126
126
|
export declare function validateReviewQueueAgainstExtractionImport(items: readonly ReviewItem[], importResult: ExtractionEnvelopeImportResult): ReviewQueueExtractionIssue[];
|
|
127
|
+
/**
|
|
128
|
+
* What a reload path can say about a stored queue and the extraction import it
|
|
129
|
+
* came from:
|
|
130
|
+
*
|
|
131
|
+
* - `attested`: an import record was supplied and the queue matches it (see
|
|
132
|
+
* {@link validateReviewQueueAgainstExtractionImport}).
|
|
133
|
+
* - `diverges`: an import record was supplied and the queue does not match it,
|
|
134
|
+
* or the record itself is invalid. The queue must not be reviewed.
|
|
135
|
+
* - `unverified`: items carry the extraction-envelope binding but no import
|
|
136
|
+
* record was supplied, so nothing checked them. A rival value deleted from
|
|
137
|
+
* the stored queue would not show; every surface says so.
|
|
138
|
+
* - `not-extraction`: no item carries the binding and no record was supplied;
|
|
139
|
+
* there is no import to check against.
|
|
140
|
+
*
|
|
141
|
+
* Detection reads the binding the import writes on every item and candidate.
|
|
142
|
+
* A writer that strips it from every item also strips every other extraction
|
|
143
|
+
* fact from the queue; keeping the stored import record beside the queue is
|
|
144
|
+
* what makes a reload path able to check it.
|
|
145
|
+
*/
|
|
146
|
+
export type ReviewQueueExtractionAttestation = {
|
|
147
|
+
readonly state: "attested";
|
|
148
|
+
} | {
|
|
149
|
+
readonly state: "not-extraction";
|
|
150
|
+
} | {
|
|
151
|
+
readonly state: "unverified";
|
|
152
|
+
readonly itemNames: readonly string[];
|
|
153
|
+
readonly message: string;
|
|
154
|
+
} | {
|
|
155
|
+
readonly state: "diverges";
|
|
156
|
+
readonly issues: readonly ReviewQueueExtractionIssue[];
|
|
157
|
+
readonly message: string;
|
|
158
|
+
};
|
|
159
|
+
/** Whether a ReviewItem, or any of its candidates, carries the extraction-envelope binding. */
|
|
160
|
+
export declare function reviewItemCarriesExtractionBinding(item: ReviewItem): boolean;
|
|
161
|
+
/**
|
|
162
|
+
* Check a queue a reload path is about to present against its extraction
|
|
163
|
+
* import. Pass the import record (or import result) stored beside the queue;
|
|
164
|
+
* pass `undefined` when the path has none, and the result says whether that
|
|
165
|
+
* leaves the queue unverified.
|
|
166
|
+
*/
|
|
167
|
+
export declare function attestReviewQueueExtraction(items: readonly ReviewItem[], extractionImport: ExtractionEnvelopeImport | ExtractionEnvelopeImportResult | undefined): ReviewQueueExtractionAttestation;
|
|
168
|
+
/** The notice every surface shows for an `unverified` queue. */
|
|
169
|
+
export declare function unverifiedExtractionQueueSentence(itemCount: number): string;
|
|
127
170
|
export declare function assertReviewQueueAgainstExtractionImport(items: readonly ReviewItem[], importResult: ExtractionEnvelopeImportResult): void;
|
|
@@ -276,6 +276,51 @@ export function validateReviewQueueAgainstExtractionImport(items, importResult)
|
|
|
276
276
|
}
|
|
277
277
|
return issues;
|
|
278
278
|
}
|
|
279
|
+
const EXTRACTION_ENVELOPE_PRODUCER = "survey.kontourai.io/extraction-envelope";
|
|
280
|
+
/** Whether a ReviewItem, or any of its candidates, carries the extraction-envelope binding. */
|
|
281
|
+
export function reviewItemCarriesExtractionBinding(item) {
|
|
282
|
+
const bound = (producer) => producer !== undefined && producer !== null && typeof producer === "object" && Object.hasOwn(producer, EXTRACTION_ENVELOPE_PRODUCER);
|
|
283
|
+
return bound(item.metadata?.producer) || (Array.isArray(item.spec?.candidates) && item.spec.candidates.some((candidate) => bound(candidate?.producer)));
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Check a queue a reload path is about to present against its extraction
|
|
287
|
+
* import. Pass the import record (or import result) stored beside the queue;
|
|
288
|
+
* pass `undefined` when the path has none, and the result says whether that
|
|
289
|
+
* leaves the queue unverified.
|
|
290
|
+
*/
|
|
291
|
+
export function attestReviewQueueExtraction(items, extractionImport) {
|
|
292
|
+
if (extractionImport === undefined || extractionImport === null) {
|
|
293
|
+
const itemNames = items.filter(reviewItemCarriesExtractionBinding).map((item) => item.metadata.name);
|
|
294
|
+
return itemNames.length === 0
|
|
295
|
+
? { state: "not-extraction" }
|
|
296
|
+
: { state: "unverified", itemNames, message: unverifiedExtractionQueueSentence(itemNames.length) };
|
|
297
|
+
}
|
|
298
|
+
const record = isImportResult(extractionImport) ? extractionImport.record : extractionImport;
|
|
299
|
+
let issues;
|
|
300
|
+
try {
|
|
301
|
+
issues = validateReviewQueueAgainstExtractionImport(items, { record, reviewItems: [] });
|
|
302
|
+
}
|
|
303
|
+
catch (error) {
|
|
304
|
+
issues = [{
|
|
305
|
+
code: "import-invalid",
|
|
306
|
+
message: `The stored extraction import is invalid: ${error instanceof Error ? error.message : String(error)}`,
|
|
307
|
+
}];
|
|
308
|
+
}
|
|
309
|
+
return issues.length === 0
|
|
310
|
+
? { state: "attested" }
|
|
311
|
+
: {
|
|
312
|
+
state: "diverges",
|
|
313
|
+
issues,
|
|
314
|
+
message: `This review queue does not match the extraction import stored with it, so it cannot be reviewed: ${issues.map((issue) => issue.message).join(" ")}`,
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
/** The notice every surface shows for an `unverified` queue. */
|
|
318
|
+
export function unverifiedExtractionQueueSentence(itemCount) {
|
|
319
|
+
return `Unverified queue: ${itemCount === 1 ? "1 item came" : `${itemCount} items came`} from an extraction import, but no import record was stored with this queue, so its items were not checked against it. A rival value removed from the stored queue would not be shown.`;
|
|
320
|
+
}
|
|
321
|
+
function isImportResult(value) {
|
|
322
|
+
return typeof value === "object" && value !== null && "record" in value && !("kind" in value);
|
|
323
|
+
}
|
|
279
324
|
export function assertReviewQueueAgainstExtractionImport(items, importResult) {
|
|
280
325
|
const issues = validateReviewQueueAgainstExtractionImport(items, importResult);
|
|
281
326
|
if (issues.length > 0) {
|
|
@@ -57,6 +57,14 @@ export interface ReviewItemPresentation {
|
|
|
57
57
|
* disproven: every decision surface shows them. Empty for other items.
|
|
58
58
|
*/
|
|
59
59
|
readonly excludedProposals: readonly ExcludedProposalPresentation[];
|
|
60
|
+
/**
|
|
61
|
+
* Stored `excludedProposals` entries that {@link excludedProposals} cannot
|
|
62
|
+
* show: malformed entries, or every entry of an item whose envelope binding
|
|
63
|
+
* is not intact. Hiding a rival is the unsafe direction, so every surface
|
|
64
|
+
* that shows excluded proposals shows this notice too. Absent when every
|
|
65
|
+
* stored entry is shown.
|
|
66
|
+
*/
|
|
67
|
+
readonly excludedProposalsUnreadable?: ExcludedProposalsUnreadable;
|
|
60
68
|
/**
|
|
61
69
|
* For an envelope-imported item, whether its import checked excerpts
|
|
62
70
|
* against the prepared artifact. Absent for other items, and for an item
|
|
@@ -71,6 +79,16 @@ export interface ExcludedProposalPresentation {
|
|
|
71
79
|
readonly locator: string;
|
|
72
80
|
readonly excerpt: string;
|
|
73
81
|
}
|
|
82
|
+
export interface ExcludedProposalsUnreadable {
|
|
83
|
+
/**
|
|
84
|
+
* `malformed-entries`: some stored entries are not well-formed.
|
|
85
|
+
* `binding-broken`: the item's extraction binding is not intact, so none of
|
|
86
|
+
* its stored entries can be vouched for.
|
|
87
|
+
*/
|
|
88
|
+
readonly reason: "malformed-entries" | "binding-broken";
|
|
89
|
+
/** How many stored entries are not shown; absent when the stored field is not a list. */
|
|
90
|
+
readonly count?: number;
|
|
91
|
+
}
|
|
74
92
|
export interface ReviewResultPresentation {
|
|
75
93
|
readonly result: ReviewWorkbenchResult;
|
|
76
94
|
readonly item?: ReviewItem;
|
|
@@ -131,6 +149,11 @@ export declare function buildInterpretationReadingPresentation(source: Interpret
|
|
|
131
149
|
* text (the MCP item, the recorded decision prompt). Undefined when none.
|
|
132
150
|
*/
|
|
133
151
|
export declare function excludedProposalsSentence(excluded: readonly ExcludedProposalPresentation[]): string | undefined;
|
|
152
|
+
/**
|
|
153
|
+
* One sentence saying that stored excluded proposals cannot be shown, for
|
|
154
|
+
* every surface that shows excluded proposals. Undefined when none are hidden.
|
|
155
|
+
*/
|
|
156
|
+
export declare function excludedProposalsUnreadableSentence(unreadable: ExcludedProposalsUnreadable | undefined): string | undefined;
|
|
134
157
|
export interface CandidateVerificationRecordPresentation {
|
|
135
158
|
readonly recordId: string;
|
|
136
159
|
readonly verifierId: string;
|
|
@@ -69,20 +69,39 @@ function extractionEnvelopeBinding(item) {
|
|
|
69
69
|
});
|
|
70
70
|
return bound ? binding : undefined;
|
|
71
71
|
}
|
|
72
|
-
/**
|
|
72
|
+
/**
|
|
73
|
+
* The well-formed `excludedProposals` entries of a bound envelope item, and
|
|
74
|
+
* what the item stores but cannot show: malformed entries, or every entry
|
|
75
|
+
* when the binding is not intact. Nothing stored is dropped silently.
|
|
76
|
+
*/
|
|
73
77
|
function excludedProposalsOf(item, binding, adapter) {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
78
|
+
const metadata = item.metadata?.producer?.[EXTRACTION_ENVELOPE_PRODUCER];
|
|
79
|
+
if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
|
|
80
|
+
// Candidates that still carry the envelope binding came from an import,
|
|
81
|
+
// whose item metadata is gone or replaced: what it stored is unknown.
|
|
82
|
+
const candidatesBound = item.spec.candidates.some((candidate) => candidate.producer?.[EXTRACTION_ENVELOPE_PRODUCER] !== undefined);
|
|
83
|
+
return candidatesBound ? { excludedProposals: [], excludedProposalsUnreadable: { reason: "binding-broken" } } : { excludedProposals: [] };
|
|
84
|
+
}
|
|
85
|
+
const stored = metadata.excludedProposals;
|
|
86
|
+
if (stored === undefined)
|
|
87
|
+
return { excludedProposals: [] };
|
|
88
|
+
const count = Array.isArray(stored) ? stored.length : undefined;
|
|
89
|
+
if (!binding)
|
|
90
|
+
return count === 0 ? { excludedProposals: [] } : { excludedProposals: [], excludedProposalsUnreadable: { reason: "binding-broken", ...(count !== undefined ? { count } : {}) } };
|
|
91
|
+
if (!Array.isArray(stored))
|
|
92
|
+
return { excludedProposals: [], excludedProposalsUnreadable: { reason: "malformed-entries" } };
|
|
93
|
+
const proposed = item.spec.candidates.find((candidate) => candidate.role === "proposed");
|
|
94
|
+
const excludedProposals = stored.flatMap((entry) => {
|
|
77
95
|
if (!entry || typeof entry !== "object" || Array.isArray(entry))
|
|
78
96
|
return [];
|
|
79
97
|
const e = entry;
|
|
80
98
|
if (!Number.isSafeInteger(e.proposalIndex) || typeof e.locator !== "string" || typeof e.excerpt !== "string" || !("value" in e))
|
|
81
99
|
return [];
|
|
82
|
-
const proposed = item.spec.candidates.find((candidate) => candidate.role === "proposed");
|
|
83
100
|
const valueText = (proposed ? adapter.summarizeValue?.(e.value, { item, candidate: proposed, value: e.value }) : undefined) ?? formatValue(e.value);
|
|
84
101
|
return [{ proposalIndex: e.proposalIndex, value: e.value, valueText, locator: e.locator, excerpt: e.excerpt }];
|
|
85
102
|
});
|
|
103
|
+
const unreadable = stored.length - excludedProposals.length;
|
|
104
|
+
return { excludedProposals, ...(unreadable > 0 ? { excludedProposalsUnreadable: { reason: "malformed-entries", count: unreadable } } : {}) };
|
|
86
105
|
}
|
|
87
106
|
/**
|
|
88
107
|
* One sentence naming every excluded proposal, for surfaces that speak in
|
|
@@ -94,6 +113,20 @@ export function excludedProposalsSentence(excluded) {
|
|
|
94
113
|
const named = excluded.map((entry) => `${entry.valueText} (proposal ${entry.proposalIndex}, ${entry.locator})`).join(", ");
|
|
95
114
|
return `${excluded.length === 1 ? "Another proposed value was" : `${excluded.length} other proposed values were`} excluded at import because the source text at the cited span is not the excerpt: ${named}. Unverifiable is not disproven.`;
|
|
96
115
|
}
|
|
116
|
+
/**
|
|
117
|
+
* One sentence saying that stored excluded proposals cannot be shown, for
|
|
118
|
+
* every surface that shows excluded proposals. Undefined when none are hidden.
|
|
119
|
+
*/
|
|
120
|
+
export function excludedProposalsUnreadableSentence(unreadable) {
|
|
121
|
+
if (!unreadable)
|
|
122
|
+
return undefined;
|
|
123
|
+
const { count } = unreadable;
|
|
124
|
+
const entries = count === undefined ? "Stored excluded proposals are" : `${count} stored excluded ${count === 1 ? "entry is" : "entries are"}`;
|
|
125
|
+
const why = unreadable.reason === "binding-broken"
|
|
126
|
+
? "not shown because this item's extraction binding is broken"
|
|
127
|
+
: "unreadable and not shown";
|
|
128
|
+
return `${entries} ${why}. Another value may have been proposed for this field.`;
|
|
129
|
+
}
|
|
97
130
|
const VERIFICATION_RESULT_TEXT = {
|
|
98
131
|
supported: "supported",
|
|
99
132
|
contradicted: "contradicted",
|
|
@@ -167,7 +200,7 @@ export function buildReviewItemPresentation(item, adapter = {}) {
|
|
|
167
200
|
function extractionPresentation(item, adapter) {
|
|
168
201
|
const binding = extractionEnvelopeBinding(item);
|
|
169
202
|
return {
|
|
170
|
-
|
|
203
|
+
...excludedProposalsOf(item, binding, adapter),
|
|
171
204
|
...(binding ? { excerptVerification: binding.excerptVerification === "verified" ? "verified" : "unverified" } : {}),
|
|
172
205
|
};
|
|
173
206
|
}
|
|
@@ -224,12 +257,20 @@ export function buildReviewResultPresentation(result, item, adapter = {}) {
|
|
|
224
257
|
selectedValueText: selectedCandidate && item
|
|
225
258
|
? buildReviewCandidatePresentation(item, selectedCandidate, adapter, targetLabel).valueText
|
|
226
259
|
: result.selectedDisplayValue,
|
|
227
|
-
applyMeaning: result.
|
|
228
|
-
? "Saved decision applies proposed
|
|
229
|
-
:
|
|
260
|
+
applyMeaning: result.decision === "select-proposed"
|
|
261
|
+
? "Saved decision applies the chosen value; the other proposed values were seen and not chosen"
|
|
262
|
+
: result.selectedCandidateRole === "proposed"
|
|
263
|
+
? "Saved decision applies proposed value"
|
|
264
|
+
: "Saved decision keeps current value",
|
|
230
265
|
reviewItemLink: item && itemContext ? adapter.linkForReviewItem?.(item, itemContext) : undefined,
|
|
231
266
|
traceRefs: item
|
|
232
|
-
?
|
|
267
|
+
? [
|
|
268
|
+
...traceRefsForResult(item, result, selectedCandidate, adapter),
|
|
269
|
+
// A choice between conflicting values names the values it passed over.
|
|
270
|
+
...(result.decision === "select-proposed"
|
|
271
|
+
? result.unselectedCandidates.filter((candidate) => candidate.role === "proposed").flatMap((candidate) => withTraceLinks([{ label: "Not chosen candidate", value: candidate.id, kind: "candidate" }], { item, candidate }, adapter))
|
|
272
|
+
: []),
|
|
273
|
+
]
|
|
233
274
|
: [{ label: "Survey ReviewItem", value: result.reviewItemName, kind: "review-item" }],
|
|
234
275
|
};
|
|
235
276
|
}
|
|
@@ -1,5 +1,16 @@
|
|
|
1
|
-
import { type ReviewCandidate, type ReviewItem, type ReviewSession, type ReviewSessionEvent, type ReviewSessionEventSpec } from "../../src/review-resource.js";
|
|
2
|
-
export type
|
|
1
|
+
import { type ReviewCandidate, type ReviewItem, type ReviewSession, type ReviewSessionEvent, type ReviewSessionEventSpec, type ReviewSessionPresentation, type ReviewSessionSampling } from "../../src/review-resource.js";
|
|
2
|
+
export type { ReviewSessionPresentation, ReviewSessionSampling } from "../../src/review-resource.js";
|
|
3
|
+
/**
|
|
4
|
+
* The decisions a reviewer can record on a ReviewItem.
|
|
5
|
+
*
|
|
6
|
+
* `select-proposed` chooses one value of a conflict (two or more `proposed`
|
|
7
|
+
* candidates) by candidate id, which the state carries beside the decision
|
|
8
|
+
* (`selectedCandidateId` / `selectedCandidateIdsByItemName`). It is its own
|
|
9
|
+
* kind rather than `accept-proposed` with an id so that the record always says
|
|
10
|
+
* a rival was seen and not chosen, and `accept-proposed` keeps its meaning:
|
|
11
|
+
* accept the one proposed value, refused on a conflict.
|
|
12
|
+
*/
|
|
13
|
+
export type ReviewWorkbenchDecision = "accept-proposed" | "select-proposed" | "keep-current" | "reject-proposed" | "could-not-confirm";
|
|
3
14
|
export type ReviewQueueRowStatus = "pending" | "in-review" | "resolved" | "rejected" | "could-not-confirm" | "escalated";
|
|
4
15
|
export declare const reviewWorkbenchSessionStorageKey = "kontourai.survey.review-workbench.session-events.v1";
|
|
5
16
|
export declare const defaultReviewSessionName = "review-workbench-session";
|
|
@@ -16,6 +27,15 @@ export interface ReviewWorkbenchState {
|
|
|
16
27
|
*/
|
|
17
28
|
readonly editedValue?: unknown;
|
|
18
29
|
readonly attemptEvidenceIds?: readonly string[];
|
|
30
|
+
/**
|
|
31
|
+
* The candidate a `select-proposed` decision chose. Required for that
|
|
32
|
+
* decision and ignored for every other one.
|
|
33
|
+
*/
|
|
34
|
+
readonly selectedCandidateId?: string;
|
|
35
|
+
/** The session's presentation; recorded on the decision. See {@link ReviewSessionPresentation}. */
|
|
36
|
+
readonly presentation?: ReviewSessionPresentation;
|
|
37
|
+
/** How the session's items were chosen; recorded on the decision. */
|
|
38
|
+
readonly sampling?: ReviewSessionSampling;
|
|
19
39
|
}
|
|
20
40
|
export interface ReviewQueueSessionState {
|
|
21
41
|
readonly items: readonly ReviewItem[];
|
|
@@ -32,7 +52,53 @@ export interface ReviewQueueSessionState {
|
|
|
32
52
|
*/
|
|
33
53
|
readonly editedValuesByItemName?: Readonly<Record<string, unknown>>;
|
|
34
54
|
readonly attemptEvidenceIdsByItemName?: Readonly<Record<string, readonly string[]>>;
|
|
55
|
+
/**
|
|
56
|
+
* The candidate each `select-proposed` decision chose, keyed by ReviewItem
|
|
57
|
+
* name. Additive/optional: sessions stored before this field existed have no
|
|
58
|
+
* `select-proposed` decision and never read it.
|
|
59
|
+
*/
|
|
60
|
+
readonly selectedCandidateIdsByItemName?: Readonly<Record<string, string>>;
|
|
61
|
+
/**
|
|
62
|
+
* Session-level presentation, set when the session is opened. Every
|
|
63
|
+
* decision event is stamped with it and replay refuses a mismatch; a
|
|
64
|
+
* `ReviewQueueBinding` also covers it. Without a binding, an edit to both the
|
|
65
|
+
* snapshot and every event stamp is not detectable. Absent: scores are shown
|
|
66
|
+
* and decisions record no presentation.
|
|
67
|
+
*/
|
|
68
|
+
readonly presentation?: ReviewSessionPresentation;
|
|
69
|
+
/** How the session's items were chosen. Absent: decisions record no sampling. */
|
|
70
|
+
readonly sampling?: ReviewSessionSampling;
|
|
35
71
|
}
|
|
72
|
+
/** Whether the session hides confidence and verifier results from its reviewer. */
|
|
73
|
+
export declare function isScoreBlind(session: {
|
|
74
|
+
readonly presentation?: ReviewSessionPresentation;
|
|
75
|
+
}): boolean;
|
|
76
|
+
/**
|
|
77
|
+
* Throws unless the session's presentation and sampling are well formed:
|
|
78
|
+
* `scoreBlind` a boolean, and a `random-audit` sampling with a rate in (0, 1]
|
|
79
|
+
* and a non-empty seed. Unknown keys are refused so a decision never records
|
|
80
|
+
* a condition nothing reads.
|
|
81
|
+
*/
|
|
82
|
+
export declare function assertReviewSessionConditions(session: {
|
|
83
|
+
readonly presentation?: unknown;
|
|
84
|
+
readonly sampling?: unknown;
|
|
85
|
+
}): void;
|
|
86
|
+
/**
|
|
87
|
+
* The session conditions a decision event carries, or `undefined` for a
|
|
88
|
+
* session that declares none. Stamped on every decision event so replay can
|
|
89
|
+
* refuse events recorded under other conditions than the snapshot's: sighted
|
|
90
|
+
* events replayed over a blind snapshot, or a snapshot whose presentation was
|
|
91
|
+
* changed after decisions were recorded.
|
|
92
|
+
*/
|
|
93
|
+
export declare function sessionConditionsStamp(session: {
|
|
94
|
+
readonly presentation?: ReviewSessionPresentation;
|
|
95
|
+
readonly sampling?: ReviewSessionSampling;
|
|
96
|
+
}): {
|
|
97
|
+
presentation?: ReviewSessionPresentation;
|
|
98
|
+
sampling?: ReviewSessionSampling;
|
|
99
|
+
} | undefined;
|
|
100
|
+
export declare function assertAuditRate(rate: unknown): asserts rate is number;
|
|
101
|
+
export declare function assertAuditSeed(seed: unknown): asserts seed is string;
|
|
36
102
|
export interface ReviewSessionSummary {
|
|
37
103
|
readonly accepted: number;
|
|
38
104
|
readonly keptCurrent: number;
|
|
@@ -48,6 +114,12 @@ export declare const workbenchDecisionDefinitions: {
|
|
|
48
114
|
candidateRole: "proposed";
|
|
49
115
|
status: "verified";
|
|
50
116
|
};
|
|
117
|
+
"select-proposed": {
|
|
118
|
+
label: string;
|
|
119
|
+
effect: string;
|
|
120
|
+
candidateRole: "proposed";
|
|
121
|
+
status: "verified";
|
|
122
|
+
};
|
|
51
123
|
"keep-current": {
|
|
52
124
|
label: string;
|
|
53
125
|
effect: string;
|
|
@@ -109,14 +181,30 @@ export declare function keepActionDecision(item: ReviewItem, flaggedWrong: boole
|
|
|
109
181
|
* {@link decisionCandidateId} is `undefined` for it, so the decision, its
|
|
110
182
|
* session events and the canonical projection name no candidate.
|
|
111
183
|
*/
|
|
112
|
-
export declare function candidateForDecision(item: ReviewItem, decision: ReviewWorkbenchDecision): ReviewCandidate;
|
|
184
|
+
export declare function candidateForDecision(item: ReviewItem, decision: ReviewWorkbenchDecision, selectedCandidateId?: string): ReviewCandidate;
|
|
185
|
+
/**
|
|
186
|
+
* Why a `select-proposed` decision naming `candidateId` cannot be recorded on
|
|
187
|
+
* this item, or `undefined` when it can.
|
|
188
|
+
*
|
|
189
|
+
* The choice must name, by a unique id, one of the item's `proposed`
|
|
190
|
+
* candidates, and the item must hold at least two of them: choosing "over a
|
|
191
|
+
* rival" on an item with no rival would record a conflict that never existed
|
|
192
|
+
* (use `accept-proposed` there). Proposals excluded at import are not
|
|
193
|
+
* candidates, so they can never be chosen.
|
|
194
|
+
*/
|
|
195
|
+
export declare function conflictSelectionIssue(item: ReviewItem, candidateId: string | undefined): string | undefined;
|
|
196
|
+
/**
|
|
197
|
+
* The proposed candidates a `select-proposed` decision saw and did not choose,
|
|
198
|
+
* in item order. Empty for every other decision.
|
|
199
|
+
*/
|
|
200
|
+
export declare function unchosenProposedCandidates(item: ReviewItem, decision: ReviewWorkbenchDecision, selectedCandidateId: string | undefined): ReviewCandidate[];
|
|
113
201
|
/** Whether a decision on this item selects no candidate at all (see {@link candidateForDecision}). */
|
|
114
202
|
export declare function decisionSelectsNoCandidate(item: ReviewItem, decision: ReviewWorkbenchDecision): boolean;
|
|
115
203
|
/**
|
|
116
204
|
* The candidate id a decision records: the selected candidate's, or
|
|
117
205
|
* `undefined` when the decision selects no candidate.
|
|
118
206
|
*/
|
|
119
|
-
export declare function decisionCandidateId(item: ReviewItem, decision: ReviewWorkbenchDecision): string | undefined;
|
|
207
|
+
export declare function decisionCandidateId(item: ReviewItem, decision: ReviewWorkbenchDecision, selectedCandidateId?: string): string | undefined;
|
|
120
208
|
/**
|
|
121
209
|
* The value that should actually be applied for a decision: the reviewer's inline
|
|
122
210
|
* edit when one was made for an accept-proposed decision, otherwise the selected
|
|
@@ -125,7 +213,7 @@ export declare function decisionCandidateId(item: ReviewItem, decision: ReviewWo
|
|
|
125
213
|
* this rule; this helper exists for callers deriving the value from raw session
|
|
126
214
|
* state directly.
|
|
127
215
|
*/
|
|
128
|
-
export declare function effectiveValueForDecision(item: ReviewItem, decision: ReviewWorkbenchDecision, editedValue?: unknown): unknown;
|
|
216
|
+
export declare function effectiveValueForDecision(item: ReviewItem, decision: ReviewWorkbenchDecision, editedValue?: unknown, selectedCandidateId?: string): unknown;
|
|
129
217
|
export declare function selectedCandidateRole(state: ReviewWorkbenchState): ReviewCandidate["role"] | undefined;
|
|
130
218
|
export declare function buildReviewSessionResource(session: ReviewQueueSessionState, events?: readonly ReviewSessionEvent[], sessionName?: string): ReviewSession;
|
|
131
219
|
export declare function buildReviewSessionEvents(session: ReviewQueueSessionState, sessionName?: string): ReviewSessionEvent[];
|