@openpond/evals 0.11.16 → 0.11.17

package/dist/types/human-review/service.d.ts

@@ -0,0 +1,878 @@
+import { type HumanReview, type HumanReviewCommand, type HumanReviewRequest } from "./contracts.js";
+import type { HumanReviewAuthority, HumanReviewContext, HumanReviewRepository } from "./repository.js";
+export declare function createHumanReviewService(repository: HumanReviewRepository, authority: HumanReviewAuthority, now?: () => string): {
+    command: (context: HumanReviewContext, raw: HumanReviewCommand) => Promise<HumanReview>;
+    get: (context: HumanReviewContext, id: string, revision?: number) => Promise<{
+        schemaVersion: "openpond.humanReview.v1";
+        id: string;
+        revision: number;
+        scope: string;
+        projectId: string;
+        createdBy: string;
+        kind: "author" | "execute" | "grade";
+        title: string;
+        authorPublication: {
+            proposalHash: string;
+            dataset: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            };
+            operationId: string;
+            publishedBy: string;
+            publishedAt: string;
+        } | null;
+        evidence: {
+            dataset: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            };
+            grader: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            };
+            graderBinding: {
+                id: string;
+                version: string;
+                contentHash: string;
+            };
+            rubric: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            };
+            attempts: {
+                experimentId: string;
+                attemptId: string;
+                taskId: string;
+                targetId: string;
+                output: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                };
+                trace: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                } | null;
+                familyKey: string;
+                split: "train" | "validation" | "test" | "frozen_eval";
+                localSource?: {
+                    kind: "owner_attested_local";
+                    ownerActorId: string;
+                    teamId: string;
+                    executionHash: string;
+                    configurationHash: string;
+                    retainedAttemptHash: string;
+                    traceHash: string;
+                    publicationCutoffHash: string;
+                } | {
+                    ownerActorId: string;
+                    teamId: string;
+                    executionHash: string;
+                    configurationHash: string;
+                    retainedAttemptHash: string;
+                    traceHash: string;
+                    kind: "native_local";
+                } | undefined;
+                recordedSource?: {
+                    id: string;
+                    snapshotHash: string;
+                    boundaryId: string;
+                    boundaryRevisionHash: string;
+                } | undefined;
+            }[];
+            publication?: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            } | undefined;
+            taskIds?: string[] | undefined;
+        };
+        form: {
+            schemaVersion: "openpond.humanForm.v1";
+            mode: "individual" | "pairwise";
+            instructions: string;
+            criteria: ({
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "score";
+                minimum: number;
+                maximum: number;
+                step: number;
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+            } | {
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "category";
+                options: string[];
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+            } | {
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "boolean";
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+            } | {
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "text";
+                maximumLength: number;
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+                minimumLength?: number | undefined;
+            } | {
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "preference";
+                allowTie: boolean;
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+                requireStrength?: boolean | undefined;
+            })[];
+        };
+        policy: {
+            approval: "none" | "manager";
+            minimumRaters: number;
+            managerIds: string[];
+            reviewerIds: string[];
+            queueClaim: boolean;
+            teamVisible: boolean;
+            distinctManager?: boolean | undefined;
+            independentRaters?: boolean | undefined;
+            blindPair?: boolean | undefined;
+        };
+        status: "queued" | "assigned" | "in_progress" | "submitted" | "returned" | "accepted" | "rejected" | "cancelled";
+        assigneeId: string | null;
+        generation: number;
+        drafts: {
+            actorId: string;
+            generation: number;
+            answers: {
+                criterionId: string;
+                value: string | number | boolean | null;
+                abstain: boolean;
+                note: string;
+                strength?: "slight" | "moderate" | "strong" | undefined;
+            }[];
+            note: string;
+            updatedAt: string;
+            taskProposal?: {
+                schemaVersion: "openpond.humanTaskProposal.v1";
+                tasks: {
+                    taskId: string;
+                    input: Record<string, unknown>;
+                    policyVisibleContext: Record<string, unknown>;
+                    expectedOutput?: Record<string, unknown> | null | undefined;
+                    tags?: string[] | undefined;
+                }[];
+                contentHash: string;
+            } | undefined;
+        }[];
+        submissions: {
+            id: string;
+            revision: number;
+            actorId: string;
+            generation: number;
+            evidenceHash: string;
+            formHash: string;
+            answers: {
+                criterionId: string;
+                value: string | number | boolean | null;
+                abstain: boolean;
+                note: string;
+                strength?: "slight" | "moderate" | "strong" | undefined;
+            }[];
+            note: string;
+            submittedAt: string;
+            supersedes: string | null;
+            contentHash: string;
+            taskProposal?: {
+                schemaVersion: "openpond.humanTaskProposal.v1";
+                tasks: {
+                    taskId: string;
+                    input: Record<string, unknown>;
+                    policyVisibleContext: Record<string, unknown>;
+                    expectedOutput?: Record<string, unknown> | null | undefined;
+                    tags?: string[] | undefined;
+                }[];
+                contentHash: string;
+            } | undefined;
+        }[];
+        decisions: {
+            id: string;
+            revision: number;
+            actorId: string;
+            outcome: "accept" | "reject" | "request_changes";
+            submissions: {
+                id: string;
+                contentHash: string;
+            }[];
+            answers: {
+                criterionId: string;
+                value: string | number | boolean | null;
+                abstain: boolean;
+                note: string;
+                strength?: "slight" | "moderate" | "strong" | undefined;
+            }[];
+            note: string;
+            decidedAt: string;
+            contentHash: string;
+            taskProposalHash?: string | undefined;
+        }[];
+        audit: {
+            actorId: string;
+            action: "claim" | "reassign" | "cancel";
+            priorAssigneeId: string | null;
+            assigneeId: string | null;
+            generation: number;
+            note: string;
+            occurredAt: string;
+        }[];
+        execution: {
+            executionId: string | null;
+            operationId: string;
+            boundBy: string;
+            configurationHash: string;
+            generation: number;
+            result: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            } | null;
+        } | null;
+        createdAt: string;
+        updatedAt: string;
+        contentHash: string;
+        dueAt?: string | undefined;
+    }>;
+    inbox: (context: HumanReviewContext, query: Extract<HumanReviewRequest, {
+        endpoint: "inbox";
+    }>) => Promise<{
+        items: {
+            schemaVersion: "openpond.humanReview.v1";
+            id: string;
+            revision: number;
+            scope: string;
+            projectId: string;
+            createdBy: string;
+            kind: "author" | "execute" | "grade";
+            title: string;
+            authorPublication: {
+                proposalHash: string;
+                dataset: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                };
+                operationId: string;
+                publishedBy: string;
+                publishedAt: string;
+            } | null;
+            evidence: {
+                dataset: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                };
+                grader: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                };
+                graderBinding: {
+                    id: string;
+                    version: string;
+                    contentHash: string;
+                };
+                rubric: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                };
+                attempts: {
+                    experimentId: string;
+                    attemptId: string;
+                    taskId: string;
+                    targetId: string;
+                    output: {
+                        id: string;
+                        revision: number;
+                        contentHash: string;
+                    };
+                    trace: {
+                        id: string;
+                        revision: number;
+                        contentHash: string;
+                    } | null;
+                    familyKey: string;
+                    split: "train" | "validation" | "test" | "frozen_eval";
+                    localSource?: {
+                        kind: "owner_attested_local";
+                        ownerActorId: string;
+                        teamId: string;
+                        executionHash: string;
+                        configurationHash: string;
+                        retainedAttemptHash: string;
+                        traceHash: string;
+                        publicationCutoffHash: string;
+                    } | {
+                        ownerActorId: string;
+                        teamId: string;
+                        executionHash: string;
+                        configurationHash: string;
+                        retainedAttemptHash: string;
+                        traceHash: string;
+                        kind: "native_local";
+                    } | undefined;
+                    recordedSource?: {
+                        id: string;
+                        snapshotHash: string;
+                        boundaryId: string;
+                        boundaryRevisionHash: string;
+                    } | undefined;
+                }[];
+                publication?: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                } | undefined;
+                taskIds?: string[] | undefined;
+            };
+            form: {
+                schemaVersion: "openpond.humanForm.v1";
+                mode: "individual" | "pairwise";
+                instructions: string;
+                criteria: ({
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "score";
+                    minimum: number;
+                    maximum: number;
+                    step: number;
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                } | {
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "category";
+                    options: string[];
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                } | {
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "boolean";
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                } | {
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "text";
+                    maximumLength: number;
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                    minimumLength?: number | undefined;
+                } | {
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "preference";
+                    allowTie: boolean;
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                    requireStrength?: boolean | undefined;
+                })[];
+            };
+            policy: {
+                approval: "none" | "manager";
+                minimumRaters: number;
+                managerIds: string[];
+                reviewerIds: string[];
+                queueClaim: boolean;
+                teamVisible: boolean;
+                distinctManager?: boolean | undefined;
+                independentRaters?: boolean | undefined;
+                blindPair?: boolean | undefined;
+            };
+            status: "queued" | "assigned" | "in_progress" | "submitted" | "returned" | "accepted" | "rejected" | "cancelled";
+            assigneeId: string | null;
+            generation: number;
+            drafts: {
+                actorId: string;
+                generation: number;
+                answers: {
+                    criterionId: string;
+                    value: string | number | boolean | null;
+                    abstain: boolean;
+                    note: string;
+                    strength?: "slight" | "moderate" | "strong" | undefined;
+                }[];
+                note: string;
+                updatedAt: string;
+                taskProposal?: {
+                    schemaVersion: "openpond.humanTaskProposal.v1";
+                    tasks: {
+                        taskId: string;
+                        input: Record<string, unknown>;
+                        policyVisibleContext: Record<string, unknown>;
+                        expectedOutput?: Record<string, unknown> | null | undefined;
+                        tags?: string[] | undefined;
+                    }[];
+                    contentHash: string;
+                } | undefined;
+            }[];
+            submissions: {
+                id: string;
+                revision: number;
+                actorId: string;
+                generation: number;
+                evidenceHash: string;
+                formHash: string;
+                answers: {
+                    criterionId: string;
+                    value: string | number | boolean | null;
+                    abstain: boolean;
+                    note: string;
+                    strength?: "slight" | "moderate" | "strong" | undefined;
+                }[];
+                note: string;
+                submittedAt: string;
+                supersedes: string | null;
+                contentHash: string;
+                taskProposal?: {
+                    schemaVersion: "openpond.humanTaskProposal.v1";
+                    tasks: {
+                        taskId: string;
+                        input: Record<string, unknown>;
+                        policyVisibleContext: Record<string, unknown>;
+                        expectedOutput?: Record<string, unknown> | null | undefined;
+                        tags?: string[] | undefined;
+                    }[];
+                    contentHash: string;
+                } | undefined;
+            }[];
+            decisions: {
+                id: string;
+                revision: number;
+                actorId: string;
+                outcome: "accept" | "reject" | "request_changes";
+                submissions: {
+                    id: string;
+                    contentHash: string;
+                }[];
+                answers: {
+                    criterionId: string;
+                    value: string | number | boolean | null;
+                    abstain: boolean;
+                    note: string;
+                    strength?: "slight" | "moderate" | "strong" | undefined;
+                }[];
+                note: string;
+                decidedAt: string;
+                contentHash: string;
+                taskProposalHash?: string | undefined;
+            }[];
+            audit: {
+                actorId: string;
+                action: "claim" | "reassign" | "cancel";
+                priorAssigneeId: string | null;
+                assigneeId: string | null;
+                generation: number;
+                note: string;
+                occurredAt: string;
+            }[];
+            execution: {
+                executionId: string | null;
+                operationId: string;
+                boundBy: string;
+                configurationHash: string;
+                generation: number;
+                result: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                } | null;
+            } | null;
+            createdAt: string;
+            updatedAt: string;
+            contentHash: string;
+            dueAt?: string | undefined;
+        }[];
+        nextCursor: string | null;
+        actionCount: number;
+        approvalAvailable: boolean;
+    }>;
+    view: (context: HumanReviewContext, record: HumanReview) => Promise<{
+        evidence: {
+            dataset: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            };
+            grader: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            };
+            graderBinding: {
+                id: string;
+                version: string;
+                contentHash: string;
+            };
+            rubric: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            };
+            attempts: {
+                experimentId: string;
+                attemptId: string;
+                taskId: string;
+                targetId: string;
+                output: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                };
+                trace: {
+                    id: string;
+                    revision: number;
+                    contentHash: string;
+                } | null;
+                familyKey: string;
+                split: "train" | "validation" | "test" | "frozen_eval";
+                localSource?: {
+                    kind: "owner_attested_local";
+                    ownerActorId: string;
+                    teamId: string;
+                    executionHash: string;
+                    configurationHash: string;
+                    retainedAttemptHash: string;
+                    traceHash: string;
+                    publicationCutoffHash: string;
+                } | {
+                    ownerActorId: string;
+                    teamId: string;
+                    executionHash: string;
+                    configurationHash: string;
+                    retainedAttemptHash: string;
+                    traceHash: string;
+                    kind: "native_local";
+                } | undefined;
+                recordedSource?: {
+                    id: string;
+                    snapshotHash: string;
+                    boundaryId: string;
+                    boundaryRevisionHash: string;
+                } | undefined;
+            }[];
+            publication?: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            } | undefined;
+            taskIds?: string[] | undefined;
+        } | null;
+        evidenceHash: string;
+        formHash: string;
+        drafts: {
+            taskProposal: {
+                schemaVersion: "openpond.humanTaskProposal.v1";
+                tasks: {
+                    taskId: string;
+                    input: Record<string, unknown>;
+                    policyVisibleContext: Record<string, unknown>;
+                    expectedOutput?: Record<string, unknown> | null | undefined;
+                    tags?: string[] | undefined;
+                }[];
+                contentHash: string;
+            } | {
+                referenceRedacted: true;
+                tasks: {
+                    taskId: string;
+                    input: Record<string, unknown>;
+                    policyVisibleContext: Record<string, unknown>;
+                    tags?: string[] | undefined;
+                }[];
+                schemaVersion: "openpond.humanTaskProposal.v1";
+                contentHash: string;
+            } | undefined;
+            actorId: string;
+            generation: number;
+            answers: {
+                criterionId: string;
+                value: string | number | boolean | null;
+                abstain: boolean;
+                note: string;
+                strength?: "slight" | "moderate" | "strong" | undefined;
+            }[];
+            note: string;
+            updatedAt: string;
+        }[];
+        submissions: {
+            taskProposal: {
+                schemaVersion: "openpond.humanTaskProposal.v1";
+                tasks: {
+                    taskId: string;
+                    input: Record<string, unknown>;
+                    policyVisibleContext: Record<string, unknown>;
+                    expectedOutput?: Record<string, unknown> | null | undefined;
+                    tags?: string[] | undefined;
+                }[];
+                contentHash: string;
+            } | {
+                referenceRedacted: true;
+                tasks: {
+                    taskId: string;
+                    input: Record<string, unknown>;
+                    policyVisibleContext: Record<string, unknown>;
+                    tags?: string[] | undefined;
+                }[];
+                schemaVersion: "openpond.humanTaskProposal.v1";
+                contentHash: string;
+            } | undefined;
+            id: string;
+            revision: number;
+            actorId: string;
+            generation: number;
+            evidenceHash: string;
+            formHash: string;
+            answers: {
+                criterionId: string;
+                value: string | number | boolean | null;
+                abstain: boolean;
+                note: string;
+                strength?: "slight" | "moderate" | "strong" | undefined;
+            }[];
+            note: string;
+            submittedAt: string;
+            supersedes: string | null;
+            contentHash: string;
+        }[];
+        decisions: {
+            id: string;
+            revision: number;
+            actorId: string;
+            outcome: "accept" | "reject" | "request_changes";
+            submissions: {
+                id: string;
+                contentHash: string;
+            }[];
+            answers: {
+                criterionId: string;
+                value: string | number | boolean | null;
+                abstain: boolean;
+                note: string;
+                strength?: "slight" | "moderate" | "strong" | undefined;
+            }[];
+            note: string;
+            decidedAt: string;
+            contentHash: string;
+            taskProposalHash?: string | undefined;
+        }[];
+        audit: {
+            actorId: string;
+            action: "claim" | "reassign" | "cancel";
+            priorAssigneeId: string | null;
+            assigneeId: string | null;
+            generation: number;
+            note: string;
+            occurredAt: string;
+        }[];
+        createdBy: string;
+        permissions: {
+            claim: boolean;
+            owner: boolean;
+            manager: boolean;
+            reviewer: boolean;
+        };
+        schemaVersion: "openpond.humanReview.v1";
+        id: string;
+        revision: number;
+        scope: string;
+        projectId: string;
+        kind: "author" | "execute" | "grade";
+        title: string;
+        authorPublication: {
+            proposalHash: string;
+            dataset: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            };
+            operationId: string;
+            publishedBy: string;
+            publishedAt: string;
+        } | null;
+        form: {
+            schemaVersion: "openpond.humanForm.v1";
+            mode: "individual" | "pairwise";
+            instructions: string;
+            criteria: ({
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "score";
+                minimum: number;
+                maximum: number;
+                step: number;
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+            } | {
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "category";
+                options: string[];
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+            } | {
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "boolean";
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+            } | {
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "text";
+                maximumLength: number;
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+                minimumLength?: number | undefined;
+            } | {
+                id: string;
+                label: string;
+                instructions: string;
+                required: boolean;
+                allowAbstain: boolean;
+                kind: "preference";
+                allowTie: boolean;
+                requireRationale?: boolean | undefined;
+                anchors?: {
+                    value: string | number | boolean;
+                    description: string;
+                }[] | undefined;
+                requireStrength?: boolean | undefined;
+            })[];
+        };
+        policy: {
+            approval: "none" | "manager";
+            minimumRaters: number;
+            managerIds: string[];
+            reviewerIds: string[];
+            queueClaim: boolean;
+            teamVisible: boolean;
+            distinctManager?: boolean | undefined;
+            independentRaters?: boolean | undefined;
+            blindPair?: boolean | undefined;
+        };
+        status: "queued" | "assigned" | "in_progress" | "submitted" | "returned" | "accepted" | "rejected" | "cancelled";
+        assigneeId: string | null;
+        generation: number;
+        execution: {
+            executionId: string | null;
+            operationId: string;
+            boundBy: string;
+            configurationHash: string;
+            generation: number;
+            result: {
+                id: string;
+                revision: number;
+                contentHash: string;
+            } | null;
+        } | null;
+        createdAt: string;
+        updatedAt: string;
+        contentHash: string;
+        dueAt?: string | undefined;
+    }>;
+};
+//# sourceMappingURL=service.d.ts.map