@altnovatrail/contracts 0.7.8 → 0.7.9

package/novatrail/eval/v1/report_pb.d.ts

@@ -0,0 +1,724 @@
+// @generated by protoc-gen-es v2.16.0 with parameter "target=js+dts,import_extension=js"
+// @generated from file novatrail/eval/v1/report.proto (package novatrail.eval.v1, syntax proto3)
+/* eslint-disable */
+
+// The runner's report of the eval protocol: what the eval runner of `tutor` measured in one run of
+// an eval set, sample by sample — the verdict on each answer slot's truth and on what the models
+// read, an essay's marks by people and by AI, or why a sample was refused —, with no value, text,
+// stroke or crop of any work. What it holds, and how the measures are counted from it, are the
+// rules of contracts/README.md, section 14.
+
+import type { GenEnum, GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
+import type { Message } from "@bufbuild/protobuf";
+import type { AnswerSlotPlace, Unreadable } from "../../observation/v1/observation_pb.js";
+import type { CriterionMark } from "./sample_pb.js";
+
+/**
+ * Describes the file novatrail/eval/v1/report.proto.
+ */
+export declare const file_novatrail_eval_v1_report: GenFile;
+
+/**
+ * One run of an eval set by the eval runner: who ran it, what it measured, which models read, by
+ * which code the verdicts came, and each sample's result.
+ *
+ * @generated from message novatrail.eval.v1.Report
+ */
+export declare type Report = Message<"novatrail.eval.v1.Report"> & {
+  /**
+   * The protocol the file is written in: "novatrail.eval.v1", the package of this message.
+   *
+   * @generated from field: string protocol = 1;
+   */
+  protocol: string;
+
+  /**
+   * The tag of the release of `tutor` whose runner made the run.
+   *
+   * @generated from field: string runner = 2;
+   */
+  runner: string;
+
+  /**
+   * The capability of AI that the run measured.
+   *
+   * @generated from field: novatrail.eval.v1.Capability capability = 3;
+   */
+  capability: Capability;
+
+  /**
+   * The models that read the work, in the order of the pair: two or one in a run of reading, one in
+   * a run of assessment.
+   *
+   * @generated from field: repeated novatrail.eval.v1.Model readers = 4;
+   */
+  readers: Model[];
+
+  /**
+   * The model that assessed the essays by their rubric: only in a run of assessment.
+   *
+   * @generated from field: novatrail.eval.v1.Model assessor = 5;
+   */
+  assessor?: Model | undefined;
+
+  /**
+   * The code that gave the verdicts.
+   *
+   * @generated from field: novatrail.eval.v1.VerdictSource verdicts = 6;
+   */
+  verdicts?: VerdictSource | undefined;
+
+  /**
+   * The eval set the run measured.
+   *
+   * @generated from oneof novatrail.eval.v1.Report.eval_set
+   */
+  evalSet: {
+    /**
+     * A module's eval set, in `modules`.
+     *
+     * @generated from field: novatrail.eval.v1.ModuleEvalSet module = 7;
+     */
+    value: ModuleEvalSet;
+    case: "module";
+  } | {
+    /**
+     * An eval set of a capability of AI, which `tutor` holds.
+     *
+     * @generated from field: novatrail.eval.v1.CapabilityEvalSet capability_set = 8;
+     */
+    value: CapabilityEvalSet;
+    case: "capabilitySet";
+  } | { case: undefined; value?: undefined };
+
+  /**
+   * How many times each sample was read: 1 or more.
+   *
+   * @generated from field: int32 repeats = 9;
+   */
+  repeats: number;
+
+  /**
+   * Each sample of the set that the capability measures, ascending by ID.
+   *
+   * @generated from field: repeated novatrail.eval.v1.SampleResult samples = 10;
+   */
+  samples: SampleResult[];
+};
+
+/**
+ * Describes the message novatrail.eval.v1.Report.
+ * Use `create(ReportSchema)` to create a new message.
+ */
+export declare const ReportSchema: GenMessage<Report>;
+
+/**
+ * A model of a provider of AI.
+ *
+ * @generated from message novatrail.eval.v1.Model
+ */
+export declare type Model = Message<"novatrail.eval.v1.Model"> & {
+  /**
+   * The provider, as the adapters of `tutor` name it.
+   *
+   * @generated from field: string provider = 1;
+   */
+  provider: string;
+
+  /**
+   * The model, as the list of admitted models names it, with its setting.
+   *
+   * @generated from field: string model = 2;
+   */
+  model: string;
+};
+
+/**
+ * Describes the message novatrail.eval.v1.Model.
+ * Use `create(ModelSchema)` to create a new message.
+ */
+export declare const ModelSchema: GenMessage<Model>;
+
+/**
+ * The code that gave the verdicts of a run, and its version.
+ *
+ * @generated from message novatrail.eval.v1.VerdictSource
+ */
+export declare type VerdictSource = Message<"novatrail.eval.v1.VerdictSource"> & {
+  /**
+   * Whose code.
+   *
+   * @generated from field: novatrail.eval.v1.VerdictCode code = 1;
+   */
+  code: VerdictCode;
+
+  /**
+   * The tag of the release of that code.
+   *
+   * @generated from field: string version = 2;
+   */
+  version: string;
+};
+
+/**
+ * Describes the message novatrail.eval.v1.VerdictSource.
+ * Use `create(VerdictSourceSchema)` to create a new message.
+ */
+export declare const VerdictSourceSchema: GenMessage<VerdictSource>;
+
+/**
+ * A module's eval set, as a commit of `modules` holds it.
+ *
+ * @generated from message novatrail.eval.v1.ModuleEvalSet
+ */
+export declare type ModuleEvalSet = Message<"novatrail.eval.v1.ModuleEvalSet"> & {
+  /**
+   * The module's ID.
+   *
+   * @generated from field: string module = 1;
+   */
+  module: string;
+
+  /**
+   * The commit of `modules` whose `eval/` the run read: 40 lowercase hexadecimal digits.
+   *
+   * @generated from field: string commit = 2;
+   */
+  commit: string;
+};
+
+/**
+ * Describes the message novatrail.eval.v1.ModuleEvalSet.
+ * Use `create(ModuleEvalSetSchema)` to create a new message.
+ */
+export declare const ModuleEvalSetSchema: GenMessage<ModuleEvalSet>;
+
+/**
+ * An eval set of a capability of AI, which `tutor` holds.
+ *
+ * @generated from message novatrail.eval.v1.CapabilityEvalSet
+ */
+export declare type CapabilityEvalSet = Message<"novatrail.eval.v1.CapabilityEvalSet"> & {
+  /**
+   * The set's name, as `tutor` names it.
+   *
+   * @generated from field: string name = 1;
+   */
+  name: string;
+};
+
+/**
+ * Describes the message novatrail.eval.v1.CapabilityEvalSet.
+ * Use `create(CapabilityEvalSetSchema)` to create a new message.
+ */
+export declare const CapabilityEvalSetSchema: GenMessage<CapabilityEvalSet>;
+
+/**
+ * What the run made of one sample.
+ *
+ * @generated from message novatrail.eval.v1.SampleResult
+ */
+export declare type SampleResult = Message<"novatrail.eval.v1.SampleResult"> & {
+  /**
+   * The sample's ID.
+   *
+   * @generated from field: string sample = 1;
+   */
+  sample: string;
+
+  /**
+   * Whether it was measured.
+   *
+   * @generated from oneof novatrail.eval.v1.SampleResult.result
+   */
+  result: {
+    /**
+     * The runner refused it, and why: nothing of it was sent to AI.
+     *
+     * @generated from field: novatrail.eval.v1.Refusal refused = 2;
+     */
+    value: Refusal;
+    case: "refused";
+  } | {
+    /**
+     * It was read and measured.
+     *
+     * @generated from field: novatrail.eval.v1.Measurement measured = 3;
+     */
+    value: Measurement;
+    case: "measured";
+  } | { case: undefined; value?: undefined };
+};
+
+/**
+ * Describes the message novatrail.eval.v1.SampleResult.
+ * Use `create(SampleResultSchema)` to create a new message.
+ */
+export declare const SampleResultSchema: GenMessage<SampleResult>;
+
+/**
+ * What a run measured on one sample: its answer slots, or its essay.
+ *
+ * @generated from message novatrail.eval.v1.Measurement
+ */
+export declare type Measurement = Message<"novatrail.eval.v1.Measurement"> & {
+  /**
+   * Every answer slot that a model reads and whose check gives a verdict, in the task's order: on
+   * a workbook, the filled boxes. None on an essay.
+   *
+   * @generated from field: repeated novatrail.eval.v1.AnswerSlotResult answer_slots = 1;
+   */
+  answerSlots: AnswerSlotResult[];
+
+  /**
+   * The essay: only on a sample of an essay.
+   *
+   * @generated from field: novatrail.eval.v1.EssayResult essay = 2;
+   */
+  essay?: EssayResult | undefined;
+};
+
+/**
+ * Describes the message novatrail.eval.v1.Measurement.
+ * Use `create(MeasurementSchema)` to create a new message.
+ */
+export declare const MeasurementSchema: GenMessage<Measurement>;
+
+/**
+ * One answer slot: the verdict on its truth, and on what the models read in each repeat.
+ *
+ * @generated from message novatrail.eval.v1.AnswerSlotResult
+ */
+export declare type AnswerSlotResult = Message<"novatrail.eval.v1.AnswerSlotResult"> & {
+  /**
+   * The answer slot.
+   *
+   * @generated from field: novatrail.observation.v1.AnswerSlotPlace answer_slot = 1;
+   */
+  answerSlot?: AnswerSlotPlace | undefined;
+
+  /**
+   * The verdict that the truth of the markup gets.
+   *
+   * @generated from field: novatrail.eval.v1.Verdict truth = 2;
+   */
+  truth?: Verdict | undefined;
+
+  /**
+   * The markup notes a mirrored digit in it.
+   *
+   * @generated from field: bool mirrored = 3;
+   */
+  mirrored: boolean;
+
+  /**
+   * The markup notes something crossed out and written again in it.
+   *
+   * @generated from field: bool crossed_out = 4;
+   */
+  crossedOut: boolean;
+
+  /**
+   * What each repeat gave, in order: as many as the report's repeats.
+   *
+   * @generated from field: repeated novatrail.eval.v1.Reading readings = 5;
+   */
+  readings: Reading[];
+};
+
+/**
+ * Describes the message novatrail.eval.v1.AnswerSlotResult.
+ * Use `create(AnswerSlotResultSchema)` to create a new message.
+ */
+export declare const AnswerSlotResultSchema: GenMessage<AnswerSlotResult>;
+
+/**
+ * A verdict of an answer slot, as the test vectors of verdicts of the module format write it.
+ *
+ * @generated from message novatrail.eval.v1.Verdict
+ */
+export declare type Verdict = Message<"novatrail.eval.v1.Verdict"> & {
+  /**
+   * "correct", "not_yet", "unreadable" or "unchecked".
+   *
+   * @generated from field: string verdict = 1;
+   */
+  verdict: string;
+
+  /**
+   * The reason of the verdict: only beside "unreadable", and empty beside the others.
+   *
+   * @generated from field: string reason = 2;
+   */
+  reason: string;
+
+  /**
+   * The trap that the value shows, by its local ID: only beside "not_yet", and empty when the
+   * value shows none.
+   *
+   * @generated from field: string trap = 3;
+   */
+  trap: string;
+};
+
+/**
+ * Describes the message novatrail.eval.v1.Verdict.
+ * Use `create(VerdictSchema)` to create a new message.
+ */
+export declare const VerdictSchema: GenMessage<Verdict>;
+
+/**
+ * What one repeat gave an answer slot.
+ *
+ * @generated from message novatrail.eval.v1.Reading
+ */
+export declare type Reading = Message<"novatrail.eval.v1.Reading"> & {
+  /**
+   * The verdict on the observation the readers gave, joined when they are two.
+   *
+   * @generated from field: novatrail.eval.v1.Verdict verdict = 1;
+   */
+  verdict?: Verdict | undefined;
+
+  /**
+   * Why the verdict is "unchecked": only beside it.
+   *
+   * @generated from field: novatrail.eval.v1.UncheckedCause unchecked = 2;
+   */
+  unchecked: UncheckedCause;
+};
+
+/**
+ * Describes the message novatrail.eval.v1.Reading.
+ * Use `create(ReadingSchema)` to create a new message.
+ */
+export declare const ReadingSchema: GenMessage<Reading>;
+
+/**
+ * An essay: its truth, the people's marks, and what each repeat gave.
+ *
+ * @generated from message novatrail.eval.v1.EssayResult
+ */
+export declare type EssayResult = Message<"novatrail.eval.v1.EssayResult"> & {
+  /**
+   * Why the truth holds no text: blank or illegible. None when it holds the essay's text.
+   *
+   * @generated from field: optional novatrail.observation.v1.Unreadable truth = 1;
+   */
+  truth?: Unreadable | undefined;
+
+  /**
+   * The marks the people agreed on, as the markup gives them.
+   *
+   * @generated from field: repeated novatrail.eval.v1.CriterionMark people = 2;
+   */
+  people: CriterionMark[];
+
+  /**
+   * What each repeat gave, in order: as many as the report's repeats.
+   *
+   * @generated from field: repeated novatrail.eval.v1.EssayReading readings = 3;
+   */
+  readings: EssayReading[];
+};
+
+/**
+ * Describes the message novatrail.eval.v1.EssayResult.
+ * Use `create(EssayResultSchema)` to create a new message.
+ */
+export declare const EssayResultSchema: GenMessage<EssayResult>;
+
+/**
+ * What one repeat gave an essay.
+ *
+ * @generated from message novatrail.eval.v1.EssayReading
+ */
+export declare type EssayReading = Message<"novatrail.eval.v1.EssayReading"> & {
+  /**
+   * How its submission ended.
+   *
+   * @generated from oneof novatrail.eval.v1.EssayReading.outcome
+   */
+  outcome: {
+    /**
+     * The essay was read and assessed.
+     *
+     * @generated from field: novatrail.eval.v1.AssessedEssay assessment = 1;
+     */
+    value: AssessedEssay;
+    case: "assessment";
+  } | {
+    /**
+     * The essay was read as blank or illegible, and was not assessed.
+     *
+     * @generated from field: novatrail.observation.v1.Unreadable unreadable = 2;
+     */
+    value: Unreadable;
+    case: "unreadable";
+  } | {
+    /**
+     * The reading or the assessment gave nothing, and why.
+     *
+     * @generated from field: novatrail.eval.v1.UncheckedCause unchecked = 3;
+     */
+    value: UncheckedCause;
+    case: "unchecked";
+  } | { case: undefined; value?: undefined };
+};
+
+/**
+ * Describes the message novatrail.eval.v1.EssayReading.
+ * Use `create(EssayReadingSchema)` to create a new message.
+ */
+export declare const EssayReadingSchema: GenMessage<EssayReading>;
+
+/**
+ * The assessment of an essay by AI.
+ *
+ * @generated from message novatrail.eval.v1.AssessedEssay
+ */
+export declare type AssessedEssay = Message<"novatrail.eval.v1.AssessedEssay"> & {
+  /**
+   * The mark that AI gave each criterion the people marked, in their order.
+   *
+   * @generated from field: repeated novatrail.eval.v1.AssessedCriterion criteria = 1;
+   */
+  criteria: AssessedCriterion[];
+};
+
+/**
+ * Describes the message novatrail.eval.v1.AssessedEssay.
+ * Use `create(AssessedEssaySchema)` to create a new message.
+ */
+export declare const AssessedEssaySchema: GenMessage<AssessedEssay>;
+
+/**
+ * The mark that AI gave one criterion.
+ *
+ * @generated from message novatrail.eval.v1.AssessedCriterion
+ */
+export declare type AssessedCriterion = Message<"novatrail.eval.v1.AssessedCriterion"> & {
+  /**
+   * The criterion's place in the rubric, from 1.
+   *
+   * @generated from field: int32 criterion = 1;
+   */
+  criterion: number;
+
+  /**
+   * The mark.
+   *
+   * @generated from field: novatrail.eval.v1.CriterionAssessment assessment = 2;
+   */
+  assessment: CriterionAssessment;
+};
+
+/**
+ * Describes the message novatrail.eval.v1.AssessedCriterion.
+ * Use `create(AssessedCriterionSchema)` to create a new message.
+ */
+export declare const AssessedCriterionSchema: GenMessage<AssessedCriterion>;
+
+/**
+ * A capability of AI that the protocol measures.
+ *
+ * @generated from enum novatrail.eval.v1.Capability
+ */
+export enum Capability {
+  /**
+   * No capability: never written. A report that holds it is not valid.
+   *
+   * @generated from enum value: CAPABILITY_UNSPECIFIED = 0;
+   */
+  UNSPECIFIED = 0,
+
+  /**
+   * Reading handwriting: the values of the answer slots on a sheet or a workbook, which get a
+   * verdict.
+   *
+   * @generated from enum value: CAPABILITY_HANDWRITING_READING = 1;
+   */
+  HANDWRITING_READING = 1,
+
+  /**
+   * Assessing an essay by its rubric, the whole path: the essay read, and what was read assessed.
+   *
+   * @generated from enum value: CAPABILITY_RUBRIC_ASSESSMENT = 2;
+   */
+  RUBRIC_ASSESSMENT = 2,
+}
+
+/**
+ * Describes the enum novatrail.eval.v1.Capability.
+ */
+export declare const CapabilitySchema: GenEnum<Capability>;
+
+/**
+ * Whose code gave the verdicts of a run.
+ *
+ * @generated from enum novatrail.eval.v1.VerdictCode
+ */
+export enum VerdictCode {
+  /**
+   * Nobody's: never written. A report that holds it is not valid.
+   *
+   * @generated from enum value: VERDICT_CODE_UNSPECIFIED = 0;
+   */
+  UNSPECIFIED = 0,
+
+  /**
+   * `module-kit`'s, as in the platform's work: only such a report admits a model.
+   *
+   * @generated from enum value: VERDICT_CODE_MODULE_KIT = 1;
+   */
+  MODULE_KIT = 1,
+
+  /**
+   * The runner's own: a measurement, which admits no model.
+   *
+   * @generated from enum value: VERDICT_CODE_RUNNER = 2;
+   */
+  RUNNER = 2,
+}
+
+/**
+ * Describes the enum novatrail.eval.v1.VerdictCode.
+ */
+export declare const VerdictCodeSchema: GenEnum<VerdictCode>;
+
+/**
+ * Why the runner refused a sample.
+ *
+ * @generated from enum novatrail.eval.v1.Refusal
+ */
+export enum Refusal {
+  /**
+   * No reason: never written. A report that holds it is not valid.
+   *
+   * @generated from enum value: REFUSAL_UNSPECIFIED = 0;
+   */
+  UNSPECIFIED = 0,
+
+  /**
+   * The sample breaks a rule of sections 11 and 12, or its markup does not fit its task.
+   *
+   * @generated from enum value: REFUSAL_INVALID = 1;
+   */
+  INVALID = 1,
+
+  /**
+   * The sample has no markup.
+   *
+   * @generated from enum value: REFUSAL_NO_MARKUP = 2;
+   */
+  NO_MARKUP = 2,
+
+  /**
+   * Its task is not the one it names: no such task, a task of another module, or an open view of
+   * another digest.
+   *
+   * @generated from enum value: REFUSAL_OTHER_TASK = 3;
+   */
+  OTHER_TASK = 3,
+
+  /**
+   * A sample written by a child, which goes to no provider until a contract with it admits
+   * children.
+   *
+   * @generated from enum value: REFUSAL_CHILD = 4;
+   */
+  CHILD = 4,
+}
+
+/**
+ * Describes the enum novatrail.eval.v1.Refusal.
+ */
+export declare const RefusalSchema: GenEnum<Refusal>;
+
+/**
+ * Why a reading gave no verdict.
+ *
+ * @generated from enum novatrail.eval.v1.UncheckedCause
+ */
+export enum UncheckedCause {
+  /**
+   * No cause: the reading gave a verdict.
+   *
+   * @generated from enum value: UNCHECKED_CAUSE_UNSPECIFIED = 0;
+   */
+  UNSPECIFIED = 0,
+
+  /**
+   * AI failed: an error, a refusal of the provider, a timeout.
+   *
+   * @generated from enum value: UNCHECKED_CAUSE_FAILURE = 1;
+   */
+  FAILURE = 1,
+
+  /**
+   * A limit or a quota ran out.
+   *
+   * @generated from enum value: UNCHECKED_CAUSE_LIMIT = 2;
+   */
+  LIMIT = 2,
+
+  /**
+   * An observation is not valid (contracts/README.md, section 10).
+   *
+   * @generated from enum value: UNCHECKED_CAUSE_INVALID = 3;
+   */
+  INVALID = 3,
+
+  /**
+   * The assessment of an essay marked no criterion.
+   *
+   * @generated from enum value: UNCHECKED_CAUSE_NO_MARK = 4;
+   */
+  NO_MARK = 4,
+}
+
+/**
+ * Describes the enum novatrail.eval.v1.UncheckedCause.
+ */
+export declare const UncheckedCauseSchema: GenEnum<UncheckedCause>;
+
+/**
+ * The mark that AI gave a criterion.
+ *
+ * @generated from enum novatrail.eval.v1.CriterionAssessment
+ */
+export enum CriterionAssessment {
+  /**
+   * No mark: never written. A report that holds it is not valid.
+   *
+   * @generated from enum value: CRITERION_ASSESSMENT_UNSPECIFIED = 0;
+   */
+  UNSPECIFIED = 0,
+
+  /**
+   * The essay meets it.
+   *
+   * @generated from enum value: CRITERION_ASSESSMENT_MET = 1;
+   */
+  MET = 1,
+
+  /**
+   * The essay does not meet it.
+   *
+   * @generated from enum value: CRITERION_ASSESSMENT_NOT_MET = 2;
+   */
+  NOT_MET = 2,
+
+  /**
+   * AI gave it no mark.
+   *
+   * @generated from enum value: CRITERION_ASSESSMENT_NONE = 3;
+   */
+  NONE = 3,
+}
+
+/**
+ * Describes the enum novatrail.eval.v1.CriterionAssessment.
+ */
+export declare const CriterionAssessmentSchema: GenEnum<CriterionAssessment>;
+