modelrig 0.8.1 → 0.10.0

package/templates/medical-classification.json

@@ -0,0 +1,71 @@
+{
+  "use_case": "medical-classification",
+  "title": "Medical classification",
+  "summary": "Classify a clinical document into your own categories — document type, specialty, coding bucket — and hand off the ones a qualified person must review. It sorts documents; it does not make clinical decisions.",
+  "fit_note": "Medical classification fits the shipped teach → calibrate → grade → optimize loop when four conditions hold. (1) Your categories and the documents you classify are stable enough that last quarter's examples still describe this quarter's — if the taxonomy is still moving, settle it before you calibrate. (2) Someone who owns the process can write down what a correct classification is, which is the grading guide below. (3) A qualified reviewer can grade a batch of past documents against the calls they would have made, so a grader can be calibrated against them. (4) You process enough documents that calibration and optimization pay back. This task handles protected health information, so the custody terms come first — quoted verbatim from the published content custody terms: \"Do not send protected health information or other regulated data through ModelRig until a BAA or DPA is in place. This service is not offered for regulated data today, and nothing on this page should be read as agreeing to handle it.\" Until those terms are in place, the today-path is to run this on a zero-retention route (or with content custody turned off), so ModelRig holds no content and grading is limited to the scores and metadata your own systems send. The full terms are at https://modelrig.ai/terms/content-custody. This template sorts documents into your categories; it is not a diagnostic or clinical-decision tool, and it makes no claim about how well any model classifies your documents — that is what your goldens and grades measure — and no compliance claim of any other kind.",
+  "starter_schema": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$comment": "STARTER SCHEMA. Replace every TODO with your own taxonomy before use. The template runs as-is: free-text fields validate immediately. TODO markers show where adding your own enum values tightens grading to your categories; do not copy example words from the docs.",
+    "title": "MedicalClassification",
+    "type": "object",
+    "additionalProperties": false,
+    "required": ["classification", "needs_human"],
+    "properties": {
+      "classification": {
+        "type": "string",
+        "description": "The category this clinical document belongs in. TODO(enum): constrain this to YOUR own category labels by adding an \"enum\": [ ... ] here. It runs as free text immediately; add your own enum values here to tighten validation and grading to your taxonomy.",
+        "$comment": "TODO(enum): add your document categories as the enum for this field."
+      },
+      "needs_human": {
+        "type": "boolean",
+        "description": "True when the document is ambiguous, clinically significant, or otherwise a call a qualified person must make. Pairs with the escalation band in the grading guide."
+      },
+      "rationale": {
+        "type": "string",
+        "description": "One or two sentences grounding the classification in what the document actually contains. Optional; keep it if you want the grader to check the decision is explained."
+      }
+    }
+  },
+  "starter_guide": {
+    "criteria": [
+      {
+        "id": "classification-correct",
+        "title": "Classification is correct",
+        "pass": "the document gets the category a qualified reviewer on your team would have chosen for it, by your own rules.",
+        "fail": "The document is classified into a category that does not match what it actually is, or into a catch-all when a specific category fits."
+      },
+      {
+        "id": "grounded-no-invention",
+        "title": "Decision is grounded in the document",
+        "pass": "every part of the classification is supported by something the document actually contains; no clinical detail is invented.",
+        "fail": "The output asserts a diagnosis, a code, or a finding that the document does not contain."
+      },
+      {
+        "id": "stays-a-classification",
+        "title": "Output stays a classification, not a clinical judgment",
+        "pass": "the output sorts the document into your categories and stops there, without offering a diagnosis or a treatment decision.",
+        "fail": "The output makes a clinical or diagnostic call the task is not meant to make, rather than classifying and deferring to a person."
+      },
+      {
+        "id": "human-flag-when-unsure",
+        "title": "Ambiguous or clinically significant documents are flagged for a human",
+        "pass": "a document that is genuinely ambiguous or clinically significant sets needs_human rather than being auto-classified.",
+        "fail": "A document a qualified reviewer would have handled is classified automatically with false confidence."
+      }
+    ],
+    "teach_grade_ids": [],
+    "escalation": {
+      "epsilon": 0.1
+    }
+  },
+  "golden_checklist": [
+    "One clear example per category in your taxonomy — the ordinary, unambiguous document for each category.",
+    "The two or three categories your team most often confuses, side by side, so the guide learns the boundary between them.",
+    "A document that spans two categories at once — the golden fixes which one wins and why.",
+    "A clinically significant document that should reach a qualified person regardless of category — the golden shows needs_human set.",
+    "A document that tempts a clinical judgment — the golden shows it classified and deferred, not diagnosed.",
+    "An illegible, truncated, or off-type document — the golden shows it flagged for a human rather than force-classified.",
+    "Aim for roughly twenty goldens total, weighted toward the ambiguous, cross-category, and clinically significant documents rather than the routine centre."
+  ],
+  "escalation_starter": "Start by escalating to a qualified person whenever the model sets needs_human, and always for clinically significant documents regardless of the classification. The machine-side band lives in starter_guide.escalation.epsilon: a grade within that distance of your route's pass threshold is treated as judge-uncertain and sent to the review queue. The 0.1 here is a starting band — keep it wide until a qualified reviewer has graded enough documents to trust the grader, then narrow it."
+}

package/templates/ticket-triage.json

@@ -0,0 +1,76 @@
+{
+  "use_case": "ticket-triage",
+  "title": "Ticket triage",
+  "summary": "Classify an incoming support ticket into your own categories and a priority, and flag the ones a human should look at.",
+  "fit_note": "Ticket triage fits the shipped teach → calibrate → grade → optimize loop when four conditions hold. (1) Your ticket stream is stable enough that last month's examples still describe this month's — if your categories churn weekly, tighten them before you calibrate. (2) Your support team can write down what a correct routing decision is, which is the grading guide below. (3) Someone who knows the queue can grade a batch of past tickets so a grader can be calibrated against them. (4) The queue carries enough volume that calibration and optimization pay back. This template is a starting point for each of those, not a claim that triage is solved — fill in your taxonomy and your definitions before you rely on it.",
+  "starter_schema": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$comment": "STARTER SCHEMA. Replace every TODO with your own taxonomy before use. The template runs as-is: free-text fields validate immediately. TODO markers show where adding your own enum values tightens grading to your taxonomy; do not copy example words from the docs.",
+    "title": "TicketTriage",
+    "type": "object",
+    "additionalProperties": false,
+    "required": ["category", "priority", "needs_human"],
+    "properties": {
+      "category": {
+        "type": "string",
+        "description": "Which queue or team the ticket belongs to. TODO(enum): constrain this to YOUR own category labels by adding an \"enum\": [ ... ] here. It runs as free text immediately; add your own enum values here to tighten validation and grading to your taxonomy.",
+        "$comment": "TODO(enum): add your category labels as the enum for this field."
+      },
+      "priority": {
+        "type": "string",
+        "description": "How urgently the ticket needs attention. TODO(enum): add an \"enum\": [ ... ] with YOUR priority levels (whatever your team already uses).",
+        "$comment": "TODO(enum): add your priority levels as the enum for this field."
+      },
+      "needs_human": {
+        "type": "boolean",
+        "description": "True when the model is not confident enough to route this automatically and a person should decide. Pairs with the escalation band in the grading guide."
+      },
+      "reason": {
+        "type": "string",
+        "description": "One short sentence grounding the category and priority in the ticket text. Optional; keep it if you want the grader to check that the decision is explained."
+      }
+    }
+  },
+  "starter_guide": {
+    "criteria": [
+      {
+        "id": "category-correct",
+        "title": "Category is correct",
+        "pass": "the ticket is assigned to the category a human on your queue would have chosen for it.",
+        "fail": "The ticket is assigned to a category that does not match its actual topic, or to a catch-all when a specific category fits."
+      },
+      {
+        "id": "priority-justified",
+        "title": "Priority matches the stated impact",
+        "pass": "the priority reflects the urgency and impact the ticket actually describes, by your team's own priority rules.",
+        "fail": "The priority is higher or lower than the ticket's stated impact warrants."
+      },
+      {
+        "id": "grounded-no-invention",
+        "title": "Decision is grounded in the ticket",
+        "pass": "Every field is supported by something the ticket actually says; nothing is invented.",
+        "fail": "The output asserts a detail (a product, an account state, an error) that the ticket text does not contain."
+      },
+      {
+        "id": "human-flag-when-unsure",
+        "title": "Ambiguous tickets are flagged for a human",
+        "pass": "a ticket that is genuinely ambiguous, mixed-topic, or high-risk sets needs_human rather than guessing a single category.",
+        "fail": "A ticket that a human would have hesitated on is auto-routed with false confidence."
+      }
+    ],
+    "teach_grade_ids": [],
+    "escalation": {
+      "epsilon": 0.1
+    }
+  },
+  "golden_checklist": [
+    "One clear example per category in your taxonomy — the ordinary, unambiguous case for each queue.",
+    "The two or three categories your team most often confuses, side by side, so the guide learns the boundary between them.",
+    "A multi-topic ticket that could plausibly land in two categories — the golden records which one wins and why.",
+    "A high-priority case and a low-priority case that use similar words, so priority is judged on impact, not vocabulary.",
+    "An edge case: an empty, spam, or unintelligible ticket — the golden shows it flagged for a human rather than force-categorized.",
+    "A sensitive case (security, legal, billing dispute, or churn risk) that should escalate regardless of category.",
+    "Aim for roughly twenty goldens total, weighted toward the boundaries above rather than the easy centre of each category."
+  ],
+  "escalation_starter": "Start by escalating to a human whenever the model sets needs_human, and whenever a ticket is sensitive (security, legal, or a churn/at-risk signal) regardless of its category. The machine-side band lives in starter_guide.escalation.epsilon: a grade within that distance of your route's pass threshold is treated as judge-uncertain and sent to the human queue. The 0.1 here is a starting band — widen it while you build trust, narrow it as your grader calibrates."
+}

package/dist/feedback/feedback.d.ts

@@ -1,35 +0,0 @@
-/**
- * rig.feedback() — Phase 4 WS2 (spec: judgment starts accumulating).
- * The SDK surface normalizes a target (inference id or {episode}) + opts
- * into a FeedbackRow; the write is a synchronous local SQLite insert
- * (fire-and-forget from the caller's view — no network, never throws for
- * sink reasons). The Supabase mirror rides the exporter, dual-store style.
- */
-import type { FeedbackRow } from "../types/telemetry";
-export type FeedbackTarget = string | {
-    readonly episode: string;
-};
-export interface FeedbackOpts {
-    readonly verdict: "up" | "down";
-    /** Optional graded signal in [0, 1]; verdict remains the primary axis. */
-    readonly score?: number;
-    readonly note?: string;
-    /** Who judged: "sdk" (default), a review-agent name, "human", … */
-    readonly source?: string;
-    /**
-     * The org this verdict belongs to (Phase 6 WS2). Omit from application code
-     * — a rig has one org and the exporter knows it.
-     *
-     * Supplied by modelrig-server, and only ever with an org it has ALREADY
-     * authorized the caller into: a value that arrived in a request body and was
-     * passed through here would be a tenant chosen by the client (T3). Setting
-     * it is what lets one hosted server hold verdicts for many orgs without
-     * misfiling any of them (QA-P6-010).
-     */
-    readonly orgId?: string;
-    /** Authenticated user who recorded it. Same provenance rule as `orgId`:
-     * resolved from a session, never read off a request body. */
-    readonly createdBy?: string | null;
-}
-export declare function buildFeedbackRow(target: FeedbackTarget, opts: FeedbackOpts): FeedbackRow;
-//# sourceMappingURL=feedback.d.ts.map

package/dist/feedback/feedback.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"feedback.d.ts","sourceRoot":"","sources":["../../src/feedback/feedback.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnE,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC;IAChC,0EAA0E;IAC1E,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB;iEAC6D;IAC7D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,GAAG,WAAW,CA0BxF"}

package/dist/feedback/feedback.js

@@ -1,39 +0,0 @@
-"use strict";
-/**
- * rig.feedback() — Phase 4 WS2 (spec: judgment starts accumulating).
- * The SDK surface normalizes a target (inference id or {episode}) + opts
- * into a FeedbackRow; the write is a synchronous local SQLite insert
- * (fire-and-forget from the caller's view — no network, never throws for
- * sink reasons). The Supabase mirror rides the exporter, dual-store style.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildFeedbackRow = buildFeedbackRow;
-const node_crypto_1 = require("node:crypto");
-function buildFeedbackRow(target, opts) {
-    let inferenceId = null;
-    let episodeKey = null;
-    if (typeof target === "string") {
-        inferenceId = target;
-    }
-    else {
-        episodeKey = target.episode;
-    }
-    if ((inferenceId === null || inferenceId === "") && (episodeKey === null || episodeKey === "")) {
-        throw new Error("feedback target must be a non-empty inference id or {episode}");
-    }
-    if (opts.score !== undefined && (!Number.isFinite(opts.score) || opts.score < 0 || opts.score > 1)) {
-        throw new Error(`feedback score must be in [0, 1], got ${opts.score}`);
-    }
-    return {
-        id: (0, node_crypto_1.randomUUID)(),
-        ts: new Date().toISOString(),
-        inferenceId,
-        episodeKey,
-        verdict: opts.verdict,
-        score: opts.score ?? null,
-        note: opts.note ?? null,
-        source: opts.source ?? "sdk",
-        orgId: opts.orgId !== undefined && opts.orgId !== "" ? opts.orgId : null,
-        createdBy: opts.createdBy ?? null,
-    };
-}