create-safest-tools 0.2.0

package/template/src/shadow-quality.ts

@@ -0,0 +1,278 @@
+import { canonicalJson } from "./audit";
+import { sha256 } from "./report-crypto";
+import { ApiError } from "./report-http";
+import type { OperatorSession } from "./report-types";
+
+interface ShadowCandidate {
+  shadowRunId: string;
+  reportId: string;
+  primaryRunId: string | null;
+  primaryState: string | null;
+  decisionId: string | null;
+  decisionCode: string | null;
+  policyCode: string | null;
+  findingId: string | null;
+  findingDataJson: string | null;
+  findingStepRunId: string | null;
+}
+
+function now(): string { return new Date().toISOString(); }
+
+function parsedObject(value: string | null): Record<string, unknown> {
+  try {
+    const parsed = value ? JSON.parse(value) : {};
+    return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record<string, unknown> : {};
+  } catch {
+    return {};
+  }
+}
+
+function recommendationValues(data: Record<string, unknown>): string[] {
+  return [data.decision_code, data.policy_code, data.result_code, data.outcome, data.conclusion, data.classification]
+    .filter((value): value is string => typeof value === "string")
+    .map((value) => value.trim().toLowerCase())
+    .filter(Boolean);
+}
+
+async function qualityQueue(db: D1Database): Promise<{ id: string } | null> {
+  return db.prepare(`SELECT id FROM queue_definitions WHERE purpose = 'quality' AND status = 'active' ORDER BY position, id LIMIT 1`)
+    .first<{ id: string }>();
+}
+
+async function insertQualityTask(
+  db: D1Database,
+  input: {
+    comparisonId: string;
+    reportId: string;
+    shadowRunId: string;
+    stepRunId: string;
+    queueId: string;
+    codes: string[];
+    severity: string;
+    actorId: string;
+    createdAt: string;
+  },
+): Promise<string> {
+  const taskId = `${input.comparisonId}:quality-task`;
+  const context = {
+    schema_version: "1",
+    comparison_id: input.comparisonId,
+    shadow_run_id: input.shadowRunId,
+    comparison_codes: input.codes,
+    severity: input.severity,
+    purpose: "Review a non-authoritative shadow recommendation against the authoritative decision.",
+  };
+  const contextJson = canonicalJson(context);
+  const contextDigest = `sha256:${await sha256(contextJson)}`;
+  await db.batch([
+    db.prepare(`
+      INSERT OR IGNORE INTO human_tasks (
+        id, installation_id, report_id, run_id, step_run_id, task_type,
+        state, queue_id, context_json, context_digest, response_schema_json,
+        due_at, escalation_at, created_at, updated_at
+      ) VALUES (?1, 'default', ?2, ?3, ?4, 'quality_review', 'open', ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?11)
+    `).bind(
+      taskId, input.reportId, input.shadowRunId, input.stepRunId, input.queueId,
+      contextJson, contextDigest,
+      canonicalJson({
+        type: "object",
+        additionalProperties: false,
+        required: ["outcome", "notes"],
+        properties: {
+          outcome: { type: "string", enum: ["agree_primary", "agree_shadow", "inconclusive"] },
+          notes: { type: "string", minLength: 10, maxLength: 4000 },
+        },
+      }),
+      new Date(Date.now() + 24 * 60 * 60_000).toISOString(),
+      new Date(Date.now() + 12 * 60 * 60_000).toISOString(), input.createdAt,
+    ),
+    db.prepare(`UPDATE shadow_comparisons SET status = 'quality_task_created' WHERE id = ?1 AND status = 'pending'`).bind(input.comparisonId),
+    db.prepare(`
+      INSERT OR IGNORE INTO audit_events (
+        id, idempotency_key, action, actor_type, actor_id,
+        target_type, target_id, details_json, created_at
+      ) VALUES (?1, ?2, 'shadow.quality_task_created', 'system', ?3,
+        'shadow_comparison', ?4, ?5, ?6)
+    `).bind(
+      crypto.randomUUID(), `shadow-quality-task:${input.comparisonId}`,
+      input.actorId, input.comparisonId, canonicalJson({ taskId, queueId: input.queueId }), input.createdAt,
+    ),
+  ]);
+  return taskId;
+}
+
+export async function compareCompletedShadowRuns(db: D1Database, limit = 50): Promise<{ compared: number; tasksCreated: number }> {
+  const rows = await db.prepare(`
+    SELECT shadow.id AS shadowRunId, shadow.report_id AS reportId,
+      primary_run.id AS primaryRunId, primary_run.state AS primaryState,
+      decision.id AS decisionId, decision.decision_code AS decisionCode,
+      decision.policy_code AS policyCode, finding.id AS findingId,
+      finding.data_json AS findingDataJson, finding.step_run_id AS findingStepRunId
+    FROM workflow_runs shadow
+    LEFT JOIN workflow_runs primary_run ON primary_run.id = (
+      SELECT candidate.id FROM workflow_runs candidate
+      WHERE candidate.report_id = shadow.report_id AND candidate.run_kind = 'primary'
+      ORDER BY candidate.created_at DESC, candidate.id DESC LIMIT 1
+    )
+    LEFT JOIN decisions decision ON decision.id = (
+      SELECT candidate.id FROM decisions candidate WHERE candidate.report_id = shadow.report_id
+      ORDER BY candidate.created_at DESC, candidate.id DESC LIMIT 1
+    )
+    LEFT JOIN findings finding ON finding.id = (
+      SELECT candidate.id FROM findings candidate
+      WHERE candidate.run_id = shadow.id AND candidate.status = 'available'
+      ORDER BY candidate.created_at DESC, candidate.id DESC LIMIT 1
+    )
+    WHERE shadow.run_kind = 'shadow' AND shadow.state = 'succeeded'
+      AND NOT EXISTS (SELECT 1 FROM shadow_comparisons comparison WHERE comparison.shadow_run_id = shadow.id)
+      AND (decision.id IS NOT NULL OR primary_run.state IN ('succeeded','failed','terminated','superseded'))
+    ORDER BY shadow.finished_at, shadow.id LIMIT ?1
+  `).bind(Math.max(1, Math.min(100, limit))).all<ShadowCandidate>();
+  const queue = await qualityQueue(db);
+  let compared = 0;
+  let tasksCreated = 0;
+  for (const row of rows.results) {
+    const data = parsedObject(row.findingDataJson);
+    const recommendations = recommendationValues(data);
+    const expected = [row.decisionCode, row.policyCode]
+      .filter((value): value is string => typeof value === "string")
+      .map((value) => value.toLowerCase());
+    let codes: string[];
+    let severity: "info" | "low" | "medium" | "high";
+    if (!row.decisionId) {
+      codes = ["authoritative_decision_missing"];
+      severity = "medium";
+    } else if (!row.findingId || recommendations.length === 0) {
+      codes = ["shadow_recommendation_missing"];
+      severity = "low";
+    } else if (expected.some((value) => recommendations.includes(value))) {
+      codes = ["shadow_agrees_with_authoritative_decision"];
+      severity = "info";
+    } else {
+      codes = ["shadow_decision_disagreement"];
+      severity = "high";
+    }
+    const comparisonId = `${row.shadowRunId}:comparison`;
+    const timestamp = now();
+    await db.batch([
+      db.prepare(`
+        INSERT OR IGNORE INTO shadow_comparisons (
+          id, report_id, primary_run_id, shadow_run_id, primary_decision_id,
+          shadow_finding_id, comparison_codes_json, severity, status, created_at
+        ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'pending', ?9)
+      `).bind(
+        comparisonId, row.reportId, row.primaryRunId, row.shadowRunId, row.decisionId,
+        row.findingId, canonicalJson(codes), severity, timestamp,
+      ),
+      db.prepare(`
+        INSERT OR IGNORE INTO audit_events (
+          id, idempotency_key, action, actor_type, actor_id,
+          target_type, target_id, details_json, created_at
+        ) VALUES (?1, ?2, 'shadow.comparison_created', 'system', 'shadow-quality-controller',
+          'shadow_comparison', ?3, ?4, ?5)
+      `).bind(
+        crypto.randomUUID(), `shadow-comparison:${row.shadowRunId}`, comparisonId,
+        canonicalJson({ reportId: row.reportId, shadowRunId: row.shadowRunId, codes, severity }), timestamp,
+      ),
+    ]);
+    compared += 1;
+    if ((severity === "high" || severity === "medium") && queue && row.findingStepRunId) {
+      await insertQualityTask(db, {
+        comparisonId, reportId: row.reportId, shadowRunId: row.shadowRunId,
+        stepRunId: row.findingStepRunId, queueId: queue.id, codes, severity,
+        actorId: "shadow-quality-controller", createdAt: timestamp,
+      });
+      tasksCreated += 1;
+    }
+  }
+  return { compared, tasksCreated };
+}
+
+export async function listShadowComparisons(
+  db: D1Database,
+  input: { status?: string | null; limit?: number; allowedQueueIds?: string[] | null },
+): Promise<Record<string, unknown>[]> {
+  const rows = await db.prepare(`
+    SELECT comparison.id, comparison.report_id AS reportId,
+      report.public_reference AS reportReference,
+      comparison.primary_run_id AS primaryRunId, comparison.shadow_run_id AS shadowRunId,
+      comparison.primary_decision_id AS primaryDecisionId,
+      comparison.shadow_finding_id AS shadowFindingId,
+      comparison.comparison_codes_json AS comparisonCodesJson,
+      comparison.severity, comparison.status, comparison.created_at AS createdAt,
+      comparison.reviewed_at AS reviewedAt
+    FROM shadow_comparisons comparison JOIN reports report ON report.id = comparison.report_id
+    WHERE (?1 IS NULL OR comparison.status = ?1)
+      AND (?3 = 1 OR report.queue_id IN (SELECT value FROM json_each(?4)))
+    ORDER BY CASE comparison.severity WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2 WHEN 'low' THEN 3 ELSE 4 END,
+      comparison.created_at DESC, comparison.id DESC LIMIT ?2
+  `).bind(
+    input.status ?? null,
+    Math.max(1, Math.min(100, input.limit ?? 50)),
+    input.allowedQueueIds === null || input.allowedQueueIds === undefined ? 1 : 0,
+    canonicalJson(input.allowedQueueIds ?? []),
+  ).all<Record<string, unknown>>();
+  return rows.results.map((row) => ({
+    ...row,
+    comparisonCodes: typeof row.comparisonCodesJson === "string" ? JSON.parse(row.comparisonCodesJson) : [],
+    comparisonCodesJson: undefined,
+  }));
+}
+
+export async function createShadowQualityTask(
+  db: D1Database,
+  comparisonId: string,
+  session: OperatorSession,
+  idempotencyKey: string,
+): Promise<{ comparisonId: string; taskId: string; idempotentReplay: boolean }> {
+  const existing = await db.prepare(`SELECT id FROM human_tasks WHERE id = ?1 LIMIT 1`).bind(`${comparisonId}:quality-task`).first();
+  if (existing) return { comparisonId, taskId: `${comparisonId}:quality-task`, idempotentReplay: true };
+  const row = await db.prepare(`
+    SELECT comparison.report_id AS reportId, comparison.shadow_run_id AS shadowRunId,
+      comparison.comparison_codes_json AS codesJson, comparison.severity,
+      COALESCE(finding.step_run_id, (
+        SELECT step.id FROM workflow_step_runs step WHERE step.run_id = comparison.shadow_run_id
+        ORDER BY step.finished_at DESC, step.id DESC LIMIT 1
+      )) AS stepRunId
+    FROM shadow_comparisons comparison
+    LEFT JOIN findings finding ON finding.id = comparison.shadow_finding_id
+    WHERE comparison.id = ?1 AND comparison.status IN ('pending','quality_task_created') LIMIT 1
+  `).bind(comparisonId).first<{ reportId: string; shadowRunId: string; codesJson: string; severity: string; stepRunId: string | null }>();
+  if (!row) throw new ApiError(404, "shadow_comparison_not_found", "The pending shadow comparison does not exist.");
+  if (!row.stepRunId) throw new ApiError(409, "shadow_comparison_step_unavailable", "The shadow run has no step to anchor a quality task.");
+  const queue = await qualityQueue(db);
+  if (!queue) throw new ApiError(409, "quality_queue_unavailable", "No active quality queue is configured.");
+  const taskId = await insertQualityTask(db, {
+    comparisonId, reportId: row.reportId, shadowRunId: row.shadowRunId,
+    stepRunId: row.stepRunId, queueId: queue.id,
+    codes: JSON.parse(row.codesJson) as string[], severity: row.severity,
+    actorId: session.actor.id, createdAt: now(),
+  });
+  await db.prepare(`INSERT OR IGNORE INTO audit_events (id, idempotency_key, action, actor_type, actor_id, target_type, target_id, details_json, created_at) VALUES (?1, ?2, 'shadow.quality_task_requested', ?3, ?4, 'shadow_comparison', ?5, ?6, ?7)`)
+    .bind(crypto.randomUUID(), `shadow-quality-request:${idempotencyKey}`, session.actor.type, session.actor.id, comparisonId, canonicalJson({ taskId }), now()).run();
+  return { comparisonId, taskId, idempotentReplay: false };
+}
+
+export async function dismissShadowComparison(
+  db: D1Database,
+  comparisonId: string,
+  reason: string,
+  session: OperatorSession,
+  idempotencyKey: string,
+): Promise<{ comparisonId: string; status: string; idempotentReplay: boolean }> {
+  if (reason.trim().length < 5 || reason.length > 2_000) throw new ApiError(400, "shadow_dismissal_reason_invalid", "A dismissal reason is required.");
+  const key = `shadow-dismiss:${idempotencyKey}`;
+  const prior = await db.prepare(`SELECT target_id AS targetId FROM audit_events WHERE idempotency_key = ?1 AND action = 'shadow.comparison_dismissed' LIMIT 1`)
+    .bind(key).first<{ targetId: string }>();
+  if (prior) {
+    if (prior.targetId !== comparisonId) throw new ApiError(409, "idempotency_key_reused", "This Idempotency-Key was already used for another comparison.");
+    return { comparisonId, status: "dismissed", idempotentReplay: true };
+  }
+  const timestamp = now();
+  const changed = await db.prepare(`UPDATE shadow_comparisons SET status = 'dismissed', reviewed_at = ?2 WHERE id = ?1 AND status IN ('pending','quality_task_created')`)
+    .bind(comparisonId, timestamp).run();
+  if (changed.meta.changes !== 1) throw new ApiError(404, "shadow_comparison_not_pending", "The pending shadow comparison does not exist.");
+  await db.prepare(`INSERT INTO audit_events (id, idempotency_key, action, actor_type, actor_id, target_type, target_id, details_json, created_at) VALUES (?1, ?2, 'shadow.comparison_dismissed', ?3, ?4, 'shadow_comparison', ?5, ?6, ?7)`)
+    .bind(crypto.randomUUID(), key, session.actor.type, session.actor.id, comparisonId, canonicalJson({ reason: reason.trim() }), timestamp).run();
+  return { comparisonId, status: "dismissed", idempotentReplay: false };
+}