@balpal4495/quorum 0.1.10 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +118 -9
  2. package/SETUP.md +6 -0
  3. package/evals/__tests__/eval.test.ts +31 -0
  4. package/evals/cases/auth_hs256_rejected.json +46 -0
  5. package/evals/cases/auth_rs256_valid.json +30 -0
  6. package/evals/cases/cache_missing_lock.json +31 -0
  7. package/evals/cases/db_naive_not_null.json +32 -0
  8. package/evals/cases/logging_pii_leak.json +32 -0
  9. package/evals/cases/migration_with_rollback.json +43 -0
  10. package/evals/cases/no_evidence_novel_design.json +16 -0
  11. package/evals/cases/payment_no_idempotency.json +33 -0
  12. package/evals/cases/redis_session_rejected.json +32 -0
  13. package/evals/cases/safe_refactor.json +17 -0
  14. package/evals/runner.ts +226 -0
  15. package/modules/AGENTS.md +9 -5
  16. package/modules/CLAUDE.md +25 -2
  17. package/modules/README.md +153 -6
  18. package/modules/council/advisors.ts +4 -1
  19. package/modules/council/chairman.ts +86 -15
  20. package/modules/council/deliberate.ts +28 -3
  21. package/modules/council/index.ts +6 -1
  22. package/modules/council/reviewers.ts +2 -1
  23. package/modules/council/risk.ts +89 -0
  24. package/modules/council/types.ts +63 -1
  25. package/modules/jury/evaluate.ts +35 -10
  26. package/modules/jury/index.ts +3 -1
  27. package/modules/jury/preflight.ts +101 -0
  28. package/modules/jury/schema.ts +9 -0
  29. package/modules/jury/types.ts +20 -1
  30. package/modules/oracle/propose.ts +19 -3
  31. package/modules/oracle/query.ts +3 -2
  32. package/modules/oracle/summary.ts +2 -1
  33. package/modules/sentinel/drift.ts +7 -3
  34. package/modules/sentinel/review.ts +2 -1
  35. package/modules/setup.ts +2 -1
  36. package/modules/shared/types.ts +47 -2
  37. package/package.json +2 -2
@@ -4,9 +4,14 @@ import { frameQuestion } from "./frame"
4
4
  import { fanOutAdvisors } from "./advisors"
5
5
  import { fanOutReviewers } from "./reviewers"
6
6
  import { chairman } from "./chairman"
7
+ import { classifyRisk } from "./risk"
7
8
 
8
9
  const DEFAULT_ADVISOR_COUNT = 5
9
10
  const DEFAULT_REVIEWER_COUNT = 5
11
+ const LITE_ADVISOR_COUNT = 1
12
+ const LITE_REVIEWER_COUNT = 2
13
+ const JURY_ONLY_ADVISOR_COUNT = 1
14
+ const JURY_ONLY_REVIEWER_COUNT = 1
10
15
 
11
16
  /**
12
17
  * Run the Council deliberation pipeline.
@@ -34,11 +39,23 @@ export async function deliberate(
34
39
  const {
35
40
  llm,
36
41
  oracle,
37
- advisorCount = DEFAULT_ADVISOR_COUNT,
38
- reviewerCount = DEFAULT_REVIEWER_COUNT,
39
42
  models = {},
40
43
  } = deps
41
44
 
45
+ // Classify risk to determine Council mode and advisor/reviewer counts
46
+ const risk = classifyRisk(input.outcome, input.design, input.evidence)
47
+ let defaultAdvisors = DEFAULT_ADVISOR_COUNT
48
+ let defaultReviewers = DEFAULT_REVIEWER_COUNT
49
+ if (risk.council_mode === "lite") {
50
+ defaultAdvisors = LITE_ADVISOR_COUNT
51
+ defaultReviewers = LITE_REVIEWER_COUNT
52
+ } else if (risk.council_mode === "jury-only") {
53
+ defaultAdvisors = JURY_ONLY_ADVISOR_COUNT
54
+ defaultReviewers = JURY_ONLY_REVIEWER_COUNT
55
+ }
56
+ const advisorCount = deps.advisorCount ?? defaultAdvisors
57
+ const reviewerCount = deps.reviewerCount ?? defaultReviewers
58
+
42
59
  // Select personas — cycle DEFAULT_PERSONAS if advisorCount > 5
43
60
  const personas = Array.from(
44
61
  { length: advisorCount },
@@ -82,12 +99,20 @@ export async function deliberate(
82
99
  .slice(0, 200)
83
100
 
84
101
  await oracle.propose({
102
+ schema_version: 2,
103
+ topic: input.outcome.slice(0, 80),
104
+ decision: keyInsight,
85
105
  key_insight: keyInsight,
86
106
  affected_areas: extractAffectedAreas(input.outcome, input.design),
107
+ alternatives_considered: verdict.challenges,
108
+ rejected_reason: verdict.satisfied
109
+ ? []
110
+ : verdict.blockers.map(b => b.issue).slice(0, 3),
87
111
  status: "open",
88
112
  confidence: input.jury_output.confidence,
89
113
  source_module: "council",
90
- evidence_cited: verdict.evidence_cited,
114
+ evidence_cited: verdict.citation_validation.valid_ids,
115
+ scope: risk.reasons.slice(0, 3),
91
116
  })
92
117
 
93
118
  return verdict
@@ -1,4 +1,9 @@
1
1
  export { deliberate } from "./deliberate"
2
- export type { CouncilInput, CouncilOutput, CouncilDeps, CouncilModels } from "./types"
2
+ export type {
3
+ CouncilInput, CouncilOutput, CouncilDeps, CouncilModels,
4
+ BlockerItem, WarningItem, CitationValidation, AdvisorSplit,
5
+ RiskLevel, CouncilMode, RiskAssessment,
6
+ } from "./types"
3
7
  export { DEFAULT_PERSONAS } from "./personas"
4
8
  export type { AdvisorPersona } from "./personas"
9
+ export { classifyRisk } from "./risk"
@@ -1,4 +1,5 @@
1
1
  import type { LLMProvider, OracleResult } from "../shared/types"
2
+ import { entryText } from "../shared/types"
2
3
  import type { AdvisorResponse } from "./advisors"
3
4
 
4
5
  export interface ReviewerResponse {
@@ -20,7 +21,7 @@ function anonymise(responses: AdvisorResponse[]): string {
20
21
  function formatEvidenceSummary(evidence: OracleResult[]): string {
21
22
  if (evidence.length === 0) return "No Oracle evidence available."
22
23
  return evidence
23
- .map(e => `[${e.id}] (${e.status}) ${e.key_insight}`)
24
+ .map(e => `[${e.id}] (${e.status}) ${entryText(e)}`)
24
25
  .join("\n")
25
26
  }
26
27
 
@@ -0,0 +1,89 @@
1
+ import type { OracleResult } from "../shared/types"
2
+ import type { RiskLevel, CouncilMode, RiskAssessment } from "./types"
3
+
4
+ /**
5
+ * Patterns that trigger risk escalation.
6
+ * Each entry has a level (the minimum risk level it triggers) and a reason label.
7
+ */
8
+ const RISK_RULES: Array<{ pattern: RegExp; level: RiskLevel; reason: string }> = [
9
+ // Critical — always run full Council + flag for human architecture review
10
+ { pattern: /\b(auth(?:entication|orization)?|jwt|token|session|password|oauth|credential|bearer)\b/i, level: "critical", reason: "authentication or authorisation logic" },
11
+ { pattern: /\b(payment|stripe|charge|billing|checkout|refund|subscription)\b/i, level: "critical", reason: "payment or billing logic" },
12
+ { pattern: /\b(encrypt|decrypt|private\s+key|certificate|tls|ssl|hmac|cipher)\b/i, level: "critical", reason: "cryptography or key management" },
13
+ { pattern: /\b(delete\s+all|drop\s+table|truncate|wipe|destroy.*data|hard\s+delete)\b/i, level: "critical", reason: "irreversible data deletion" },
14
+
15
+ // High — full Council
16
+ { pattern: /\b(migrat(?:ion|e)|alter\s+table|schema\s+change|not\s+null|backfill|pg_repack|shadow\s+column)\b/i, level: "high", reason: "database schema migration" },
17
+ { pattern: /\b(permission|role(?:s)?|acl|rbac|access\s+control|entitlement)\b/i, level: "high", reason: "permissions or access control" },
18
+ { pattern: /\b(pii|personal\s+data|gdpr|ccpa|email(?:\s+address)?|phone(?:\s+number)?|ssn|passport)\b/i, level: "high", reason: "PII or compliance-regulated data" },
19
+ { pattern: /\b(api\s+key|secret(?:s)?|private\s+key|credentials?)\b/i, level: "high", reason: "secrets or credentials handling" },
20
+
21
+ // Medium — Jury + lite Council
22
+ { pattern: /\b(cache|redis|memcached|invalidat(?:e|ion))\b/i, level: "medium", reason: "cache strategy" },
23
+ { pattern: /\b(rate\s*limit|throttl(?:e|ing)|quota)\b/i, level: "medium", reason: "rate limiting or throttling" },
24
+ { pattern: /\b(webhook|event|queue|pubsub|kafka|rabbitmq|sns|sqs)\b/i, level: "medium", reason: "async event or messaging" },
25
+ { pattern: /\b(deploy(?:ment)?|ci(?:\/cd)?|docker|kubernetes|infra(?:structure)?)\b/i, level: "medium", reason: "deployment or infrastructure" },
26
+ ]
27
+
28
+ const RISK_ORDER: RiskLevel[] = ["low", "medium", "high", "critical"]
29
+
30
+ function maxLevel(a: RiskLevel, b: RiskLevel): RiskLevel {
31
+ return RISK_ORDER.indexOf(a) >= RISK_ORDER.indexOf(b) ? a : b
32
+ }
33
+
34
+ function councilModeForLevel(level: RiskLevel): CouncilMode {
35
+ switch (level) {
36
+ case "low": return "jury-only"
37
+ case "medium": return "lite"
38
+ case "high": return "full"
39
+ case "critical": return "full"
40
+ }
41
+ }
42
+
43
+ /**
44
+ * Classify the risk level of a proposed change from its text and evidence.
45
+ *
46
+ * Risk determines Council mode — avoid running full fan-out on low-risk changes:
47
+ * low → jury-only (no advisor/reviewer fan-out)
48
+ * medium → lite (Jury + 2 reviewers)
49
+ * high → full (standard 5 advisors + 5 reviewers)
50
+ * critical → full (same as high, but Chronicle entry flags for human architecture review)
51
+ *
52
+ * Refuted Oracle entries also elevate risk — a known failure mode in the evidence pack
53
+ * means the design is repeating something that already went wrong.
54
+ */
55
+ export function classifyRisk(
56
+ outcome: string,
57
+ design: string,
58
+ evidence: OracleResult[],
59
+ ): RiskAssessment {
60
+ const text = `${outcome} ${design}`
61
+ let level: RiskLevel = "low"
62
+ const reasons: string[] = []
63
+
64
+ for (const rule of RISK_RULES) {
65
+ if (rule.pattern.test(text)) {
66
+ const matched = maxLevel(level, rule.level)
67
+ if (matched !== level || !reasons.includes(rule.reason)) {
68
+ level = matched
69
+ reasons.push(rule.reason)
70
+ }
71
+ }
72
+ }
73
+
74
+ // Refuted entries in the evidence pack are a direct risk signal
75
+ const refutedCount = evidence.filter(e => e.status === "refuted").length
76
+ if (refutedCount > 0) {
77
+ const refutedRisk: RiskLevel = refutedCount >= 2 ? "high" : "medium"
78
+ if (RISK_ORDER.indexOf(refutedRisk) > RISK_ORDER.indexOf(level)) {
79
+ level = maxLevel(level, refutedRisk)
80
+ }
81
+ reasons.push(`${refutedCount} refuted Chronicle ${refutedCount === 1 ? "entry" : "entries"} in evidence pack`)
82
+ }
83
+
84
+ return {
85
+ level,
86
+ reasons: reasons.length > 0 ? reasons : ["no sensitive patterns detected"],
87
+ council_mode: councilModeForLevel(level),
88
+ }
89
+ }
@@ -12,14 +12,57 @@ export interface CouncilInput {
12
12
  jury_output: JuryOutput
13
13
  }
14
14
 
15
+ /** A finding that must be resolved before the design can proceed. */
16
+ export interface BlockerItem {
17
+ issue: string
18
+ /** Oracle entry IDs that evidence this blocker. */
19
+ evidence: string[]
20
+ /** What must change in the design to resolve this. */
21
+ required_fix: string
22
+ }
23
+
24
+ /** A finding that should be addressed but does not block proceeding. */
25
+ export interface WarningItem {
26
+ issue: string
27
+ suggested_fix?: string
28
+ }
29
+
30
+ /** Validates that cited Oracle IDs actually appeared in the evidence pack. */
31
+ export interface CitationValidation {
32
+ /** IDs that were cited and exist in the evidence pack. */
33
+ valid_ids: string[]
34
+ /** IDs that were cited but were NOT in the evidence pack — likely hallucinated. */
35
+ hallucinated_ids: string[]
36
+ }
37
+
38
+ /** How advisors split on their recommendation. Signals disagreement level. */
39
+ export interface AdvisorSplit {
40
+ proceed: number
41
+ redesign: number
42
+ "investigate-more": number
43
+ }
44
+
15
45
  export interface CouncilOutput {
16
46
  satisfied: boolean
17
47
  /** Chairman synthesis — every material conclusion cites Oracle entry IDs. */
18
48
  verdict: string
19
- /** What was challenged or could not be validated. */
49
+ /**
50
+ * Findings that MUST be resolved before the design proceeds.
51
+ * Each blocker names the issue, the Oracle evidence behind it, and the required fix.
52
+ */
53
+ blockers: BlockerItem[]
54
+ /**
55
+ * Findings that SHOULD be addressed but don't block execution.
56
+ */
57
+ warnings: WarningItem[]
58
+ /** Flat list of all issues raised — backwards compatible with existing consumers. */
20
59
  challenges: string[]
21
60
  /** Oracle entry IDs referenced in the verdict. */
22
61
  evidence_cited: string[]
62
+ /** Validation of whether cited IDs exist in the evidence pack. */
63
+ citation_validation: CitationValidation
64
+ /** How advisors split on recommendation — high disagreement = escalate. */
65
+ advisor_split: AdvisorSplit
23
66
  recommendation: "proceed" | "redesign" | "investigate-more"
24
67
  }
25
68
 
@@ -43,3 +86,22 @@ export interface CouncilDeps {
43
86
  reviewerCount?: number
44
87
  models?: CouncilModels
45
88
  }
89
+
90
+ // ── Risk classifier types ─────────────────────────────────────────────────────
91
+
92
+ export type RiskLevel = "low" | "medium" | "high" | "critical"
93
+
94
+ /**
95
+ * Determines which Council mode to use.
96
+ * skip → Oracle query only, no LLM validation
97
+ * jury-only → Jury scores, no Council fan-out
98
+ * lite → Jury + 1–2 reviewers (no full advisor fan-out)
99
+ * full → Full Council (default 5 advisors + 5 reviewers + Chairman)
100
+ */
101
+ export type CouncilMode = "skip" | "jury-only" | "lite" | "full"
102
+
103
+ export interface RiskAssessment {
104
+ level: RiskLevel
105
+ reasons: string[]
106
+ council_mode: CouncilMode
107
+ }
@@ -1,6 +1,8 @@
1
1
  import type { JuryInput, JuryOutput, JuryDeps } from "./types"
2
2
  import type { OracleResult } from "../shared/types"
3
+ import { entryText } from "../shared/types"
3
4
  import { JuryOutputSchema } from "./schema"
5
+ import { runPreflight, formatPreflight } from "./preflight"
4
6
 
5
7
  const CONFIDENCE_THRESHOLD = 0.6
6
8
 
@@ -12,8 +14,8 @@ function formatEvidence(evidence: OracleResult[]): string {
12
14
  .map(e =>
13
15
  [
14
16
  `[${e.id}] status=${e.status} confidence=${e.confidence.toFixed(2)} score=${e.score.toFixed(3)}`,
15
- `Insight: ${e.key_insight}`,
16
- `Areas: ${e.affected_areas.join(", ")}`,
17
+ `Insight: ${entryText(e)}`,
18
+ `Areas: ${e.affected_areas.join(", ")}${e.scope ? " | " + e.scope.join(", ") : ""}`,
17
19
  e.outcome ? `Outcome: ${e.outcome}` : null,
18
20
  ]
19
21
  .filter(Boolean)
@@ -24,14 +26,21 @@ function formatEvidence(evidence: OracleResult[]): string {
24
26
 
25
27
  const SYSTEM_PROMPT = `You are the Jury — an evidence-based evaluator for agentic development workflows.
26
28
 
27
- Your job is to evaluate a proposed design against Oracle evidence and produce a structured confidence score.
29
+ Your job is to evaluate a proposed design against Oracle evidence and produce a calibrated confidence score.
28
30
  You do NOT make decisions. You assess and score. Your output determines the Council's brief.
29
31
 
30
- Score the design across these four dimensions (equally weighted to produce a final confidence in [0, 1]):
31
- 1. Evidence support — do validated Oracle entries confirm this approach works in this codebase?
32
- 2. Feasibility — do Oracle entries (or their absence) suggest this is achievable?
33
- 3. Risk what do refuted entries reveal about failure modes? Has this been tried and failed?
34
- 4. Completeness — does the design address the full outcome, or only part of it?
32
+ Score the design across four dimensions, each 01:
33
+ 1. evidence_support — do validated Oracle entries confirm this approach works in this codebase?
34
+ 2. feasibility — do Oracle entries (or their absence) suggest this is achievable?
35
+ 3. risk how well does the design address known failure modes? (1 = fully addressed, 0 = ignored)
36
+ 4. completeness — does the design cover the full outcome, or only part of it?
37
+
38
+ confidence = average of the four scores (you must compute this yourself — do not round or adjust it).
39
+
40
+ Gaps fall into two categories:
41
+ - gaps: any missing evidence that would improve confidence
42
+ - blocking_gaps: a SUBSET of gaps that are hard blockers — must be resolved before proceeding
43
+ (examples: no rollback plan for a destructive change, no auth strategy for a security-sensitive feature)
35
44
 
36
45
  council_brief is determined by confidence only (do not invent a value):
37
46
  confidence < 0.6 → council_brief = "challenge"
@@ -40,8 +49,15 @@ council_brief is determined by confidence only (do not invent a value):
40
49
  Return ONLY valid JSON that matches this schema exactly — no markdown fences, no explanation:
41
50
  {
42
51
  "confidence": <number 0–1>,
52
+ "confidence_breakdown": {
53
+ "evidence_support": <number 0–1>,
54
+ "feasibility": <number 0–1>,
55
+ "risk": <number 0–1>,
56
+ "completeness": <number 0–1>
57
+ },
43
58
  "assessment": <string — what the evidence supports or contradicts>,
44
- "gaps": [<string — each missing piece of evidence from Oracle>],
59
+ "gaps": [<string — each missing piece of evidence>],
60
+ "blocking_gaps": [<string — gaps that are hard blockers only>],
45
61
  "council_brief": "challenge" | "pressure-test",
46
62
  "recommendation": "proceed" | "investigate-more" | "redesign"
47
63
  }`
@@ -62,6 +78,8 @@ export async function evaluate(
62
78
  ): Promise<JuryOutput> {
63
79
  const { llm, model } = deps
64
80
  const evidenceText = formatEvidence(input.evidence)
81
+ const preflight = runPreflight(input.outcome, input.design, input.evidence)
82
+ const preflightText = formatPreflight(preflight)
65
83
 
66
84
  const userPrompt = [
67
85
  "## Outcome",
@@ -70,6 +88,8 @@ export async function evaluate(
70
88
  "## Proposed Design",
71
89
  input.design,
72
90
  "",
91
+ preflightText,
92
+ "",
73
93
  "## Oracle Evidence",
74
94
  evidenceText,
75
95
  ].join("\n")
@@ -104,7 +124,12 @@ export async function evaluate(
104
124
 
105
125
  const output = result.data
106
126
 
107
- // Enforce council_brief from confidence do not trust the LLM to compute this correctly
127
+ // Recompute confidence as the exact average of breakdown dimensions
128
+ // This makes confidence deterministic and calibrated regardless of what the LLM returned
129
+ const { evidence_support, feasibility, risk, completeness } = output.confidence_breakdown
130
+ output.confidence = Math.round(((evidence_support + feasibility + risk + completeness) / 4) * 100) / 100
131
+
132
+ // Enforce council_brief from recomputed confidence — do not trust the LLM to compute this correctly
108
133
  output.council_brief =
109
134
  output.confidence < CONFIDENCE_THRESHOLD ? "challenge" : "pressure-test"
110
135
 
@@ -1,3 +1,5 @@
1
1
  export { evaluate } from "./evaluate"
2
- export type { JuryInput, JuryOutput, JuryDeps } from "./types"
2
+ export type { JuryInput, JuryOutput, JuryDeps, ConfidenceBreakdown } from "./types"
3
3
  export { JuryOutputSchema } from "./schema"
4
+ export { runPreflight, formatPreflight } from "./preflight"
5
+ export type { PreflightResult } from "./preflight"
@@ -0,0 +1,101 @@
1
+ import type { OracleResult } from "../shared/types"
2
+ import { entryText } from "../shared/types"
3
+
4
+ /** Areas that warrant elevated scrutiny. */
5
+ const SENSITIVE_PATTERNS: Record<string, RegExp> = {
6
+ auth: /\b(auth(?:entication|orization)?|jwt|token|session|password|oauth|login|logout|credential|bearer)\b/i,
7
+ database: /\b(migrat(?:ion|e)|alter\s+table|schema\s+change|postgres|mysql|sqlite|prisma|drizzle|knex|sequelize)\b/i,
8
+ crypto: /\b(encrypt|decrypt|cipher|hash(?:ing)?|hmac|sign(?:ing)?|verify|private\s+key|certificate|tls|ssl)\b/i,
9
+ payments: /\b(payment|stripe|charge|billing|invoice|subscription|price|checkout|refund)\b/i,
10
+ permissions: /\b(permission|role(?:s)?|acl|access\s+control|rbac|authorization|entitlement)\b/i,
11
+ pii: /\b(pii|personal\s+data|gdpr|ccpa|email(?:\s+address)?|phone(?:\s+number)?|postal\s+address|ssn|passport)\b/i,
12
+ data_deletion: /\b(delete(?:\s+all)?|drop\s+table|truncate|purge|wipe|destroy.*data|hard\s+delete)\b/i,
13
+ secrets: /\b(api\s+key|secret(?:s)?|env(?:ironment)?\s+var(?:iable)?|\.env|private\s+key|credentials?)\b/i,
14
+ }
15
+
16
+ const ROLLBACK_PATTERNS = /\b(rollback|roll\s+back|revert|undo|restore|recovery|fallback|backward[- ]compat)\b/i
17
+ const TEST_PATTERNS = /\b(test(?:ing|s)?|spec(?:ification)?|unit\s+test|integration\s+test|coverage|vitest|jest|mocha)\b/i
18
+
19
+ export interface PreflightResult {
20
+ touches_sensitive_area: boolean
21
+ /** Which sensitive area categories were detected. */
22
+ sensitive_areas: string[]
23
+ /** Whether the design mentions a rollback or recovery strategy. */
24
+ rollback_mentioned: boolean
25
+ /** Whether the design mentions testing. */
26
+ test_strategy_mentioned: boolean
27
+ /**
28
+ * IDs of refuted Chronicle entries that semantically overlap with the design text.
29
+ * These are potential conflicts — Jury should surface them.
30
+ */
31
+ chronicle_conflicts: string[]
32
+ }
33
+
34
+ /**
35
+ * Static preflight analysis — no LLM required.
36
+ *
37
+ * Runs deterministic checks on the outcome + design text and the evidence pack
38
+ * before any LLM call. Results are injected into the Jury prompt so the LLM
39
+ * reasons over concrete signals rather than discovering them itself.
40
+ */
41
+ export function runPreflight(
42
+ outcome: string,
43
+ design: string,
44
+ evidence: OracleResult[],
45
+ ): PreflightResult {
46
+ const text = `${outcome} ${design}`
47
+
48
+ const sensitive_areas = Object.entries(SENSITIVE_PATTERNS)
49
+ .filter(([, pattern]) => pattern.test(text))
50
+ .map(([area]) => area)
51
+
52
+ // Refuted entries whose primary text shares at least one significant word with the design
53
+ const designWords = new Set(
54
+ text
55
+ .toLowerCase()
56
+ .split(/\W+/)
57
+ .filter(w => w.length > 4),
58
+ )
59
+
60
+ const chronicle_conflicts = evidence
61
+ .filter(e => {
62
+ if (e.status !== "refuted") return false
63
+ const entryWords = entryText(e)
64
+ .toLowerCase()
65
+ .split(/\W+/)
66
+ .filter(w => w.length > 4)
67
+ return entryWords.some(w => designWords.has(w))
68
+ })
69
+ .map(e => e.id)
70
+
71
+ return {
72
+ touches_sensitive_area: sensitive_areas.length > 0,
73
+ sensitive_areas,
74
+ rollback_mentioned: ROLLBACK_PATTERNS.test(text),
75
+ test_strategy_mentioned: TEST_PATTERNS.test(text),
76
+ chronicle_conflicts,
77
+ }
78
+ }
79
+
80
+ /** Format preflight result for injection into the Jury prompt. */
81
+ export function formatPreflight(preflight: PreflightResult): string {
82
+ const lines: string[] = ["## Deterministic Preflight (machine-checked, not LLM-inferred)"]
83
+
84
+ if (preflight.touches_sensitive_area) {
85
+ lines.push(`⚠ Sensitive areas detected: ${preflight.sensitive_areas.join(", ")}`)
86
+ } else {
87
+ lines.push("✓ No sensitive areas detected")
88
+ }
89
+
90
+ lines.push(preflight.rollback_mentioned ? "✓ Rollback strategy mentioned" : "✗ No rollback strategy mentioned")
91
+ lines.push(preflight.test_strategy_mentioned ? "✓ Test strategy mentioned" : "✗ No test strategy mentioned")
92
+
93
+ if (preflight.chronicle_conflicts.length > 0) {
94
+ lines.push(`⚠ Refuted Chronicle entries potentially conflicting: ${preflight.chronicle_conflicts.join(", ")}`)
95
+ lines.push(" These entries were previously tried and failed — verify the design addresses the documented failure reason.")
96
+ } else {
97
+ lines.push("✓ No conflicting refuted Chronicle entries")
98
+ }
99
+
100
+ return lines.join("\n")
101
+ }
@@ -1,13 +1,22 @@
1
1
  import { z } from "zod"
2
2
 
3
+ const ConfidenceBreakdownSchema = z.object({
4
+ evidence_support: z.number().min(0).max(1),
5
+ feasibility: z.number().min(0).max(1),
6
+ risk: z.number().min(0).max(1),
7
+ completeness: z.number().min(0).max(1),
8
+ })
9
+
3
10
  /**
4
11
  * Zod schema for the Jury's structured LLM output.
5
12
  * evaluate() validates all LLM responses against this before returning.
6
13
  */
7
14
  export const JuryOutputSchema = z.object({
8
15
  confidence: z.number().min(0).max(1),
16
+ confidence_breakdown: ConfidenceBreakdownSchema,
9
17
  assessment: z.string().min(1),
10
18
  gaps: z.array(z.string()),
19
+ blocking_gaps: z.array(z.string()),
11
20
  council_brief: z.enum(["challenge", "pressure-test"]),
12
21
  recommendation: z.enum(["proceed", "investigate-more", "redesign"]),
13
22
  })
@@ -9,13 +9,32 @@ export interface JuryInput {
9
9
  evidence: OracleResult[]
10
10
  }
11
11
 
12
+ /** Per-dimension breakdown of the 0–1 confidence score. */
13
+ export interface ConfidenceBreakdown {
14
+ /** Do validated Oracle entries confirm this approach works here? */
15
+ evidence_support: number
16
+ /** Do Oracle entries suggest this is achievable in this codebase? */
17
+ feasibility: number
18
+ /** How well does the design address known failure modes? (1 = fully addressed) */
19
+ risk: number
20
+ /** Does the design cover the full outcome, or only part of it? */
21
+ completeness: number
22
+ }
23
+
12
24
  export interface JuryOutput {
13
- /** 0–1 confidence score. Drives the Council brief. */
25
+ /** 0–1 confidence score. Average of the four breakdown dimensions. */
14
26
  confidence: number
27
+ /** Per-dimension breakdown of the confidence score. */
28
+ confidence_breakdown: ConfidenceBreakdown
15
29
  /** What the evidence supports or contradicts. */
16
30
  assessment: string
17
31
  /** Evidence missing from Oracle that would improve confidence. */
18
32
  gaps: string[]
33
+ /**
34
+ * Gaps that are hard blockers — must be resolved before Council should proceed.
35
+ * Subset of gaps where the missing information is critical (auth, rollback, data safety).
36
+ */
37
+ blocking_gaps: string[]
19
38
  /**
20
39
  * Council brief derived from confidence:
21
40
  * < 0.6 → "challenge" (find what is wrong — broader scope)
@@ -4,6 +4,7 @@ import { randomUUID } from "crypto"
4
4
  import { exec } from "child_process"
5
5
  import { promisify } from "util"
6
6
  import type { ChronicleEntry, SimilarityWarning } from "../shared/types"
7
+ import { entryText } from "../shared/types"
7
8
  import type { OracleDeps } from "./types"
8
9
  import { updateSummary } from "./summary"
9
10
 
@@ -27,6 +28,21 @@ function validateEntry(entry: Omit<ChronicleEntry, "id" | "timestamp">): void {
27
28
  `Distil to a single clear sentence.`,
28
29
  )
29
30
  }
31
+ if (entry.decision !== undefined) {
32
+ const d = entry.decision.trim()
33
+ if (d.length < INSIGHT_MIN_LENGTH) {
34
+ throw new Error(
35
+ `decision too short (${d.length} chars, min ${INSIGHT_MIN_LENGTH}). ` +
36
+ `Write a specific, complete sentence describing the decision.`,
37
+ )
38
+ }
39
+ if (d.length > INSIGHT_MAX_LENGTH) {
40
+ throw new Error(
41
+ `decision too long (${d.length} chars, max ${INSIGHT_MAX_LENGTH}). ` +
42
+ `Distil to a single clear sentence.`,
43
+ )
44
+ }
45
+ }
30
46
  if (!entry.affected_areas || entry.affected_areas.filter(a => a.trim()).length === 0) {
31
47
  throw new Error(`affected_areas must contain at least one non-empty entry.`)
32
48
  }
@@ -40,7 +56,7 @@ async function checkSimilarity(
40
56
  deps: OracleDeps,
41
57
  ): Promise<SimilarityWarning | undefined> {
42
58
  try {
43
- const text = [entry.key_insight, ...entry.affected_areas].join(" ")
59
+ const text = [entryText(entry), ...entry.affected_areas, ...(entry.scope ?? [])].join(" ")
44
60
  const vector = await deps.embedder(text)
45
61
  const results = await deps.vectorStore.search(vector, 3)
46
62
  if (results.length === 0) return undefined
@@ -116,8 +132,8 @@ export async function commit(
116
132
  timestamp: new Date().toISOString(),
117
133
  }
118
134
 
119
- // Embed the key insight (plus affected areas for richer retrieval)
120
- const embeddingText = [entry.key_insight, ...entry.affected_areas].join(" ")
135
+ // Embed the primary text + areas + scope tags for richer retrieval
136
+ const embeddingText = [entryText(entry), ...entry.affected_areas, ...(entry.scope ?? [])].join(" ")
121
137
  const vector = await deps.embedder(embeddingText)
122
138
  await deps.vectorStore.upsert(entry.id, vector, entry)
123
139
 
@@ -1,4 +1,5 @@
1
1
  import type { ChronicleEntry, OracleResult, QueryOptions } from "../shared/types"
2
+ import { entryText } from "../shared/types"
2
3
  import type { OracleDeps } from "./types"
3
4
  import { bm25Score, extractDomainTerms } from "./bm25"
4
5
  import { appendQueryLog } from "./log"
@@ -62,13 +63,13 @@ export async function query(
62
63
  // ── Pass 2: BM25 re-ranking with query enrichment ─────────────────────────
63
64
  const topInsights = candidates
64
65
  .slice(0, Math.min(5, candidates.length))
65
- .map(c => c.entry.key_insight)
66
+ .map(c => entryText(c.entry))
66
67
  const domainTerms = extractDomainTerms(topInsights)
67
68
  const enrichedQuery =
68
69
  domainTerms.length > 0 ? `${text} ${domainTerms.join(" ")}` : text
69
70
 
70
71
  const documents = candidates.map(c =>
71
- [c.entry.key_insight, ...c.entry.affected_areas].join(" "),
72
+ [entryText(c.entry), ...c.entry.affected_areas, ...(c.entry.scope ?? [])].join(" "),
72
73
  )
73
74
  const bm25Scores = bm25Score(enrichedQuery, documents)
74
75
 
@@ -1,6 +1,7 @@
1
1
  import { promises as fs } from "fs"
2
2
  import path from "path"
3
3
  import type { ChronicleEntry } from "../shared/types"
4
+ import { entryText } from "../shared/types"
4
5
 
5
6
  const SUMMARY_WEEKS = 12
6
7
  const DIRECTIVE =
@@ -29,7 +30,7 @@ function workRefLabel(entry: ChronicleEntry): string {
29
30
  function renderEntry(entry: ChronicleEntry): string {
30
31
  const areas = entry.affected_areas.join(", ")
31
32
  const id = entry.id.slice(0, 8)
32
- return `- **[${id}]** ${areas} — \`${entry.status}\` (${entry.confidence.toFixed(2)}) — ${entry.key_insight}`
33
+ return `- **[${id}]** ${areas} — \`${entry.status}\` (${entry.confidence.toFixed(2)}) — ${entryText(entry)}`
33
34
  }
34
35
 
35
36
  /**
@@ -1,6 +1,7 @@
1
1
  import { promises as fs } from "fs"
2
2
  import path from "path"
3
3
  import type { ChronicleEntry, DriftFlag, DriftReport, LLMProvider } from "../shared/types"
4
+ import { entryText } from "../shared/types"
4
5
 
5
6
  const FILE_CONTENT_LIMIT = 3000
6
7
 
@@ -73,7 +74,10 @@ async function evaluateDrift(
73
74
  {
74
75
  role: "user",
75
76
  content:
76
- `Documented insight:\n"${entry.key_insight}"\n\n` +
77
+ `Documented insight:
78
+ "${entryText(entry)}"
79
+
80
+ ` +
77
81
  `Current source:\n${fileSection}\n\n` +
78
82
  `Does this insight still accurately describe the code above?\n` +
79
83
  `{"stillValid": boolean, "confidence": number, "reasoning": "one sentence"}`,
@@ -86,7 +90,7 @@ async function evaluateDrift(
86
90
  const parsed = JSON.parse(match[0]) as { stillValid?: unknown; confidence?: unknown; reasoning?: unknown }
87
91
  return {
88
92
  entryId: entry.id,
89
- keyInsight: entry.key_insight,
93
+ keyInsight: entryText(entry),
90
94
  affectedFiles: files.map(f => f.filePath),
91
95
  stillValid: Boolean(parsed.stillValid),
92
96
  confidence: typeof parsed.confidence === "number" ? Math.max(0, Math.min(1, parsed.confidence)) : 0.5,
@@ -96,7 +100,7 @@ async function evaluateDrift(
96
100
  // Parse failure → conservative: flag for human review
97
101
  return {
98
102
  entryId: entry.id,
99
- keyInsight: entry.key_insight,
103
+ keyInsight: entryText(entry),
100
104
  affectedFiles: files.map(f => f.filePath),
101
105
  stillValid: false,
102
106
  confidence: 0,