@balpal4495/quorum 0.1.0

package/modules/oracle/propose.ts

@@ -0,0 +1,148 @@
+import { promises as fs } from "fs"
+import path from "path"
+import { randomUUID } from "crypto"
+import { exec } from "child_process"
+import { promisify } from "util"
+import type { ChronicleEntry, SimilarityWarning } from "../shared/types"
+import type { OracleDeps } from "./types"
+import { updateSummary } from "./summary"
+
+const execAsync = promisify(exec)
+
+const INSIGHT_MIN_LENGTH = 20
+const INSIGHT_MAX_LENGTH = 200
+const SIMILARITY_WARNING_THRESHOLD = 0.85
+
+function validateEntry(entry: Omit<ChronicleEntry, "id" | "timestamp">): void {
+  const insight = entry.key_insight?.trim() ?? ""
+  if (insight.length < INSIGHT_MIN_LENGTH) {
+    throw new Error(
+      `key_insight too short (${insight.length} chars, min ${INSIGHT_MIN_LENGTH}). ` +
+      `Write a specific, complete sentence naming the module or area affected.`,
+    )
+  }
+  if (insight.length > INSIGHT_MAX_LENGTH) {
+    throw new Error(
+      `key_insight too long (${insight.length} chars, max ${INSIGHT_MAX_LENGTH}). ` +
+      `Distil to a single clear sentence.`,
+    )
+  }
+  if (!entry.affected_areas || entry.affected_areas.filter(a => a.trim()).length === 0) {
+    throw new Error(`affected_areas must contain at least one non-empty entry.`)
+  }
+  if (entry.confidence < 0 || entry.confidence > 1) {
+    throw new Error(`confidence must be between 0 and 1, got ${entry.confidence}.`)
+  }
+}
+
+async function checkSimilarity(
+  entry: Omit<ChronicleEntry, "id" | "timestamp">,
+  deps: OracleDeps,
+): Promise<SimilarityWarning | undefined> {
+  try {
+    const text = [entry.key_insight, ...entry.affected_areas].join(" ")
+    const vector = await deps.embedder(text)
+    const results = await deps.vectorStore.search(vector, 3)
+    if (results.length === 0) return undefined
+    const top = results[0]
+    if (top.score < SIMILARITY_WARNING_THRESHOLD) return undefined
+    return {
+      entry: top.entry,
+      score: top.score,
+      // If the existing entry is validated, a near-duplicate is likely a correction
+      warning: top.entry.status === "validated" ? "potential-supersession" : "potential-duplicate",
+    }
+  } catch {
+    // Similarity check is best-effort — never block a proposal because of it
+    return undefined
+  }
+}
+
+/**
+ * Propose a new Chronicle entry for human review.
+ * Validates entry quality and checks for similar existing entries before writing.
+ * Writes the entry to .chronicle/proposals/<id>.json — NOT yet indexed.
+ * The proposal sits pending until a human calls commit() to approve it.
+ *
+ * Throws if key_insight is too short/long or affected_areas is empty.
+ * Returns a SimilarityWarning if a near-identical entry already exists —
+ * the human gate should surface this before approving the commit.
+ */
+export async function propose(
+  entry: Omit<ChronicleEntry, "id" | "timestamp">,
+  deps: OracleDeps,
+): Promise<{ proposalId: string; similarity?: SimilarityWarning }> {
+  validateEntry(entry)
+
+  const similarity = await checkSimilarity(entry, deps)
+
+  const chronicleDir = deps.chronicleDir ?? ".chronicle"
+  const proposalsDir = path.join(chronicleDir, "proposals")
+  await fs.mkdir(proposalsDir, { recursive: true })
+
+  const proposalId = randomUUID()
+  const proposalPath = path.join(proposalsDir, `${proposalId}.json`)
+  await fs.writeFile(proposalPath, JSON.stringify(entry, null, 2), "utf8")
+
+  return { proposalId, ...(similarity ? { similarity } : {}) }
+}
+
+/**
+ * Commit a pending proposal after human approval.
+ * Reads the proposal file, assigns an ID and timestamp, embeds the entry,
+ * upserts it into the vector store, and deletes the proposal file.
+ *
+ * Throws if the proposal does not exist.
+ */
+export async function commit(
+  proposalId: string,
+  deps: OracleDeps,
+): Promise<ChronicleEntry> {
+  const chronicleDir = deps.chronicleDir ?? ".chronicle"
+  const proposalPath = path.join(chronicleDir, "proposals", `${proposalId}.json`)
+
+  let raw: string
+  try {
+    raw = await fs.readFile(proposalPath, "utf8")
+  } catch {
+    throw new Error(`Proposal not found: ${proposalId}`)
+  }
+
+  const partial = JSON.parse(raw) as Omit<ChronicleEntry, "id" | "timestamp">
+
+  const entry: ChronicleEntry = {
+    ...partial,
+    id: randomUUID(),
+    timestamp: new Date().toISOString(),
+  }
+
+  // Embed the key insight (plus affected areas for richer retrieval)
+  const embeddingText = [entry.key_insight, ...entry.affected_areas].join(" ")
+  const vector = await deps.embedder(embeddingText)
+  await deps.vectorStore.upsert(entry.id, vector, entry)
+
+  // Write to committed/ — the git-tracked source of truth shared across the team
+  const committedDir = path.join(chronicleDir, "committed")
+  await fs.mkdir(committedDir, { recursive: true })
+  const committedPath = path.join(committedDir, `${entry.id}.json`)
+  await fs.writeFile(committedPath, JSON.stringify(entry, null, 2), "utf8")
+
+  // Stage the committed entry for the next git commit — best-effort
+  try {
+    await execAsync(`git add "${committedPath}"`)
+  } catch {
+    // Not in a git repo, or git is unavailable — silently continue
+  }
+
+  // Rebuild SUMMARY.md — best-effort, never fail a commit
+  try {
+    await updateSummary(chronicleDir)
+  } catch {
+    // Summary generation failure must not fail a commit
+  }
+
+  // Remove the proposal — it has been committed
+  await fs.unlink(proposalPath)
+
+  return entry
+}

package/modules/oracle/query.ts

@@ -0,0 +1,145 @@
+import type { ChronicleEntry, OracleResult, QueryOptions } from "../shared/types"
+import type { OracleDeps } from "./types"
+import { bm25Score, extractDomainTerms } from "./bm25"
+import { appendQueryLog } from "./log"
+
+const DEFAULT_LIMIT = 10
+const DEFAULT_SCORE_THRESHOLD = 0.031
+const RRF_K = 60
+/** Retrieve this many vector candidates before BM25 re-ranking. */
+const CANDIDATE_MULTIPLIER = 3
+
+/**
+ * Reciprocal Rank Fusion score.
+ * score = Σ 1 / (k + rank_i) summed across all rank lists.
+ * k = 60 (standard constant).
+ */
+function rrfScore(ranks: number[]): number {
+  return ranks.reduce((sum, rank) => sum + 1 / (RRF_K + rank), 0)
+}
+
+/**
+ * Two-pass retrieval with Reciprocal Rank Fusion.
+ *
+ * Pass 1 — vector similarity:
+ *   Embed the query, retrieve top (limit × CANDIDATE_MULTIPLIER) candidates.
+ *
+ * Pass 2 — BM25 re-ranking with query enrichment:
+ *   Extract domain terms from Pass 1 key insights, enrich the query,
+ *   score candidates with BM25, fuse ranks via RRF.
+ *
+ * Results below scoreThreshold are dropped entirely.
+ * All queries are appended to .chronicle/query-log.jsonl.
+ */
+export async function query(
+  text: string,
+  options: QueryOptions = {},
+  deps: OracleDeps,
+): Promise<OracleResult[]> {
+  const {
+    statusFilter,
+    limit = DEFAULT_LIMIT,
+    scoreThreshold = DEFAULT_SCORE_THRESHOLD,
+  } = options
+
+  const startTime = Date.now()
+
+  // ── Pass 1: vector similarity ──────────────────────────────────────────────
+  const queryVector = await deps.embedder(text)
+  const candidateLimit = limit * CANDIDATE_MULTIPLIER
+  let candidates = await deps.vectorStore.search(queryVector, candidateLimit)
+
+  // Status filter applied before BM25 to avoid scoring irrelevant entries
+  if (statusFilter && statusFilter.length > 0) {
+    candidates = candidates.filter(c => statusFilter.includes(c.entry.status))
+  }
+
+  if (candidates.length === 0) {
+    await tryLogQuery(text, [], startTime, deps)
+    return []
+  }
+
+  // ── Pass 2: BM25 re-ranking with query enrichment ─────────────────────────
+  const topInsights = candidates
+    .slice(0, Math.min(5, candidates.length))
+    .map(c => c.entry.key_insight)
+  const domainTerms = extractDomainTerms(topInsights)
+  const enrichedQuery =
+    domainTerms.length > 0 ? `${text} ${domainTerms.join(" ")}` : text
+
+  const documents = candidates.map(c =>
+    [c.entry.key_insight, ...c.entry.affected_areas].join(" "),
+  )
+  const bm25Scores = bm25Score(enrichedQuery, documents)
+
+  // Build BM25 rank lookup (index → rank)
+  const bm25RankOf: number[] = new Array(candidates.length)
+  bm25Scores
+    .map((score, i) => ({ i, score }))
+    .sort((a, b) => b.score - a.score)
+    .forEach(({ i }, rank) => {
+      bm25RankOf[i] = rank
+    })
+
+  // ── RRF fusion ─────────────────────────────────────────────────────────────
+  const fused: Array<ChronicleEntry & { score: number }> = candidates.map(
+    (candidate, vectorRank) => ({
+      ...candidate.entry,
+      score: rrfScore([vectorRank, bm25RankOf[vectorRank]]),
+    }),
+  )
+
+  fused.sort((a, b) => b.score - a.score)
+
+  const filtered = fused
+    .filter(r => r.score >= scoreThreshold)
+    .slice(0, limit)
+
+  const results = assignTiers(filtered)
+
+  await tryLogQuery(text, results, startTime, deps)
+  return results
+}
+
+/**
+ * Assign relevance tiers within the result set using relative rank.
+ * Top ~30% → primary, next ~40% → supporting, remainder → background.
+ * Thresholds are relative so they self-calibrate as Chronicle grows.
+ */
+function assignTiers(
+  results: Array<ChronicleEntry & { score: number }>,
+): OracleResult[] {
+  const n = results.length
+  if (n === 0) return []
+  const primaryCount = Math.max(1, Math.ceil(n * 0.3))
+  const supportingCount = Math.max(1, Math.ceil(n * 0.4))
+  return results.map((r, i) => ({
+    ...r,
+    tier:
+      i < primaryCount ? "primary"
+      : i < primaryCount + supportingCount ? "supporting"
+      : "background",
+  }))
+}
+
+async function tryLogQuery(
+  text: string,
+  results: OracleResult[],
+  startTime: number,
+  deps: OracleDeps,
+): Promise<void> {
+  try {
+    await appendQueryLog(
+      {
+        query: text,
+        results: results.map(r => ({ id: r.id, score: r.score, status: r.status })),
+        resultCount: results.length,
+        durationMs: Date.now() - startTime,
+        timestamp: new Date().toISOString(),
+      },
+      deps.chronicleDir ?? ".chronicle",
+    )
+  } catch {
+    // Query logging is best-effort — never fail a query because of a log write failure
+  }
+}

package/modules/oracle/summary.ts

@@ -0,0 +1,115 @@
+import { promises as fs } from "fs"
+import path from "path"
+import type { ChronicleEntry } from "../shared/types"
+
+const SUMMARY_WEEKS = 12
+const DIRECTIVE =
+  "<!-- Chronicle Summary v1 — temporal orientation for agents. " +
+  "Use for sequence context; query Oracle by entry ID for full reasoning. -->"
+
+/**
+ * Returns the ISO week string (YYYY-Www) for a given date.
+ * Uses the ISO 8601 definition: week 1 is the week containing the first Thursday.
+ */
+function isoWeekKey(date: Date): string {
+  const d = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()))
+  const day = d.getUTCDay() || 7
+  d.setUTCDate(d.getUTCDate() + 4 - day)
+  const yearStart = new Date(Date.UTC(d.getUTCFullYear(), 0, 1))
+  const week = Math.ceil(((d.getTime() - yearStart.getTime()) / 86_400_000 + 1) / 7)
+  return `${d.getUTCFullYear()}-W${String(week).padStart(2, "0")}`
+}
+
+function workRefLabel(entry: ChronicleEntry): string {
+  if (!entry.work_ref) return "__none__"
+  const { type, ref } = entry.work_ref
+  return ref ? `[${type} ${ref}]` : `[${type}]`
+}
+
+function renderEntry(entry: ChronicleEntry): string {
+  const areas = entry.affected_areas.join(", ")
+  const id = entry.id.slice(0, 8)
+  return `- **[${id}]** ${areas} — \`${entry.status}\` (${entry.confidence.toFixed(2)}) — ${entry.key_insight}`
+}
+
+/**
+ * Rebuild .chronicle/SUMMARY.md from all committed entries.
+ *
+ * Groups entries by ISO week (most-recent first), then by work_ref within
+ * each week. Shows the last SUMMARY_WEEKS weeks; older entries are omitted
+ * (still fully queryable via Oracle).
+ *
+ * Called by commit() as a best-effort side-effect — never throws.
+ */
+export async function updateSummary(chronicleDir: string): Promise<void> {
+  const committedDir = path.join(chronicleDir, "committed")
+
+  let files: string[]
+  try {
+    files = await fs.readdir(committedDir)
+  } catch {
+    return
+  }
+
+  const entries: ChronicleEntry[] = []
+  for (const file of files) {
+    if (!file.endsWith(".json")) continue
+    try {
+      const raw = await fs.readFile(path.join(committedDir, file), "utf8")
+      entries.push(JSON.parse(raw) as ChronicleEntry)
+    } catch {
+      // Skip malformed entries
+    }
+  }
+
+  if (entries.length === 0) return
+
+  entries.sort((a, b) => b.timestamp.localeCompare(a.timestamp))
+
+  // Group by ISO week
+  const byWeek = new Map<string, ChronicleEntry[]>()
+  for (const entry of entries) {
+    const week = isoWeekKey(new Date(entry.timestamp))
+    const bucket = byWeek.get(week) ?? []
+    bucket.push(entry)
+    byWeek.set(week, bucket)
+  }
+
+  const weeks = [...byWeek.keys()].sort().reverse().slice(0, SUMMARY_WEEKS)
+
+  const lines: string[] = [DIRECTIVE, ""]
+
+  for (const week of weeks) {
+    lines.push(`## Week ${week}`, "")
+
+    // Group entries within week by work_ref label
+    const weekEntries = byWeek.get(week)!
+    const byWork = new Map<string, ChronicleEntry[]>()
+    for (const entry of weekEntries) {
+      const key = workRefLabel(entry)
+      const bucket = byWork.get(key) ?? []
+      bucket.push(entry)
+      byWork.set(key, bucket)
+    }
+
+    // Labelled work groups first, then ungrouped
+    const workKeys = [...byWork.keys()].sort((a, b) =>
+      a === "__none__" ? 1 : b === "__none__" ? -1 : a.localeCompare(b),
+    )
+
+    for (const key of workKeys) {
+      if (key === "__none__") {
+        lines.push(`### (no work context — query Oracle by entry ID for details)`)
+      } else {
+        lines.push(`### ${key}`)
+      }
+      for (const entry of byWork.get(key)!) {
+        lines.push(renderEntry(entry))
+      }
+      lines.push("")
+    }
+  }
+
+  const summaryPath = path.join(chronicleDir, "SUMMARY.md")
+  await fs.writeFile(summaryPath, lines.join("\n"), "utf8")
+}

package/modules/oracle/types.ts

@@ -0,0 +1,32 @@
+import type { ChronicleEntry } from "../shared/types"
+
+/**
+ * Abstract vector store interface.
+ * Swap implementations without changing Oracle logic.
+ * Default implementation: LanceDB (see adapters/lance-db.ts).
+ */
+export interface VectorStore {
+  /**
+   * Upsert a Chronicle entry with its embedding vector.
+   * If an entry with this ID already exists, it is replaced.
+   */
+  upsert: (id: string, vector: number[], metadata: ChronicleEntry) => Promise<void>
+  /**
+   * Return the top-K most similar entries to the given query vector.
+   * Scores should be in [0, 1] (higher = more similar).
+   */
+  search: (
+    vector: number[],
+    limit: number,
+  ) => Promise<Array<{ entry: ChronicleEntry; score: number }>>
+  /** Return all stored entries (used for full-corpus BM25 if needed). */
+  getAll: () => Promise<ChronicleEntry[]>
+}
+
+export interface OracleDeps {
+  /** Converts text to a numeric embedding vector. */
+  embedder: (text: string) => Promise<number[]>
+  vectorStore: VectorStore
+  /** Root directory for Chronicle data. Default: ".chronicle" */
+  chronicleDir?: string
+}

package/modules/sentinel/assert.ts

@@ -0,0 +1,95 @@
+import { coverage } from "./coverage"
+import { detectDrift } from "./drift"
+import type { LLMProvider } from "../shared/types"
+
+export interface SentinelAssertOptions {
+  chronicleDir?: string
+  codebasePath?: string
+  /** When provided, drift detection runs. When absent, drift tests are skipped. */
+  llm?: LLMProvider
+  extensions?: string[]
+  /**
+   * Chronicle coverage must reach this percentage for the CI test to pass.
+   * Default 0 = report gaps as advisory output without failing the build.
+   * Raise this as the project matures (e.g. 50 for an established codebase).
+   */
+  minCoveragePercent?: number
+}
+
+/**
+ * Returns a set of named assertions designed to be called inside a Vitest
+ * describe block. Coverage assertions are deterministic and always run.
+ * Drift assertions skip gracefully when no LLM is provided.
+ *
+ * @example
+ * import { describe } from "vitest"
+ * import { sentinelAssertions } from "../modules/sentinel/assert"
+ *
+ * const assertions = sentinelAssertions({ chronicleDir: ".chronicle", codebasePath: "modules" })
+ * describe("sentinel", () => { assertions.forEach(a => a()) })
+ */
+export function sentinelAssertions(options: SentinelAssertOptions = {}): Array<() => void> {
+  const {
+    chronicleDir = ".chronicle",
+    codebasePath = ".",
+    llm,
+    extensions,
+    minCoveragePercent = 0,
+  } = options
+
+  // Import vitest lazily so this file is usable outside of a test context too
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const { it, expect, describe: _describe } = require("vitest") as typeof import("vitest")
+
+  const assertions: Array<() => void> = []
+
+  // ── Coverage (deterministic, always run) ──────────────────────────────────
+  assertions.push(() => {
+    const label = minCoveragePercent > 0
+      ? `coverage: Chronicle coverage ≥ ${minCoveragePercent}%`
+      : "coverage: Chronicle coverage report [advisory]"
+    it(label, async () => {
+      const report = await coverage(chronicleDir, codebasePath, { extensions })
+      if (report.uncoveredFiles.length > 0) {
+        const list = report.uncoveredFiles.slice(0, 10).join("\n  ")
+        const msg = `${report.uncoveredFiles.length} source file(s) have no Chronicle coverage (${report.percentage}% covered):\n  ${list}`
+        if (minCoveragePercent > 0) {
+          expect(report.percentage, msg).toBeGreaterThanOrEqual(minCoveragePercent)
+        } else {
+          // New project or no threshold set — surface gaps without failing the build
+          console.info(`[sentinel] ${msg}`)
+        }
+      }
+    })
+  })
+
+  assertions.push(() => {
+    it("coverage: report is readable and well-formed", async () => {
+      const report = await coverage(chronicleDir, codebasePath, { extensions })
+      expect(report.totalFiles).toBeGreaterThanOrEqual(0)
+      expect(report.percentage).toBeGreaterThanOrEqual(0)
+      expect(report.percentage).toBeLessThanOrEqual(100)
+    })
+  })
+
+  // ── Drift (advisory, skips when no LLM configured) ────────────────────────
+  assertions.push(() => {
+    it.skipIf(!llm)(
+      "drift: no Chronicle entries flagged as potentially stale [advisory]",
+      async () => {
+        const report = await detectDrift(chronicleDir, codebasePath, llm!)
+        if (report.flags.length > 0) {
+          const detail = report.flags
+            .map(f => `  [${f.entryId.slice(0, 8)}] ${f.keyInsight}\n    → ${f.reasoning}`)
+            .join("\n")
+          expect(
+            report.flags,
+            `${report.flags.length} Chronicle entry/entries may have drifted (advisory — review before marking refuted):\n${detail}`,
+          ).toHaveLength(0)
+        }
+      },
+    )
+  })
+
+  return assertions
+}

package/modules/sentinel/coverage.ts

@@ -0,0 +1,106 @@
+import { promises as fs, Dirent } from "fs"
+import path from "path"
+import type { ChronicleEntry, CoverageReport, FileCoverage } from "../shared/types"
+
+const IGNORED_DIRS = new Set(["node_modules", "dist", ".git", ".chronicle", "coverage", "__tests__"])
+const TEST_SUFFIXES = [".test.ts", ".spec.ts", ".test.js", ".spec.js"]
+
+async function walkFiles(
+  dir: string,
+  extensions: string[],
+  excludeTestFiles: boolean,
+): Promise<string[]> {
+  const results: string[] = []
+
+  async function recurse(current: string): Promise<void> {
+    let entries: Dirent<string>[]
+    try {
+      entries = await fs.readdir(current, { withFileTypes: true, encoding: "utf8" })
+    } catch {
+      return
+    }
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        if (!IGNORED_DIRS.has(entry.name)) await recurse(path.join(current, entry.name))
+      } else if (extensions.some(ext => entry.name.endsWith(ext))) {
+        if (excludeTestFiles && TEST_SUFFIXES.some(s => entry.name.endsWith(s))) continue
+        results.push(path.join(current, entry.name))
+      }
+    }
+  }
+
+  await recurse(dir)
+  return results
+}
+
+async function readCommittedEntries(chronicleDir: string): Promise<ChronicleEntry[]> {
+  const committedDir = path.join(chronicleDir, "committed")
+  let files: string[]
+  try {
+    files = await fs.readdir(committedDir)
+  } catch {
+    return []
+  }
+  const entries: ChronicleEntry[] = []
+  for (const file of files) {
+    if (!file.endsWith(".json")) continue
+    try {
+      const raw = await fs.readFile(path.join(committedDir, file), "utf8")
+      entries.push(JSON.parse(raw) as ChronicleEntry)
+    } catch {
+      // skip malformed
+    }
+  }
+  return entries
+}
+
+function isCovered(relativePath: string, entries: ChronicleEntry[]): { covered: boolean; entryIds: string[] } {
+  const matched: string[] = []
+  const normalised = relativePath.replace(/\\/g, "/")
+  for (const entry of entries) {
+    const hits = entry.affected_areas.some(area => {
+      const normArea = area.replace(/\\/g, "/")
+      return normalised.includes(normArea) || normArea.includes(normalised)
+    })
+    if (hits) matched.push(entry.id)
+  }
+  return { covered: matched.length > 0, entryIds: matched }
+}
+
+/**
+ * Scan the codebase and report which files have Chronicle entries referencing
+ * them in affected_areas and which do not.
+ *
+ * Matching is substring-based — "oracle/propose.ts" in affected_areas covers
+ * "modules/oracle/propose.ts" in the codebase. Treat percentage as directional
+ * signal, not a precision metric.
+ */
+export async function coverage(
+  chronicleDir: string,
+  codebasePath: string,
+  options: { extensions?: string[]; excludeTestFiles?: boolean } = {},
+): Promise<CoverageReport> {
+  const extensions = options.extensions ?? [".ts"]
+  const excludeTestFiles = options.excludeTestFiles ?? true
+  const [entries, files] = await Promise.all([
+    readCommittedEntries(chronicleDir),
+    walkFiles(codebasePath, extensions, excludeTestFiles),
+  ])
+
+  const coverageByFile: FileCoverage[] = files.map(absolute => {
+    const relative = path.relative(codebasePath, absolute).replace(/\\/g, "/")
+    const { covered, entryIds } = isCovered(relative, entries)
+    return { file: relative, covered, entryIds }
+  })
+
+  const covered = coverageByFile.filter(f => f.covered)
+  const uncovered = coverageByFile.filter(f => !f.covered)
+
+  return {
+    totalFiles: files.length,
+    coveredFiles: covered.length,
+    uncoveredFiles: uncovered.map(f => f.file),
+    coverageByFile,
+    percentage: files.length === 0 ? 0 : Math.round((covered.length / files.length) * 100),
+  }
+}