@balpal4495/quorum 3.3.3 → 3.4.0

package/README.md

@@ -141,6 +141,9 @@ Every PR merge posts a growth comment showing what Chronicle learned. `quorum ev
 | Approve what the agent should remember | `quorum commit <id>` |
 | See whether memory is growing | `quorum growth` |
 | Consolidate stale or duplicate entries | `quorum evolve` |
+| Seed Chronicle from git history | `quorum bootstrap --from-git --propose` |
+| Ingest docs and source files as evidence | `quorum ingest docs/ --recurse` |
+| Ingest URLs as evidence | `quorum ingest-url https://example.com/rfc` |
 | Find undocumented areas | `quorum sentinel coverage` |
 | Understand what the product currently does | `quorum compass map` |
 | Generate product pathways toward a goal | `quorum compass pathways --goal "..."` |
@@ -151,6 +154,14 @@ Every PR merge posts a growth comment showing what Chronicle learned. `quorum ev
 
 ## Start small, then add guardrails
 
+### Level 0 — Cold start
+New repo with no Chronicle yet? Seed it from git history:
+```bash
+quorum bootstrap --from-git --since P90D --propose
+quorum commit --list   # review, then approve entries
+```
+Each commit becomes a low-trust draft proposal (`confidence: 0.4`, `needs_human_summary: true`). Nothing is indexed until you run `quorum commit <id>`.
+
 ### Level 1 — Local memory
 Use `quorum advisor brief` and `quorum advisor query` at the start of AI sessions. No setup beyond `init`.
 
@@ -230,6 +241,66 @@ For most host-project use cases the CLI is sufficient and requires no loader. Se
 
 ## Command reference
 
+### `quorum ingest` — ingest files and folders
+
+```bash
+quorum ingest README.md SETUP.md
+quorum ingest docs/ --recurse
+quorum ingest docs/ --recurse --propose   # also stage as proposals
+```
+
+Writes low-trust evidence to `.chronicle/sources/` and `.chronicle/evidence/`. Content-hash deduplication skips files that have not changed since the last ingest. Use `--propose` to also write to `.chronicle/proposals/` for review with `quorum commit --list`.
+
+Supported extensions: `.md`, `.txt`, `.js`, `.ts`, `.json`, `.yaml`, `.html`, and other plain-text formats. Binary and unsupported files are recorded as sources with a fallback summary.
+
+---
+
+### `quorum ingest-git` — ingest git history
+
+```bash
+quorum ingest-git --since P90D
+quorum ingest-git --since P6M --propose   # also stage commits as proposals
+```
+
+`--since` accepts ISO 8601 durations: `P30D`, `P6M`, `P1Y`. Defaults to `P90D`. Each commit is stored as a source record and an evidence record containing the commit subject and changed files. Already-ingested commits are skipped by hash.
+
+---
+
+### `quorum ingest-url` — ingest URLs
+
+```bash
+quorum ingest-url https://example.com/internal-rfc
+quorum ingest-url https://example.com/rfc https://example.com/spec --propose
+```
+
+Fetches each URL (http/https only), strips HTML, and stores a low-trust evidence record. Follows redirects. Deduplicates by URL on re-runs.
+
+---
+
+### `quorum bootstrap` — cold-start Chronicle from history
+
+```bash
+quorum bootstrap --from-git
+quorum bootstrap --from-git --since P6M --propose
+```
+
+Convenience wrapper around `ingest-git`. Seeds a new Chronicle from the project's commit history. Without `--propose`, evidence sits in `.chronicle/evidence/` until you promote it. With `--propose`, every commit becomes a draft proposal ready for `quorum commit`.
+
+All ingested evidence uses the same low-trust format as PR-merge proposals:
+
+```json
+{
+  "source_quality": "metadata-derived",
+  "needs_human_summary": true,
+  "status": "open",
+  "confidence": 0.4
+}
+```
+
+This solves the cold-start problem: a repo gets candidate memory from real history without bypassing the human gate.
+
+---
+
 ### `quorum advisor` — ask Chronicle a question
 
 ```bash
@@ -300,6 +371,8 @@ Writes to `.chronicle/committed/`, updates `SUMMARY.md`, removes the proposal. A
 
 ```
 .chronicle/
+  sources/      ← raw ingested source records (files, URLs, git commits)
+  evidence/     ← low-trust extracted insights, not yet Chronicle
   proposals/    ← AI-staged entries waiting for your approval
   committed/    ← approved entries, indexed and searchable
   SUMMARY.md    ← auto-generated context for your AI to read

package/bin/__tests__/ingest.test.js

@@ -0,0 +1,220 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest"
+import { promises as fs } from "fs"
+import path from "path"
+import os from "os"
+
+// ── helpers ───────────────────────────────────────────────────────────────────
+
+async function makeTmpDir() {
+  return fs.mkdtemp(path.join(os.tmpdir(), "quorum-ingest-"))
+}
+
+async function rmrf(dir) {
+  await fs.rm(dir, { recursive: true, force: true })
+}
+
+async function makeChronicle(root) {
+  const chronicleDir = path.join(root, ".chronicle")
+  await fs.mkdir(chronicleDir, { recursive: true })
+  return chronicleDir
+}
+
+async function readJsonDir(dir) {
+  let files
+  try { files = await fs.readdir(dir) } catch { return [] }
+  const results = []
+  for (const f of files) {
+    if (!f.endsWith(".json")) continue
+    results.push(JSON.parse(await fs.readFile(path.join(dir, f), "utf8")))
+  }
+  return results
+}
+
+// ── parseDurationToGitSince ───────────────────────────────────────────────────
+
+import { parseDurationToGitSince } from "../commands/ingest-git.js"
+
+describe("parseDurationToGitSince", () => {
+  it("parses P90D to '90 days ago'", () => {
+    expect(parseDurationToGitSince("P90D")).toBe("90 days ago")
+  })
+  it("parses P30D to '30 days ago'", () => {
+    expect(parseDurationToGitSince("P30D")).toBe("30 days ago")
+  })
+  it("parses P1Y to '365 days ago'", () => {
+    expect(parseDurationToGitSince("P1Y")).toBe("365 days ago")
+  })
+  it("parses P6M to '180 days ago'", () => {
+    expect(parseDurationToGitSince("P6M")).toBe("180 days ago")
+  })
+  it("parses P1Y6M to '545 days ago'", () => {
+    expect(parseDurationToGitSince("P1Y6M")).toBe("545 days ago")
+  })
+  it("falls back to '90 days ago' for invalid input", () => {
+    expect(parseDurationToGitSince("not-valid")).toBe("90 days ago")
+    expect(parseDurationToGitSince("")).toBe("90 days ago")
+    expect(parseDurationToGitSince("P")).toBe("90 days ago")
+  })
+})
+
+// ── ingest command ────────────────────────────────────────────────────────────
+
+import { run as runIngest } from "../commands/ingest.js"
+import { findChronicleDir, readEvidence, readSources } from "../shared/chronicle.js"
+
+describe("quorum ingest", () => {
+  let tmpDir, chronicleDir, origCwd
+
+  beforeEach(async () => {
+    tmpDir = await makeTmpDir()
+    chronicleDir = await makeChronicle(tmpDir)
+    origCwd = process.cwd()
+    process.chdir(tmpDir)
+  })
+
+  afterEach(async () => {
+    process.chdir(origCwd)
+    await rmrf(tmpDir)
+  })
+
+  it("ingests a single text file — writes source + evidence records", async () => {
+    const file = path.join(tmpDir, "README.md")
+    await fs.writeFile(file, "# My Project\n\nAn example project.")
+
+    await runIngest(["README.md"])
+
+    const sources  = await readSources(chronicleDir)
+    const evidence = await readEvidence(chronicleDir)
+
+    expect(sources).toHaveLength(1)
+    expect(sources[0].type).toBe("file")
+    expect(sources[0].ref).toBe("README.md")
+    expect(sources[0].content_hash).toMatch(/^sha256:/)
+
+    expect(evidence).toHaveLength(1)
+    expect(evidence[0].source_quality).toBe("metadata-derived")
+    expect(evidence[0].needs_human_summary).toBe(true)
+    expect(evidence[0].confidence).toBe(0.4)
+    expect(evidence[0].status).toBe("open")
+    expect(evidence[0].source_module).toBe("ingest")
+    expect(evidence[0].affected_areas).toContain("README.md")
+  })
+
+  it("skips unchanged files on second ingest (content-hash dedup)", async () => {
+    const file = path.join(tmpDir, "notes.md")
+    await fs.writeFile(file, "# Notes\nSome content.")
+
+    await runIngest(["notes.md"])
+    await runIngest(["notes.md"])
+
+    const sources  = await readSources(chronicleDir)
+    const evidence = await readEvidence(chronicleDir)
+    expect(sources).toHaveLength(1)
+    expect(evidence).toHaveLength(1)
+  })
+
+  it("writes proposals when --propose flag is set", async () => {
+    const file = path.join(tmpDir, "design.md")
+    await fs.writeFile(file, "# Design\nDecision records go here.")
+
+    await runIngest(["design.md", "--propose"])
+
+    const proposals = await readJsonDir(path.join(chronicleDir, "proposals"))
+    expect(proposals).toHaveLength(1)
+    expect(proposals[0].source_quality).toBe("metadata-derived")
+    expect(proposals[0].confidence).toBe(0.4)
+    expect(proposals[0].needs_human_summary).toBe(true)
+    // proposal must NOT contain id or ingested_at (stripped before writing)
+    expect(proposals[0].id).toBeUndefined()
+    expect(proposals[0].ingested_at).toBeUndefined()
+  })
+
+  it("does not write proposals without --propose", async () => {
+    const file = path.join(tmpDir, "spec.md")
+    await fs.writeFile(file, "# Spec")
+
+    await runIngest(["spec.md"])
+
+    const proposalDir = path.join(chronicleDir, "proposals")
+    const files = await fs.readdir(proposalDir).catch(() => [])
+    expect(files.filter(f => f.endsWith(".json"))).toHaveLength(0)
+  })
+
+  it("skips non-text files but writes a source record without content", async () => {
+    // Binary-like file with unsupported extension
+    const file = path.join(tmpDir, "icon.png")
+    await fs.writeFile(file, Buffer.from([0x89, 0x50, 0x4e, 0x47]))
+
+    await runIngest(["icon.png"])
+
+    // Should have skipped writing (no TEXT_EXTS match → null content)
+    // The command writes evidence regardless, with a fallback summary
+    const sources  = await readSources(chronicleDir)
+    const evidence = await readEvidence(chronicleDir)
+    // .png not in TEXT_EXTS → content null → evidence summary is fallback
+    expect(sources).toHaveLength(1)
+    expect(evidence[0].key_insight).toContain("icon.png")
+  })
+
+  it("walks directories recursively with --recurse", async () => {
+    const subDir = path.join(tmpDir, "docs")
+    await fs.mkdir(subDir)
+    await fs.writeFile(path.join(subDir, "api.md"), "# API")
+    await fs.writeFile(path.join(subDir, "guide.md"), "# Guide")
+
+    await runIngest(["docs", "--recurse"])
+
+    const evidence = await readEvidence(chronicleDir)
+    expect(evidence).toHaveLength(2)
+  })
+})
+
+// ── readEvidence / readSources helpers ───────────────────────────────────────
+
+describe("readEvidence and readSources", () => {
+  let tmpDir, chronicleDir
+
+  beforeEach(async () => {
+    tmpDir = await makeTmpDir()
+    chronicleDir = await makeChronicle(tmpDir)
+  })
+
+  afterEach(async () => {
+    await rmrf(tmpDir)
+  })
+
+  it("readEvidence returns empty array when directory does not exist", async () => {
+    const result = await readEvidence(chronicleDir)
+    expect(result).toEqual([])
+  })
+
+  it("readSources returns empty array when directory does not exist", async () => {
+    const result = await readSources(chronicleDir)
+    expect(result).toEqual([])
+  })
+
+  it("readEvidence reads written evidence records", async () => {
+    const dir = path.join(chronicleDir, "evidence")
+    await fs.mkdir(dir, { recursive: true })
+    await fs.writeFile(
+      path.join(dir, "abc123.json"),
+      JSON.stringify({ id: "abc123", key_insight: "test", ingested_at: "2026-01-01T00:00:00.000Z" }),
+    )
+    const result = await readEvidence(chronicleDir)
+    expect(result).toHaveLength(1)
+    expect(result[0].id).toBe("abc123")
+  })
+
+  it("readEvidence skips malformed JSON files", async () => {
+    const dir = path.join(chronicleDir, "evidence")
+    await fs.mkdir(dir, { recursive: true })
+    await fs.writeFile(path.join(dir, "bad.json"), "not json {{{{")
+    await fs.writeFile(
+      path.join(dir, "good.json"),
+      JSON.stringify({ id: "good", key_insight: "ok", ingested_at: "2026-01-01T00:00:00.000Z" }),
+    )
+    const result = await readEvidence(chronicleDir)
+    expect(result).toHaveLength(1)
+    expect(result[0].id).toBe("good")
+  })
+})