@hippo-digital/hippocampus 1.0.0-rc.1

package/scripts/audit-provenance.js

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+const fs = require('fs')
+const path = require('path')
+const { createContextOrExit } = require('../lib/context')
+
+// "Where did this come from?" asked of every record, and answered honestly.
+//
+// Provenance in this schema lives in three places - evidence, research rounds
+// and screens - and everything else reaches an original through them. So this
+// walks those three, then asks which records reach nothing.
+//
+// A record with no source is not automatically a defect: an assumption is
+// explicitly an assertion the design rests on, and saying so is provenance. The
+// defect is a record that reads as a finding but cannot be traced, and a source
+// path too vague to find the file it names.
+
+const argv = process.argv.slice(2)
+const strict = argv.includes('--strict')
+const ctx = createContextOrExit({ argv })
+
+// A path that names a real file someone could go and open, versus one that
+// gestures at a folder. `Round 4 - ...` is the shape that fails: it looks like a
+// citation and identifies nothing.
+const VAGUE = [/\.\.\./, /^\s*$/, /\*/]
+const ASSERTION_TYPES = /assumption|principle|decision|local repo reference/i
+
+function main () {
+  const project = ctx.requireProjects()[0]
+  const knowledge = project.store.getKnowledge()
+  const index = readSourceIndex()
+  const findings = { unsourced: [], vague: [], unindexed: [], noOrigin: [], unreachable: [], asserted: [] }
+
+  for (const evidence of knowledge.evidence) {
+    const source = evidence.sourcePath
+    if (!source) {
+      const bucket = ASSERTION_TYPES.test(evidence.sourceType || '') ? 'asserted' : 'unsourced'
+      findings[bucket].push(`${evidence.id} (${evidence.sourceType || 'no type'})`)
+      continue
+    }
+    if (VAGUE.some((pattern) => pattern.test(source))) {
+      findings.vague.push(`${evidence.id}: ${source}`)
+      continue
+    }
+    // Research filenames have spaces in them - stopping at the first one turns
+    // "R16 Location-only search research insights.xlsx" into "R16" and then
+    // reports it as missing. Take everything up to a comma or the end instead.
+    const artefact = source.match(/source-artefacts\/([^,]+)/)
+    if (!artefact) continue
+    const indexed = index.get(artefact[1].trim())
+    if (!indexed) findings.unindexed.push(`${evidence.id}: ${artefact[1]}`)
+    else if (!indexed.origin.stated) findings.noOrigin.push(`${evidence.id}: ${artefact[1]}`)
+  }
+
+  for (const round of knowledge.researchRounds) {
+    if (!round.sourcePath) findings.unsourced.push(`${round.id} (research round)`)
+    else if (VAGUE.some((pattern) => pattern.test(round.sourcePath))) {
+      findings.vague.push(`${round.id}: ${round.sourcePath}`)
+    }
+  }
+
+  // A need or insight is a claim about users, and how well it is backed is a
+  // spectrum rather than a yes or no. An insight that cites a named file is
+  // strongly traced; one that only belongs to a research round is traced to a
+  // folder; one that reaches neither is an opinion in a database that looks
+  // like a finding. Reporting only the last of those flattered the data.
+  const evidenceIds = new Set(knowledge.evidence.map((e) => e.id))
+  const roundIds = new Set(knowledge.researchRounds.map((r) => r.id))
+  const reachesEvidence = new Set()
+  for (const collection of ['needs', 'insights']) {
+    for (const record of knowledge[collection]) {
+      if ((record.evidenceIds || []).some((id) => evidenceIds.has(id))) reachesEvidence.add(record.id)
+    }
+  }
+  for (const relationship of knowledge.relationships) {
+    if (evidenceIds.has(relationship.to)) reachesEvidence.add(relationship.from)
+  }
+
+  findings.strength = []
+  for (const collection of ['needs', 'insights']) {
+    const records = knowledge[collection]
+    const toFile = records.filter((r) => reachesEvidence.has(r.id))
+    const toRound = records.filter((r) => !reachesEvidence.has(r.id) && roundIds.has(r.roundId))
+    const toNothing = records.filter((r) => !reachesEvidence.has(r.id) && !roundIds.has(r.roundId))
+    findings.strength.push(
+      `${collection}: ${toFile.length} cite a named file, ${toRound.length} reach only their research round, ` +
+      `${toNothing.length} reach nothing`)
+    if (toNothing.length) {
+      findings.unreachable.push(`${toNothing.length} of ${records.length} ${collection} reach neither evidence nor a round`)
+    }
+  }
+
+  report(findings, index)
+
+  const blocking = findings.unsourced.length + findings.vague.length +
+    findings.unindexed.length + findings.noOrigin.length
+  if (strict && blocking) process.exit(1)
+}
+
+function readSourceIndex () {
+  const indexPath = path.join(ctx.paths.manifestsDir, 'source-index.json')
+  if (!fs.existsSync(indexPath)) return new Map()
+  const index = JSON.parse(fs.readFileSync(indexPath, 'utf8'))
+  return new Map(index.artefacts.map((a) => [a.path, a]))
+}
+
+function report (findings, index) {
+  const section = (title, items, explanation) => {
+    if (!items.length) return
+    console.log(`\n${title} (${items.length})`)
+    console.log(`  ${explanation}`)
+    for (const item of items.slice(0, 20)) console.log(`    ${item}`)
+    if (items.length > 20) console.log(`    ... and ${items.length - 20} more`)
+  }
+
+  console.log(`Provenance audit — ${index.size} artefacts indexed`)
+
+  if (findings.strength && findings.strength.length) {
+    console.log('\nHow well claims are backed')
+    for (const line of findings.strength) console.log(`    ${line}`)
+  }
+
+  section('No source at all', findings.unsourced,
+    'Reads as a finding but names nothing. Needs a source or an honest sourceType.')
+  section('Source too vague to follow', findings.vague,
+    'Names a folder rather than a file, or contains a placeholder.')
+  section('Cites an artefact that is not indexed', findings.unindexed,
+    'Run `hippocampus source index` while the file is still present.')
+  section('Cites an artefact whose origin was never recorded', findings.noOrigin,
+    'The copy can be hashed but nobody wrote down where it came from.')
+  section('Claims that reach nothing at all', findings.unreachable,
+    'No evidence record and no research round. Nothing backs these.')
+  section('Asserted rather than sourced', findings.asserted,
+    'Assumptions and principles. Having no upstream file is correct for these.')
+
+  const blocking = findings.unsourced.length + findings.vague.length +
+    findings.unindexed.length + findings.noOrigin.length
+  console.log(blocking
+    ? `\n${blocking} records cannot be traced to a named original.`
+    : '\nEvery record traces to a named original.')
+}
+
+try {
+  main()
+} catch (error) {
+  console.error(error.message)
+  process.exit(1)
+}

package/scripts/doctor.js

@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+const fs = require('fs')
+const path = require('path')
+const { spawnSync } = require('child_process')
+const { resolveRoot, CONFIG_FILENAME } = require('../lib/resolve-root')
+const { readConfig } = require('../lib/config')
+const { createContext } = require('../lib/context')
+const { lintKnowledge } = require('../lib/lint-knowledge')
+const { findBlock } = require('../lib/host-edit')
+const packagePaths = require('../lib/package-paths')
+
+// What doctor is for: answering "why is it not working?" without guessing.
+// The resolved root is the first line of output, because it is the answer to
+// most of those questions.
+
+const argv = process.argv.slice(2)
+const problems = []
+const notes = []
+
+function main () {
+  const { root, source } = resolveRoot({ argv })
+  console.log(`Project root: ${root}`)
+  console.log(`Resolved from: ${source}`)
+  console.log(`Package:      ${packagePaths.packageRoot}\n`)
+
+  const config = readConfig(root)
+  report('config', config.exists, `${CONFIG_FILENAME} present`,
+    `${CONFIG_FILENAME} missing - run \`npx hippocampus init\``)
+
+  const ctx = createContext({ argv })
+  let projects = []
+  try {
+    projects = ctx.projects()
+  } catch (error) {
+    problems.push(error.message)
+  }
+
+  report('knowledge base', projects.length > 0,
+    `${projects.length} knowledge base${projects.length === 1 ? '' : 's'}: ${projects.map((p) => p.name).join(', ')}`,
+    'no knowledge base found - run `npx hippocampus init`')
+
+  checkHostFile(root, 'app/routes.js', 'hippocampus', 'the viewer is mounted')
+  checkViewsPath(root)
+
+  for (const project of projects) {
+    const validation = project.store.validateKnowledge()
+    report(`${project.name} integrity`, validation.valid,
+      'references intact',
+      `${validation.errors.length} broken references - run \`npx hippocampus validate\``)
+
+    if (validation.valid) {
+      const { warnings, infos } = lintKnowledge(project.store.getKnowledge())
+      if (warnings.length) notes.push(`${project.name}: ${warnings.length} warnings - run \`npx hippocampus validate\``)
+      if (infos.length) notes.push(`${project.name}: ${infos.length} coverage notes - run \`npx hippocampus validate\``)
+    }
+  }
+
+  if (!argv.includes('--no-smoke') && projects.length && !problems.length) {
+    const smoke = spawnSync(process.execPath, [path.join(packagePaths.scriptsDir, 'smoke-routes.js'), '--root', root],
+      { encoding: 'utf8' })
+    report('routes', smoke.status === 0, 'every route serves',
+      (smoke.stderr || smoke.stdout || '').trim().split('\n').slice(0, 4).join('\n           '))
+  }
+
+  for (const note of notes) console.log(`  note  ${note}`)
+
+  if (problems.length) {
+    console.log(`\n${problems.length} problem${problems.length === 1 ? '' : 's'} to fix.`)
+    process.exit(1)
+  }
+
+  console.log('\nAll good.')
+}
+
+function report (label, ok, okMessage, problemMessage) {
+  if (ok) {
+    console.log(`  ok    ${label}: ${okMessage}`)
+    return
+  }
+  console.log(`  FAIL  ${label}: ${problemMessage}`)
+  problems.push(`${label}: ${problemMessage}`)
+}
+
+function checkHostFile (root, relPath, id, description) {
+  const filePath = path.join(root, relPath)
+  if (!fs.existsSync(filePath)) {
+    report(relPath, false, '', 'not found')
+    return
+  }
+  const block = findBlock(fs.readFileSync(filePath, 'utf8'), id)
+  report(relPath, Boolean(block) && !block.unterminated, description,
+    block && block.unterminated
+      ? 'the hippocampus block has no end marker - fix it by hand'
+      : 'the viewer is not mounted - run `npx hippocampus init`, or add the two lines it prints')
+}
+
+function checkViewsPath (root) {
+  const appPath = path.join(root, 'app.js')
+  if (!fs.existsSync(appPath)) {
+    report('app.js', false, '', 'not found')
+    return
+  }
+  const source = fs.readFileSync(appPath, 'utf8')
+  report('app.js', source.includes('@hippo-digital/hippocampus'),
+    'the package views are on the views path',
+    'viewsPath does not include the package views - add `require(\'@hippo-digital/hippocampus\').createHippocampus().viewsPath`')
+}
+
+try {
+  main()
+} catch (error) {
+  console.error(error.message)
+  process.exit(1)
+}

package/scripts/eject.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+const fs = require('fs')
+const path = require('path')
+const { resolveRoot } = require('../lib/resolve-root')
+const { removeBlock } = require('../lib/host-edit')
+
+// Removes the host edits and nothing else. All data stays: the knowledge base
+// is the team's work, not this package's, and a command that deletes it because
+// somebody wanted to stop using the viewer would be indefensible.
+
+const argv = process.argv.slice(2)
+const dryRun = argv.includes('--dry-run')
+
+function main () {
+  const { root } = resolveRoot({ argv })
+  console.log(`Project root: ${root}\n`)
+
+  let changed = 0
+
+  const routesPath = path.join(root, 'app/routes.js')
+  if (fs.existsSync(routesPath)) {
+    const before = fs.readFileSync(routesPath, 'utf8')
+    const result = removeBlock(before, 'hippocampus')
+    if (result.action === 'removed') {
+      changed++
+      console.log(`${dryRun ? 'would remove' : 'removed'}  the hippocampus block from app/routes.js`)
+      if (!dryRun) fs.writeFileSync(routesPath, result.source)
+    }
+  }
+
+  const appPath = path.join(root, 'app.js')
+  if (fs.existsSync(appPath)) {
+    const source = fs.readFileSync(appPath, 'utf8')
+    if (source.includes('@hippo-digital/hippocampus')) {
+      console.log('\napp.js still references the package. Remove the viewsPath entry by hand:')
+      console.log("  require('@hippo-digital/hippocampus').createHippocampus().viewsPath")
+    }
+  }
+
+  if (!changed) console.log('Nothing to remove.')
+
+  console.log('\nYour knowledge base, artefacts and config are untouched.')
+  console.log('Uninstall the package with: npm uninstall @hippo-digital/hippocampus')
+}
+
+try {
+  main()
+} catch (error) {
+  console.error(error.message)
+  process.exit(1)
+}

package/scripts/import-blueprint-spreadsheet.js

@@ -0,0 +1,242 @@
+const fs = require('fs')
+const path = require('path')
+const { openWorkbook, readSheetGrid } = require('./lib/xlsx-grid')
+const { createContextOrExit } = require('../lib/context')
+const ctx = createContextOrExit()
+
+// Generic triage mapping from common service-blueprint swimlane vocabulary to
+// where it belongs in Hippocampus's knowledge model (see
+// docs/source-model.md). Anything that doesn't match falls through as
+// "unmapped" and is flagged for a manual decision - this list intentionally
+// holds only generic blueprint/service-design terms, not any one project's
+// specific persona or team names, so the same rules apply to any spreadsheet.
+const SWIMLANE_TRIAGE = [
+  // Open questions and WIP markers are checked first so they take priority
+  // over the more generic keyword matches below (e.g. "Do we need a row for
+  // future steps?" is a question about the row, not future-state content).
+  { match: /^do we need/i, destination: 'assumption', note: 'An open question row, not step content.' },
+  { match: /unhappy path/i, destination: 'assumption', note: 'Likely deserves its own scenario/journey once detailed, rather than a lane on this one.' },
+
+  { match: /^user groups$/i, destination: 'users', note: 'One User record per distinct persona named in this row.' },
+  { match: /^(external )?user steps/i, destination: 'journeyStep.userAction', note: 'What the user actually does at this step.' },
+  { match: /exception journeys/i, destination: 'journeyStep.exceptionHandling', note: '' },
+  { match: /^back ?office/i, destination: 'journeyStep.backstage', note: '' },
+  { match: /future steps?$/i, destination: 'journeyStep.futureState', note: 'Proposed/target-state, not current fact.' },
+  { match: /^pain points/i, destination: 'journeyStep.painPoints', note: '' },
+  { match: /^data$/i, destination: 'journeyStep.dataDependencies', note: '' },
+  { match: /^processes/i, destination: 'journeyStep.supportProcesses', note: '' },
+  { match: /needs? linked to (jira|a ticket)/i, destination: 'needs', note: 'Need.statement + Need.reference (ticket ID).' },
+  { match: /^backlog status/i, destination: 'projectContext.facts', note: '' },
+  { match: /^notes$/i, destination: 'projectContext.facts', note: '' },
+  { match: /^opportunities/i, destination: 'assumption', note: 'One Assumption per opportunity.' },
+  { match: /^systems/i, destination: 'projectContext.channels', note: '' },
+  { match: /^sources$/i, destination: 'evidence', note: '' }
+]
+
+async function main () {
+  const args = parseArgs(process.argv.slice(2))
+  if (!args.document || !args.projectDir) {
+    throw new Error('Usage: node scripts/import-blueprint-spreadsheet.js --document path/to/blueprint.xlsx --project-dir path/to/hippocampus-style-project [--output-dir dir]')
+  }
+
+  const documentPath = path.resolve(args.document)
+  const workbook = openWorkbook(documentPath)
+
+  const draft = {
+    sourceDocument: path.relative(ctx.root, documentPath),
+    importedAt: new Date().toISOString(),
+    note: 'Review artefact only - nothing here has been written to the project source yet. Decide, sheet by sheet and row by row, whether any identifying details should be genericized before promoting content into <project>/source/*.json.',
+    sheets: Object.fromEntries(
+      [...workbook.sheetFiles.keys()].map((sheetName) => [sheetName, extractSheet(workbook, sheetName)])
+    )
+  }
+
+  const outputDir = args.outputDir ? path.resolve(args.outputDir) : path.join(path.resolve(args.projectDir), 'imports/project')
+  fs.mkdirSync(outputDir, { recursive: true })
+
+  const outputPath = path.join(outputDir, 'blueprint-extract.json')
+  const reviewPath = path.join(outputDir, 'blueprint-extract.review.md')
+  fs.writeFileSync(outputPath, `${JSON.stringify(draft, null, 2)}\n`)
+  fs.writeFileSync(reviewPath, buildReviewMarkdown(draft))
+
+  console.log(`Extracted ${Object.keys(draft.sheets).length} sheets`)
+  console.log(`Draft written to ${outputPath}`)
+  console.log(`Review written to ${reviewPath}`)
+  console.log('This is a review artefact, not source data - see docs/source-model.md before promoting anything from it.')
+}
+
+/**
+ * Extracts one sheet generically: try reading it as a step x swimlane
+ * blueprint (a header row of step names, then rows labelled in column 0,
+ * where a blank column 0 continues the previous label onto more rows). If no
+ * column-0 labels are found at all, the sheet doesn't follow that
+ * convention - fall back to a flat dump of every non-empty cell instead of
+ * silently dropping its content.
+ */
+function extractSheet (workbook, sheetName) {
+  const grid = readSheetGrid(workbook, sheetName)
+  const { steps, headerRowIndex } = buildSteps(grid)
+  const swimlanes = groupIntoSwimlanes(grid, steps, headerRowIndex + 2)
+
+  // A handful of stray column-0 values isn't enough evidence that a sheet
+  // follows the swimlane convention - most of its actual content needs to
+  // have landed inside a swimlane, or this isn't really a step x lane
+  // blueprint and forcing that shape on it would silently drop everything
+  // that came before the first (possibly late, possibly sparse) label.
+  const contentCells = flattenCells(grid.slice(headerRowIndex + 2)).length
+  const capturedCells = swimlanes.reduce(
+    (sum, swimlane) => sum + Object.values(swimlane.cellsByStep).reduce((n, v) => n + asList(v).length, 0),
+    0
+  )
+
+  if (swimlanes.length && contentCells > 0 && capturedCells / contentCells >= 0.5) {
+    return { shape: 'blueprint', steps: steps.map((step) => step.label), swimlanes }
+  }
+
+  return { shape: 'flat', cells: flattenCells(grid) }
+}
+
+function buildSteps (grid) {
+  // The step-header row is whichever of the first few rows has the most
+  // populated columns - generic enough to not assume a fixed row number.
+  const candidateRows = grid.slice(0, 6).map((row, index) => ({ row: row || [], index }))
+  const headerRowIndex = candidateRows.reduce((best, candidate) =>
+    (candidate.row.filter(Boolean).length > (candidateRows[best]?.row.filter(Boolean).length || 0) ? candidate.index : best), 0)
+
+  const headerRow = grid[headerRowIndex] || []
+  const nextRow = grid[headerRowIndex + 1] || []
+  const steps = []
+
+  for (let col = 1; col < Math.max(headerRow.length, nextRow.length); col++) {
+    const label = nextRow[col] || headerRow[col]
+    if (label) steps.push({ column: col, label: String(label).trim() })
+  }
+
+  return { steps, headerRowIndex }
+}
+
+function groupIntoSwimlanes (grid, steps, startRow) {
+  const swimlanes = []
+  let current = null
+
+  for (let rowIndex = startRow; rowIndex < grid.length; rowIndex++) {
+    const row = grid[rowIndex]
+    if (!row) continue
+
+    if (row[0]) {
+      if (current) swimlanes.push(current)
+      const label = firstLine(String(row[0]))
+      const triage = triageFor(label)
+      current = { label, destination: triage.destination, triageNote: triage.note, cellsByStep: {} }
+    }
+
+    if (!current || !steps.length) continue
+
+    for (const step of steps) {
+      const value = row[step.column]
+      if (!value) continue
+      const existing = current.cellsByStep[step.label]
+      const entry = String(value).trim()
+      current.cellsByStep[step.label] = existing ? [...asList(existing), entry] : entry
+    }
+  }
+
+  if (current) swimlanes.push(current)
+  return swimlanes
+}
+
+function flattenCells (grid) {
+  const cells = []
+  for (let rowIndex = 0; rowIndex < grid.length; rowIndex++) {
+    const row = grid[rowIndex]
+    if (!row) continue
+    for (let col = 0; col < row.length; col++) {
+      if (row[col]) cells.push({ row: rowIndex, column: col, text: String(row[col]).trim() })
+    }
+  }
+  return cells
+}
+
+function asList (value) {
+  return Array.isArray(value) ? value : [value]
+}
+
+function triageFor (label) {
+  const match = SWIMLANE_TRIAGE.find((entry) => entry.match.test(label))
+  return match || { destination: 'unmapped', note: 'No triage rule matched this label - needs a manual decision before promoting into source.' }
+}
+
+function firstLine (value) {
+  return value.split('\n')[0].trim()
+}
+
+function buildReviewMarkdown (draft) {
+  const lines = [
+    '# Blueprint spreadsheet import - review',
+    '',
+    `- **Source:** ${draft.sourceDocument}`,
+    `- **Imported:** ${draft.importedAt}`,
+    '',
+    `> ${draft.note}`,
+    ''
+  ]
+
+  for (const [sheetName, sheet] of Object.entries(draft.sheets)) {
+    lines.push(`## ${sheetName}`)
+    lines.push('')
+
+    if (sheet.shape === 'blueprint') {
+      lines.push(`Steps: ${sheet.steps.join(' -> ')}`)
+      lines.push('')
+      lines.push('| Swimlane | Destination | Note |')
+      lines.push('| --- | --- | --- |')
+      for (const swimlane of sheet.swimlanes) {
+        const flag = swimlane.destination === 'unmapped' ? '**UNMAPPED** ' : ''
+        lines.push(`| ${flag}${swimlane.label} | ${swimlane.destination} | ${swimlane.triageNote} |`)
+      }
+    } else {
+      lines.push('Does not follow the step x swimlane convention - captured as a flat list of cells for manual review.')
+      lines.push('')
+      lines.push(`Cells captured: ${sheet.cells.length}`)
+    }
+
+    lines.push('')
+  }
+
+  const unmapped = Object.values(draft.sheets)
+    .flatMap((sheet) => sheet.swimlanes || [])
+    .filter((swimlane) => swimlane.destination === 'unmapped')
+
+  if (unmapped.length) {
+    lines.push('## Needs a manual decision before promoting into source')
+    lines.push('')
+    for (const swimlane of unmapped) lines.push(`- ${swimlane.label}`)
+    lines.push('')
+  }
+
+  return `${lines.join('\n')}\n`
+}
+
+function parseArgs (args) {
+  const parsed = {}
+  for (let index = 0; index < args.length; index++) {
+    const arg = args[index]
+    if (!arg.startsWith('--')) continue
+    const [rawKey, value] = arg.slice(2).split('=', 2)
+    const key = rawKey.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())
+    if (value !== undefined) {
+      parsed[key] = value
+    } else if (args[index + 1] && !args[index + 1].startsWith('--')) {
+      parsed[key] = args[index + 1]
+      index++
+    } else {
+      parsed[key] = true
+    }
+  }
+  return parsed
+}
+
+main().catch((error) => {
+  console.error(error.message)
+  process.exit(1)
+})