@hippo-digital/hippocampus 1.0.0-rc.1

package/scripts/lib/artefact-text.js

@@ -0,0 +1,347 @@
+/**
+ * Shared artefact text extraction.
+ *
+ * The original importer understood PowerPoint and PDF only, and dropped
+ * PowerPoint speaker notes. Real research folders are mostly Word and Excel,
+ * and playback decks carry a tenth of their content in the notes pane.
+ *
+ * Every extractor returns the same shape so downstream code does not care
+ * which format the artefact was:
+ *
+ *   { sourceType, sections: [{ number, title, text[], notes[] }] }
+ *
+ * A "section" is a slide, a PDF page, a Word heading block, or a worksheet.
+ */
+
+const fs = require('fs')
+const path = require('path')
+const AdmZip = require('adm-zip')
+const { XMLParser } = require('fast-xml-parser')
+
+const parser = new XMLParser({ ignoreAttributes: false, removeNSPrefix: true })
+const attrParser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@_', textNodeName: '#text' })
+
+const extensions = {
+  '.pptx': 'powerpoint deck',
+  '.pdf': 'pdf document',
+  '.docx': 'word document',
+  '.dotx': 'word template',
+  '.xlsx': 'excel workbook',
+  '.csv': 'csv extract',
+  '.txt': 'text note',
+  '.md': 'markdown note',
+  '.url': 'external link'
+}
+
+function supports (filePath) {
+  return Object.prototype.hasOwnProperty.call(extensions, path.extname(filePath).toLowerCase())
+}
+
+function sourceTypeFor (filePath) {
+  return extensions[path.extname(filePath).toLowerCase()] || 'unknown artefact'
+}
+
+async function extractArtefact (filePath) {
+  const extension = path.extname(filePath).toLowerCase()
+  const sourceType = sourceTypeFor(filePath)
+
+  if (extension === '.pptx') return { sourceType, sections: extractPowerPoint(filePath) }
+  if (extension === '.pdf') return { sourceType, sections: await extractPdf(filePath) }
+  if (extension === '.docx' || extension === '.dotx') return { sourceType, sections: extractWord(filePath) }
+  if (extension === '.xlsx') return { sourceType, sections: extractWorkbook(filePath) }
+  if (extension === '.csv') return { sourceType, sections: extractDelimited(filePath) }
+  if (extension === '.txt' || extension === '.md') return { sourceType, sections: extractPlainText(filePath) }
+  if (extension === '.url') return { sourceType, sections: extractShortcut(filePath) }
+
+  throw new Error(`Unsupported research artefact extension ${extension || '(none)'}`)
+}
+
+/* ---------------------------------------------------------------- PowerPoint */
+
+function extractPowerPoint (deckPath) {
+  const zip = new AdmZip(deckPath)
+  const notesByNumber = readSpeakerNotes(zip)
+
+  return zip.getEntries()
+    .filter((entry) => /^ppt\/slides\/slide\d+\.xml$/.test(entry.entryName))
+    .sort((a, b) => numberIn(a.entryName) - numberIn(b.entryName))
+    .map((entry) => {
+      const number = numberIn(entry.entryName)
+      const text = collectText(parser.parse(entry.getData().toString('utf8'))).map(normalise).filter(Boolean)
+      return {
+        number,
+        title: text[0] || `Slide ${number}`,
+        text,
+        notes: notesByNumber.get(number) || []
+      }
+    })
+}
+
+function readSpeakerNotes (zip) {
+  const notes = new Map()
+  for (const entry of zip.getEntries()) {
+    if (!/^ppt\/notesSlides\/notesSlide\d+\.xml$/.test(entry.entryName)) continue
+    const lines = collectText(parser.parse(entry.getData().toString('utf8')))
+      .map(normalise)
+      // the slide-number placeholder is repeated into every notes page
+      .filter((line) => line && !/^\d{1,3}$/.test(line))
+    if (lines.length) notes.set(numberIn(entry.entryName), lines)
+  }
+  return notes
+}
+
+/* ----------------------------------------------------------------------- PDF */
+
+/**
+ * pdf-parse 2.x reaches for DOMMatrix at module load, which normally arrives with
+ * the optional @napi-rs/canvas native binding. That binding is platform-specific,
+ * so a repo installed on macOS cannot load it on Linux CI and every PDF fails with
+ * "DOMMatrix is not defined". Text extraction never rasterises anything, so a
+ * minimal stand-in is enough and keeps PDF support working everywhere.
+ */
+function ensurePdfGlobals () {
+  if (typeof globalThis.DOMMatrix !== 'undefined') return
+  globalThis.DOMMatrix = class DOMMatrix {
+    constructor (init) {
+      this.a = 1; this.b = 0; this.c = 0; this.d = 1; this.e = 0; this.f = 0
+      if (Array.isArray(init) && init.length >= 6) [this.a, this.b, this.c, this.d, this.e, this.f] = init
+    }
+
+    translate () { return this }
+    scale () { return this }
+    multiply () { return this }
+    inverse () { return this }
+    transformPoint (point) { return point }
+  }
+}
+
+async function extractPdf (pdfPath) {
+  ensurePdfGlobals()
+  const { PDFParse } = require('pdf-parse')
+  const pdf = new PDFParse({ data: fs.readFileSync(pdfPath) })
+  try {
+    const result = await pdf.getText({ pageJoiner: '' })
+    return result.pages.map((page) => {
+      const text = page.text.split(/\n+/).map(normalise).filter(Boolean)
+      return { number: page.num, title: text[0] || `Page ${page.num}`, text, notes: [] }
+    })
+  } finally {
+    await pdf.destroy()
+  }
+}
+
+/* ---------------------------------------------------------------------- Word */
+
+/**
+ * Word has no page concept we can recover from the XML, so sections are cut at
+ * heading paragraphs. That keeps a finding attached to the heading it sat under,
+ * which is what the structuring step needs in order to scope an insight.
+ */
+function extractWord (docPath) {
+  const zip = new AdmZip(docPath)
+  const entry = zip.getEntry('word/document.xml')
+  if (!entry) return []
+
+  const xml = entry.getData().toString('utf8')
+  const paragraphs = []
+
+  for (const match of xml.match(/<w:p[ >][\s\S]*?<\/w:p>|<w:p\/>/g) || []) {
+    const runs = [...match.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/g)].map((m) => m[1])
+    const text = normalise(decodeXml(runs.join('')))
+    if (!text) continue
+    const style = (match.match(/<w:pStyle w:val="([^"]+)"/) || [])[1] || 'Normal'
+    paragraphs.push({ style, text, heading: /^Heading[1-6]$|^Title$/i.test(style) })
+  }
+
+  const sections = []
+  let current = null
+  for (const paragraph of paragraphs) {
+    if (paragraph.heading || !current) {
+      current = { number: sections.length + 1, title: paragraph.text, text: [], notes: [] }
+      sections.push(current)
+      if (paragraph.heading) continue
+    }
+    current.text.push(paragraph.text)
+  }
+
+  // A document with no heading styles still deserves a usable title.
+  if (sections.length === 1 && sections[0].title === sections[0].text[0]) {
+    sections[0].title = path.basename(docPath, path.extname(docPath))
+  }
+
+  return sections
+}
+
+/* --------------------------------------------------------------------- Excel */
+
+/**
+ * Research teams keep insight and design-action trackers in Excel. A flat cell
+ * dump loses the column a value came from, so each row is emitted as
+ * "Header: value" pairs and the header row is kept as its own line.
+ */
+function extractWorkbook (workbookPath) {
+  const zip = new AdmZip(workbookPath)
+  const shared = readSharedStrings(zip)
+  const sheetNames = readSheetNames(zip)
+
+  return zip.getEntries()
+    .filter((entry) => /^xl\/worksheets\/sheet\d+\.xml$/.test(entry.entryName))
+    .sort((a, b) => numberIn(a.entryName) - numberIn(b.entryName))
+    .map((entry, index) => {
+      const grid = readGrid(entry.getData().toString('utf8'), shared)
+      const title = sheetNames[index] || `Sheet ${index + 1}`
+      const headers = grid.length ? grid[0] : []
+      const text = []
+
+      if (headers.filter(Boolean).length) text.push(`Columns: ${headers.filter(Boolean).join(' | ')}`)
+
+      for (const row of grid.slice(1)) {
+        const pairs = row
+          .map((value, column) => {
+            if (!value) return null
+            const header = headers[column] || `Column ${column + 1}`
+            return `${header}: ${readableCell(header, value)}`
+          })
+          .filter(Boolean)
+        if (pairs.length) text.push(pairs.join(' | '))
+      }
+
+      return { number: index + 1, title, text, notes: [] }
+    })
+    .filter((sheet) => sheet.text.length)
+}
+
+function readSheetNames (zip) {
+  const entry = zip.getEntry('xl/workbook.xml')
+  if (!entry) return []
+  const parsed = attrParser.parse(entry.getData().toString('utf8'))
+  const sheets = parsed?.workbook?.sheets?.sheet
+  return asArray(sheets).map((sheet) => String(sheet['@_name'] || '').trim())
+}
+
+function readSharedStrings (zip) {
+  const entry = zip.getEntry('xl/sharedStrings.xml')
+  if (!entry) return []
+  const xml = entry.getData().toString('utf8')
+  return (xml.match(/<si>[\s\S]*?<\/si>/g) || []).map((si) => {
+    const runs = [...si.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((m) => m[1])
+    return normalise(decodeXml(runs.join('')))
+  })
+}
+
+function readGrid (xml, shared) {
+  const grid = []
+  for (const rowXml of xml.match(/<row[ >][\s\S]*?<\/row>|<row[^>]*\/>/g) || []) {
+    const cells = []
+    for (const cellMatch of rowXml.match(/<c[ >][\s\S]*?<\/c>|<c[^>]*\/>/g) || []) {
+      const reference = (cellMatch.match(/\br="([A-Z]+)\d+"/) || [])[1]
+      const type = (cellMatch.match(/\bt="([^"]+)"/) || [])[1] || 'n'
+      const column = reference ? columnIndex(reference) : cells.length
+      cells[column] = cellValue(cellMatch, type, shared)
+    }
+    grid.push([...cells].map((value) => value || ''))
+  }
+  return grid
+}
+
+function cellValue (cellXml, type, shared) {
+  if (type === 'inlineStr') {
+    const runs = [...cellXml.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((m) => m[1])
+    return normalise(decodeXml(runs.join('')))
+  }
+  const raw = (cellXml.match(/<v>([\s\S]*?)<\/v>/) || [])[1]
+  if (raw === undefined) return ''
+  if (type === 'e') return ''
+  if (type === 's') return shared[Number(decodeXml(raw))] || ''
+  return normalise(decodeXml(raw))
+}
+
+/**
+ * Excel stores dates as serial numbers. A tracker column called "Date Received"
+ * holding 46174 is a date, not a quantity, and a research round needs the real
+ * date, so serials in date-ish columns are rendered back as YYYY-MM-DD.
+ */
+function readableCell (header, value) {
+  if (!/date|received|agreed|when|day/i.test(header)) return value
+  const serial = Number(value)
+  if (!Number.isFinite(serial) || serial < 20000 || serial > 60000) return value
+  const epoch = Date.UTC(1899, 11, 30)
+  return new Date(epoch + Math.round(serial) * 86400000).toISOString().slice(0, 10)
+}
+
+function columnIndex (letters) {
+  let index = 0
+  for (const character of letters) index = index * 26 + (character.charCodeAt(0) - 64)
+  return index - 1
+}
+
+/* ------------------------------------------------------------ plain formats */
+
+function extractDelimited (filePath) {
+  const rows = fs.readFileSync(filePath, 'utf8').split(/\r?\n/).map(normalise).filter(Boolean)
+  if (!rows.length) return []
+  return [{ number: 1, title: path.basename(filePath), text: rows, notes: [] }]
+}
+
+function extractPlainText (filePath) {
+  const lines = fs.readFileSync(filePath, 'utf8').split(/\r?\n/).map(normalise).filter(Boolean)
+  if (!lines.length) return []
+  return [{ number: 1, title: lines[0], text: lines, notes: [] }]
+}
+
+/**
+ * Windows .url shortcuts are how this team records MURAL boards, Confluence
+ * pages and survey links. They hold no findings, but they are the pointer to
+ * where the finding lives, so they belong in the graph as evidence.
+ */
+function extractShortcut (filePath) {
+  const target = (fs.readFileSync(filePath, 'utf8').match(/^URL=(.+)$/m) || [])[1]
+  if (!target) return []
+  return [{
+    number: 1,
+    title: path.basename(filePath, path.extname(filePath)),
+    text: [`External reference: ${target.trim()}`],
+    notes: []
+  }]
+}
+
+/* ------------------------------------------------------------------- helpers */
+
+function collectText (value, includePrimitive = false) {
+  if (value === null || value === undefined) return []
+  if (typeof value === 'string' || typeof value === 'number') return includePrimitive ? [String(value)] : []
+  if (Array.isArray(value)) return value.flatMap((child) => collectText(child, includePrimitive))
+  if (typeof value !== 'object') return []
+
+  const text = []
+  for (const [key, child] of Object.entries(value)) {
+    if (key === 't') text.push(...collectText(child, true))
+    else text.push(...collectText(child, false))
+  }
+  return text
+}
+
+function decodeXml (value) {
+  return String(value)
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
+    .replace(/&amp;/g, '&')
+}
+
+function normalise (value) {
+  return String(value ?? '').replace(/\s+/g, ' ').trim()
+}
+
+function numberIn (entryName) {
+  return Number((entryName.match(/(\d+)\.xml$/) || [])[1] || 0)
+}
+
+function asArray (value) {
+  if (value === null || value === undefined) return []
+  return Array.isArray(value) ? value : [value]
+}
+
+module.exports = { extractArtefact, supports, sourceTypeFor, extensions }

package/scripts/lib/xlsx-grid.js

@@ -0,0 +1,114 @@
+const AdmZip = require('adm-zip')
+const { XMLParser } = require('fast-xml-parser')
+
+const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@_', textNodeName: '#text' })
+
+function openWorkbook (filePath) {
+  const zip = new AdmZip(filePath)
+  const sharedStrings = readSharedStrings(zip)
+  const sheetFiles = readSheetFileMap(zip)
+  return { zip, sharedStrings, sheetFiles }
+}
+
+function readSheetFileMap (zip) {
+  const workbookDoc = parser.parse(zip.readAsText('xl/workbook.xml'))
+  const relsDoc = parser.parse(zip.readAsText('xl/_rels/workbook.xml.rels'))
+
+  const relTargets = new Map(
+    asArray(relsDoc.Relationships.Relationship).map((rel) => [rel['@_Id'], rel['@_Target']])
+  )
+
+  const sheets = asArray(workbookDoc.workbook.sheets.sheet)
+  return new Map(sheets.map((sheet) => [
+    sheet['@_name'].trim(),
+    resolveTarget(relTargets.get(sheet['@_r:id']))
+  ]))
+}
+
+/**
+ * Excel writes sheet relationship targets relative to xl/ ("worksheets/sheet1.xml"),
+ * openpyxl writes them absolute from the package root ("/xl/worksheets/sheet1.xml").
+ * Both are valid, so handle both rather than only the files Excel produced.
+ */
+function resolveTarget (target) {
+  if (!target) return null
+  const clean = String(target).replace(/^\.\//, '')
+  return clean.startsWith('/') ? clean.slice(1) : `xl/${clean}`
+}
+
+function readSharedStrings (zip) {
+  if (!zip.getEntry('xl/sharedStrings.xml')) return []
+  const doc = parser.parse(zip.readAsText('xl/sharedStrings.xml'))
+  const siList = asArray(doc.sst?.si)
+  return siList.map(extractRichText)
+}
+
+function extractRichText (si) {
+  if (si == null) return ''
+  if (typeof si === 'string') return si
+  if (si.t != null) return textValue(si.t)
+  if (si.r) return asArray(si.r).map((run) => textValue(run.t)).join('')
+  return ''
+}
+
+function textValue (node) {
+  if (node == null) return ''
+  return typeof node === 'string' ? node : (node['#text'] ?? String(node))
+}
+
+function colLetterToIndex (letters) {
+  let index = 0
+  for (const char of letters) index = index * 26 + (char.charCodeAt(0) - 64)
+  return index - 1
+}
+
+function parseCellRef (ref) {
+  const match = /^([A-Z]+)(\d+)$/.exec(ref)
+  return { col: colLetterToIndex(match[1]), row: parseInt(match[2], 10) }
+}
+
+/**
+ * Reads a sheet (by its display name in the workbook) into a 2D grid of
+ * plain-text cell values, keyed [rowNumber][columnIndex] (both 1-based row,
+ * 0-based column, matching how spreadsheet UIs label columns A, B, C...).
+ */
+function readSheetGrid (workbook, sheetName) {
+  const sheetFile = workbook.sheetFiles.get(sheetName)
+  if (!sheetFile) {
+    throw new Error(`Sheet "${sheetName}" not found. Available sheets: ${[...workbook.sheetFiles.keys()].join(', ')}`)
+  }
+
+  const doc = parser.parse(workbook.zip.readAsText(sheetFile))
+  const rows = asArray(doc.worksheet.sheetData.row)
+  const grid = []
+
+  for (const row of rows) {
+    const rowIndex = parseInt(row['@_r'], 10)
+    for (const cell of asArray(row.c)) {
+      const { col } = parseCellRef(cell['@_r'])
+      const value = readCellValue(cell, workbook.sharedStrings)
+      if (value === '') continue
+      grid[rowIndex] = grid[rowIndex] || []
+      grid[rowIndex][col] = value
+    }
+  }
+
+  return grid
+}
+
+function readCellValue (cell, sharedStrings) {
+  if (cell.v != null) {
+    const raw = (typeof cell.v === 'object') ? cell.v['#text'] : cell.v
+    if (cell['@_t'] === 's') return sharedStrings[parseInt(raw, 10)] || ''
+    return String(raw)
+  }
+  if (cell.is) return extractRichText(cell.is)
+  return ''
+}
+
+function asArray (value) {
+  if (value == null) return []
+  return Array.isArray(value) ? value : [value]
+}
+
+module.exports = { openWorkbook, readSheetGrid }

package/scripts/link-insights-to-needs.js

@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+/**
+ * Link insights to the user needs they bear on.
+ *
+ * The first attempt matched the tracker's short Need phrase ("Need location-only
+ * search") against need statements using a hand-written synonym list. It found 80
+ * links and missed seven phrases entirely - not because those needs were absent,
+ * but because a four-word phrase is too thin to match on and the vocabularies
+ * differ. All seven were already stated by the personas.
+ *
+ * This does two things differently:
+ *
+ * 1. It matches on the whole insight - title and summary - not the Need phrase
+ *    alone, so there is enough text to score against.
+ * 2. It only considers needs belonging to the users the tracker says the insight
+ *    affects. That is recorded data, not a guess, and it stops a safeguarding
+ *    finding attaching itself to a midwife's need for sharing a keyword.
+ *
+ * Rare words carry more weight than common ones, so "safeguarding" counts for
+ * more than "patient", which almost every need contains.
+ *
+ *   node scripts/link-insights-to-needs.js [--round round-16] [--dry-run]
+ */
+
+const fs = require('fs')
+const path = require('path')
+const { createContextOrExit } = require('../lib/context')
+const ctx = createContextOrExit()
+const { knowledgeSourceDir } = ctx.paths
+
+// The tracker names user groups; the knowledge base names users.
+const GROUP_TO_USERS = [
+  [/safeguarding/i, ['admin-safeguarding', 'clinical-safeguarding']],
+  [/urgent\s*(&|and)\s*emergency|u&e/i, ['clinical-advisor', 'clinical-paramedic', 'clinical-ed-consultant']],
+  [/child health/i, ['admin-child-health']],
+  [/data quality/i, ['admin-data-quality']],
+  [/policy admin|overseas|funding|chargeable/i, ['admin-overseas-visitor-manager']],
+  [/pharmac/i, ['clinical-ed-consultant']],
+  [/screen(ing)? admin/i, ['admin-screening']],
+  [/booking|reception/i, ['admin-booking-and-reception']],
+  [/midwife|maternity/i, ['clinical-midwife']],
+  [/nurs/i, ['clinical-nursing']],
+  [/gp|out.of.hours/i, ['clinical-out-of-hours-gp']]
+]
+
+const STOP = new Set(`a an and are as at be but by can for from has have i if in is it its me my need needs not of on or our so that the their them then there these they this to use used user users want we what when where which who will with would you your also more most only other same some such than very
+nhs ncrs patient patients record records search searching system systems data information staff people person`.split(/\s+/))
+
+function tokens (text) {
+  return (String(text).toLowerCase().match(/[a-z][a-z-]{2,}/g) || [])
+    .map((t) => t.replace(/-$/, ''))
+    .filter((t) => t.length > 2 && !STOP.has(t))
+}
+
+function main () {
+  const args = parseArgs(process.argv.slice(2))
+  const read = (f) => JSON.parse(fs.readFileSync(path.join(knowledgeSourceDir, f), 'utf8'))
+
+  const needs = read('needs.json')
+  const participants = read('participants.json')
+  const insights = read('insights.json').filter((i) => !args.round || i.roundId === args.round)
+  const relationships = read('relationships.json')
+  const insightIds = new Set(read('insights.json').map((i) => i.id))
+
+  // Rare words are informative; words in every need are not.
+  const df = new Map()
+  const needTokens = new Map()
+  for (const need of needs) {
+    const set = new Set(tokens(need.statement))
+    needTokens.set(need.id, set)
+    for (const t of set) df.set(t, (df.get(t) || 0) + 1)
+  }
+  const weight = (t) => Math.log(needs.length / ((df.get(t) || 0) + 1)) + 1
+
+  const links = []
+  const report = []
+
+  for (const insight of insights) {
+    // Round 16's tracker names the affected user groups on every row. Other
+    // rounds have no such column, so the candidate users come from who actually
+    // took part in that round - an insight cannot bear on the needs of a user
+    // group that was not in the room.
+    const affects = (insight.summary.match(/Affects ([^.]+)\./) || [])[1] || ''
+    const users = new Set(
+      GROUP_TO_USERS.filter(([re]) => re.test(affects)).flatMap(([, ids]) => ids)
+    )
+    if (!users.size) {
+      const roles = participants
+        .filter((p) => p.roundIds.includes(insight.roundId))
+        .map((p) => p.role)
+        .join(' | ')
+      for (const [re, ids] of GROUP_TO_USERS) if (re.test(roles)) for (const id of ids) users.add(id)
+    }
+    if (!users.size) { report.push({ insight, users: 0, matched: [], reason: 'no user groups recorded and no participants in this round' }); continue }
+
+    const query = new Set(tokens(`${insight.title} ${insight.summary}`))
+    const candidates = needs.filter((n) => users.has(n.userId))
+
+    const scored = candidates.map((need) => {
+      const have = needTokens.get(need.id)
+      let score = 0
+      let distinctive = 0
+      for (const t of query) {
+        if (!have.has(t)) continue
+        const w = weight(t)
+        score += w
+        if (w >= 2) distinctive++
+      }
+      // Normalise by the need's own length so a long statement does not win by
+      // sheer surface area.
+      return { need, score: score / Math.sqrt(have.size || 1), distinctive }
+    }).sort((a, b) => b.score - a.score)
+
+    const best = scored[0]?.score || 0
+    // Keep needs close to the strongest match for this insight rather than
+    // applying one absolute cut-off across insights of very different length.
+    // A high cumulative score built from many common words is not a match. Two
+    // genuinely uncommon words in common is. Without this an insight about
+    // responding to a 999 call at an address attached itself to a need about
+    // allergy information, on general clinical vocabulary alone.
+    const kept = scored.filter((s) => s.score >= best * 0.70 && s.distinctive >= 2).slice(0, 5)
+
+    for (const { need } of kept) {
+      links.push({ id: `rel-${insight.id}-tests-${need.id}`, from: insight.id, type: 'tests-need', to: need.id })
+    }
+    report.push({ insight, users: users.size, matched: kept })
+  }
+
+  // Replace only the links this script owns: insight -> need. Scenario and screen
+  // links to needs are somebody else's and must survive.
+  const kept = relationships.filter((r) => !(r.type === 'tests-need' && insightIds.has(r.from)))
+  const merged = [...new Map([...kept, ...links].map((r) => [r.id, r])).values()]
+
+  const before = relationships.filter((r) => r.type === 'tests-need' && insightIds.has(r.from)).length
+  console.log(`insight tests-need links: ${before} -> ${links.length}`)
+  console.log(`insights linked: ${report.filter((r) => r.matched.length).length} of ${insights.length}`)
+  console.log(`distinct needs reached: ${new Set(links.map((l) => l.to)).size} of ${needs.length}`)
+
+  const none = report.filter((r) => !r.matched.length)
+  if (none.length) {
+    console.log(`\nunlinked insights (${none.length}):`)
+    for (const r of none) console.log(`  ${r.reason || 'no need scored'}: ${r.insight.title.slice(0, 90)}`)
+  }
+
+  if (args.dryRun) {
+    console.log('\nDry run. Source files were not changed.')
+    console.log('\nweakest kept link per insight (precision check):')
+    for (const r of report.filter((x) => x.matched.length)) {
+      const last = r.matched[r.matched.length - 1]
+      console.log(`  ${last.score.toFixed(2)} [${last.need.userId.padEnd(30)}] ${last.need.statement.slice(0, 80)}`)
+      console.log(`        for: ${r.insight.title.slice(0, 78)}`)
+    }
+    console.log('\nsample:')
+    for (const r of report.filter((x) => x.matched.length).slice(0, 3)) {
+      console.log(`\n  ${r.insight.title.slice(0, 100)}`)
+      for (const m of r.matched) console.log(`     ${m.score.toFixed(2)}  [${m.need.userId}] ${m.need.statement.slice(0, 110)}`)
+    }
+    return
+  }
+
+  fs.writeFileSync(path.join(knowledgeSourceDir, 'relationships.json'), `${JSON.stringify(merged, null, 2)}\n`)
+  console.log(`\nrelationships.json: ${relationships.length} -> ${merged.length}`)
+
+  // Similarity matching has a fuzzy boundary by nature. Rather than pretend a
+  // threshold settles it, every link is written out with its score so a person
+  // can prune the weak ones - the same review-first habit the ingestion
+  // workflow uses for drafts.
+  const reviewDir = path.join(ctx.paths.artefactsDir, 'research-triage')
+  fs.mkdirSync(reviewDir, { recursive: true })
+  const reviewPath = path.join(reviewDir, `${new Date().toISOString().slice(0, 10)}-insight-need-links${args.round ? `-${args.round}` : ''}.md`)
+
+  const lines = [
+    `# Insight to need links${args.round ? `: ${args.round}` : ''}`,
+    '',
+    `Generated by \`scripts/link-insights-to-needs.js\`. ${links.length} links across ${report.filter((r) => r.matched.length).length} insights.`,
+    '',
+    'Scores are relative within an insight, not across insights. The lowest-scoring link',
+    'under each insight is the one most worth checking. Delete a link from',
+    '`relationships.json` if it does not hold.',
+    ''
+  ]
+  for (const r of report) {
+    lines.push(`## ${r.insight.title}`, '')
+    if (!r.matched.length) {
+      lines.push(`_No link. ${r.reason || 'No need shared two distinctive terms with this insight.'}_`, '')
+      continue
+    }
+    lines.push('| Score | User | Need |', '| ---: | --- | --- |')
+    for (const m of r.matched) {
+      lines.push(`| ${m.score.toFixed(2)} | \`${m.need.userId}\` | ${m.need.statement.replace(/\|/g, '\\|').slice(0, 200)} |`)
+    }
+    lines.push('')
+  }
+  fs.writeFileSync(reviewPath, `${lines.join('\n')}\n`)
+  console.log(`review pack: ${path.relative(ctx.root, reviewPath)}`)
+}
+
+function parseArgs (argv) {
+  const parsed = {}
+  for (let i = 0; i < argv.length; i++) {
+    if (!argv[i].startsWith('--')) continue
+    const [k, v] = argv[i].slice(2).split('=', 2)
+    const key = k.replace(/-([a-z])/g, (_, c) => c.toUpperCase())
+    parsed[key] = v !== undefined ? v : (argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true)
+  }
+  return parsed
+}
+
+main()