@hippo-digital/hippocampus 1.0.0-rc.1

package/scripts/import-design-actions.js

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+/**
+ * Import design actions and decisions from the round tracker workbook.
+ *
+ * The design process is: analyse insights, agree design actions with the team -
+ * which stay deliberately open and can carry several directions - then decide on
+ * one, which proceeds into production.
+ *
+ * The knowledge model already had the shape for that and was not being used for
+ * it. A design action is the open brief, with `insightIds` recording what
+ * prompted it. Each option weighed against that action is a decision record, and
+ * `DecisionSchema` already carries proposed / accepted / rejected / superseded -
+ * so the options ledger, and which one won, is expressible without a new record
+ * type. What was considered and turned down survives, which is the part that
+ * otherwise lives only in someone's memory.
+ *
+ *   node scripts/import-design-actions.js --workbook <path> [--dry-run]
+ */
+
+const fs = require('fs')
+const path = require('path')
+const { openWorkbook, readSheetGrid } = require('./lib/xlsx-grid')
+const { KnowledgeSchema } = require('../lib/schema')
+const { createContextOrExit } = require('../lib/context')
+const ctx = createContextOrExit()
+const { knowledgeSourceDir } = ctx.paths
+
+const ACTION_STATUS = ['proposed', 'in-progress', 'done', 'parked']
+const DECISION_STATUS = ['proposed', 'accepted', 'rejected', 'superseded']
+
+// Row 2 of every sheet is a worked example. It is there to show the format, and
+// deleting it is optional, so it is skipped rather than imported.
+const EXAMPLE_IDS = /^(R17-05|R17-10|DA-R17-|DEC-R17-)/i
+
+function main () {
+  const args = parseArgs(process.argv.slice(2))
+  if (!args.workbook) throw new Error('Usage: npm run import:design-actions -- --workbook path/to/tracker.xlsx [--dry-run]')
+
+  const workbook = openWorkbook(path.resolve(args.workbook))
+  const actionRows = rows(workbook, 'Design Actions')
+  const decisionRows = rows(workbook, 'Decisions')
+
+  const read = (f) => JSON.parse(fs.readFileSync(path.join(knowledgeSourceDir, f), 'utf8'))
+  const insightIds = new Set(read('insights.json').map((i) => i.id))
+  const existingActions = read('design-actions.json')
+  const existingDecisions = read('decisions.json')
+  const relationships = read('relationships.json')
+
+  const problems = []
+  const actions = []
+  const actionIdByTracker = new Map()
+
+  for (const row of actionRows) {
+    const tracker = row['ID']
+    if (!tracker || EXAMPLE_IDS.test(tracker)) continue
+    const id = kebab(tracker)
+    actionIdByTracker.set(tracker.toLowerCase(), id)
+
+    const status = normalise(row['Status'], ACTION_STATUS)
+    if (!status) problems.push(`Design action ${tracker}: status "${row['Status'] || ''}" is not one of ${ACTION_STATUS.join(', ')}`)
+
+    const linked = splitIds(row['Insight IDs']).map((ref) => resolveInsight(ref, insightIds))
+    const missing = splitIds(row['Insight IDs']).filter((ref) => !resolveInsight(ref, insightIds))
+    if (missing.length) problems.push(`Design action ${tracker}: insight IDs not found in the knowledge base: ${missing.join(', ')}`)
+    if (!linked.filter(Boolean).length) problems.push(`Design action ${tracker}: needs at least one insight ID that exists`)
+
+    actions.push({
+      id,
+      title: text(row['Title']) || text(row['Action']).slice(0, 110),
+      status: status || 'proposed',
+      action: text(row['Action']),
+      rationale: [text(row['Rationale']), row['Owner'] && `Owner: ${text(row['Owner'])}.`, row['Dependency'] && `Dependency: ${text(row['Dependency'])}.`]
+        .filter(Boolean).join(' ') || `Recorded on tracker row ${tracker}.`,
+      insightIds: [...new Set(linked.filter(Boolean))]
+    })
+  }
+
+  const decisions = []
+  for (const row of decisionRows) {
+    const tracker = row['ID']
+    if (!tracker || EXAMPLE_IDS.test(tracker)) continue
+
+    const status = normalise(row['Status'], DECISION_STATUS)
+    if (!status) problems.push(`Decision ${tracker}: status "${row['Status'] || ''}" is not one of ${DECISION_STATUS.join(', ')}`)
+
+    const actionRef = text(row['Design action ID'])
+    const actionId = actionIdByTracker.get(actionRef.toLowerCase()) ||
+      (existingActions.some((a) => a.id === kebab(actionRef)) ? kebab(actionRef) : null)
+    if (actionRef && !actionId) problems.push(`Decision ${tracker}: design action "${actionRef}" not found in this workbook or in the knowledge base`)
+
+    decisions.push({
+      record: {
+        id: kebab(tracker),
+        title: text(row['Option']) || text(row['Decision']).slice(0, 110),
+        status: status || 'proposed',
+        decision: text(row['Decision']),
+        consequence: [text(row['Consequence']), row['Date agreed'] && `Agreed ${text(row['Date agreed'])}.`, row['Decided by'] && `Decided by ${text(row['Decided by'])}.`]
+          .filter(Boolean).join(' ') || 'Consequence not yet recorded.'
+      },
+      actionId
+    })
+  }
+
+  // One accepted option per action, or the ledger is ambiguous.
+  const acceptedByAction = {}
+  for (const d of decisions) {
+    if (d.record.status !== 'accepted' || !d.actionId) continue
+    acceptedByAction[d.actionId] = (acceptedByAction[d.actionId] || 0) + 1
+  }
+  for (const [actionId, count] of Object.entries(acceptedByAction)) {
+    if (count > 1) problems.push(`Design action ${actionId}: ${count} decisions are marked accepted. Only one option can be the one that proceeds.`)
+  }
+
+  for (const list of [[actions, existingActions, 'design action'], [decisions.map((d) => d.record), existingDecisions, 'decision']]) {
+    const [incoming, current, label] = list
+    const ids = new Set(current.map((x) => x.id))
+    for (const item of incoming) if (ids.has(item.id)) problems.push(`${label} ${item.id} already exists in source data`)
+  }
+
+  const links = []
+  for (const action of actions) {
+    for (const insightId of action.insightIds) {
+      links.push({ id: `rel-${action.id}-responds-to-${insightId}`, from: action.id, type: 'responds-to-insight', to: insightId })
+    }
+  }
+  for (const d of decisions) {
+    if (!d.actionId) continue
+    links.push({ id: `rel-${d.record.id}-resolves-${d.actionId}`, from: d.record.id, type: 'resolves-action', to: d.actionId })
+  }
+
+  console.log(`Design actions: ${actions.length}  (${ACTION_STATUS.map((s) => `${s} ${actions.filter((a) => a.status === s).length}`).join(', ')})`)
+  console.log(`Decisions:      ${decisions.length}  (${DECISION_STATUS.map((s) => `${s} ${decisions.filter((d) => d.record.status === s).length}`).join(', ')})`)
+  console.log(`Relationships:  ${links.length}`)
+
+  if (problems.length) {
+    console.error(`\n${problems.length} problem${problems.length === 1 ? '' : 's'} to fix in the workbook:`)
+    for (const p of problems) console.error(`- ${p}`)
+    process.exit(1)
+  }
+
+  if (args.dryRun) {
+    console.log('\nDry run. Source files were not changed.')
+    for (const a of actions) {
+      console.log(`\n  ${a.id}  [${a.status}]  from ${a.insightIds.length} insight(s)`)
+      console.log(`     ${a.action.slice(0, 120)}`)
+      for (const d of decisions.filter((x) => x.actionId === a.id)) {
+        console.log(`     - ${d.record.status.padEnd(10)} ${d.record.title.slice(0, 90)}`)
+      }
+    }
+    return
+  }
+
+  const proposed = {
+    'design-actions.json': [...existingActions, ...actions],
+    'decisions.json': [...existingDecisions, ...decisions.map((d) => d.record)],
+    'relationships.json': [...new Map([...relationships, ...links].map((r) => [r.id, r])).values()]
+  }
+
+  // Validate the whole knowledge base before writing, not after, so a bad
+  // workbook cannot leave source data broken.
+  // Files beginning with _ are the knowledge base's own metadata, not one of its
+  // collections - _meta.json carries the schema version. KnowledgeSchema is
+  // strict, so feeding it one fails the parse and blocks every import.
+  const preview = Object.fromEntries(fs.readdirSync(knowledgeSourceDir)
+    .filter((f) => f.endsWith('.json') && !f.startsWith('_'))
+    .map((f) => [key(f), proposed[f] || JSON.parse(fs.readFileSync(path.join(knowledgeSourceDir, f), 'utf8'))]))
+  const result = KnowledgeSchema.safeParse(preview)
+  if (!result.success) {
+    console.error('\nThe import would make the knowledge base invalid:')
+    for (const issue of result.error.issues.slice(0, 10)) console.error(`- ${issue.path.join('.')}: ${issue.message}`)
+    process.exit(1)
+  }
+
+  for (const [file, data] of Object.entries(proposed)) {
+    fs.writeFileSync(path.join(knowledgeSourceDir, file), `${JSON.stringify(data, null, 2)}\n`)
+  }
+  console.log(`\nImported from ${path.relative(ctx.root, path.resolve(args.workbook))}. Run npm run validate:knowledge to confirm.`)
+}
+
+function rows (workbook, sheetName) {
+  if (!workbook.sheetFiles.has(sheetName)) {
+    throw new Error(`The workbook has no "${sheetName}" sheet. It has: ${[...workbook.sheetFiles.keys()].join(', ')}. Start from hippocampus/templates/ncrs-insight-design-action-tracker.xlsx.`)
+  }
+  const grid = readSheetGrid(workbook, sheetName)
+  const header = grid.find((r) => r && r.some(Boolean)) || []
+  const headerRow = grid.indexOf(header)
+  const out = []
+  for (let i = headerRow + 1; i < grid.length; i++) {
+    const row = grid[i]
+    if (!row || !row.some(Boolean)) continue
+    out.push(Object.fromEntries(header.map((h, c) => [String(h || '').trim(), row[c] || ''])))
+  }
+  return out
+}
+
+const text = (v) => String(v ?? '').replace(/\s+/g, ' ').trim()
+const splitIds = (v) => text(v).split(/[,;]/).map((s) => s.trim()).filter(Boolean)
+
+/** Tracker IDs are written R16-15; insight IDs are round-16-insight-15. */
+function resolveInsight (ref, insightIds) {
+  const direct = kebab(ref)
+  if (insightIds.has(direct)) return direct
+  const m = ref.match(/^R-?\s*(\d+)\s*-\s*(.+)$/i)
+  if (m) {
+    const candidate = `round-${m[1]}-insight-${kebab(m[2])}`
+    if (insightIds.has(candidate)) return candidate
+  }
+  return [...insightIds].find((id) => id.endsWith(`-${kebab(ref)}`)) || null
+}
+
+function normalise (value, allowed) {
+  const v = text(value).toLowerCase().replace(/\s+/g, '-')
+  return allowed.includes(v) ? v : null
+}
+
+const key = (file) => file.replace('.json', '').replace(/-([a-z])/g, (_, c) => c.toUpperCase())
+const kebab = (v) => text(v).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '')
+
+function parseArgs (argv) {
+  const parsed = {}
+  for (let i = 0; i < argv.length; i++) {
+    if (!argv[i].startsWith('--')) continue
+    const [k, v] = argv[i].slice(2).split('=', 2)
+    const name = k.replace(/-([a-z])/g, (_, c) => c.toUpperCase())
+    parsed[name] = v !== undefined ? v : (argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true)
+  }
+  return parsed
+}
+
+main()

package/scripts/import-project-context.js

@@ -0,0 +1,427 @@
+const fs = require('fs')
+const path = require('path')
+const AdmZip = require('adm-zip')
+const { PDFParse } = require('pdf-parse')
+const { XMLParser } = require('fast-xml-parser')
+const { ProjectContextImportDraftSchema } = require('../lib/schema')
+const { createContextOrExit } = require('../lib/context')
+const ctx = createContextOrExit()
+const { projectImportsDir } = ctx.paths
+
+async function main () {
+  const args = parseArgs(process.argv.slice(2))
+  if (!args.document || !args.contextId || !args.title) {
+    throw new Error('Usage: npm run import:project-context -- --document path/to/overview.pdf --context-id context-id --title "Project overview"')
+  }
+
+  const sections = await extractSections(args.document)
+  const draft = buildDraft(args, sections)
+  validateDraft(draft)
+
+  const outputDir = args.outputDir ? path.resolve(args.outputDir) : projectImportsDir
+  fs.mkdirSync(outputDir, { recursive: true })
+
+  const outputPath = path.join(outputDir, `${args.contextId}.json`)
+  const reviewPath = path.join(outputDir, `${args.contextId}.review.md`)
+  fs.writeFileSync(outputPath, `${JSON.stringify(draft, null, 2)}\n`)
+  fs.writeFileSync(reviewPath, buildReviewMarkdown(draft))
+
+  console.log(`Imported ${sections.length} ${sectionLabel(args.document).toLowerCase()}`)
+  reportFit(draft)
+  console.log(`Draft written to ${outputPath}`)
+  console.log(`Review written to ${reviewPath}`)
+}
+
+function parseArgs (args) {
+  const parsed = {}
+  for (let index = 0; index < args.length; index++) {
+    const arg = args[index]
+    if (!arg.startsWith('--')) continue
+    const [rawKey, value] = arg.slice(2).split('=', 2)
+    const key = camelCase(rawKey)
+    if (value !== undefined) {
+      parsed[key] = value
+    } else if (args[index + 1] && !args[index + 1].startsWith('--')) {
+      parsed[key] = args[index + 1]
+      index++
+    } else {
+      parsed[key] = true
+    }
+  }
+  return parsed
+}
+
+async function extractSections (documentPath) {
+  const extension = path.extname(documentPath).toLowerCase()
+  if (extension === '.pptx') return extractPowerPointSlides(documentPath)
+  if (extension === '.pdf') return extractPdfPages(documentPath)
+  throw new Error(`Unsupported project context artefact extension ${extension || '(none)'}`)
+}
+
+function extractPowerPointSlides (deckPath) {
+  const zip = new AdmZip(deckPath)
+  const parser = new XMLParser({
+    ignoreAttributes: false,
+    removeNSPrefix: true
+  })
+
+  return zip.getEntries()
+    .filter((entry) => /^ppt\/slides\/slide\d+\.xml$/.test(entry.entryName))
+    .sort((a, b) => slideNumber(a.entryName) - slideNumber(b.entryName))
+    .map((entry) => {
+      const xml = entry.getData().toString('utf8')
+      const parsed = parser.parse(xml)
+      const text = collectText(parsed).map(normaliseText).filter(Boolean)
+      return {
+        number: slideNumber(entry.entryName),
+        title: text[0] || `Slide ${slideNumber(entry.entryName)}`,
+        text
+      }
+    })
+}
+
+async function extractPdfPages (pdfPath) {
+  const parser = new PDFParse({ data: fs.readFileSync(pdfPath) })
+
+  try {
+    const result = await parser.getText({ pageJoiner: '' })
+    return result.pages.map((page) => {
+      const text = page.text.split(/\n+/).map(normaliseText).filter(Boolean)
+      return {
+        number: page.num,
+        title: text[0] || `Page ${page.num}`,
+        text
+      }
+    })
+  } finally {
+    await parser.destroy()
+  }
+}
+
+function buildDraft (args, sections) {
+  const sourceDocument = path.relative(ctx.root, path.resolve(args.document))
+  const lines = sections.flatMap((section) => section.text)
+  const context = {
+    id: args.contextId,
+    title: args.title,
+    summary: args.summary || firstUsefulLine(lines) || `Imported project context from ${path.basename(args.document)}.`,
+    purpose: extractPurpose(lines),
+    vision: extractStartingAt(lines, 'Our vision'),
+    problemStatement: extractStartingAt(lines, 'Users find'),
+    serviceProposition: extractServiceProposition(lines),
+    users: extractUsers(lines),
+    channels: extractChannels(lines),
+    policies: extractPolicies(lines),
+    constraints: extractConstraints(lines),
+    risks: extractRisks(lines),
+    milestones: extractMilestones(lines),
+    facts: extractFacts(lines),
+    sourceArtefacts: [
+      {
+        id: kebab(`${args.contextId} source`),
+        title: args.title,
+        sourcePath: sourceDocument,
+        sourceType: sourceType(args.document),
+        importedAt: new Date().toISOString()
+      }
+    ]
+  }
+
+  return {
+    sourceDocument,
+    importedAt: new Date().toISOString(),
+    warnings: assessFit(context),
+    projectContext: context,
+    sections
+  }
+}
+
+function extractPurpose (lines) {
+  const purposeLines = lines.filter((line) => {
+    const lower = line.toLowerCase()
+    return lower.includes('statutory obligation') ||
+      lower.includes('dhsc are responsible') ||
+      lower.includes('digital service provided by dhsc')
+  })
+  return sentence(purposeLines)
+}
+
+function extractServiceProposition (lines) {
+  const propositionLines = lines.filter((line) => {
+    const lower = line.toLowerCase()
+    return lower.includes('similar but enhanced service') ||
+      lower.includes('robust, fast, reliable') ||
+      lower.includes('user friendly webform') ||
+      lower.includes('audit trail') ||
+      lower.includes('case management')
+  })
+  return sentence(propositionLines)
+}
+
+function extractStartingAt (lines, prefix) {
+  const index = lines.findIndex((line) => line.toLowerCase().startsWith(prefix.toLowerCase()))
+  if (index === -1) return undefined
+  return sentence(lines.slice(index, index + 3))
+}
+
+function extractUsers (lines) {
+  const start = lines.findIndex((line) => /this service is for/i.test(line))
+  const end = lines.findIndex((line, index) => index > start && /under the terms/i.test(line))
+  const userLines = start === -1 ? [] : lines.slice(start + 1, end === -1 ? start + 30 : end)
+  return unique(userLines.filter(isUsefulUserLine).map((line) => ({
+    id: kebab(line),
+    label: line,
+    group: userGroup(line)
+  })))
+}
+
+function extractChannels (lines) {
+  return ['API', 'Paper', 'Webform'].flatMap((label) => {
+    const index = lines.findIndex((line) => line.toLowerCase() === label.toLowerCase())
+    if (index === -1) return []
+    const detail = lines.slice(index + 1, index + 4).filter((line) => !['API', 'Paper', 'Webform'].includes(line)).join(' ')
+    const usage = lines.find((line) => line.toLowerCase().includes(`${label.toLowerCase()},`)) ||
+      lines.find((line) => line.toLowerCase().includes(`${label.toLowerCase()} `) && line.includes('%'))
+    return [{
+      id: kebab(label),
+      label,
+      description: detail || undefined,
+      usage
+    }]
+  })
+}
+
+function extractPolicies (lines) {
+  return unique(lines.filter((line) => {
+    const lower = line.toLowerCase()
+    return lower.includes('abortion act') ||
+      lower.includes('abortion regulations') ||
+      lower.includes('amendment regulations') ||
+      lower.includes('legal requirement') ||
+      lower.includes('statutory obligation')
+  }).map((line) => ({
+    id: kebab(line).slice(0, 80).replace(/-+$/g, ''),
+    title: line,
+    summary: line
+  })))
+}
+
+function extractConstraints (lines) {
+  return unique(lines.filter((line) => {
+    const lower = line.toLowerCase()
+    return lower.includes('contract') ||
+      lower.includes('cannot') ||
+      lower.includes('not engaged') ||
+      lower.includes('legal requirement')
+  }).map((line) => ({
+    id: kebab(line).slice(0, 80).replace(/-+$/g, ''),
+    label: line,
+    detail: line
+  })))
+}
+
+function extractRisks (lines) {
+  return unique(lines.filter((line) => {
+    const lower = line.toLowerCase()
+    return lower.includes('risk') ||
+      lower.includes('low engagement') ||
+      lower.includes('low tech maturity') ||
+      lower.includes('unreliable') ||
+      lower.includes('prone to errors')
+  }).map((line) => ({
+    id: kebab(line).slice(0, 80).replace(/-+$/g, ''),
+    label: line,
+    detail: line,
+    severity: lowerSeverity(line)
+  })))
+}
+
+function extractMilestones (lines) {
+  return unique(lines.filter((line) => {
+    const lower = line.toLowerCase()
+    return lower.includes('may 2027') ||
+      lower.includes('alpha') ||
+      lower.includes('re-procure')
+  }).map((line) => ({
+    id: kebab(line).slice(0, 80).replace(/-+$/g, ''),
+    label: line,
+    detail: line
+  })))
+}
+
+function extractFacts (lines) {
+  const facts = []
+  for (const line of lines) {
+    if (/\d/.test(line) && (line.includes('%') || /users|forms|submissions/i.test(line))) {
+      facts.push({
+        id: kebab(line).slice(0, 80).replace(/-+$/g, ''),
+        label: line,
+        value: line
+      })
+    }
+  }
+  return unique(facts)
+}
+
+function assessFit (context) {
+  const warnings = []
+  if (!context.purpose) warnings.push('No clear purpose was detected. Add one before promotion if the overview contains it.')
+  if (!context.problemStatement) warnings.push('No problem statement was detected. Review the source text and add it if present.')
+  if (!context.users.length) warnings.push('No project-level users were detected.')
+  if (!context.channels.length) warnings.push('No service channels were detected.')
+  if (!context.policies.length) warnings.push('No policy or regulation context was detected.')
+  return warnings
+}
+
+function validateDraft (draft) {
+  const result = ProjectContextImportDraftSchema.safeParse(draft)
+  if (result.success) return
+
+  const issues = result.error.issues.map((issue) => {
+    const fieldPath = issue.path.length ? issue.path.join('.') : 'draft'
+    return `${fieldPath}: ${issue.message}`
+  })
+  throw new Error(`Imported draft does not match ProjectContextImportDraftSchema:\n- ${issues.join('\n- ')}`)
+}
+
+function reportFit (draft) {
+  const warnings = draft.warnings || []
+  const fit = warnings.length ? 'needs review' : 'good'
+  console.log(`Import fit: ${fit}`)
+  if (warnings.length) {
+    console.log('Import warnings:')
+    for (const warning of warnings) console.log(`- ${warning}`)
+  }
+}
+
+function buildReviewMarkdown (draft) {
+  const context = draft.projectContext
+  const lines = [
+    `# ${context.title}`,
+    '',
+    `- **Context ID:** ${context.id}`,
+    `- **Source:** ${draft.sourceDocument}`,
+    `- **Imported:** ${draft.importedAt}`,
+    '',
+    '## Import Fit',
+    ''
+  ]
+
+  if (draft.warnings && draft.warnings.length) {
+    lines.push('Needs review.')
+    lines.push('')
+    for (const warning of draft.warnings) lines.push(`- ${warning}`)
+  } else {
+    lines.push('Good. No fit warnings detected.')
+  }
+
+  lines.push('')
+  lines.push('## Overview')
+  lines.push('')
+  if (context.purpose) lines.push(`- **Purpose:** ${context.purpose}`)
+  if (context.vision) lines.push(`- **Vision:** ${context.vision}`)
+  if (context.problemStatement) lines.push(`- **Problem statement:** ${context.problemStatement}`)
+  if (context.serviceProposition) lines.push(`- **Service proposition:** ${context.serviceProposition}`)
+  lines.push('')
+  lines.push('## Counts')
+  lines.push('')
+  lines.push(`- Users: ${context.users.length}`)
+  lines.push(`- Channels: ${context.channels.length}`)
+  lines.push(`- Policies: ${context.policies.length}`)
+  lines.push(`- Constraints: ${context.constraints.length}`)
+  lines.push(`- Risks: ${context.risks.length}`)
+  lines.push(`- Milestones: ${context.milestones.length}`)
+  lines.push(`- Facts: ${context.facts.length}`)
+  lines.push('')
+  lines.push('## Extracted Text')
+  lines.push('')
+
+  for (const section of draft.sections) {
+    lines.push(`### ${section.title}`)
+    lines.push('')
+    for (const text of section.text) lines.push(`- ${text}`)
+    lines.push('')
+  }
+
+  return `${lines.join('\n')}\n`
+}
+
+function slideNumber (entryName) {
+  return Number(entryName.match(/slide(\d+)\.xml$/)[1])
+}
+
+function collectText (value, includePrimitive = false) {
+  if (value === null || value === undefined) return []
+  if (typeof value === 'string' || typeof value === 'number') return includePrimitive ? [String(value)] : []
+  if (Array.isArray(value)) return value.flatMap((child) => collectText(child, includePrimitive))
+  if (typeof value !== 'object') return []
+
+  const text = []
+  for (const [key, child] of Object.entries(value)) {
+    if (key === 't') text.push(...collectText(child, true))
+    else text.push(...collectText(child, false))
+  }
+  return text
+}
+
+function isUsefulUserLine (line) {
+  if (!line || line.length < 3) return false
+  if (/^\d/.test(line)) return false
+  if (/purpose|problem|vision|channels|usage/i.test(line)) return false
+  return true
+}
+
+function userGroup (line) {
+  if (/dhsc|analyst|researcher|data entry|medical advisor/i.test(line)) return 'Internal'
+  if (/healthcare|policy|education|media|police/i.test(line)) return 'Data consumer'
+  return 'External'
+}
+
+function lowerSeverity (line) {
+  if (/low engagement|low tech|risk/i.test(line)) return 'medium'
+  return undefined
+}
+
+function firstUsefulLine (lines) {
+  return lines.find((line) => line.length > 30)
+}
+
+function sentence (lines) {
+  const value = lines.filter(Boolean).join(' ').replace(/\s+/g, ' ').trim()
+  return value || undefined
+}
+
+function sourceType (filePath) {
+  const extension = path.extname(filePath).toLowerCase()
+  if (extension === '.pdf') return 'pdf document'
+  if (extension === '.pptx') return 'presentation deck'
+  return extension.replace('.', '') || 'document'
+}
+
+function sectionLabel (filePath) {
+  return path.extname(filePath).toLowerCase() === '.pdf' ? 'pages' : 'slides'
+}
+
+function normaliseText (value) {
+  return value.replace(/\s+/g, ' ').trim()
+}
+
+function unique (items) {
+  return Object.values(Object.fromEntries(items.map((item) => [item.id, item])))
+}
+
+function kebab (value) {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+}
+
+function camelCase (value) {
+  return value.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())
+}
+
+main().catch((error) => {
+  console.error(error.message)
+  process.exit(1)
+})