@hippo-digital/hippocampus 1.0.0-rc.1

package/scripts/import-research-batch.js

@@ -0,0 +1,160 @@
+const fs = require('fs')
+const path = require('path')
+const { spawn } = require('child_process')
+const { createContextOrExit } = require('../lib/context')
+const ctx = createContextOrExit()
+const { researchImportsDir } = ctx.paths
+
+const supportedExtensions = new Set(['.pptx', '.pdf'])
+
+async function main () {
+  const args = parseArgs(process.argv.slice(2))
+  if (!args.source || !args.batchId) {
+    throw new Error('Usage: npm run import:research-batch -- --source path/to/files --batch-id batch-id [--date YYYY-MM-DD] [--concurrency 4]')
+  }
+
+  const sourceDir = path.resolve(args.source)
+  const batchDir = path.join(researchImportsDir, 'batches', args.batchId)
+  const draftsDir = path.join(batchDir, 'drafts')
+  const files = listFiles(sourceDir)
+  const supported = files.filter((file) => supportedExtensions.has(path.extname(file).toLowerCase()))
+  const unsupported = files.filter((file) => !supportedExtensions.has(path.extname(file).toLowerCase()))
+  const concurrency = Number(args.concurrency || 4)
+  const date = args.date || new Date().toISOString().slice(0, 10)
+
+  fs.mkdirSync(draftsDir, { recursive: true })
+
+  const results = await runPool(supported, concurrency, (file) => importDeck({
+    file,
+    batchId: args.batchId,
+    draftsDir,
+    date
+  }))
+
+  const manifest = {
+    batchId: args.batchId,
+    source: path.relative(ctx.root, sourceDir),
+    importedAt: new Date().toISOString(),
+    concurrency,
+    filesFound: files.length,
+    filesProcessed: results.filter((result) => result.ok).length,
+    draftsCreated: results.filter((result) => result.ok).length,
+    failed: results.filter((result) => !result.ok),
+    unsupported: unsupported.map((file) => ({
+      path: path.relative(ctx.root, file),
+      reason: `Unsupported extension ${path.extname(file) || '(none)'}`
+    })),
+    drafts: results.filter((result) => result.ok).map((result) => ({
+      draft: result.draft,
+      review: result.review
+    }))
+  }
+
+  const manifestPath = path.join(batchDir, 'manifest.json')
+  fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`)
+
+  console.log(`Files found: ${manifest.filesFound}`)
+  console.log(`Drafts created: ${manifest.draftsCreated}`)
+  console.log(`Unsupported files: ${manifest.unsupported.length}`)
+  console.log(`Failed files: ${manifest.failed.length}`)
+  console.log(`Manifest written to ${manifestPath}`)
+
+  if (manifest.failed.length) process.exitCode = 1
+}
+
+function parseArgs (args) {
+  const parsed = {}
+  for (let index = 0; index < args.length; index++) {
+    const arg = args[index]
+    if (!arg.startsWith('--')) continue
+    const [rawKey, value] = arg.slice(2).split('=', 2)
+    const key = camelCase(rawKey)
+    if (value !== undefined) {
+      parsed[key] = value
+    } else if (args[index + 1] && !args[index + 1].startsWith('--')) {
+      parsed[key] = args[index + 1]
+      index++
+    } else {
+      parsed[key] = true
+    }
+  }
+  return parsed
+}
+
+function listFiles (dir) {
+  return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+    const fullPath = path.join(dir, entry.name)
+    if (entry.isDirectory()) return listFiles(fullPath)
+    if (entry.isFile()) return [fullPath]
+    return []
+  })
+}
+
+async function runPool (items, concurrency, worker) {
+  const results = []
+  let nextIndex = 0
+
+  async function runNext () {
+    const index = nextIndex
+    nextIndex++
+    if (index >= items.length) return
+    results[index] = await worker(items[index])
+    await runNext()
+  }
+
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, runNext)
+  await Promise.all(workers)
+  return results
+}
+
+function importDeck ({ file, batchId, draftsDir, date }) {
+  const roundId = kebab(`${batchId} ${path.basename(file, path.extname(file))}`)
+  const title = path.basename(file, path.extname(file))
+
+  return new Promise((resolve) => {
+    const child = spawn(process.execPath, [
+      path.join(ctx.package.scriptsDir, 'import-research-deck.js'),
+      `--deck=${file}`,
+      `--round-id=${roundId}`,
+      `--title=${title}`,
+      `--date=${date}`,
+      `--output-dir=${draftsDir}`
+    ], {
+      cwd: ctx.root,
+      stdio: ['ignore', 'pipe', 'pipe']
+    })
+
+    let stdout = ''
+    let stderr = ''
+    child.stdout.on('data', (chunk) => { stdout += chunk.toString() })
+    child.stderr.on('data', (chunk) => { stderr += chunk.toString() })
+
+    child.on('close', (code) => {
+      const draftPath = path.join(draftsDir, `${roundId}.json`)
+      resolve({
+        ok: code === 0,
+        source: path.relative(ctx.root, file),
+        draft: path.relative(ctx.root, draftPath),
+        review: path.relative(ctx.root, path.join(draftsDir, `${roundId}.review.md`)),
+        stdout: stdout.trim(),
+        stderr: stderr.trim()
+      })
+    })
+  })
+}
+
+function kebab (value) {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+}
+
+function camelCase (value) {
+  return value.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())
+}
+
+main().catch((error) => {
+  console.error(error.message)
+  process.exit(1)
+})

package/scripts/import-research-deck.js

@@ -0,0 +1,448 @@
+const fs = require('fs')
+const path = require('path')
+const { ResearchImportDraftSchema } = require('../lib/schema')
+const { createContextOrExit } = require('../lib/context')
+const ctx = createContextOrExit()
+const { researchImportsDir } = ctx.paths
+const { extractArtefact, sourceTypeFor } = require('./lib/artefact-text')
+
+const defaultOutputDir = researchImportsDir
+
+async function main () {
+  const args = parseArgs(process.argv.slice(2))
+  // Several artefacts can belong to one round, so insight and action IDs are
+  // namespaced per artefact while participants stay round-scoped: the same
+  // participant code in two files from one round is one person.
+  args.artefactId = kebab(args.artefactId || args.roundId || '')
+
+  if (!args.deck || !args.roundId || !args.title || !args.date) {
+    throw new Error('Usage: npm run import:research-deck -- --deck path/to/artefact --round-id round-id [--artefact-id artefact-id] --title "Round title" --date YYYY-MM-DD')
+  }
+
+  const slides = await extractSlides(args.deck)
+  const draft = buildDraft(args, slides)
+  validateDraft(draft)
+
+  const outputDir = args.outputDir ? path.resolve(args.outputDir) : defaultOutputDir
+  fs.mkdirSync(outputDir, { recursive: true })
+  const outputPath = path.join(outputDir, `${args.artefactId}.json`)
+  const reviewPath = path.join(outputDir, `${args.artefactId}.review.md`)
+  fs.writeFileSync(outputPath, `${JSON.stringify(draft, null, 2)}\n`)
+  fs.writeFileSync(reviewPath, buildReviewMarkdown(draft))
+
+  console.log(`Imported ${slides.length} ${sectionLabel(args.deck).toLowerCase()}`)
+  reportFit(draft)
+  console.log(`Draft written to ${outputPath}`)
+  console.log(`Review written to ${reviewPath}`)
+}
+
+function parseArgs (args) {
+  const parsed = {}
+  for (let index = 0; index < args.length; index++) {
+    const arg = args[index]
+    if (!arg.startsWith('--')) continue
+    const [rawKey, value] = arg.slice(2).split('=', 2)
+    const key = camelCase(rawKey)
+    if (value !== undefined) {
+      parsed[key] = value
+    } else {
+      parsed[key] = args[index + 1]
+      index++
+    }
+  }
+  return parsed
+}
+
+function camelCase (value) {
+  return value.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())
+}
+
+/**
+ * Sections come back in one shape whatever the format. Speaker notes are folded
+ * into the section text - the notes pane on a playback deck is often where the
+ * finding is actually stated, and dropping it lost roughly a tenth of the words
+ * in this team's decks.
+ */
+async function extractSlides (deckPath) {
+  const { sections } = await extractArtefact(deckPath)
+  return sections.map((section) => ({
+    number: section.number,
+    title: section.title,
+    text: [...section.text, ...(section.notes || []).map((note) => `Speaker note: ${note}`)]
+  }))
+}
+
+/**
+ * Heading-scoped detection.
+ *
+ * The first version of this importer only recognised lines literally prefixed
+ * "Insight:", "Participant:" or "Action:". No real research deck is written
+ * that way, so a 69-slide round-16 playback full of findings imported as zero
+ * insights and zero participants.
+ *
+ * Real artefacts signal intent in the section heading - a slide titled
+ * "Insights - Safeguarding", a worksheet column called "Design Action", a
+ * persona heading called "Goals" - and then list the content beneath it. So the
+ * heading sets a mode, and substantive lines under that heading are captured in
+ * that mode. Explicit line prefixes still work, and still win where present.
+ */
+const SECTION_MODES = [
+  { mode: 'insight', pattern: /\b(insights?|findings?|what we (?:learned|found)|key takeaways?|themes?|pain ?points?|problems?)\b/i },
+  { mode: 'action', pattern: /\b(design actions?|actions?|recommendations?|next steps?|opportunit(?:y|ies)|what we(?:'| a)re doing)\b/i },
+  { mode: 'participant', pattern: /\b(participants?|who we (?:spoke to|interviewed)|our (?:participants|users)|sample|cohort|the people)\b/i },
+  { mode: 'assumption', pattern: /\b(assumptions?|hypothes(?:is|es)|riskiest)\b/i }
+]
+
+const PARTICIPANT_TOKEN = /\bP\d{1,3}\b/g
+const QUOTE = /\u201c([^\u201d]{25,})\u201d/g
+
+function buildDraft (args, slides) {
+  const participants = []
+  const insights = []
+  const designActions = []
+  const relationships = []
+  const seenParticipants = new Map()
+  const quotes = []
+
+  const addParticipant = (label, role) => {
+    const key = label.toLowerCase()
+    if (seenParticipants.has(key)) return seenParticipants.get(key)
+    const item = {
+      // A participant code identifies a person, not a person-within-a-round.
+      // P36 appears in four rounds and is one participant with four roundIds;
+      // namespacing the id by round created a second record for the same person
+      // every time a new round mentioned them.
+      id: kebab(`participant ${label}`),
+      label,
+      role: role || 'Research participant',
+      roundIds: [args.roundId]
+    }
+    participants.push(item)
+    seenParticipants.set(key, item)
+    relationships.push(relationship(`${args.roundId}-has-participant-${item.id}`, args.roundId, 'has-participant', item.id))
+    return item
+  }
+
+  const addInsight = (statement, sourceSection, participantIds) => {
+    if (!isSubstantive(statement)) return null
+    const item = {
+      id: kebab(`${args.artefactId} insight ${insights.length + 1}`),
+      roundId: args.roundId,
+      title: truncate(statement, 120),
+      summary: `${statement} (source: ${sourceSection})`,
+      confidence: 'medium',
+      participantIds: participantIds || [],
+      evidenceIds: []
+    }
+    insights.push(item)
+    relationships.push(relationship(`${args.roundId}-produced-insight-${item.id}`, args.roundId, 'produced-insight', item.id))
+    return item
+  }
+
+  const addAction = (statement, sourceSection) => {
+    if (!isSubstantive(statement)) return null
+    const item = {
+      id: kebab(`${args.artefactId} action ${designActions.length + 1}`),
+      title: truncate(statement, 120),
+      status: 'proposed',
+      action: statement,
+      rationale: `Stated under "${sourceSection}". Confirm against the extracted text before promoting.`,
+      insightIds: insights.length ? [insights[insights.length - 1].id] : []
+    }
+    designActions.push(item)
+    if (item.insightIds.length) {
+      // The action points at the insight, not the other way round, so evidence
+      // impact can walk from research through to the design work it drives.
+      relationships.push(relationship(`${args.artefactId}-action-${item.id}`, item.id, 'responds-to-insight', item.insightIds[0]))
+    }
+    return item
+  }
+
+  for (const slide of slides) {
+    const mode = modeFor(slide.title)
+
+    for (const line of slide.text) {
+      // Any participant code mentioned anywhere is a real participant.
+      for (const token of line.match(PARTICIPANT_TOKEN) || []) {
+        addParticipant(token, roleNear(token, line))
+      }
+      for (const match of line.matchAll(QUOTE)) {
+        quotes.push({ section: slide.title, participants: line.match(PARTICIPANT_TOKEN) || [], text: match[1].trim() })
+      }
+
+      // Explicit prefixes still win.
+      const explicitParticipant = parsePrefixedLine(line, ['Participant', 'Attendee'])
+      if (explicitParticipant) {
+        const [label, role] = explicitParticipant.split(/\s+[\u2013-]\s+/, 2).map((part) => part.trim())
+        addParticipant(label || explicitParticipant, role)
+        continue
+      }
+
+      const explicitInsight = parsePrefixedLine(line, ['Insight', 'Finding', 'Observation'])
+      if (explicitInsight) { addInsight(explicitInsight, slide.title, []); continue }
+
+      const explicitAction = parsePrefixedLine(line, ['Action', 'Design action', 'Recommendation'])
+      if (explicitAction) { addAction(explicitAction, slide.title); continue }
+
+      // Otherwise fall back to the heading's mode.
+      if (line === slide.title || isSpeakerNote(line)) continue
+      if (mode === 'insight') addInsight(line, slide.title, (line.match(PARTICIPANT_TOKEN) || []).map((t) => kebab(`${args.roundId} participant ${t}`)))
+      else if (mode === 'action') addAction(line, slide.title)
+      else if (mode === 'participant' && /\b\d+\s+\w/.test(line)) addInsight(`Sample composition: ${line}`, slide.title, [])
+    }
+  }
+
+  // Only keep participant links that point at participants we actually created.
+  const participantIds = new Set(participants.map((item) => item.id))
+  for (const insight of insights) {
+    insight.participantIds = [...new Set(insight.participantIds)].filter((id) => participantIds.has(id))
+  }
+
+  return {
+    sourceDeck: path.relative(ctx.root, path.resolve(args.deck)),
+    importedAt: new Date().toISOString(),
+    warnings: [
+      ...assessFit({ participants, insights, designActions, slides, quotes }),
+      ...(args.dateDerived === 'false'
+        ? [`The round date ${args.date} is today's date, not the date of the research. No date was found in the file or folder name. Set it before promoting.`]
+        : [])
+    ],
+    researchRounds: [
+      {
+        id: args.roundId,
+        title: args.title,
+        date: args.date,
+        sourceType: sourceType(args.deck),
+        sourcePath: path.relative(ctx.root, path.resolve(args.deck)),
+        summary: args.summary || `Imported from ${path.basename(args.deck)} (${slides.length} ${sectionLabel(args.deck).toLowerCase()}). A round may span several artefacts.`
+      }
+    ],
+    participants,
+    insights,
+    designActions,
+    relationships,
+    slides
+  }
+}
+
+/** A heading only sets a mode if it reads as a label, not as a sentence. */
+function modeFor (title) {
+  if (!title || title.split(/\s+/).length > 12) return null
+  const match = SECTION_MODES.find((entry) => entry.pattern.test(title))
+  return match ? match.mode : null
+}
+
+/**
+ * Bullet fragments like "Time-critical" are labels, not findings, and a deck's
+ * notes pane wraps mid-sentence, so a line starting lower-case or with stray
+ * punctuation is the tail of a thought rather than a statement of one.
+ */
+function isSubstantive (line) {
+  const words = line.split(/\s+/).filter(Boolean)
+  if (words.length < 6 || words.length > 90) return false
+  if (/^(agenda|contents|thank you|questions|next slide|any questions)\b/i.test(line)) return false
+  if (!/^[A-Z\u201c(]/.test(line)) return false
+  if (/^[)\],.;:]/.test(line)) return false
+  return /[a-z]/.test(line)
+}
+
+/**
+ * Speaker notes are kept in the extracted text so a reviewer and the
+ * structuring step can read them, but they are commentary on a slide rather
+ * than a stated finding, so they do not seed insights on their own.
+ */
+function isSpeakerNote (line) {
+  return /^Speaker note:/.test(line)
+}
+
+/**
+ * "P44 - Data Quality Manager" names a role. "P48 - NCA Request for Information -"
+ * is the start of a sentence, so anything left dangling on a dash is rejected.
+ */
+function roleNear (token, line) {
+  const match = line.match(new RegExp(`${token}\\s*[\\u2013-]\\s*([A-Z][^\\u201c.,"]{3,50})`))
+  if (!match) return null
+  const role = match[1].trim().replace(/[,;\u2013-]+$/, '').trim()
+  if (!role || role.split(/\s+/).length > 6) return null
+  if (!/[a-z]/.test(role)) return null
+  return role
+}
+
+function truncate (value, limit) {
+  return value.length <= limit ? value : `${value.slice(0, limit - 1).trimEnd()}\u2026`
+}
+
+function validateDraft (draft) {
+  const result = ResearchImportDraftSchema.safeParse(draft)
+  if (result.success) return
+
+  const issues = result.error.issues.map((issue) => {
+    const fieldPath = issue.path.length ? issue.path.join('.') : 'draft'
+    return `${fieldPath}: ${issue.message}`
+  })
+  throw new Error(`Imported draft does not match ResearchImportDraftSchema:\n- ${issues.join('\n- ')}`)
+}
+
+function assessFit ({ participants, insights, designActions, slides, quotes }) {
+  const warnings = []
+  if (!participants.length) warnings.push('No participants were detected. Use participant codes such as P12 in the artefact, or lines like "Participant: P1 - Policy administrator".')
+  if (!insights.length) warnings.push('No insights were detected. Put findings under a section titled "Insights", "Findings" or "What we learned", or prefix lines with "Insight:".')
+  if (!designActions.length) warnings.push('No design actions were detected. Put them under a section titled "Design actions" or "Recommendations", or prefix lines with "Action:".')
+  if (slides.length && insights.length / slides.length < 0.2) warnings.push('Low insight density for the number of sections. Check whether findings use unsupported headings.')
+  if (quotes && quotes.length && !quotes.some((quote) => quote.participants.length)) {
+    warnings.push(`${quotes.length} quotes were found but none carry a participant code. Attribute quotes as 'P12 - "quote"' so evidence can be traced to a person.`)
+  }
+  if (insights.length > 60) warnings.push('More than 60 insights were detected. The heading-scoped pass is probably capturing narrative text; tighten the draft before promoting.')
+  return warnings
+}
+
+function reportFit (draft) {
+  const warnings = draft.warnings || []
+  const fit = warnings.length ? 'needs review' : 'good'
+  console.log(`Import fit: ${fit}`)
+  if (warnings.length) {
+    console.log('Import warnings:')
+    for (const warning of warnings) console.log(`- ${warning}`)
+  }
+}
+
+function buildReviewMarkdown (draft) {
+  const round = draft.researchRounds[0]
+  const unit = { 'pdf document': 'Page', 'powerpoint deck': 'Slide', 'word document': 'Section', 'word template': 'Section', 'excel workbook': 'Sheet' }[round.sourceType] || 'Section'
+  const sectionHeading = `${unit} Text`
+  const sectionUnit = `${unit}s`
+  const sectionItem = unit
+  const lines = [
+    `# ${round.title}`,
+    '',
+    `- **Round ID:** ${round.id}`,
+    `- **Date:** ${round.date}`,
+    `- **Source:** ${draft.sourceDeck}`,
+    `- **Imported:** ${draft.importedAt}`,
+    '',
+    '## Import Fit',
+    ''
+  ]
+
+  if (draft.warnings && draft.warnings.length) {
+    lines.push('Needs review.')
+    lines.push('')
+    for (const warning of draft.warnings) lines.push(`- ${warning}`)
+  } else {
+    lines.push('Good. No fit warnings detected.')
+  }
+
+  lines.push('')
+  lines.push('## Summary')
+  lines.push('')
+  lines.push(`- Participants: ${draft.participants.length}`)
+  lines.push(`- Insights: ${draft.insights.length}`)
+  lines.push(`- Design actions: ${draft.designActions.length}`)
+  lines.push(`- Relationships: ${draft.relationships.length}`)
+  lines.push(`- ${sectionUnit}: ${draft.slides.length}`)
+  lines.push('')
+
+  appendSection(lines, 'Participants', draft.participants, (participant) => [
+    `### ${participant.label}`,
+    '',
+    `- **ID:** ${participant.id}`,
+    `- **Role:** ${participant.role}`,
+    participant.organisation ? `- **Organisation:** ${participant.organisation}` : null
+  ])
+
+  appendSection(lines, 'Insights', draft.insights, (insight) => [
+    `### ${insight.title}`,
+    '',
+    `- **ID:** ${insight.id}`,
+    `- **Confidence:** ${insight.confidence}`,
+    '',
+    insight.summary
+  ])
+
+  appendSection(lines, 'Design Actions', draft.designActions, (action) => [
+    `### ${action.title}`,
+    '',
+    `- **ID:** ${action.id}`,
+    `- **Status:** ${action.status}`,
+    '',
+    action.action,
+    '',
+    `**Rationale:** ${action.rationale}`
+  ])
+
+  lines.push(`## ${sectionHeading}`)
+  lines.push('')
+  for (const slide of draft.slides) {
+    lines.push(`### ${sectionItem} ${slide.number}: ${slide.title}`)
+    lines.push('')
+    if (slide.text.length) {
+      for (const text of slide.text) lines.push(`- ${text}`)
+    } else {
+      lines.push('_No text extracted._')
+    }
+    lines.push('')
+  }
+
+  return `${lines.filter((line) => line !== null).join('\n')}\n`
+}
+
+function appendSection (lines, title, items, renderItem) {
+  lines.push(`## ${title}`)
+  lines.push('')
+  if (!items.length) {
+    lines.push('_None detected._')
+    lines.push('')
+    return
+  }
+
+  for (const item of items) {
+    lines.push(...renderItem(item).filter((line) => line !== null))
+    lines.push('')
+  }
+}
+
+function parsePrefixedLine (line, prefixes) {
+  for (const prefix of prefixes) {
+    const match = line.match(new RegExp(`^${escapeRegExp(prefix)}\\s*[:\\-]\\s*(.+)$`, 'i'))
+    if (match) return match[1].trim()
+  }
+  return null
+}
+
+function sourceType (sourcePath) {
+  return sourceTypeFor(sourcePath)
+}
+
+function sectionLabel (sourcePath) {
+  const extension = path.extname(sourcePath).toLowerCase()
+  if (extension === '.pdf') return 'PDF pages'
+  if (extension === '.pptx') return 'PowerPoint slides'
+  if (extension === '.docx' || extension === '.dotx') return 'document sections'
+  if (extension === '.xlsx') return 'worksheets'
+  return 'sections'
+}
+
+function relationship (id, from, type, to) {
+  return {
+    id: kebab(id),
+    from,
+    type,
+    to
+  }
+}
+
+function kebab (value) {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+}
+
+function escapeRegExp (value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+main().catch((error) => {
+  console.error(error.message)
+  process.exit(1)
+})