@hippo-digital/hippocampus 1.0.0-rc.1

package/scripts/migrate-knowledge.js

@@ -0,0 +1,73 @@
+const fs = require('fs')
+const path = require('path')
+const { createContextOrExit } = require('../lib/context')
+const {
+  CURRENT_SCHEMA_VERSION,
+  META_FILENAME,
+  readMeta,
+  migrateInMemory,
+  writeMeta
+} = require('../lib/schema-version')
+
+// The loader migrates in memory and warns on every read. This is the command
+// that makes it permanent, writing the upgraded collections and stamping
+// _meta.json, so the warning stops and the files on disk match the code.
+
+const ctx = createContextOrExit()
+const dryRun = process.argv.includes('--dry-run')
+
+const collectionFiles = {
+  users: 'users.json',
+  needs: 'needs.json',
+  scenarios: 'scenarios.json',
+  journeys: 'journeys.json',
+  screens: 'screens.json',
+  evidence: 'evidence.json',
+  decisions: 'decisions.json',
+  assumptions: 'assumptions.json',
+  researchRounds: 'research-rounds.json',
+  participants: 'participants.json',
+  insights: 'insights.json',
+  designActions: 'design-actions.json',
+  relationships: 'relationships.json'
+}
+
+function main () {
+  for (const project of ctx.requireProjects()) {
+    const sourceDir = path.join(project.dir, 'source')
+    const meta = readMeta(sourceDir)
+
+    if (meta.schemaVersion === CURRENT_SCHEMA_VERSION && !meta.implicit) {
+      console.log(`${project.name}: already at schemaVersion ${CURRENT_SCHEMA_VERSION}`)
+      continue
+    }
+
+    const raw = {}
+    for (const [key, filename] of Object.entries(collectionFiles)) {
+      const filePath = path.join(sourceDir, filename)
+      raw[key] = fs.existsSync(filePath) ? JSON.parse(fs.readFileSync(filePath, 'utf8')) : []
+    }
+    raw.project = JSON.parse(fs.readFileSync(path.join(sourceDir, 'project.json'), 'utf8'))
+    raw.projectContext = JSON.parse(fs.readFileSync(path.join(sourceDir, 'project-context.json'), 'utf8'))
+
+    const { data, version } = migrateInMemory(raw, meta)
+
+    if (dryRun) {
+      console.log(`${project.name}: would write schemaVersion ${version} and stamp ${META_FILENAME}`)
+      continue
+    }
+
+    for (const [key, filename] of Object.entries(collectionFiles)) {
+      fs.writeFileSync(path.join(sourceDir, filename), JSON.stringify(data[key], null, 2) + '\n')
+    }
+    writeMeta(sourceDir, { generator: 'hippocampus migrate:knowledge' })
+    console.log(`${project.name}: migrated ${meta.schemaVersion} -> ${version}, ${META_FILENAME} stamped`)
+  }
+}
+
+try {
+  main()
+} catch (error) {
+  console.error(error.message)
+  process.exit(1)
+}

package/scripts/promote-project-context-draft.js

@@ -0,0 +1,125 @@
+const fs = require('fs')
+const path = require('path')
+const { spawnSync } = require('child_process')
+const { ProjectContextImportDraftSchema } = require('../lib/schema')
+const { createContextOrExit } = require('../lib/context')
+const ctx = createContextOrExit()
+const { knowledgeSourceDir, projectImportsDir } = ctx.paths
+
+const projectContextPath = path.join(knowledgeSourceDir, 'project-context.json')
+
+function main () {
+  const args = parseArgs(process.argv.slice(2))
+  if (!args.draft) {
+    throw new Error(`Usage: npm run promote:project-context-draft -- --draft ${path.relative(ctx.root, path.join(projectImportsDir, 'context-id.json'))} [--dry-run]`)
+  }
+
+  const draft = validateDraft(readJson(path.resolve(args.draft)))
+  const current = readJson(projectContextPath)
+  const proposed = mergeProjectContext(current, draft.projectContext)
+
+  console.log([
+    `users: +${newCount(current.users, draft.projectContext.users)}`,
+    `channels: +${newCount(current.channels, draft.projectContext.channels)}`,
+    `policies: +${newCount(current.policies, draft.projectContext.policies)}`,
+    `constraints: +${newCount(current.constraints, draft.projectContext.constraints)}`,
+    `risks: +${newCount(current.risks, draft.projectContext.risks)}`,
+    `milestones: +${newCount(current.milestones, draft.projectContext.milestones)}`,
+    `facts: +${newCount(current.facts, draft.projectContext.facts)}`,
+    `sourceArtefacts: +${newCount(current.sourceArtefacts, draft.projectContext.sourceArtefacts)}`
+  ].join(', '))
+
+  if (args.dryRun) {
+    console.log('Dry run only. Source files were not changed.')
+    return
+  }
+
+  writeJson(projectContextPath, proposed)
+
+  const validation = spawnSync('npm', ['run', 'validate:knowledge'], {
+    cwd: ctx.root,
+    stdio: 'inherit'
+  })
+
+  if (validation.status !== 0) {
+    throw new Error('Promotion wrote project context, but knowledge validation failed. Review the promoted data.')
+  }
+
+  console.log('Project context draft promoted and knowledge validation passed.')
+}
+
+function validateDraft (draft) {
+  const result = ProjectContextImportDraftSchema.safeParse(draft)
+  if (result.success) return result.data
+
+  const issues = result.error.issues.map((issue) => {
+    const fieldPath = issue.path.length ? issue.path.join('.') : 'draft'
+    return `${fieldPath}: ${issue.message}`
+  })
+  throw new Error(`Project context draft does not match ProjectContextImportDraftSchema:\n- ${issues.join('\n- ')}`)
+}
+
+function mergeProjectContext (current, incoming) {
+  return {
+    ...current,
+    title: incoming.title || current.title,
+    summary: incoming.summary || current.summary,
+    purpose: incoming.purpose || current.purpose,
+    vision: incoming.vision || current.vision,
+    problemStatement: incoming.problemStatement || current.problemStatement,
+    serviceProposition: incoming.serviceProposition || current.serviceProposition,
+    users: mergeById(current.users, incoming.users),
+    channels: mergeById(current.channels, incoming.channels),
+    policies: mergeById(current.policies, incoming.policies),
+    constraints: mergeById(current.constraints, incoming.constraints),
+    risks: mergeById(current.risks, incoming.risks),
+    milestones: mergeById(current.milestones, incoming.milestones),
+    facts: mergeById(current.facts, incoming.facts),
+    sourceArtefacts: mergeById(current.sourceArtefacts, incoming.sourceArtefacts)
+  }
+}
+
+function mergeById (current, incoming) {
+  return Object.values(Object.fromEntries([
+    ...current.map((item) => [item.id, item]),
+    ...incoming.map((item) => [item.id, item])
+  ]))
+}
+
+function newCount (current, incoming) {
+  const existingIds = new Set(current.map((item) => item.id))
+  return incoming.filter((item) => !existingIds.has(item.id)).length
+}
+
+function parseArgs (args) {
+  const parsed = {}
+  for (let index = 0; index < args.length; index++) {
+    const arg = args[index]
+    if (!arg.startsWith('--')) continue
+    const [rawKey, value] = arg.slice(2).split('=', 2)
+    const key = camelCase(rawKey)
+    if (value !== undefined) {
+      parsed[key] = value
+    } else if (args[index + 1] && !args[index + 1].startsWith('--')) {
+      parsed[key] = args[index + 1]
+      index++
+    } else {
+      parsed[key] = true
+    }
+  }
+  return parsed
+}
+
+function readJson (filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'))
+}
+
+function writeJson (filePath, data) {
+  fs.writeFileSync(filePath, `${JSON.stringify(data, null, 2)}\n`)
+}
+
+function camelCase (value) {
+  return value.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())
+}
+
+main()

package/scripts/promote-research-draft.js

@@ -0,0 +1,236 @@
+const fs = require('fs')
+const path = require('path')
+const { spawnSync } = require('child_process')
+const { ResearchImportDraftSchema } = require('../lib/schema')
+const { createContextOrExit } = require('../lib/context')
+const ctx = createContextOrExit()
+const { knowledgeSourceDir, researchImportsDir } = ctx.paths
+
+const dataDir = knowledgeSourceDir
+
+const collections = {
+  researchRounds: 'research-rounds.json',
+  participants: 'participants.json',
+  insights: 'insights.json',
+  designActions: 'design-actions.json',
+  relationships: 'relationships.json'
+}
+
+function main () {
+  const args = parseArgs(process.argv.slice(2))
+  if (!args.draft) {
+    throw new Error(`Usage: npm run promote:research-draft -- --draft ${path.relative(ctx.root, path.join(researchImportsDir, 'round-id.json'))} [--dry-run]`)
+  }
+
+  const draft = validateDraft(readJson(path.resolve(args.draft)))
+  const current = readCurrentCollections()
+  const reused = reconcileSharedRecords(current, draft)
+  const proposed = mergeCollections(current, draft)
+  validateNoDuplicateIds(current, draft)
+  validateDraftReferences(proposed)
+
+  if (reused.length) console.log(`Reusing existing records from earlier artefacts in this round: ${reused.join(', ')}`)
+
+  const counts = Object.keys(collections).map((key) => `${key}: +${(draft[key] || []).length}`)
+  console.log(`Promotion checked: ${counts.join(', ')}`)
+
+  if (args.dryRun) {
+    console.log('Dry run only. Source files were not changed.')
+    return
+  }
+
+  for (const [key, filename] of Object.entries(collections)) {
+    writeJson(path.join(dataDir, filename), proposed[key])
+  }
+
+  const validation = spawnSync('npm', ['run', 'validate:knowledge'], {
+    cwd: ctx.root,
+    stdio: 'inherit'
+  })
+
+  if (validation.status !== 0) {
+    throw new Error('Promotion wrote source files, but knowledge validation failed. Review the promoted data.')
+  }
+
+  console.log('Research draft promoted and knowledge validation passed.')
+}
+
+/**
+ * A round is fieldwork, not a file, so several artefacts promote into the same
+ * round and meet the same people. Re-promoting an identical round record, or a
+ * participant already seen in a sibling artefact, is expected rather than a
+ * collision - the duplicate check should only fire on genuinely conflicting data.
+ *
+ * Returns the ids that were absorbed rather than added, so the run says so out loud.
+ */
+function reconcileSharedRecords (current, draft) {
+  const reused = []
+
+  const existingRounds = new Map(current.researchRounds.map((round) => [round.id, round]))
+  draft.researchRounds = (draft.researchRounds || []).filter((round) => {
+    const existing = existingRounds.get(round.id)
+    if (!existing) return true
+    if (existing.title !== round.title || existing.date !== round.date) {
+      throw new Error(`research round ${round.id} already exists with a different title or date. Reconcile the draft against hippocampus/source/research-rounds.json before promoting.`)
+    }
+    reused.push(`round ${round.id}`)
+    return false
+  })
+
+  const existingParticipants = new Map(current.participants.map((participant) => [participant.id, participant]))
+  draft.participants = (draft.participants || []).filter((participant) => {
+    const existing = existingParticipants.get(participant.id)
+    if (!existing) return true
+    existing.roundIds = [...new Set([...existing.roundIds, ...participant.roundIds])]
+    // "Research participant" is the placeholder the importer falls back to, so a
+    // named role from any artefact in the round wins over it.
+    if (existing.role === 'Research participant' && participant.role !== 'Research participant') existing.role = participant.role
+    if (!existing.organisation && participant.organisation) existing.organisation = participant.organisation
+    reused.push(`participant ${participant.id}`)
+    return false
+  })
+
+  const draftIds = new Set([
+    ...draft.researchRounds.map((item) => item.id),
+    ...draft.participants.map((item) => item.id),
+    ...current.researchRounds.map((item) => item.id),
+    ...current.participants.map((item) => item.id)
+  ])
+  const relationshipIds = new Set(current.relationships.map((item) => item.id))
+  draft.relationships = (draft.relationships || []).filter((link) => {
+    if (!relationshipIds.has(link.id)) return true
+    reused.push(`relationship ${link.id}`)
+    return false
+  })
+  void draftIds
+
+  return reused
+}
+
+function validateDraft (draft) {
+  const result = ResearchImportDraftSchema.safeParse(draft)
+  if (result.success) return result.data
+
+  const issues = result.error.issues.map((issue) => {
+    const fieldPath = issue.path.length ? issue.path.join('.') : 'draft'
+    return `${fieldPath}: ${issue.message}`
+  })
+  throw new Error(`Research draft does not match ResearchImportDraftSchema:\n- ${issues.join('\n- ')}`)
+}
+
+function parseArgs (args) {
+  const parsed = {}
+  for (let index = 0; index < args.length; index++) {
+    const arg = args[index]
+    if (!arg.startsWith('--')) continue
+    const [rawKey, value] = arg.slice(2).split('=', 2)
+    const key = camelCase(rawKey)
+    if (value !== undefined) {
+      parsed[key] = value
+    } else if (args[index + 1] && !args[index + 1].startsWith('--')) {
+      parsed[key] = args[index + 1]
+      index++
+    } else {
+      parsed[key] = true
+    }
+  }
+  return parsed
+}
+
+function readCurrentCollections () {
+  return Object.fromEntries(
+    Object.entries(collections).map(([key, filename]) => [key, readJson(path.join(dataDir, filename))])
+  )
+}
+
+function readJson (filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'))
+}
+
+function writeJson (filePath, data) {
+  fs.writeFileSync(filePath, `${JSON.stringify(data, null, 2)}\n`)
+}
+
+function mergeCollections (current, draft) {
+  return Object.fromEntries(
+    Object.keys(collections).map((key) => [key, [...current[key], ...(draft[key] || [])]])
+  )
+}
+
+function validateNoDuplicateIds (current, draft) {
+  for (const key of Object.keys(collections)) {
+    const existingIds = new Set(current[key].map((item) => item.id))
+    const draftIds = new Set()
+
+    for (const item of draft[key] || []) {
+      if (existingIds.has(item.id)) throw new Error(`${key}: ${item.id} already exists in source data`)
+      if (draftIds.has(item.id)) throw new Error(`${key}: ${item.id} appears more than once in the draft`)
+      draftIds.add(item.id)
+    }
+  }
+}
+
+function validateDraftReferences (proposed) {
+  const ids = {
+    researchRounds: idSet(proposed.researchRounds),
+    participants: idSet(proposed.participants),
+    insights: idSet(proposed.insights),
+    designActions: idSet(proposed.designActions),
+    relationships: idSet(proposed.relationships)
+  }
+
+  for (const participant of proposed.participants) {
+    for (const roundId of participant.roundIds || []) requireId(ids.researchRounds, 'participant', participant.id, 'roundIds', roundId)
+  }
+
+  for (const insight of proposed.insights) {
+    requireId(ids.researchRounds, 'insight', insight.id, 'roundId', insight.roundId)
+    for (const participantId of insight.participantIds || []) requireId(ids.participants, 'insight', insight.id, 'participantIds', participantId)
+  }
+
+  for (const action of proposed.designActions) {
+    for (const insightId of action.insightIds || []) requireId(ids.insights, 'design action', action.id, 'insightIds', insightId)
+  }
+
+  const entityIds = new Set([
+    ...ids.researchRounds,
+    ...ids.participants,
+    ...ids.insights,
+    ...ids.designActions,
+    ...readCanonicalEntityIds()
+  ])
+
+  for (const relationship of proposed.relationships) {
+    requireId(entityIds, 'relationship', relationship.id, 'from', relationship.from)
+    requireId(entityIds, 'relationship', relationship.id, 'to', relationship.to)
+  }
+}
+
+function readCanonicalEntityIds () {
+  const filenames = [
+    'users.json',
+    'needs.json',
+    'scenarios.json',
+    'journeys.json',
+    'screens.json',
+    'evidence.json',
+    'decisions.json',
+    'assumptions.json'
+  ]
+
+  return filenames.flatMap((filename) => readJson(path.join(dataDir, filename)).map((item) => item.id))
+}
+
+function idSet (items) {
+  return new Set(items.map((item) => item.id))
+}
+
+function requireId (ids, type, id, field, value) {
+  if (!ids.has(value)) throw new Error(`${type} ${id}: ${field} references missing id ${value}`)
+}
+
+function camelCase (value) {
+  return value.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())
+}
+
+main()

package/scripts/smoke-routes.js

@@ -0,0 +1,157 @@
+const { spawn } = require('child_process')
+const { createContextOrExit } = require('../lib/context')
+const { listDocs } = require('../lib/docs')
+const ctx = createContextOrExit()
+
+const port = process.env.SMOKE_PORT || 4587
+const baseUrl = `http://127.0.0.1:${port}`
+// A cold Prototype Kit boot compiles sass before it listens, which on a fresh
+// install is a good deal slower than the 15s this used to allow - and the
+// failure looked like "your routes are broken" rather than "it had not finished
+// starting".
+const timeoutMs = Number(process.env.SMOKE_TIMEOUT_MS || 60000)
+
+async function main () {
+  // Kept rather than discarded, so a server that dies during boot can say why
+  // instead of timing out silently.
+  let serverOutput = ''
+  const server = spawn(process.execPath, ['app.js'], {
+    cwd: ctx.root,
+    env: { ...process.env, PORT: String(port) },
+    stdio: ['ignore', 'pipe', 'pipe']
+  })
+  server.stdout.on('data', (chunk) => { serverOutput += chunk })
+  server.stderr.on('data', (chunk) => { serverOutput += chunk })
+
+  try {
+    await waitForServer(server, () => serverOutput)
+    for (const project of ctx.requireProjects()) {
+      await smokeRoutes(buildRoutes(project.basePath, project.store.getKnowledge()))
+    }
+    console.log('Route smoke test passed')
+  } finally {
+    // The pipes have to go explicitly. Browsersync leaves descendants holding
+    // the other end, so killing the direct child is not enough - Node keeps the
+    // handles open and the process never exits, which looks exactly like a
+    // hung smoke test.
+    server.stdout.destroy()
+    server.stderr.destroy()
+    server.kill('SIGKILL')
+  }
+}
+
+function buildRoutes (basePath, knowledge) {
+  const routes = new Set([
+    basePath,
+    `${basePath}/project`,
+    `${basePath}/users`,
+    `${basePath}/user-needs`,
+    `${basePath}/scenarios`,
+    `${basePath}/research`,
+    `${basePath}/graph`,
+    `${basePath}/trace`,
+    `${basePath}/handover`,
+    `${basePath}/docs`,
+    // The viewer serves these itself. If they 404 the pages load unstyled and
+    // the blueprint stops working, which is exactly the kind of breakage a
+    // smoke test should catch rather than a person noticing later.
+    `${basePath}/assets/hippocampus.css`,
+    `${basePath}/assets/blueprint.js`
+  ])
+
+  for (const doc of listDocs()) {
+    routes.add(`${basePath}/docs/${doc.slug}`)
+  }
+
+  for (const journey of knowledge.journeys) {
+    routes.add(`${basePath}/journeys/${journey.id}`)
+    routes.add(`${basePath}/journeys/${journey.id}/blueprint`)
+    routes.add(`${basePath}/journeys/${journey.id}/blueprint/classic`)
+    routes.add(`${basePath}/journeys/${journey.id}/flow`)
+    routes.add(`${basePath}/journeys/${journey.id}/handover`)
+  }
+
+  for (const user of knowledge.users) {
+    routes.add(`${basePath}/users/${user.id}`)
+  }
+
+  for (const screen of knowledge.screens) {
+    // Screens marked external live in the real NCRS prototype, not here, so
+    // this app has nothing to serve at their route.
+    if (screen.external) continue
+    routes.add(screen.route)
+  }
+
+  for (const evidence of knowledge.evidence) {
+    routes.add(`${basePath}/evidence/${evidence.id}/impact`)
+  }
+
+  if (knowledge.relationships.length) {
+    routes.add(`${basePath}/trace?entity=${knowledge.relationships[0].from}`)
+  }
+
+  return [...routes]
+}
+
+async function waitForServer (server, output) {
+  const startedAt = Date.now()
+
+  while (Date.now() - startedAt < timeoutMs) {
+    if (server.exitCode !== null) {
+      throw new Error(
+        `The prototype exited with code ${server.exitCode} before it started listening.\n` +
+        indent(output())
+      )
+    }
+
+    try {
+      const response = await fetch(`${baseUrl}/hippocampus`)
+      if (response.ok) return
+    } catch (error) {
+      // Keep polling until the Prototype Kit has finished booting.
+    }
+
+    await wait(250)
+  }
+
+  throw new Error(
+    `The prototype did not start listening on ${baseUrl} within ${timeoutMs}ms.\n` +
+    'This command starts your app with PORT set, so if app.js hardcodes a port\n' +
+    'it will be running somewhere else. Otherwise raise SMOKE_TIMEOUT_MS.\n' +
+    indent(output())
+  )
+}
+
+async function smokeRoutes (routes) {
+  const failures = []
+
+  for (const route of routes) {
+    // `Connection: close` avoids a Node/undici keep-alive assertion bug
+    // (`assert(!this.paused)` in Parser.finish) seen with larger response
+    // bodies across many sequential fetches on Node 24.
+    const response = await fetch(`${baseUrl}${route}`, { headers: { Connection: 'close' } })
+    await response.arrayBuffer()
+    if (response.status !== 200) failures.push(`${route} returned ${response.status}`)
+  }
+
+  if (failures.length) {
+    throw new Error(`Route smoke test failed:\n- ${failures.join('\n- ')}`)
+  }
+}
+
+function indent (text) {
+  const trimmed = String(text || '').trim()
+  if (!trimmed) return '  (the prototype printed nothing)'
+  return trimmed.split('\n').slice(-12).map((line) => `  ${line}`).join('\n')
+}
+
+function wait (ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((error) => {
+    console.error(error.message)
+    process.exit(1)
+  })