@mindrian_os/install 1.13.0-beta.11

package/lib/import/person-detector.cjs

@@ -0,0 +1,229 @@
+'use strict';
+
+/**
+ * Phase 80 Stage 02 -- Person Detector (pure function).
+ *
+ * detectPeople(fileEntries, opts) runs a 5-tier detection ladder against
+ * vault-scanner file entries and returns a deduplicated array of detected
+ * people matching the MANIFEST.json people[] schema subset.
+ *
+ * Detection tiers (highest to lowest confidence):
+ *   1. frontmatter          -- attendees/author/team/contributors YAML
+ *   2. attendees_header     -- "## Attendees" or "## Participants" list
+ *   3. mentions             -- "@Name" style mentions
+ *   4. first_last_repeated  -- First Last capitalized pair >= 3 distinct files
+ *   5. ner                  -- LLM NER sweep (STUB, returns nothing)
+ *
+ * Role detection is keyword-based against a sliding window around each name
+ * occurrence. See references/import-config.md for the role bucket keywords.
+ *
+ * Pure: no fs writes, no child_process, no AI calls. Body access goes
+ * through opts.bodyLoader (default is fs.readFileSync(source_abs_path)) so
+ * tests can pass synthetic content.
+ */
+
+const fs = require('node:fs');
+
+// Keyword order matters: longer/more-specific keywords MUST come before their
+// substrings so "co-founder" beats "founder", "board member" beats "board".
+const ROLE_BUCKETS = {
+  'core-team':   ['co-founder', 'cofounder', 'founder', 'ceo', 'cto', 'coo', 'cfo', 'head of', 'vp', 'our team', 'we built', 'team lead'],
+  'consultants': ['consultant', 'contractor', 'freelance', 'hired', 'retained', 'engaged'],
+  'advisors':    ['advisor', 'mentor', 'guidance', 'advised us', 'coach'],
+  'investors':   ['investor', 'vc', 'fund', 'angel', 'seed round', 'series a', 'series b', 'series c', 'cap table', 'term sheet'],
+  'board':       ['board member', 'chairman', 'director', 'sits on the board', 'board seat']
+};
+
+const STOPWORDS_2WORD = new Set([
+  'new york', 'los angeles', 'san francisco', 'project plan', 'action items',
+  'next steps', 'key insights', 'data room', 'team sync', 'team meeting',
+  'problem definition', 'solution design', 'business model'
+]);
+
+const TIER_RANK = {
+  frontmatter: 5,
+  attendees_header: 4,
+  mentions: 3,
+  first_last_repeated: 2,
+  ner: 1
+};
+
+const TIER_CONFIDENCE = {
+  frontmatter: 0.97,
+  attendees_header: 0.88,
+  mentions: 0.75,
+  first_last_repeated: 0.60,
+  ner: 0.45
+};
+
+const ATTENDEES_HEADER_RE = /##\s*(?:Attendees|Participants)\s*\n((?:\s*[-*]\s*.+\n?)+)/i;
+const MENTION_RE = /(?:^|\s)@([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)/g;
+// Use lookahead so overlapping pairs like "Today Eli Zarchin" yield both
+// "Today Eli" AND "Eli Zarchin" -- the stopword filter discards common
+// false-positives like "Today Morning".
+const FIRST_LAST_RE = /\b([A-Z][a-z]+)(?=\s+([A-Z][a-z]+)\b)/g;
+
+function slugify(name) {
+  return String(name || '')
+    .toLowerCase()
+    .normalize('NFD')
+    .replace(/[\u0300-\u036f]/g, '')
+    .replace(/[^a-z0-9\s-]/g, '')
+    .trim()
+    .replace(/\s+/g, '-');
+}
+
+function inferRole(name, content) {
+  const haystack = String(content || '').toLowerCase();
+  const needle = name.toLowerCase();
+  const idx = haystack.indexOf(needle);
+  if (idx === -1) return { bucket: 'unassigned', evidence: 'no match' };
+  const window = haystack.slice(Math.max(0, idx - 80), idx + needle.length + 80);
+  for (const bucket of Object.keys(ROLE_BUCKETS)) {
+    for (const kw of ROLE_BUCKETS[bucket]) {
+      if (window.indexOf(kw) !== -1) {
+        return { bucket: bucket, evidence: kw };
+      }
+    }
+  }
+  return { bucket: 'unassigned', evidence: 'no match' };
+}
+
+function detectPeople(fileEntries, opts) {
+  opts = opts || {};
+  const loader = opts.bodyLoader || function (f) {
+    try { return fs.readFileSync(f.source_abs_path, 'utf8'); } catch (e) { return ''; }
+  };
+  const exclude = new Set((opts.excludeNames || []).map(slugify));
+  const registry = new Map();
+
+  function record(name, fileId, tier, content) {
+    if (!name || typeof name !== 'string') return;
+    const trimmed = name.trim();
+    if (!trimmed) return;
+    if (STOPWORDS_2WORD.has(trimmed.toLowerCase())) return;
+    const slug = slugify(trimmed);
+    if (!slug || exclude.has(slug)) return;
+
+    const existing = registry.get(slug);
+    const role = inferRole(trimmed, content);
+
+    if (!existing) {
+      registry.set(slug, {
+        name: trimmed,
+        slug: slug,
+        detection_tier: tier,
+        detection_confidence: TIER_CONFIDENCE[tier],
+        role_guess: role.bucket,
+        role_evidence: role.evidence,
+        mention_files: [fileId]
+      });
+      return;
+    }
+
+    // merge mention_files
+    if (existing.mention_files.indexOf(fileId) === -1) {
+      existing.mention_files.push(fileId);
+    }
+    // upgrade to higher tier if applicable
+    if (TIER_RANK[tier] > TIER_RANK[existing.detection_tier]) {
+      existing.detection_tier = tier;
+      existing.detection_confidence = TIER_CONFIDENCE[tier];
+    }
+    // upgrade role if existing was unassigned and new finding has real role
+    if (existing.role_guess === 'unassigned' && role.bucket !== 'unassigned') {
+      existing.role_guess = role.bucket;
+      existing.role_evidence = role.evidence;
+    }
+  }
+
+  // First pass: gather bodies + run tiers 1, 2, 3 per file
+  const bodies = new Map(); // fileId -> body string
+  for (const f of fileEntries) {
+    let body = '';
+    try { body = loader(f) || ''; } catch (e) { body = ''; }
+    bodies.set(f.id, body);
+
+    // Tier 1: frontmatter
+    const fm = f.source_frontmatter || {};
+    const fmKeys = ['attendees', 'author', 'team', 'contributors'];
+    for (const key of fmKeys) {
+      const v = fm[key];
+      if (!v) continue;
+      if (Array.isArray(v)) {
+        for (const n of v) record(String(n), f.id, 'frontmatter', body);
+      } else if (typeof v === 'string') {
+        record(v, f.id, 'frontmatter', body);
+      }
+    }
+
+    // Tier 2: ## Attendees / ## Participants header
+    const am = body.match(ATTENDEES_HEADER_RE);
+    if (am) {
+      const lines = am[1].split('\n');
+      for (const line of lines) {
+        const m = line.match(/^\s*[-*]\s*(.+?)\s*$/);
+        if (m && m[1]) record(m[1], f.id, 'attendees_header', body);
+      }
+    }
+
+    // Tier 3: @mentions
+    MENTION_RE.lastIndex = 0;
+    let mm;
+    while ((mm = MENTION_RE.exec(body)) !== null) {
+      record(mm[1], f.id, 'mentions', body);
+    }
+  }
+
+  // Tier 4: first_last_repeated -- capitalized pair across >= 3 distinct files
+  const candidateFiles = new Map(); // "First Last" -> Set(fileId)
+  for (const f of fileEntries) {
+    const body = bodies.get(f.id) || '';
+    FIRST_LAST_RE.lastIndex = 0;
+    const seenInFile = new Set();
+    let m;
+    while ((m = FIRST_LAST_RE.exec(body)) !== null) {
+      const full = m[1] + ' ' + m[2];
+      // advance at least one char to avoid infinite loop on zero-width match
+      if (FIRST_LAST_RE.lastIndex === m.index) FIRST_LAST_RE.lastIndex += 1;
+      if (STOPWORDS_2WORD.has(full.toLowerCase())) continue;
+      if (seenInFile.has(full)) continue;
+      seenInFile.add(full);
+      if (!candidateFiles.has(full)) candidateFiles.set(full, new Set());
+      candidateFiles.get(full).add(f.id);
+    }
+  }
+  for (const entry of candidateFiles.entries()) {
+    const full = entry[0];
+    const fileSet = entry[1];
+    if (fileSet.size < 3) continue;
+    const slug = slugify(full);
+    // If already in registry via a higher tier, just merge mention_files
+    if (registry.has(slug)) {
+      const existing = registry.get(slug);
+      for (const fid of fileSet) {
+        if (existing.mention_files.indexOf(fid) === -1) existing.mention_files.push(fid);
+      }
+      continue;
+    }
+    // Record once per file so mention_files captures all
+    for (const fid of fileSet) {
+      const body = bodies.get(fid) || '';
+      record(full, fid, 'first_last_repeated', body);
+    }
+  }
+
+  // Tier 5: NER stub -- intentionally empty. Deferred to a future LLM pass.
+
+  return Array.from(registry.values());
+}
+
+module.exports = {
+  detectPeople: detectPeople,
+  slugify: slugify,
+  inferRole: inferRole,
+  ROLE_BUCKETS: ROLE_BUCKETS,
+  STOPWORDS_2WORD: STOPWORDS_2WORD,
+  TIER_RANK: TIER_RANK,
+  TIER_CONFIDENCE: TIER_CONFIDENCE
+};

package/lib/import/person-detector.test.cjs

@@ -0,0 +1,149 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Unit tests for lib/import/person-detector.cjs
+ * Run: node lib/import/person-detector.test.cjs
+ */
+
+const assert = require('node:assert/strict');
+const { detectPeople, slugify } = require('./person-detector.cjs');
+
+let passed = 0;
+let failed = 0;
+function test(name, fn) {
+  try {
+    fn();
+    passed += 1;
+    process.stdout.write('  ok  ' + name + '\n');
+  } catch (err) {
+    failed += 1;
+    process.stdout.write('  FAIL ' + name + '\n    ' + err.message + '\n');
+  }
+}
+
+function fakeFile(id, frontmatter, body) {
+  return {
+    id: id,
+    source_path: 'notes/' + id + '.md',
+    source_abs_path: '/tmp/fake/' + id + '.md',
+    source_frontmatter: frontmatter || {},
+    source_wikilinks: [],
+    parent_folder: 'notes',
+    __body: body || ''
+  };
+}
+
+function makeLoader() {
+  return function (f) { return f.__body || ''; };
+}
+
+test('Test 1: frontmatter tier detects attendees with confidence >= 0.95', function () {
+  const files = [fakeFile('f_1', { attendees: ['Jane Doe', 'John Roe'] }, 'body text')];
+  const r = detectPeople(files, { bodyLoader: makeLoader() });
+  assert.equal(r.length, 2, 'expected 2 people');
+  const jane = r.find(function (p) { return p.slug === 'jane-doe'; });
+  assert.ok(jane, 'jane present');
+  assert.equal(jane.detection_tier, 'frontmatter');
+  assert.ok(jane.detection_confidence >= 0.95, 'confidence >= 0.95 got ' + jane.detection_confidence);
+});
+
+test('Test 2: attendees header tier confidence >= 0.85', function () {
+  const body = '## Attendees\n- Jane Doe\n- John Roe\n\nDiscussion...';
+  const files = [fakeFile('f_1', {}, body)];
+  const r = detectPeople(files, { bodyLoader: makeLoader() });
+  assert.equal(r.length, 2);
+  const jane = r.find(function (p) { return p.slug === 'jane-doe'; });
+  assert.ok(jane);
+  assert.equal(jane.detection_tier, 'attendees_header');
+  assert.ok(jane.detection_confidence >= 0.85, 'confidence >= 0.85');
+});
+
+test('Test 3: first_last_repeated tier across 3 files', function () {
+  const files = [
+    fakeFile('f_1', {}, 'Today Eli Zarchin spoke about plans.'),
+    fakeFile('f_2', {}, 'Eli Zarchin reviewed the deck.'),
+    fakeFile('f_3', {}, 'Note from Eli Zarchin on budget.')
+  ];
+  const r = detectPeople(files, { bodyLoader: makeLoader() });
+  const eli = r.find(function (p) { return p.slug === 'eli-zarchin'; });
+  assert.ok(eli, 'eli detected');
+  assert.equal(eli.detection_tier, 'first_last_repeated');
+  assert.equal(eli.mention_files.length, 3);
+});
+
+test('Test 4: slug normalization handles punctuation and unicode', function () {
+  assert.equal(slugify('Jane Doe'), 'jane-doe');
+  assert.equal(slugify("O'Brien"), 'obrien');
+  assert.equal(slugify('Zoë Müller'), 'zoe-muller');
+  assert.equal(slugify('  Alice   Smith  '), 'alice-smith');
+});
+
+test('Test 5: role bucket fires on co-founder keyword near name', function () {
+  const body = 'Team sync notes. Jane Doe, co-founder, opened the meeting.';
+  const files = [fakeFile('f_1', { attendees: ['Jane Doe'] }, body)];
+  const r = detectPeople(files, { bodyLoader: makeLoader() });
+  const jane = r.find(function (p) { return p.slug === 'jane-doe'; });
+  assert.ok(jane);
+  assert.equal(jane.role_guess, 'core-team');
+  assert.equal(jane.role_evidence, 'co-founder');
+});
+
+test('Test 6: role bucket fallthrough to unassigned', function () {
+  const files = [fakeFile('f_1', { attendees: ['Bob Random'] }, 'Bob Random joined.')];
+  const r = detectPeople(files, { bodyLoader: makeLoader() });
+  const bob = r.find(function (p) { return p.slug === 'bob-random'; });
+  assert.ok(bob);
+  assert.equal(bob.role_guess, 'unassigned');
+  assert.equal(bob.role_evidence, 'no match');
+});
+
+test('Test 7: dedup across tiers keeps highest tier and merges mention_files', function () {
+  const files = [
+    fakeFile('f_1', { attendees: ['Jane Doe'] }, 'Jane Doe frontmatter'),
+    fakeFile('f_2', {}, '## Attendees\n- Jane Doe\n'),
+    fakeFile('f_3', {}, 'Jane Doe appears here too.')
+  ];
+  const r = detectPeople(files, { bodyLoader: makeLoader() });
+  const janeEntries = r.filter(function (p) { return p.slug === 'jane-doe'; });
+  assert.equal(janeEntries.length, 1, 'one jane only');
+  assert.equal(janeEntries[0].detection_tier, 'frontmatter', 'highest tier kept');
+  assert.ok(janeEntries[0].mention_files.indexOf('f_1') !== -1);
+  assert.ok(janeEntries[0].mention_files.indexOf('f_2') !== -1);
+});
+
+test('Test 8: stopword filter blocks "New York" and "Project Plan"', function () {
+  const files = [
+    fakeFile('f_1', {}, 'New York New York New York'),
+    fakeFile('f_2', {}, 'New York update'),
+    fakeFile('f_3', {}, 'Project Plan Project Plan Project Plan'),
+    fakeFile('f_4', {}, 'Project Plan notes')
+  ];
+  const r = detectPeople(files, { bodyLoader: makeLoader() });
+  const ny = r.find(function (p) { return p.slug === 'new-york'; });
+  const pp = r.find(function (p) { return p.slug === 'project-plan'; });
+  assert.equal(ny, undefined, 'New York should not be a person');
+  assert.equal(pp, undefined, 'Project Plan should not be a person');
+});
+
+test('Test 9: excludeNames filters self-references', function () {
+  const files = [fakeFile('f_1', { attendees: ['Jonathan Sagir', 'Jane Doe'] }, 'body')];
+  const r = detectPeople(files, { bodyLoader: makeLoader(), excludeNames: ['Jonathan Sagir'] });
+  assert.equal(r.length, 1);
+  assert.equal(r[0].slug, 'jane-doe');
+});
+
+test('Test 10: purity -- no fs writes, result is array', function () {
+  const files = [fakeFile('f_1', { attendees: ['Jane Doe'] }, 'Jane Doe, co-founder.')];
+  const r = detectPeople(files, { bodyLoader: makeLoader() });
+  assert.ok(Array.isArray(r));
+  // shape matches manifest people[] subset
+  const p = r[0];
+  assert.ok('name' in p && 'slug' in p && 'detection_tier' in p);
+  assert.ok('detection_confidence' in p && 'role_guess' in p);
+  assert.ok('role_evidence' in p && Array.isArray(p.mention_files));
+});
+
+process.stdout.write('\n' + passed + ' passed, ' + failed + ' failed\n');
+if (failed > 0) process.exit(1);
+process.stdout.write('PASS\n');

package/lib/import/report.cjs

@@ -0,0 +1,186 @@
+'use strict';
+
+/**
+ * Phase 80 Plan 80-05 -- IMPORT-REPORT.md renderer.
+ *
+ * Consumed by scripts/vault-import.cjs at the end of the 4-stage pipeline
+ * (and by the --undo branch to re-render with UNDONE: true). Shape verbatim
+ * from 80-CONTEXT.md gray area #10.
+ *
+ * Sections (in order): frontmatter, H1, Summary, Routing Table,
+ * People Detected, Meetings Detected, Warnings, /mos: Usability Check
+ * (placeholder populated by 80-06), Manifest Reference.
+ *
+ * Main-topic-slug extraction: top-2 destination sections by count. If a
+ * single section dominates (> 60%), use that section alone. --topic flag
+ * override is applied upstream in scripts/vault-import.cjs via pickMainTopic.
+ */
+
+const { readManifest } = require('./manifest.cjs');
+
+function slugifyTopic(s) {
+  return String(s || '').toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '');
+}
+
+function extractMainTopicSlug(manifest, override) {
+  if (override) return slugifyTopic(override);
+  const counts = new Map();
+  for (const f of manifest.files || []) {
+    const sec = f.classification && f.classification.section;
+    if (!sec) continue;
+    counts.set(sec, (counts.get(sec) || 0) + 1);
+  }
+  const sorted = Array.from(counts.entries()).sort(function (a, b) { return b[1] - a[1]; });
+  if (sorted.length === 0) return 'unclassified';
+  const total = sorted.reduce(function (s, e) { return s + e[1]; }, 0);
+  const top = sorted[0];
+  if (top[1] / total > 0.6) return slugifyTopic(top[0]);
+  if (sorted.length === 1) return slugifyTopic(top[0]);
+  return slugifyTopic(top[0]) + '-' + slugifyTopic(sorted[1][0]);
+}
+
+function countDecisions(files) {
+  let auto = 0, sugg = 0, inbox = 0;
+  for (const f of files || []) {
+    const d = f.classification && f.classification.decision;
+    if (d === 'AUTO') auto++;
+    else if (d === 'SUGGEST') sugg++;
+    else if (d === 'INBOX') inbox++;
+  }
+  return { auto: auto, sugg: sugg, inbox: inbox };
+}
+
+function escapeCell(s) {
+  return String(s == null ? '' : s).replace(/\|/g, '\\|').replace(/\r?\n/g, ' ');
+}
+
+function generateImportReport(manifest, opts) {
+  opts = opts || {};
+  const date = (manifest.import_id || '').slice(0, 10);
+  const counts = countDecisions(manifest.files || []);
+  const mainTopic = manifest.main_topic || extractMainTopicSlug(manifest, opts.topic);
+  const isUndone = !!(manifest.undo && manifest.undo.is_undone);
+
+  const lines = [];
+  lines.push('---');
+  if (isUndone) lines.push('UNDONE: true');
+  lines.push('date: ' + date);
+  lines.push('main_topic: ' + mainTopic);
+  lines.push('source_vault: ' + (manifest.source_vault || ''));
+  lines.push('total_files: ' + (manifest.files || []).length);
+  lines.push('status: ' + (manifest.status || 'in_progress'));
+  lines.push('import_id: ' + (manifest.import_id || ''));
+  lines.push('---');
+  lines.push('');
+
+  const titleTopic = mainTopic.replace(/-/g, ' ').replace(/\b\w/g, function (c) { return c.toUpperCase(); });
+  lines.push('# Import Report -- ' + date + ' ' + titleTopic);
+  lines.push('');
+
+  // Summary
+  lines.push('## Summary');
+  lines.push('- ' + (manifest.files || []).length + ' files ingested');
+  lines.push('- ' + counts.auto + ' auto-routed (confidence >= 0.75)');
+  lines.push('- ' + counts.sugg + ' in inbox/suggested (0.45 - 0.74)');
+  lines.push('- ' + counts.inbox + ' in inbox/unclassified (< 0.45)');
+  lines.push('- ' + (manifest.people || []).length + ' people detected');
+  lines.push('- ' + (manifest.meetings || []).length + ' meetings detected');
+  if (isUndone) {
+    lines.push('');
+    lines.push('> This import has been UNDONE. Files were moved to `imports/' + (manifest.import_id || '') + '/undone/`. ROOM.md and MINTO.md scaffolds scoped to this import were removed. The source vault was never touched.');
+  }
+  lines.push('');
+
+  // Routing Table
+  lines.push('## Routing Table');
+  lines.push('');
+  lines.push('| Source | Destination | Confidence | Decision |');
+  lines.push('|--------|-------------|------------|----------|');
+  for (const f of manifest.files || []) {
+    const c = f.classification || {};
+    lines.push(
+      '| ' + escapeCell(f.source_path) +
+      ' | ' + escapeCell(f.destination_path || '(skipped)') +
+      ' | ' + (c.confidence != null ? c.confidence : '') +
+      ' | ' + escapeCell(c.decision || '') + ' |'
+    );
+  }
+  lines.push('');
+
+  // People Detected
+  lines.push('## People Detected');
+  lines.push('');
+  if (!(manifest.people || []).length) {
+    lines.push('(none)');
+  } else {
+    lines.push('| Name | Role Guess | Mentions | Profile |');
+    lines.push('|------|------------|----------|---------|');
+    for (const p of manifest.people) {
+      lines.push(
+        '| ' + escapeCell(p.name) +
+        ' | ' + escapeCell(p.role_guess || p.role_bucket || 'unassigned') +
+        ' | ' + ((p.mention_files || []).length) +
+        ' | ' + escapeCell(p.profile_path || '') + ' |'
+      );
+    }
+  }
+  lines.push('');
+
+  // Meetings Detected
+  lines.push('## Meetings Detected');
+  lines.push('');
+  if (!(manifest.meetings || []).length) {
+    lines.push('(none)');
+  } else {
+    lines.push('| Date | Signals | Routed To |');
+    lines.push('|------|---------|-----------|');
+    for (const m of manifest.meetings) {
+      lines.push(
+        '| ' + escapeCell(m.detected_date || '') +
+        ' | ' + escapeCell((m.detection_signals || []).join(', ')) +
+        ' | ' + escapeCell(m.routed_to || m.filed_via || '(pending)') + ' |'
+      );
+    }
+  }
+  lines.push('');
+
+  // Warnings
+  lines.push('## Warnings');
+  lines.push('');
+  if (!(manifest.warnings || []).length) {
+    lines.push('(none)');
+  } else {
+    for (const w of manifest.warnings) {
+      lines.push('- ' + escapeCell(w.type) + ': ' + escapeCell(w.context || ''));
+    }
+  }
+  lines.push('');
+
+  // /mos: Usability Check (placeholder for 80-06)
+  lines.push('## /mos: Usability Check');
+  lines.push('');
+  if (opts.smokeTestResult) {
+    lines.push('- status: ' + escapeCell(opts.smokeTestResult.status || 'unknown'));
+    if (opts.smokeTestResult.details) {
+      lines.push('- details: ' + escapeCell(opts.smokeTestResult.details));
+    }
+  } else {
+    lines.push('_Placeholder -- populated by Phase 80 Plan 06 (branding + smoke test)._');
+  }
+  lines.push('');
+
+  // Manifest Reference
+  lines.push('## Manifest Reference');
+  lines.push('');
+  lines.push('See `01-ingest/output/MANIFEST.json` for the full source -> destination mapping (undo support).');
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+module.exports = {
+  generateImportReport: generateImportReport,
+  extractMainTopicSlug: extractMainTopicSlug,
+  slugifyTopic: slugifyTopic,
+  countDecisions: countDecisions
+};