@aiwg/cli 0.0.0-bootstrap.0 → 2026.7.18

package/dist/src/artifacts/corpus-tools/sidecar-lint.js

@@ -0,0 +1,233 @@
+/**
+ * Citation-sidecar structural lint + orphan detection (#1503).
+ *
+ * TS-native port of section9 `scripts/corpus/lint_sidecars.py` (structural
+ * lint) and `scripts/corpus/find_orphans.py` (zero-edge sidecars). These are
+ * *sidecar-structural* checks — distinct from the generic `aiwg lint
+ * --ruleset research` (note-level frontmatter/orphans) that the `research-lint`
+ * skill runs. The `research-lint` skill references these for sidecar depth
+ * rather than duplicating them (reconcile, don't duplicate).
+ *
+ * @source historical: corpus/lint_sidecars.py, corpus/find_orphans.py
+ * @tests @test/unit/artifacts/sidecar-lint.test.ts
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+const REF_ID_RE = /\bREF-\d+[a-z]?\b/g;
+const CITES_DIR = ['documentation', 'citations'];
+const REFS_DIR = ['documentation', 'references'];
+/** List `documentation/citations/REF-*-citations.md` files under the corpus root. */
+function listSidecars(root) {
+    const dir = path.join(root, ...CITES_DIR);
+    if (!fs.existsSync(dir))
+        return [];
+    return fs
+        .readdirSync(dir)
+        .filter((f) => /^REF-.*-citations\.md$/.test(f))
+        .sort()
+        .map((f) => path.join(dir, f));
+}
+/** Extract the ref id (`REF-NNN`) from a sidecar filename. */
+function refOf(file) {
+    return path.basename(file).replace(/-citations\.md$/, '');
+}
+/**
+ * Structural lint for one sidecar's text. Mirrors `lint_sidecars.lint`:
+ *  - frontmatter present + has `ref:`/`title:`/`type:`
+ *  - has an Outgoing section + an Incoming section (merge-redirects exempt)
+ *  - no duplicate `| # | Title` table headers under the same sub-header
+ *    (the doubled-append bug pattern)
+ */
+export function lintSidecarText(text) {
+    const issues = [];
+    const lines = text.split('\n');
+    // Frontmatter
+    if (!text.startsWith('---')) {
+        issues.push('missing-frontmatter');
+    }
+    else {
+        const end = lines.indexOf('---', 1);
+        if (end === -1) {
+            issues.push('malformed-frontmatter');
+        }
+        else {
+            const fm = lines.slice(1, end).join('\n');
+            for (const req of ['ref:', 'title:', 'type:']) {
+                if (!fm.includes(req))
+                    issues.push(`frontmatter-missing-${req.replace(/:$/, '')}`);
+            }
+        }
+    }
+    // Section presence (H1/H2 only; sub-headers belong to the enclosing section)
+    let hasOutgoing = false;
+    let hasIncoming = false;
+    for (const line of lines) {
+        const low = line.toLowerCase().trim();
+        if (low.startsWith('## ') && !low.startsWith('### ')) {
+            const h = low.replace(/^#+/, '').trim();
+            if (h.startsWith('outgoing') ||
+                h.includes('papers this work cites') ||
+                h.includes('works / resources referenced')) {
+                hasOutgoing = true;
+            }
+            if (h.startsWith('incoming') ||
+                h.includes('cited by') ||
+                h.includes('referenced by') ||
+                h.includes('citing this') ||
+                h.includes('that cite')) {
+                hasIncoming = true;
+            }
+        }
+    }
+    const isRedirect = text.includes('MERGED INTO') || text.includes('status: merged');
+    if (!hasOutgoing && !isRedirect)
+        issues.push('missing-outgoing-section');
+    if (!hasIncoming && !isRedirect)
+        issues.push('missing-incoming-section');
+    // Duplicate table headers: two `| # | Title` headers with no intervening
+    // sub-header (a doubled-append signature).
+    let bugDupes = 0;
+    let lastSubHdr = -1;
+    let lastTable = -1;
+    for (let i = 0; i < lines.length; i++) {
+        const s = lines[i].trim();
+        if (s.startsWith('## ') || s.startsWith('### ')) {
+            lastSubHdr = i;
+            lastTable = -1;
+        }
+        else if (s.startsWith('| # | Title')) {
+            if (lastTable !== -1 && lastTable > lastSubHdr)
+                bugDupes++;
+            lastTable = i;
+        }
+    }
+    if (bugDupes > 0)
+        issues.push(`duplicate-table-headers-count=${bugDupes}`);
+    return issues;
+}
+export function lintSidecars(root) {
+    const files = listSidecars(root);
+    const byIssue = {};
+    let filesWithIssues = 0;
+    let totalIssues = 0;
+    for (const file of files) {
+        const issues = lintSidecarText(fs.readFileSync(file, 'utf-8'));
+        if (issues.length) {
+            filesWithIssues++;
+            totalIssues += issues.length;
+            for (const issue of issues)
+                (byIssue[issue] ??= []).push(refOf(file));
+        }
+    }
+    return { totalFiles: files.length, filesWithIssues, totalIssues, byIssue };
+}
+export function renderLint(r) {
+    const out = [];
+    out.push(`Sidecars linted: ${r.totalFiles}`);
+    out.push(`Files with issues: ${r.filesWithIssues}`);
+    out.push(`Total issues: ${r.totalIssues}`);
+    out.push('');
+    const sorted = Object.entries(r.byIssue).sort((a, b) => b[1].length - a[1].length);
+    for (const [issue, refs] of sorted) {
+        out.push(`  ${issue}: ${refs.length} files`);
+        for (const ref of refs.slice(0, 10))
+            out.push(`    ${ref}`);
+        if (refs.length > 10)
+            out.push(`    ... and ${refs.length - 10} more`);
+    }
+    return out.join('\n') + '\n';
+}
+/**
+ * Read the title from a reference analysis doc (`REF-NNN-*.md` or `REF-NNN.md`).
+ * Prefers frontmatter `title:`, then the first `# ` heading in the body — so
+ * docs that open with a `---` frontmatter block still yield a real title (the
+ * section9 original read the literal first line and rendered `---`).
+ */
+export function refTitle(root, ref) {
+    const refsDir = path.join(root, ...REFS_DIR);
+    if (!fs.existsSync(refsDir))
+        return '?';
+    const match = fs
+        .readdirSync(refsDir)
+        .find((f) => f === `${ref}.md` || f.startsWith(`${ref}-`));
+    if (!match)
+        return '?';
+    const text = fs.readFileSync(path.join(refsDir, match), 'utf-8');
+    // 1. frontmatter title:
+    const fm = text.match(/^---\n([\s\S]*?)\n---\n/);
+    if (fm) {
+        const t = fm[1].match(/^title:\s*(.+)$/m);
+        if (t)
+            return t[1].trim().replace(/^["']|["']$/g, '').slice(0, 80);
+    }
+    // 2. first `# ` heading (strip a `REF-NNN:` prefix if present)
+    const heading = text.match(/^#\s+(.+)$/m);
+    if (heading)
+        return heading[1].replace(/^REF-[^:]*:\s*/, '').trim().slice(0, 80) || '?';
+    return '?';
+}
+/**
+ * Scan a sidecar's Outgoing/Incoming sections for any `REF-NNN` edge token
+ * (excluding self). Faithful port of `find_orphans.parse_sections` — lenient
+ * by design: it matches any ref id appearing in an edge section regardless of
+ * table-column naming, unlike the column-specific `parseCitationSidecar`.
+ */
+function scanEdges(text, selfRef) {
+    const outgoing = new Set();
+    const incoming = new Set();
+    let current = null;
+    for (const line of text.split('\n')) {
+        const low = line.toLowerCase().trim();
+        const isSectionHeader = low.startsWith('# ') || (low.startsWith('## ') && !low.startsWith('### '));
+        if (isSectionHeader) {
+            const h = low.replace(/^#+/, '').trim();
+            if (h.startsWith('notes') || h === 'note')
+                current = null;
+            else if (h.startsWith('outgoing'))
+                current = 'out';
+            else if (h.startsWith('incoming'))
+                current = 'in';
+            else if (h.includes('cited by') || h.includes('referenced by') || h.includes('citing this') || h.includes('that cite'))
+                current = 'in';
+            else if (h.includes('papers this work cites') || h.includes('works / resources referenced') || h.startsWith('references ') || h === 'references')
+                current = 'out';
+            else
+                current = null;
+            continue;
+        }
+        if (current === null)
+            continue;
+        const matches = line.match(REF_ID_RE);
+        if (!matches)
+            continue;
+        for (const m of matches) {
+            if (m === selfRef)
+                continue;
+            if (current === 'out')
+                outgoing.add(m);
+            else
+                incoming.add(m);
+        }
+    }
+    return { outgoing, incoming };
+}
+/** Sidecars with zero incoming AND zero outgoing edges (lenient section scan). */
+export function findOrphans(root) {
+    const orphans = [];
+    for (const file of listSidecars(root)) {
+        const ref = refOf(file);
+        const { outgoing, incoming } = scanEdges(fs.readFileSync(file, 'utf-8'), ref);
+        if (outgoing.size === 0 && incoming.size === 0) {
+            orphans.push({ ref, title: refTitle(root, ref) });
+        }
+    }
+    return orphans;
+}
+export function renderOrphans(orphans) {
+    const out = [];
+    out.push(`Orphaned sidecars (no incoming AND no outgoing edges): ${orphans.length}`);
+    for (const o of orphans)
+        out.push(`  ${o.ref}: ${o.title}`);
+    return out.join('\n') + '\n';
+}
+//# sourceMappingURL=sidecar-lint.js.map

package/dist/src/artifacts/corpus-tools/sidecar-repair.js

@@ -0,0 +1,277 @@
+/**
+ * Citation-sidecar metadata repair (#1503): author backfill + affiliation
+ * normalization. TS-native port of section9 `scripts/corpus/fix_broken_authors.py`
+ * and `scripts/corpus/normalize_affiliation.py`.
+ *
+ * Both default to dry-run; pass `{ write: true }` to persist. The affiliation
+ * map is loaded from the corpus (`documentation/profiles/orgs/affiliation-map.yaml`)
+ * with the built-in `DEFAULT_AFFILIATION_MAP` as fallback — the hardcoded
+ * section9 map is externalized to config per epic #1496 principle #3.
+ *
+ * @source historical: corpus/fix_broken_authors.py, corpus/normalize_affiliation.py
+ * @tests @test/unit/artifacts/sidecar-repair.test.ts
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { load as loadYaml } from 'js-yaml';
+import { DEFAULT_AFFILIATION_MAP, buildAffiliationReverse } from '../corpus-views/corpus-config.js';
+const CITES_DIR = ['documentation', 'citations'];
+const REFS_DIR = ['documentation', 'references'];
+const AFFIL_MAP_FILE = ['documentation', 'profiles', 'orgs', 'affiliation-map.yaml'];
+function listSidecars(root) {
+    const dir = path.join(root, ...CITES_DIR);
+    if (!fs.existsSync(dir))
+        return [];
+    return fs
+        .readdirSync(dir)
+        .filter((f) => /^REF-.*-citations\.md$/.test(f))
+        .sort()
+        .map((f) => path.join(dir, f));
+}
+function refOf(file) {
+    return path.basename(file).replace(/-citations\.md$/, '');
+}
+// ── Author repair ──────────────────────────────────────────────────────────
+const INITIAL_RE = /^[A-Z]\.(?:[\s-]*[A-Z]\.)*$/;
+const INST_KEYWORDS = [
+    'labs', 'contributors', 'foundation', 'team', 'datatracker', 'improvement proposal',
+    'task force', 'consortium', 'committee', 'working group', 'society', 'council', 'inc',
+    'llc', 'corp', 'openai', 'anthropic', 'deepmind', 'meta', 'google', 'microsoft',
+    'ietf', 'ieee', 'iso', 'w3c', 'nist',
+];
+function isInstitutional(name) {
+    const nm = name.toLowerCase();
+    return INST_KEYWORDS.some((k) => nm.includes(k));
+}
+/** Extract the body of the `## Citation` section from an analysis doc. */
+export function extractCitationBlock(text) {
+    const m = text.match(/##\s+(?:1\.\s+)?Citation\s*\n\s*\n([\s\S]+?)(?=\n##\s|\n---|$)/);
+    return m ? m[1].trim() : '';
+}
+/**
+ * Parse author names from a citation string. Faithful port of
+ * `fix_broken_authors.parse_authors` — handles "Last, F. M., Last2, F. (YYYY)",
+ * "& "/" and " separators, et-al truncation, institutional single-authors, and
+ * spelled-out first names.
+ */
+export function parseAuthors(citation) {
+    if (!citation)
+        return [];
+    const firstLine = citation.split('\n')[0].trim();
+    const m = firstLine.match(/^(.+?)\s*\((\d{4})/);
+    let authorsStr;
+    if (!m) {
+        // Institutional: "Org. *Title*..." or "Org Name. Title..."
+        const m2 = firstLine.match(/^([A-Z][^.*]+?)\.\s+\*?\S/);
+        if (m2)
+            return [m2[1].trim()];
+        return [];
+    }
+    authorsStr = m[1].trim().replace(/[,.]+$/, '');
+    if (/et al/i.test(authorsStr)) {
+        authorsStr = authorsStr.split(/,?\s*et\s+al\.?/i)[0];
+    }
+    if (!authorsStr)
+        return [];
+    // Institutional single author
+    if (!authorsStr.includes(',') && !/\s+&\s+|\s+and\s+/.test(authorsStr)) {
+        return [authorsStr.trim().replace(/\.$/, '')];
+    }
+    if (isInstitutional(authorsStr.split(',')[0]) && !/^[A-Z][a-z]+,\s*[A-Z]\./.test(authorsStr)) {
+        return [authorsStr.trim().replace(/\.$/, '')];
+    }
+    // Normalize separators
+    let norm = authorsStr.replace(/ & /g, ', ');
+    norm = norm.replace(/\s+and\s+/gi, ', ');
+    norm = norm.replace(/\s*&\s*/g, ', ');
+    const parts = norm.split(/,\s*/).map((p) => p.trim()).filter(Boolean);
+    const out = [];
+    let i = 0;
+    while (i < parts.length) {
+        const cur = parts[i].trim();
+        if (!cur) {
+            i++;
+            continue;
+        }
+        if (i + 1 < parts.length) {
+            const nxt = parts[i + 1].trim();
+            if (INITIAL_RE.test(nxt) || /^[A-Z]\.\s*[A-Z]\.?$/.test(nxt)) {
+                out.push(`${cur}, ${nxt}`);
+                i += 2;
+                continue;
+            }
+            if (/^[A-Z]\.?(?:\s+[A-Z]\.?)*$/.test(nxt) && nxt.length <= 8) {
+                out.push(`${cur}, ${nxt}`);
+                i += 2;
+                continue;
+            }
+            if (/^[A-Z][a-zA-Z'-]+$/.test(cur) &&
+                /^[A-Z][a-zA-Z'-]+$/.test(nxt) &&
+                nxt.split(' ').length === 1) {
+                out.push(`${cur}, ${nxt}`);
+                i += 2;
+                continue;
+            }
+        }
+        out.push(cur);
+        i++;
+    }
+    const cleaned = [];
+    for (let a of out) {
+        a = a.trim().replace(/[,.]+$/, '').trim().replace(/\s+/g, ' ');
+        if (!a)
+            continue;
+        if (['the', 'et al', 'et al.', 'n/a', 'and'].includes(a.toLowerCase()))
+            continue;
+        if (a.length < 3 && !isInstitutional(a))
+            continue;
+        cleaned.push(a);
+    }
+    return cleaned;
+}
+function findAnalysisDoc(root, ref) {
+    const dir = path.join(root, ...REFS_DIR);
+    if (!fs.existsSync(dir))
+        return null;
+    const match = fs.readdirSync(dir).find((f) => f.startsWith(`${ref}-`));
+    return match ? path.join(dir, match) : null;
+}
+export function repairAuthors(root, opts = {}) {
+    const results = [];
+    for (const file of listSidecars(root)) {
+        const text = fs.readFileSync(file, 'utf-8');
+        if (!text.includes('(see REF doc)') && !text.includes('"see REF doc"'))
+            continue;
+        const ref = refOf(file);
+        const fmMatch = text.match(/^---\n([\s\S]*?)\n---\n/);
+        if (!fmMatch) {
+            results.push({ ref, ok: false, reason: 'no-frontmatter' });
+            continue;
+        }
+        const analysis = findAnalysisDoc(root, ref);
+        if (!analysis) {
+            results.push({ ref, ok: false, reason: 'no-analysis-doc' });
+            continue;
+        }
+        const authors = parseAuthors(extractCitationBlock(fs.readFileSync(analysis, 'utf-8')));
+        if (!authors.length) {
+            results.push({ ref, ok: false, reason: 'could-not-parse-authors' });
+            continue;
+        }
+        const newBlock = ['authors:', ...authors.map((a) => `  - name: "${a}"`)].join('\n');
+        const newFm = fmMatch[1].replace(/authors:\s*(?:\n\s*-\s*.*)+/, newBlock);
+        if (newFm === fmMatch[1]) {
+            results.push({ ref, ok: false, reason: 'frontmatter-replace-no-op' });
+            continue;
+        }
+        const newText = `---\n${newFm}\n---\n${text.slice(fmMatch[0].length)}`;
+        if (opts.write)
+            fs.writeFileSync(file, newText);
+        results.push({ ref, ok: true, reason: 'fixed', authors });
+    }
+    return results;
+}
+// ── Affiliation normalization ────────────────────────────────────────────────
+/** Load the corpus affiliation map (data-file override → built-in default). */
+export function loadAffiliationMap(root) {
+    const file = path.join(root, ...AFFIL_MAP_FILE);
+    if (fs.existsSync(file)) {
+        try {
+            const raw = loadYaml(fs.readFileSync(file, 'utf-8'));
+            if (raw && typeof raw === 'object') {
+                return buildAffiliationReverse(raw);
+            }
+        }
+        catch {
+            /* fall through to default on malformed override */
+        }
+    }
+    return buildAffiliationReverse(DEFAULT_AFFILIATION_MAP);
+}
+/** Resolve an affiliation string to a canonical PROF-O slug, or null if ambiguous. */
+export function normalizeAffiliation(value, reverse) {
+    if (!value)
+        return null;
+    const v = value.trim().replace(/^"|"$/g, '').trim();
+    if (/^PROF-O-[a-z0-9-]+$/.test(v))
+        return v; // already canonical
+    const key = v.toLowerCase().trim();
+    if (reverse.has(key))
+        return reverse.get(key);
+    // Compound (multiple orgs) — skip
+    if ([';', ' / ', ' — ', ' ↔ '].some((sep) => v.includes(sep)))
+        return null;
+    // Parenthetical — try the part before the paren
+    if (v.includes('(') && v.includes(')')) {
+        const before = v.split('(')[0].trim().replace(/,$/, '').trim().toLowerCase();
+        return reverse.get(before) ?? null;
+    }
+    return null;
+}
+export function normalizeAffiliations(root, opts = {}) {
+    const reverse = loadAffiliationMap(root);
+    const res = { normalized: 0, alreadyCanonical: 0, ambiguous: [], errors: [], changes: [] };
+    for (const file of listSidecars(root)) {
+        const ref = refOf(file);
+        const text = fs.readFileSync(file, 'utf-8');
+        const fmMatch = text.match(/^---\n([\s\S]*?)\n---\n/);
+        if (!fmMatch) {
+            res.errors.push({ ref, reason: 'no-frontmatter' });
+            continue;
+        }
+        let fm;
+        try {
+            fm = loadYaml(fmMatch[1]) ?? {};
+        }
+        catch {
+            res.errors.push({ ref, reason: 'yaml-parse-error' });
+            continue;
+        }
+        const cur = String(fm['affiliation-primary'] ?? '').trim().replace(/^"|"$/g, '');
+        if (!cur) {
+            res.alreadyCanonical++;
+            continue;
+        }
+        const canonical = normalizeAffiliation(cur, reverse);
+        if (canonical === null) {
+            res.ambiguous.push({ ref, value: cur });
+            continue;
+        }
+        if (canonical === cur) {
+            res.alreadyCanonical++;
+            continue;
+        }
+        const newFm = fmMatch[1].replace(/^(affiliation-primary:\s*)"?[^"\n]*"?\s*$/m, `$1${canonical}`);
+        if (newFm === fmMatch[1]) {
+            res.errors.push({ ref, reason: 'regex-no-op' });
+            continue;
+        }
+        if (opts.write)
+            fs.writeFileSync(file, `---\n${newFm}\n---\n${text.slice(fmMatch[0].length)}`);
+        res.normalized++;
+        res.changes.push({ ref, from: cur, to: canonical });
+    }
+    return res;
+}
+export function renderRepair(authors, affil, write) {
+    const out = [];
+    const mode = write ? 'WRITE' : 'dry-run';
+    const fixed = authors.filter((a) => a.ok && a.reason === 'fixed');
+    const failed = authors.filter((a) => !a.ok);
+    out.push(`Sidecar repair (${mode})`);
+    out.push('');
+    out.push(`Authors — fixed: ${fixed.length}, failed: ${failed.length}`);
+    for (const r of fixed.slice(0, 5))
+        out.push(`  ${r.ref}: ${(r.authors ?? []).join('; ')}`);
+    for (const r of failed.slice(0, 10))
+        out.push(`  ! ${r.ref}: ${r.reason}`);
+    out.push('');
+    out.push(`Affiliations — normalized: ${affil.normalized}, already-canonical/none: ${affil.alreadyCanonical}, ` +
+        `ambiguous: ${affil.ambiguous.length}, errors: ${affil.errors.length}`);
+    for (const c of affil.changes.slice(0, 10))
+        out.push(`  ${c.ref}: ${c.from} → ${c.to}`);
+    for (const a of affil.ambiguous.slice(0, 10))
+        out.push(`  ? ${a.ref}: ${a.value.slice(0, 80)}`);
+    return out.join('\n') + '\n';
+}
+//# sourceMappingURL=sidecar-repair.js.map