@aiwg/cli 0.0.0-bootstrap.0 → 2026.7.17

package/dist/src/artifacts/corpus-tools/source-types.js

@@ -0,0 +1,164 @@
+/**
+ * Canonical, extensible source-type registry + normalizer (#1509).
+ *
+ * A research corpus accumulates three drifting type vocabularies — frontmatter
+ * `type:` (mixes source type with doc role), `source_type:` (hyphen/underscore
+ * drift), and a body "Source Type" field. This module gives one canonical
+ * registry and a `normalizeSourceType` that folds all three (plus a venue
+ * fallback) into a single source-type dimension.
+ *
+ * The authoritative runtime default is `DEFAULT_SOURCE_TYPES` below; the
+ * human-readable + overridable form ships at
+ * `agentic/code/frameworks/research-complete/config/source-types.yaml`
+ * (a corpus may override at `documentation/source-types.yaml`). A drift test
+ * keeps the YAML and this constant in sync.
+ *
+ * Consumed by: the `by-source-type` index view, per-type induction audit
+ * (#1504), acquisition dispatch (#1507).
+ *
+ * @source historical: section9 REFERENCE-TEMPLATE*.md per-type rules
+ * @tests @test/unit/artifacts/source-types.test.ts
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { load as loadYaml } from 'js-yaml';
+const OVERRIDE_FILE = ['documentation', 'source-types.yaml'];
+/** Authoritative runtime default registry. Mirrors the framework source-types.yaml. */
+export const DEFAULT_SOURCE_TYPES = {
+    version: 1,
+    // VENUE_PATTERNS label → canonical type. Academic conferences/journals → paper;
+    // arXiv → preprint; lab/vendor research posts → blog; GitHub/Docs → repo;
+    // RFC → standard; Wikipedia → encyclopedia. Used only when frontmatter
+    // type/source_type is absent or unmatched.
+    venueFallback: {
+        NeurIPS: 'paper', ICML: 'paper', ICLR: 'paper', ACL: 'paper', EMNLP: 'paper',
+        NAACL: 'paper', CVPR: 'paper', ICCV: 'paper', ECCV: 'paper', AAAI: 'paper',
+        IJCAI: 'paper', KDD: 'paper', SIGIR: 'paper', WWW: 'paper', RSS: 'paper',
+        CoRL: 'paper', ICRA: 'paper', IROS: 'paper', OSDI: 'paper', SOSP: 'paper',
+        NSDI: 'paper', 'USENIX ATC': 'paper', 'USENIX Security': 'paper',
+        'IEEE S&P / Oakland': 'paper', CCS: 'paper', CHI: 'paper', UIST: 'paper',
+        VLDB: 'paper', SIGMOD: 'paper', FAccT: 'paper', TMLR: 'paper', JMLR: 'paper',
+        Nature: 'paper', Science: 'paper', PNAS: 'paper',
+        arXiv: 'preprint',
+        'RFC (IETF)': 'standard',
+        'Anthropic Research': 'blog', 'OpenAI Research': 'blog',
+        'DeepMind / Google Research': 'blog', 'Meta AI / FAIR': 'blog',
+        'Microsoft Research': 'blog',
+        'GitHub / Documentation': 'repo',
+        'Blog / Web Article': 'blog',
+        Wikipedia: 'encyclopedia',
+    },
+    metaRoles: ['redirect', 'stub', 'gap-note', 'merged', 'index'],
+    types: {
+        paper: { description: 'Peer-reviewed conference or journal paper.', aliases: ['paper', 'conference-paper', 'conference_paper', 'conference paper', 'journal_article', 'journal-article', 'journal article', 'reference', 'peer-reviewed'], template: 'reference-academic', requiredSections: ['Citation', 'Summary', 'Key Contributions', 'Benchmark Results', 'Comparison with Related Work'], citationFormat: 'doi-bibtex', acquisition: 'pdf-download', storage: 'sources/pdfs/full', qualityRules: 'peer-review-grade', defaultRadarCadence: 'quarterly' },
+        preprint: { description: 'arXiv / preprint server; not yet peer reviewed.', aliases: ['preprint', 'arxiv', 'arxiv-preprint', 'eprint'], template: 'reference-academic', requiredSections: ['Citation', 'Summary', 'Key Contributions', 'Benchmark Results'], citationFormat: 'arxiv-id', acquisition: 'pdf-download', storage: 'sources/pdfs/full', qualityRules: 'preprint-hedged-grade', defaultRadarCadence: 'quarterly' },
+        blog: { description: 'Blog post, lab announcement, or vendor research post.', aliases: ['blog', 'blog-post', 'blog post', 'announcement', 'lab-announcement', 'lab announcement', 'vendor-research', 'post'], template: 'reference-web', requiredSections: ['Citation', 'Executive Summary', 'Key Claim', 'Practical Relevance'], citationFormat: 'url-venue-retrieved', acquisition: 'web-snapshot', storage: 'sources/web', qualityRules: 'non-peer-reviewed-hedging', defaultRadarCadence: 'biannual' },
+        repo: { description: 'Code repository or benchmark repository.', aliases: ['repo', 'codebase', 'code', 'repository', 'benchmark-repository', 'benchmark-repo', 'benchmark repository', 'github'], template: 'reference-repo', requiredSections: ['Citation', 'Summary', 'Architecture', 'Maintenance Signals'], citationFormat: 'repo-url-commit', acquisition: 'git-clone', storage: 'sources/repos', qualityRules: 'maintenance-activity-signals', defaultRadarCadence: 'on-demand' },
+        book: { description: 'Book.', aliases: ['book', 'monograph'], template: 'reference-academic', requiredSections: ['Citation', 'Summary', 'Key Contributions'], citationFormat: 'isbn', acquisition: 'manual', storage: 'sources/pdfs/full', qualityRules: 'editorial-grade', defaultRadarCadence: 'on-demand' },
+        chapter: { description: 'Book chapter.', aliases: ['chapter', 'book_chapter', 'book-chapter', 'book chapter'], template: 'reference-academic', requiredSections: ['Citation', 'Summary', 'Key Contributions'], citationFormat: 'isbn-chapter', acquisition: 'manual', storage: 'sources/pdfs/full', qualityRules: 'editorial-grade', defaultRadarCadence: 'on-demand' },
+        standard: { description: 'RFC, IETF/ISO/W3C standard, or formal specification.', aliases: ['standard', 'rfc', 'spec', 'specification', 'ietf', 'iso', 'w3c'], template: 'reference-web', requiredSections: ['Citation', 'Summary', 'Scope', 'Key Requirements'], citationFormat: 'standard-id', acquisition: 'web-snapshot', storage: 'sources/web', qualityRules: 'normative-spec', defaultRadarCadence: 'annual' },
+        doc: { description: 'Maintainer/vendor documentation, technical report, or report.', aliases: ['doc', 'documentation', 'maintainer-doc', 'vendor-doc', 'technical-report', 'technical report', 'report', 'manual'], template: 'reference-web', requiredSections: ['Citation', 'Summary', 'Practical Relevance'], citationFormat: 'url-venue-retrieved', acquisition: 'web-snapshot', storage: 'sources/web', qualityRules: 'non-peer-reviewed-hedging', defaultRadarCadence: 'biannual' },
+        discussion: { description: 'Issue, forum thread, or discussion.', aliases: ['discussion', 'issue', 'thread', 'forum'], template: 'reference-web', requiredSections: ['Citation', 'Summary', 'Key Points'], citationFormat: 'url-venue-retrieved', acquisition: 'web-snapshot', storage: 'sources/web', qualityRules: 'anecdotal-hedging', defaultRadarCadence: 'on-demand' },
+        encyclopedia: { description: 'Encyclopedia / reference-work entry (e.g. Wikipedia).', aliases: ['encyclopedia', 'wikipedia', 'wiki'], template: 'reference-web', requiredSections: ['Citation', 'Summary'], citationFormat: 'url-venue-retrieved', acquisition: 'web-snapshot', storage: 'sources/web', qualityRules: 'tertiary-source-hedging', defaultRadarCadence: 'annual' },
+        'expert-material': { description: 'Expert commentary, talk, or interview material.', aliases: ['expert-material', 'expert material', 'talk', 'interview', 'keynote'], template: 'reference-media', requiredSections: ['Citation', 'Media Profile', 'Summary', 'Key Timestamps'], citationFormat: 'timestamp-transcript', acquisition: 'manual', storage: 'media/transcripts', qualityRules: 'expert-opinion-hedging', defaultRadarCadence: 'on-demand' },
+        video: { description: 'Video, conference recording, course recording, or platform-hosted talk.', aliases: ['video', 'recording', 'webcast', 'livestream', 'stream', 'lecture-video', 'conference-video'], template: 'reference-media', requiredSections: ['Citation', 'Media Profile', 'Summary', 'Key Timestamps', 'Evidence and Transcript Method'], citationFormat: 'timestamp-transcript', acquisition: 'media-curator', storage: 'media/video', qualityRules: 'media-grade', defaultRadarCadence: 'on-demand' },
+        audio: { description: 'Audio recording, lecture audio, or direct-hosted audio source.', aliases: ['audio', 'audio-recording', 'lecture-audio', 'recording-audio'], template: 'reference-media', requiredSections: ['Citation', 'Media Profile', 'Summary', 'Key Timestamps', 'Evidence and Transcript Method'], citationFormat: 'timestamp-transcript', acquisition: 'media-curator', storage: 'media/audio', qualityRules: 'media-grade', defaultRadarCadence: 'on-demand' },
+        podcast: { description: 'Podcast episode or feed-hosted interview.', aliases: ['podcast', 'podcast-episode', 'episode', 'interview-audio'], template: 'reference-media', requiredSections: ['Citation', 'Media Profile', 'Summary', 'Key Timestamps', 'Evidence and Transcript Method'], citationFormat: 'timestamp-transcript', acquisition: 'media-curator', storage: 'media/audio', qualityRules: 'interview-hedged-grade', defaultRadarCadence: 'on-demand' },
+        lecture: { description: 'Institutional lecture, course video, keynote, seminar, or invited talk recording.', aliases: ['lecture', 'course', 'seminar', 'keynote', 'talk', 'invited-talk', 'conference-talk'], template: 'reference-media', requiredSections: ['Citation', 'Media Profile', 'Summary', 'Key Timestamps', 'Evidence and Transcript Method'], citationFormat: 'timestamp-transcript', acquisition: 'media-curator', storage: 'media/video', qualityRules: 'lecture-grade', defaultRadarCadence: 'on-demand' },
+        'internal-review': { description: 'Internal research note or architecture review (not an external source).', aliases: ['internal-review', 'internal-research', 'internal-architecture-review', 'audit'], template: 'reference-internal', requiredSections: ['Summary', 'Findings'], citationFormat: 'internal-ref', acquisition: 'internal', storage: 'documentation/synthesis', qualityRules: 'internal-note', defaultRadarCadence: 'on-demand' },
+    },
+};
+/** Coerce a parsed YAML registry (snake_case fields) into the typed registry. */
+function coerceRegistry(raw) {
+    if (!raw || typeof raw !== 'object' || !raw.types)
+        return null;
+    const types = {};
+    for (const [k, v] of Object.entries(raw.types)) {
+        types[k] = {
+            description: String(v.description ?? ''),
+            aliases: (v.aliases ?? []).map(String),
+            template: String(v.template ?? ''),
+            requiredSections: (v.required_sections ?? v.requiredSections ?? []).map(String),
+            citationFormat: String(v.citation_format ?? v.citationFormat ?? ''),
+            acquisition: String(v.acquisition ?? ''),
+            storage: String(v.storage ?? ''),
+            qualityRules: String(v.quality_rules ?? v.qualityRules ?? ''),
+            defaultRadarCadence: String(v.default_radar_cadence ?? v.defaultRadarCadence ?? 'on-demand'),
+        };
+    }
+    return {
+        version: Number(raw.version ?? 1),
+        types,
+        venueFallback: (raw.venue_fallback ?? raw.venueFallback ?? {}),
+        metaRoles: (raw.meta_roles ?? raw.metaRoles ?? []).map(String),
+    };
+}
+/** Load the registry: corpus override (`documentation/source-types.yaml`) → built-in default. */
+export function loadSourceTypeRegistry(corpusRoot) {
+    const override = path.join(corpusRoot, ...OVERRIDE_FILE);
+    if (fs.existsSync(override)) {
+        try {
+            const reg = coerceRegistry(loadYaml(fs.readFileSync(override, 'utf-8')));
+            if (reg && Object.keys(reg.types).length)
+                return reg;
+        }
+        catch {
+            /* malformed override → default */
+        }
+    }
+    return DEFAULT_SOURCE_TYPES;
+}
+/** Build alias → canonical and meta-role lookups for fast normalization. */
+function buildLookup(reg) {
+    const alias = new Map();
+    for (const [canonical, t] of Object.entries(reg.types)) {
+        alias.set(canonical.toLowerCase(), canonical);
+        for (const a of t.aliases)
+            alias.set(a.toLowerCase().trim(), canonical);
+    }
+    return { alias, meta: new Set(reg.metaRoles.map((r) => r.toLowerCase())) };
+}
+/**
+ * Normalize any of the three drifting type vocabularies (+ venue fallback) to a
+ * canonical source type. Returns the canonical key, `'meta'` for doc-role
+ * values (redirect/stub/…), or `'other'` when nothing matches.
+ */
+export function normalizeSourceType(input, reg = DEFAULT_SOURCE_TYPES) {
+    const { alias, meta } = buildLookup(reg);
+    const candidates = [input.sourceType, input.type, input.bodySourceType]
+        .map((v) => (v ?? '').toLowerCase().trim())
+        .filter(Boolean);
+    for (const c of candidates) {
+        if (meta.has(c))
+            return 'meta';
+        if (alias.has(c))
+            return alias.get(c);
+    }
+    // No explicit type matched — try the venue classification fallback.
+    if (input.venue && reg.venueFallback[input.venue])
+        return reg.venueFallback[input.venue];
+    return 'other';
+}
+/** Look up a canonical source type's rules. */
+export function getSourceType(canonical, reg = DEFAULT_SOURCE_TYPES) {
+    return reg.types[canonical] ?? null;
+}
+/** List canonical source-type keys. */
+export function listSourceTypes(reg = DEFAULT_SOURCE_TYPES) {
+    return Object.keys(reg.types).sort();
+}
+/** Render the registry as a human-readable table. */
+export function renderSourceTypes(reg) {
+    const out = [];
+    out.push(`Source-type registry (v${reg.version}) — ${Object.keys(reg.types).length} canonical types`);
+    out.push('');
+    out.push(`${'type'.padEnd(16)} ${'template'.padEnd(20)} ${'acquisition'.padEnd(14)} ${'cadence'.padEnd(10)} aliases`);
+    for (const key of listSourceTypes(reg)) {
+        const t = reg.types[key];
+        out.push(`${key.padEnd(16)} ${t.template.padEnd(20)} ${t.acquisition.padEnd(14)} ${t.defaultRadarCadence.padEnd(10)} ${t.aliases.slice(0, 4).join(', ')}${t.aliases.length > 4 ? ', …' : ''}`);
+    }
+    out.push('');
+    out.push(`meta roles (not source types): ${reg.metaRoles.join(', ')}`);
+    return out.join('\n') + '\n';
+}
+//# sourceMappingURL=source-types.js.map

package/dist/src/artifacts/corpus-tools/vision-extract.js

@@ -0,0 +1,243 @@
+/**
+ * Scanned-page vision extraction (#1507). TS-native, provider-neutral port of
+ * section9 `scripts/corpus/codex_vision_extract_pages.sh`.
+ *
+ * Transcribes a directory of scanned page PNGs into per-page + combined Markdown
+ * via a pluggable vision adapter (the source script hardcoded the `codex` CLI;
+ * per the issue this is provider-neutral). Preserves the resumable / retry /
+ * `# Page N` contract and the `<!-- extraction-status: complete -->` markers.
+ * Optionally rasterizes a PDF → page PNGs first (poppler `pdftoppm`).
+ *
+ * Adapters:
+ *  - `codex`   — `codex exec --sandbox read-only --image … --output-last-message`
+ *  - `command` — a generic shell-command template (AIWG_VISION_COMMAND) with
+ *                {image}/{prompt_file}/{out} placeholders → works with any vision
+ *                CLI (claude, a custom script, …). Truly provider-neutral.
+ *  - a caller-supplied adapter (used by tests with a fake transcriber).
+ *
+ * @source historical: corpus/codex_vision_extract_pages.sh
+ * @tests @test/unit/artifacts/vision-extract.test.ts
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { spawnSync } from 'child_process';
+/** Strict-transcription prompt. `title` parameterizes the corpus-specific intro. */
+export function buildPrompt(page, title) {
+    const subject = title ? `from ${title}` : 'from a scanned document';
+    return [
+        `You are transcribing one scanned page image ${subject}.`,
+        '',
+        'Task:',
+        '- Extract all readable text from this image as accurately as possible.',
+        '- Preserve reading order.',
+        '- Keep headings, captions, labels, footnotes, marginal text, chart labels, table entries, and page numbers when readable.',
+        '- If text is part of a figure or chart, include it under a short marker like "[figure text]" or "[chart labels]".',
+        '- Do not summarize, interpret, modernize spelling, or add commentary.',
+        '- If a word is uncertain, write the best reading followed by "[?]".',
+        '- If a region is unreadable, write "[unreadable]" only for that region.',
+        '- Output Markdown only.',
+        `- Start exactly with: "# Page ${page}"`,
+        '',
+        'Accuracy is more important than preserving coordinates or line breaks.',
+    ].join('\n');
+}
+// ── Adapters ─────────────────────────────────────────────────────────────────
+const codexAdapter = {
+    name: 'codex',
+    transcribe(imagePath, prompt, ctx) {
+        const tmp = path.join(path.dirname(imagePath), `.codex-out-${ctx.page}.txt`);
+        const args = ['exec'];
+        if (ctx.model)
+            args.push('--model', ctx.model);
+        args.push('--sandbox', 'read-only', '--cd', ctx.root, '--image', imagePath, '--output-last-message', tmp, '-');
+        try {
+            const r = spawnSync(process.env.AIWG_CODEX_BIN || 'codex', args, {
+                input: prompt,
+                timeout: ctx.timeoutSec * 1000,
+                encoding: 'utf8',
+            });
+            if (r.status === 0 && fs.existsSync(tmp)) {
+                const text = fs.readFileSync(tmp, 'utf8');
+                fs.rmSync(tmp, { force: true });
+                return { ok: true, text };
+            }
+            fs.rmSync(tmp, { force: true });
+            return { ok: false };
+        }
+        catch {
+            fs.rmSync(tmp, { force: true });
+            return { ok: false };
+        }
+    },
+};
+/**
+ * Generic command-template adapter (provider-neutral). Runs
+ * `AIWG_VISION_COMMAND` (or the `--command` flag) with placeholders:
+ *   {image}        absolute image path
+ *   {prompt_file}  path to a file containing the prompt
+ *   {out}          path the command must write the transcription to
+ * e.g.  claude -p "$(cat {prompt_file})" --image {image} > {out}
+ */
+function commandAdapter(template) {
+    return {
+        name: 'command',
+        transcribe(imagePath, prompt, ctx) {
+            const dir = path.dirname(imagePath);
+            const promptFile = path.join(dir, `.vision-prompt-${ctx.page}.txt`);
+            const out = path.join(dir, `.vision-out-${ctx.page}.txt`);
+            fs.writeFileSync(promptFile, prompt);
+            const cmd = template
+                .replaceAll('{image}', JSON.stringify(imagePath))
+                .replaceAll('{prompt_file}', JSON.stringify(promptFile))
+                .replaceAll('{out}', JSON.stringify(out));
+            try {
+                const r = spawnSync('sh', ['-c', cmd], { timeout: ctx.timeoutSec * 1000, encoding: 'utf8' });
+                let text;
+                if (fs.existsSync(out))
+                    text = fs.readFileSync(out, 'utf8');
+                else if (r.status === 0 && r.stdout)
+                    text = r.stdout; // command wrote to stdout
+                fs.rmSync(promptFile, { force: true });
+                fs.rmSync(out, { force: true });
+                return { ok: r.status === 0 && !!text, text };
+            }
+            catch {
+                fs.rmSync(promptFile, { force: true });
+                fs.rmSync(out, { force: true });
+                return { ok: false };
+            }
+        },
+    };
+}
+/** Resolve the vision adapter from options/env. Throws when misconfigured. */
+export function resolveAdapter(opts = {}) {
+    const command = opts.command ?? process.env.AIWG_VISION_COMMAND;
+    const provider = opts.provider ?? process.env.AIWG_VISION_PROVIDER ?? (command ? 'command' : 'codex');
+    if (provider === 'command') {
+        if (!command)
+            throw new Error("vision provider 'command' requires --command or AIWG_VISION_COMMAND (template with {image}/{prompt_file}/{out})");
+        return commandAdapter(command);
+    }
+    if (provider === 'codex')
+        return codexAdapter;
+    throw new Error(`unknown vision provider '${provider}' (codex | command)`);
+}
+// ── PDF → PNG rasterization (poppler pdftoppm) ───────────────────────────────
+export function pdftoppmAvailable() {
+    try {
+        return spawnSync('pdftoppm', ['-v'], { encoding: 'utf8' }).status === 0 || spawnSync('pdftoppm', ['-h'], { encoding: 'utf8' }).status !== null;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Rasterize a PDF into zero-padded `page-NNN.png` files in `outDir` via
+ * `pdftoppm`. Returns the page count. Throws if pdftoppm is unavailable.
+ */
+export function rasterizePdf(pdfPath, outDir, dpi = 200) {
+    if (!pdftoppmAvailable()) {
+        throw new Error('PDF rasterization needs poppler `pdftoppm` on PATH (macOS: `brew install poppler`; Debian/Ubuntu: `apt install poppler-utils`).');
+    }
+    if (!fs.existsSync(pdfPath))
+        throw new Error(`PDF not found: ${pdfPath}`);
+    fs.mkdirSync(outDir, { recursive: true });
+    // pdftoppm writes <prefix>-<n>.png with its own zero-padding based on page count.
+    const prefix = path.join(outDir, 'page');
+    const r = spawnSync('pdftoppm', ['-png', '-r', String(dpi), pdfPath, prefix], { encoding: 'utf8' });
+    if (r.status !== 0)
+        throw new Error(`pdftoppm failed: ${r.stderr || r.status}`);
+    // Normalize names to page-NNN.png (3-digit zero-pad).
+    let count = 0;
+    for (const f of fs.readdirSync(outDir)) {
+        const m = f.match(/^page-(\d+)\.png$/);
+        if (!m)
+            continue;
+        count++;
+        const padded = `page-${m[1].padStart(3, '0')}.png`;
+        if (padded !== f)
+            fs.renameSync(path.join(outDir, f), path.join(outDir, padded));
+    }
+    return count;
+}
+// ── Orchestration (resumable page loop + combine) ────────────────────────────
+const COMPLETE_MARKER = '<!-- extraction-status: complete -->';
+function pageNumber(file) {
+    const m = path.basename(file).match(/^page-0*(\d+)\.png$/);
+    return m ? parseInt(m[1], 10) : null;
+}
+function isComplete(outFile) {
+    return fs.existsSync(outFile) && fs.readFileSync(outFile, 'utf8').startsWith(COMPLETE_MARKER);
+}
+export async function extractPages(opts) {
+    if (!fs.existsSync(opts.imageDir))
+        throw new Error(`missing image directory: ${opts.imageDir}`);
+    const pageDir = path.join(opts.outDir, 'pages');
+    fs.mkdirSync(pageDir, { recursive: true });
+    const log = [];
+    const logLine = (s) => log.push(s);
+    const sleep = opts.sleep ?? (() => { });
+    const start = opts.start ?? 1;
+    const end = opts.end ?? Number.MAX_SAFE_INTEGER;
+    const retries = opts.retries ?? 0;
+    const timeoutSec = opts.timeoutSec ?? 240;
+    const images = fs
+        .readdirSync(opts.imageDir)
+        .filter((f) => /^page-\d+\.png$/.test(f))
+        .map((f) => path.join(opts.imageDir, f))
+        .sort();
+    let processed = 0;
+    let completed = 0;
+    let failed = 0;
+    for (const image of images) {
+        const page = pageNumber(image);
+        if (page == null || page < start || page > end)
+            continue;
+        processed++;
+        const outFile = path.join(pageDir, `page-${String(page).padStart(3, '0')}.md`);
+        if (!opts.force && isComplete(outFile)) {
+            logLine(`skip page ${page}: already complete`);
+            completed++;
+            continue;
+        }
+        const prompt = buildPrompt(page, opts.title);
+        let ok = false;
+        for (let attempt = 1; attempt <= retries + 1; attempt++) {
+            const res = opts.adapter.transcribe(image, prompt, { root: path.dirname(opts.imageDir), page, timeoutSec, model: opts.model });
+            if (res.ok && res.text && new RegExp(`^# Page ${page}\\b`, 'm').test(res.text)) {
+                fs.writeFileSync(outFile, `${COMPLETE_MARKER}\n<!-- source-image: ${image} -->\n\n${res.text.trimEnd()}\n`);
+                logLine(`page ${page}: complete (attempt ${attempt})`);
+                ok = true;
+                completed++;
+                break;
+            }
+            logLine(`page ${page}: unusable output (attempt ${attempt})`);
+            if (attempt <= retries)
+                sleep(opts.sleepSec ?? 0);
+        }
+        if (!ok) {
+            fs.writeFileSync(outFile, `<!-- extraction-status: failed -->\n<!-- source-image: ${image} -->\n\n# Page ${page}\n\n[extraction failed]\n`);
+            logLine(`page ${page}: failed after ${retries + 1} attempts`);
+            failed++;
+            sleep(opts.sleepSec ?? 0);
+        }
+        sleep(opts.sleepSec ?? 0);
+    }
+    // Combine.
+    const slug = path.basename(opts.outDir);
+    const head = [`# ${opts.title ?? slug}`, '', `Source: vision extraction (${opts.adapter.name}) from page PNGs`, `Pages processed: ${processed}`, ''];
+    const parts = [];
+    for (const f of fs.readdirSync(pageDir).filter((x) => /^page-\d+\.md$/.test(x)).sort()) {
+        parts.push(fs.readFileSync(path.join(pageDir, f), 'utf8'), '\n---\n\n');
+    }
+    const combinedMd = head.join('\n') + parts.join('');
+    const combinedTxt = combinedMd.split('\n').filter((l) => !/^<!--.*-->$/.test(l.trim())).join('\n');
+    fs.writeFileSync(path.join(opts.outDir, `${slug}.vision.md`), combinedMd);
+    fs.writeFileSync(path.join(opts.outDir, `${slug}.vision.txt`), combinedTxt);
+    fs.writeFileSync(path.join(opts.outDir, 'run.log'), log.join('\n') + '\n');
+    return { processed, completed, failed, combinedMd, combinedTxt };
+}
+export function renderExtract(r) {
+    return `vision-extract: processed ${r.processed}, completed ${r.completed}, failed ${r.failed}\n`;
+}
+//# sourceMappingURL=vision-extract.js.map

package/dist/src/artifacts/corpus-views/build.js

@@ -0,0 +1,126 @@
+/**
+ * Research-corpus markdown-view builder (#1490).
+ *
+ * The native replacement for the retired `corpus-index-build/build.py`.
+ * `aiwg index build` invokes this in the same process that builds the JSON
+ * graphs, so the corpus is parsed once. Reads the view manifest from
+ * `index.graphs.indices.manifest` (canonical `.aiwg/aiwg.config`, #1491).
+ *
+ * @source historical: corpus-index-build/build.py (configured_graphs, main)
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { readIndexConfig, readAiwgConfig } from '../../config/aiwg-config.js';
+import { loadCorpus } from './ref-parser.js';
+import { renderView, outputForView } from './renderers.js';
+const DEFAULT_VIEWS = ['by-topic', 'by-year', 'authors', 'by-venue', 'by-method', 'training-pipeline', 'by-model-size'];
+/**
+ * Resolve which views to render from the index config, mirroring build.py
+ * configured_graphs(): manifest entries (by name), else a default set, plus
+ * citation-network when it is present as an index.graphs key.
+ */
+function configuredViews(index) {
+    const views = new Map();
+    const graphs = (index?.graphs ?? {});
+    const indices = graphs.indices;
+    const manifest = Array.isArray(indices?.manifest) ? indices.manifest : [];
+    for (const entry of manifest) {
+        if (entry && typeof entry === 'object' && typeof entry.name === 'string') {
+            const e = entry;
+            views.set(e.name, e);
+        }
+    }
+    if (views.size === 0) {
+        for (const name of DEFAULT_VIEWS)
+            views.set(name, { name });
+    }
+    if ('citation-network' in graphs && !views.has('citation-network')) {
+        views.set('citation-network', { name: 'citation-network' });
+    }
+    return views;
+}
+/** Read the Source-Checksum recorded in an existing view file (for staleness skip). */
+function existingChecksum(filePath) {
+    if (!fs.existsSync(filePath))
+        return null;
+    const m = fs.readFileSync(filePath, 'utf-8').match(/^Source-Checksum:\s*sha256:([a-f0-9]{64})\s*$/m);
+    return m ? m[1] : null;
+}
+function utcTimestamp() {
+    // Second-precision UTC, matching build.py's isoformat().replace("+00:00","Z").
+    return new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
+}
+/**
+ * Resolve the corpus root (#1497): `AIWG_CORPUS_ROOT` env > `research.corpusRoot`
+ * in `.aiwg/aiwg.config` > the project root (cwd). Relative values resolve
+ * against cwd.
+ */
+export async function resolveCorpusRoot(cwd) {
+    const env = process.env.AIWG_CORPUS_ROOT;
+    if (env && env.trim())
+        return path.isAbsolute(env) ? env : path.join(cwd, env);
+    try {
+        const cfg = await readAiwgConfig(cwd);
+        const cr = cfg?.research?.corpusRoot;
+        if (cr && cr.trim())
+            return path.isAbsolute(cr) ? cr : path.join(cwd, cr);
+    }
+    catch {
+        /* no config / unreadable — fall through to cwd */
+    }
+    return cwd;
+}
+/**
+ * Render the configured research-corpus markdown views. Returns one result per
+ * selected view. Returns `[]` (renders nothing) when the corpus root has no
+ * `documentation/references/` corpus — so this is a safe no-op in ordinary
+ * SDLC projects.
+ */
+export async function buildCorpusViews(cwd, opts = {}) {
+    // Corpus root may differ from the project root (#1497): env override >
+    // research.corpusRoot in .aiwg/aiwg.config > project root. The index config
+    // is always read from the project's .aiwg/aiwg.config (cwd); corpus content
+    // and rendered views live under the corpus root.
+    const root = await resolveCorpusRoot(cwd);
+    // No research corpus → nothing to render.
+    if (!fs.existsSync(path.join(root, 'documentation', 'references')))
+        return [];
+    const { index } = await readIndexConfig(cwd);
+    const views = configuredViews(index);
+    // `only` selects a single view when it is one we render; an unrelated JSON
+    // graph name (e.g. 'codebase') selects nothing here.
+    let selected;
+    if (opts.only) {
+        if (!views.has(opts.only))
+            return [];
+        selected = [opts.only];
+    }
+    else {
+        selected = [...views.keys()];
+    }
+    const { records, corpusRoot, checksum } = loadCorpus(root);
+    const generated = utcTimestamp();
+    const ctx = { records, corpusRoot, generated, checksum };
+    const results = [];
+    for (const name of selected) {
+        const outRel = outputForView(name, views.get(name)?.output);
+        const outPath = path.join(root, outRel);
+        if (!opts.force && existingChecksum(outPath) === checksum) {
+            results.push({ graph: name, status: 'skipped', output: outRel, papers: records.length });
+            continue;
+        }
+        let content;
+        try {
+            content = renderView(name, ctx);
+        }
+        catch (err) {
+            results.push({ graph: name, status: 'unsupported', output: outRel, error: err.message });
+            continue;
+        }
+        fs.mkdirSync(path.dirname(outPath), { recursive: true });
+        fs.writeFileSync(outPath, content, 'utf-8');
+        results.push({ graph: name, status: 'built', output: outRel, papers: records.length });
+    }
+    return results;
+}
+//# sourceMappingURL=build.js.map

package/dist/src/artifacts/corpus-views/corpus-config.js

@@ -0,0 +1,124 @@
+/**
+ * Shared research-corpus cadence + staleness config (#1498, #1502).
+ *
+ * Single source of truth for refresh-cadence → window math. Consumed by:
+ *  - renderers.ts `radar-stale-queue` view (#1492)
+ *  - the radar subsystem: radar-init GRADE→cadence default, radar-status (#1498)
+ *  - the profile subsystem: profile-status staleness (#1502)
+ *
+ * Ports `corpus/radar_staleness.py` CADENCE_DAYS, `corpus/radar_init.py`
+ * CADENCE_BY_GRADE, and `profiles/find_stale_profiles.py`'s cadence map into one
+ * module so the date-math lives in exactly one place — the explicit #1498/#1502
+ * "one staleness helper" requirement.
+ *
+ * @source historical: corpus/radar_staleness.py, corpus/radar_init.py
+ */
+/**
+ * Refresh-cadence enum word → staleness window in days.
+ *
+ * `null` = on-demand: never auto-overdue. `biennial` is accepted as a synonym
+ * for `biannual` (180d); both spellings appear in corpus sidecars. The Python
+ * sources used distinct on-demand sentinels (radar_staleness: None;
+ * find_stale_profiles: 9999) — both mean "never auto-stale", normalized to
+ * `null` here.
+ */
+export const CADENCE_DAYS = {
+    // PROF-S curator revisit-cadence vocab (#1499): daily/weekly/biweekly.
+    daily: 1,
+    weekly: 7,
+    biweekly: 14,
+    monthly: 30,
+    quarterly: 90,
+    biannual: 180,
+    biennial: 180,
+    'semi-annual': 180, // profile sidecars (#1502) spell the 180-day cadence this way
+    annual: 365,
+    'on-demand': null,
+};
+/**
+ * Default refresh cadence by GRADE letter (radar-init scaffold default).
+ * A→quarterly, B/C→biannual, D→on-demand. Ports `radar_init.py` CADENCE_BY_GRADE.
+ */
+export const CADENCE_BY_GRADE = {
+    A: 'quarterly',
+    B: 'biannual',
+    C: 'biannual',
+    D: 'on-demand',
+};
+/**
+ * Look up the staleness window for a cadence word.
+ * Returns `null` for on-demand and `undefined` for an unknown/absent cadence —
+ * callers distinguish "never stale" (null) from "can't tell" (undefined).
+ */
+export function cadenceWindowDays(cadence) {
+    if (!cadence)
+        return undefined;
+    return CADENCE_DAYS[cadence.toLowerCase()];
+}
+/** Default cadence for a GRADE letter; falls back to `biannual` for unknown/absent grades. */
+export function cadenceForGrade(grade) {
+    if (!grade)
+        return 'biannual';
+    const letter = grade.trim().toUpperCase()[0] ?? '';
+    return CADENCE_BY_GRADE[letter] ?? 'biannual';
+}
+/**
+ * Default affiliation canonicalization map (#1503): canonical `PROF-O-{slug}`
+ * → variant org-name strings that should resolve to it. Ports
+ * `corpus/normalize_affiliation.py` CANONICAL_MAP. Per epic #1496 principle #3
+ * (externalize hardcoded data), this default is overridable per-corpus via a
+ * `documentation/profiles/orgs/affiliation-map.yaml` data file (loaded by the
+ * sidecar-repair tool); this module keeps only the pure default + reverse-map
+ * builder so it stays dependency-free.
+ */
+export const DEFAULT_AFFILIATION_MAP = {
+    'PROF-O-anthropic': ['anthropic'],
+    'PROF-O-openai': ['openai'],
+    'PROF-O-google-deepmind': ['google deepmind', 'deepmind', 'google brain (now part of google deepmind)'],
+    'PROF-O-meta-fair': ['meta fair', 'facebook ai research', 'meta ai', 'meta ai research'],
+    'PROF-O-microsoft-research': ['microsoft research'],
+    'PROF-O-mit-csail': ['mit csail', 'massachusetts institute of technology — csail'],
+    'PROF-O-nvidia': ['nvidia', 'nvidia research'],
+    'PROF-O-sakana-ai': ['sakana ai'],
+    'PROF-O-stanford-hai': ['stanford hai', 'stanford institute for human-centered artificial intelligence'],
+    'PROF-O-together-ai': ['together ai', 'together.ai'],
+    'PROF-O-uc-berkeley': ['uc berkeley', 'university of california, berkeley'],
+    'PROF-O-cohere': ['cohere'],
+    'PROF-O-apollo-research': ['apollo research'],
+};
+/**
+ * Reverse a canonical map (slug → variants) into variant-name → slug, plus
+ * self-mapping each slug. All keys lowercased/trimmed for direct lookup.
+ */
+export function buildAffiliationReverse(map) {
+    const rev = new Map();
+    for (const [slug, names] of Object.entries(map)) {
+        for (const n of names)
+            rev.set(n.toLowerCase().trim(), slug);
+        rev.set(slug.toLowerCase(), slug);
+    }
+    return rev;
+}
+const MS_PER_DAY = 86_400_000;
+/**
+ * Compute staleness for a `refresh-cadence` + `last-refreshed` ISO date as of
+ * `today`. Mirrors `radar_staleness.compute_staleness`: whole-day floor,
+ * UTC-midnight anchoring, and on-demand / unknown-cadence / undatable inputs all
+ * yield `overdueDays === null` (not stale).
+ *
+ * `today` should be a UTC-midnight Date (e.g. `new Date('2026-05-26T00:00:00Z')`)
+ * so day deltas are exact.
+ */
+export function computeStaleness(cadence, lastRefreshed, today) {
+    const windowDays = cadenceWindowDays(cadence);
+    let daysSince = null;
+    if (lastRefreshed) {
+        const last = new Date(`${lastRefreshed}T00:00:00Z`);
+        if (!Number.isNaN(last.getTime())) {
+            daysSince = Math.floor((today.getTime() - last.getTime()) / MS_PER_DAY);
+        }
+    }
+    const overdueDays = daysSince !== null && typeof windowDays === 'number' ? daysSince - windowDays : null;
+    return { daysSince, windowDays, overdueDays, isStale: overdueDays !== null && overdueDays > 0 };
+}
+//# sourceMappingURL=corpus-config.js.map