@uwmd/core 1.3.0 → 1.5.0

package/dist/cli.js

@@ -2,8 +2,8 @@
 // uwmd CLI — command-line interface for .uw.md files
 // Commands: parse, validate, compact, diff, init, summary, render
 // Usage: uwmd <command> <file> [options]
-import { readFileSync, writeFileSync, existsSync } from 'node:fs';
-import { resolve, basename } from 'node:path';
+import { readFileSync, writeFileSync, existsSync, readdirSync, mkdirSync } from 'node:fs';
+import { resolve, basename, dirname } from 'node:path';
 import { parseUWFile } from './parser.js';
 import { validateUWFile } from './validator.js';
 import { compact, diff } from './compactor.js';
@@ -24,6 +24,8 @@ import { runBancroftAgent } from './agents/bancroft.js';
 import { buildContext } from './context-profiles.js';
 import { rankGaps } from './refinement.js';
 import { resolveValue } from './cascade.js';
+import { parseUWPart, resolveComposition, externalizeSection, stringifyUWPart, UWPART_EXTENSION, } from './composition.js';
+import { createDocumentMarketData, parseMarketDataDocument } from './market-data.js';
 import { getAssetClassDefaults } from './defaults.js';
 import { MULTIFAMILY_PACK, getPackForAssetClass } from './packs/index.js';
 import { issueReceipt, verifyReceipt, assertUWReceipt } from './receipts.js';
@@ -171,7 +173,9 @@ function cmdValidate(file, flags) {
 }
 async function cmdVerify(file, flags) {
     const content = readFile(file);
-    const parsed = parseUWFile(content);
+    // --resolved verifies the assembled record. Without it an externalized
+    // section verifies as a directive, which is not the document anyone means.
+    const parsed = withResolved(parseUWFile(content), file, flags);
     // No flag set → run all three.
     const onlyValidate = flags['validate'] === true;
     const onlyIntegrity = flags['integrity'] === true;
@@ -329,7 +333,11 @@ function cmdSummary(file) {
 }
 async function cmdExport(file, flags) {
     const includeSuperseded = !(flags['no-superseded'] === true || flags['compact'] === true);
-    const loaded = await loadEnvelope(file);
+    // --resolved exports the assembled record. Exporting the directive instead
+    // would ship a document whose rent roll is a list of filenames.
+    const loaded = flags['resolved']
+        ? toUWEnvelope(withResolved(parseUWFile(readFile(file)), file, flags))
+        : await loadEnvelope(file);
     const envelope = includeSuperseded ? loaded : { ...loaded, superseded: {} };
     const text = stringifyUWEnvelope(await stampEnvelopeDigest(envelope));
     // Default output path: swap the .uw.md suffix for .uw.json (fall back to appending).
@@ -419,9 +427,15 @@ async function cmdConvert(file, flags) {
         const projection = projectUWEnvelopeToLite(envelope);
         encoded = projection.content;
         extension = '.uw.md';
-        if (projection.report.lossy) {
+        // Two independent losses, reported separately. A record whose only loss is
+        // an externalized section omits zero *paths*, so folding these together
+        // would print "omitted 0 advanced path(s)" over a missing rent roll.
+        if (projection.report.omitted_paths.length > 0) {
             console.warn(`Warning: Lite projection omitted ${projection.report.omitted_paths.length} advanced path(s).`);
         }
+        if (projection.report.externalized_sections.length > 0) {
+            console.warn(`Warning: Lite projection omitted ${projection.report.externalized_sections.length} externalized section(s), unresolved here: ${projection.report.externalized_sections.join(', ')}.`);
+        }
         if (typeof flags['projection-report'] === 'string') {
             writeFileSync(resolve(flags['projection-report']), JSON.stringify(projection.report, null, 2), 'utf-8');
         }
@@ -644,6 +658,65 @@ function cmdReport(file, flags) {
     console.log(`Rendered ${label} → ${basename(outPath)} (${result.sectionsRendered.length} sections${result.sectionsSkipped.length ? `, skipped: ${result.sectionsSkipped.join(', ')}` : ''})`);
     console.log('For PDF output, use @uwmd/report (uwmd-report) or print this HTML from a browser.');
 }
+/**
+ * Load `--market-data <file>` into a cascade-ready lookup, or return null when
+ * the flag is absent. Exits with a message rather than throwing, so a bad
+ * observation set is a usage error rather than a stack trace.
+ */
+function loadMarketData(flags) {
+    const path = flags['market-data'];
+    if (!path)
+        return null;
+    try {
+        const doc = parseMarketDataDocument(parseUWFile(readFile(path)));
+        return {
+            lookup: createDocumentMarketData(doc),
+            identity: {
+                document_id: doc.document_id,
+                as_of: doc.as_of,
+                provider: doc.provider,
+                geo: doc.geo,
+            },
+        };
+    }
+    catch (e) {
+        console.error(`Market data: ${e instanceof Error ? e.message : String(e)}`);
+        process.exit(1);
+    }
+}
+function cmdMarketDataValidate(file, flags) {
+    const parsed = parseUWFile(readFile(file));
+    try {
+        const doc = parseMarketDataDocument(parsed);
+        if (flags['json']) {
+            process.stdout.write(`${JSON.stringify({ valid: true, ...doc }, null, 2)}\n`);
+        }
+        else {
+            console.log(`OK  ${doc.document_id}`);
+            console.log(`    as of ${doc.as_of} — ${doc.provider}`);
+            console.log(`    ${doc.geo}${doc.asset_class ? ` — ${doc.asset_class}` : ''}`);
+            console.log(`    ${doc.observations.length} observation(s):`);
+            for (const o of doc.observations) {
+                console.log(`      ${o.field_path} = ${String(o.value)} ${o.unit}${o.confidence ? ` (${o.confidence})` : ''}`);
+                console.log(`        basis: ${o.basis}`);
+            }
+            // Restated on every run because it is the thing people forget.
+            console.log('\n    Attributable, not verified: this says the observations');
+            console.log('    are traceable, not that they are accurate or current.');
+        }
+    }
+    catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        if (flags['json']) {
+            process.stdout.write(`${JSON.stringify({ valid: false, error: message }, null, 2)}\n`);
+        }
+        else {
+            console.error(`INVALID  ${file}`);
+            console.error(`         ${message}`);
+        }
+        process.exit(1);
+    }
+}
 function cmdScope(file, flags) {
     const content = readFile(file);
     const parsed = parseUWFile(content);
@@ -655,12 +728,19 @@ function cmdScope(file, flags) {
         console.error(`No published asset-class default table for '${assetClass}'.`);
         process.exit(1);
     }
+    const market = loadMarketData(flags);
     const out = {};
     for (const path of Object.keys(table.fields)) {
-        const r = resolveValue(path, parsed);
+        const r = resolveValue(path, parsed, {
+            asset_class: assetClass,
+            ...(market ? { market: market.lookup } : {}),
+        });
         out[path] = {
             value: r.value,
             step: r.step,
+            // Surfaced so a promoted value stays distinguishable in the output: it
+            // resolves at the `user_input` step but is tagged `market_data_accepted`.
+            source: r.source,
             range: r.range,
             resolved_from: r.resolved_from,
         };
@@ -670,6 +750,7 @@ function cmdScope(file, flags) {
         deal_stage_target: 'scope',
         asset_class: assetClass,
         defaults_table: `${assetClass}@${table.version}`,
+        ...(market ? { market_data: market.identity } : {}),
         resolved: out,
     };
     const text = JSON.stringify(payload, null, 2);
@@ -692,7 +773,16 @@ function cmdRefine(file, flags) {
         parsed.frontmatter.asset_class ??
         'multifamily';
     const pack = getPackForAssetClass(assetClass) ?? MULTIFAMILY_PACK;
-    const result = rankGaps(parsed, { targets, top, packs: [pack] });
+    // With `--market-data`, a field the observation set covers is no longer a
+    // gap the cascade cannot fill, so it drops out of the VOI ranking. That is
+    // the point: the ranking should tell you what is still worth diligencing.
+    const market = loadMarketData(flags);
+    const result = rankGaps(parsed, {
+        targets,
+        top,
+        packs: [pack],
+        ...(market ? { cascadeContext: { asset_class: assetClass, market: market.lookup } } : {}),
+    });
     if (flags['json']) {
         process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
         return;
@@ -729,6 +819,137 @@ for (let _i = 0; _i < args.length; _i++) {
     } // skip flag + its value
     positional.push(a);
 }
+// ─── Composition (RFC 0021) ──────────────────────────────────────────────────
+/** Load every `.uwpart.md` in a directory, keyed by `part_id`. */
+function loadPartsDir(dir) {
+    const parts = new Map();
+    if (!existsSync(dir))
+        return parts;
+    for (const file of readdirSync(dir).filter((f) => f.endsWith(UWPART_EXTENSION)).sort()) {
+        const path = resolve(dir, file);
+        const part = parseUWPart(parseUWFile(readFileSync(path, 'utf-8')), { filename: file });
+        if (parts.has(part.part_id)) {
+            console.error(`Duplicate part_id '${part.part_id}' in ${dir}`);
+            process.exit(1);
+        }
+        parts.set(part.part_id, part);
+    }
+    return parts;
+}
+/** Where fragments live for a record, unless told otherwise. */
+function defaultPartsDir(file) {
+    return resolve(dirname(resolve(file)), 'parts');
+}
+function cmdCompose(file, flags) {
+    const section = flags['externalize'];
+    if (typeof section !== 'string') {
+        console.error('Usage: uwmd compose <file> --externalize <section> [--collection-key <k>] [--collection-path <p>]');
+        process.exit(1);
+    }
+    const collectionKey = flags['collection-key'] ?? 'unit_id';
+    const collectionPath = flags['collection-path'] ?? 'units';
+    const parsed = parseUWFile(readFile(file));
+    let result;
+    try {
+        result = externalizeSection(parsed, {
+            section,
+            collectionKey,
+            collectionPath,
+            ...(flags['part-prefix'] ? { partIdPrefix: flags['part-prefix'] } : {}),
+        });
+    }
+    catch (e) {
+        console.error(e instanceof Error ? e.message : String(e));
+        process.exit(1);
+    }
+    const outDir = flags['out-dir'] ? resolve(flags['out-dir']) : defaultPartsDir(file);
+    const recordPath = flags['in-place']
+        ? resolve(file)
+        : flags['output']
+            ? resolve(flags['output'])
+            : resolve(replaceUWExtension(file, '.externalized.uwx.md'));
+    const recordText = stringifyUWX(toUWEnvelope(result.document));
+    if (flags['dry-run']) {
+        console.log(`Would write ${result.parts.length} fragment(s) to ${outDir}:`);
+        for (const part of result.parts)
+            console.log(`  ${part.part_id}${UWPART_EXTENSION}`);
+        console.log(`Would write the record to ${recordPath}`);
+        return;
+    }
+    mkdirSync(outDir, { recursive: true });
+    for (const part of result.parts) {
+        writeFileSync(resolve(outDir, `${part.part_id}${UWPART_EXTENSION}`), stringifyUWPart(part), 'utf-8');
+    }
+    writeFileSync(recordPath, recordText, 'utf-8');
+    console.log(`Externalized ${section} → ${result.parts.length} fragment(s) in ${basename(outDir)}/`);
+    console.log(`Record written to ${basename(recordPath)}`);
+    // Externalizing is a packaging decision, not a model change — worth saying,
+    // because the file looks dramatically different afterwards.
+    console.log('The resolved record has the same semantic digest as the original.');
+}
+function cmdResolve(file, flags) {
+    const partsDir = flags['parts'] ? resolve(flags['parts']) : defaultPartsDir(file);
+    const parsed = parseUWFile(readFile(file));
+    let parts;
+    try {
+        parts = loadPartsDir(partsDir);
+    }
+    catch (e) {
+        console.error(e instanceof Error ? e.message : String(e));
+        process.exit(1);
+    }
+    const resolution = resolveComposition(parsed, { parts });
+    if (flags['json']) {
+        process.stdout.write(`${JSON.stringify({
+            status: resolution.status,
+            externalized: resolution.externalized,
+            parts_available: parts.size,
+            issues: resolution.issues,
+        }, null, 2)}\n`);
+    }
+    else {
+        console.log(`${resolution.status === 'resolved' ? 'RESOLVED' : 'UNRESOLVED'}  ${basename(file)}`);
+        console.log(`    ${parts.size} fragment(s) available in ${basename(partsDir)}/`);
+        console.log(`    externalized section(s): ${resolution.externalized.join(', ') || '(none)'}`);
+        for (const issue of resolution.issues) {
+            console.log(`    [${issue.code}] ${issue.message}`);
+        }
+        if (resolution.status === 'unresolved') {
+            // The rule worth restating: under-resolution is never a smaller answer.
+            console.log('\n    Sections stay externalized rather than resolving partially —');
+            console.log('    a collection missing rows would still total and still validate.');
+        }
+    }
+    if (flags['output'] && resolution.status === 'resolved') {
+        const outPath = resolve(flags['output']);
+        writeFileSync(outPath, stringifyUWX(toUWEnvelope(resolution.document)), 'utf-8');
+        if (!flags['json'])
+            console.log(`    resolved record → ${basename(outPath)}`);
+    }
+    if (resolution.status !== 'resolved')
+        process.exit(1);
+}
+/**
+ * Resolve a record's externalized sections before another command reads it.
+ *
+ * Returns the record unchanged when `--resolved` was not requested, so callers
+ * can apply it unconditionally. Exits when resolution fails: verifying or
+ * exporting a partially-resolved record would report on a document that does
+ * not exist.
+ */
+function withResolved(parsed, file, flags) {
+    if (!flags['resolved'])
+        return parsed;
+    const partsDir = flags['parts'] ? resolve(flags['parts']) : defaultPartsDir(file);
+    const resolution = resolveComposition(parsed, { parts: loadPartsDir(partsDir) });
+    if (resolution.status !== 'resolved') {
+        console.error(`Cannot resolve ${basename(file)} against ${basename(partsDir)}/:`);
+        for (const issue of resolution.issues)
+            console.error(`  [${issue.code}] ${issue.message}`);
+        process.exit(1);
+    }
+    return resolution.document;
+}
 // Top-level async wrapper so `run --live` can use await
 (async () => {
     switch (command) {
@@ -748,7 +969,7 @@ for (let _i = 0; _i < args.length; _i++) {
             break;
         case 'verify':
             if (!positional[0]) {
-                console.error('Usage: uwmd verify <file> [--validate] [--integrity] [--policy] [--json]');
+                console.error('Usage: uwmd verify <file> [--validate] [--integrity] [--policy] [--resolved] [--json]');
                 process.exit(1);
             }
             await cmdVerify(positional[0], flags);
@@ -1020,7 +1241,7 @@ for (let _i = 0; _i < args.length; _i++) {
             break;
         case 'export':
             if (!positional[0]) {
-                console.error('Usage: uwmd export <file.uw.md> [--output <file.uw.json>] [--no-superseded] [--stdout]');
+                console.error('Usage: uwmd export <file.uw.md> [--output <file.uw.json>] [--no-superseded] [--resolved] [--stdout]');
                 process.exit(1);
             }
             await cmdExport(positional[0], flags);
@@ -1044,18 +1265,47 @@ for (let _i = 0; _i < args.length; _i++) {
             break;
         case 'scope':
             if (!positional[0]) {
-                console.error('Usage: uwmd scope <file> [--asset-class multifamily] [--output <file>]');
+                console.error('Usage: uwmd scope <file> [--asset-class multifamily] [--market-data <file>] [--output <file>]');
                 process.exit(1);
             }
             cmdScope(positional[0], flags);
             break;
         case 'refine':
             if (!positional[0]) {
-                console.error('Usage: uwmd refine <file> [--targets dscr,debt_yield] [--top 5] [--json]');
+                console.error('Usage: uwmd refine <file> [--targets dscr,debt_yield] [--top 5] [--market-data <file>] [--json]');
                 process.exit(1);
             }
             cmdRefine(positional[0], flags);
             break;
+        case 'compose':
+            if (!positional[0]) {
+                console.error('Usage: uwmd compose <file> --externalize <section> [--collection-key <k>] [--collection-path <p>] [--out-dir <dir>] [--in-place] [--dry-run]');
+                process.exit(1);
+            }
+            cmdCompose(positional[0], flags);
+            break;
+        case 'resolve':
+            if (!positional[0]) {
+                console.error('Usage: uwmd resolve <file> [--parts <dir>] [--output <file>] [--json]');
+                process.exit(1);
+            }
+            cmdResolve(positional[0], flags);
+            break;
+        case 'market-data': {
+            const sub = positional[0];
+            if (sub === 'validate') {
+                if (!positional[1]) {
+                    console.error('Usage: uwmd market-data validate <file> [--json]');
+                    process.exit(1);
+                }
+                cmdMarketDataValidate(positional[1], flags);
+            }
+            else {
+                console.error('Usage: uwmd market-data validate <file> [--json]');
+                process.exit(1);
+            }
+            break;
+        }
         case 'lease': {
             const sub = positional[0];
             if (sub === 'validate') {
@@ -1191,6 +1441,8 @@ Commands:
   init                         Generate a blank .uwx.md file
   summary  <file>              Print quick metrics to terminal
   export   <file>              Export a digested UW JSON 1.0 document
+  compose  <file>              Externalize a section into .uwpart.md fragments (RFC 0021)
+  resolve  <file>              Resolve externalized sections from a parts directory (RFC 0021)
   formats                       List registered machine representations
   convert  <file>              Convert Lite/UWX/JSON/XML/CSV bundle representations
   report   <file>              Render the lender package / credit memo HTML (§7.1/§7.2)
@@ -1209,6 +1461,12 @@ Options:
   --strict           Throw on parse errors instead of collecting
   --format <f>       Render format: json|csv|chat|summary (render, default: summary)
   --no-superseded    Drop append-only history from the .uw.json export (export)
+  --resolved         Resolve externalized sections first (verify, export)
+  --parts <dir>      Fragment directory (compose, resolve, --resolved; default: ./parts)
+  --externalize <s>  Section to split into fragments (compose)
+  --collection-key <k>  Row identity field (compose, default: unit_id)
+  --collection-path <p> Field the rows occupy (compose, default: units)
+  --in-place         Overwrite the input record (compose)
   --stdout           Write export/convert output to stdout
   --to <format>       Target: lite|uwx|uw-json|uw-xml|uw-csv-bundle (convert)
   --projection-report Write the UWX-to-Lite omission report as JSON (convert)