@uwmd/batch 0.2.0 → 0.8.0

package/README.md

@@ -16,3 +16,23 @@ returns deterministic asset-class, stage, and flag summaries. `projectUnderwriti
 returns every matching candidate ordered by `blocking_flags`, then error count,
 warning count, and path. Invalid candidates stay visible in the index, summary,
 and unfiltered queue; none of these helpers changes a deal file.
+
+## Corpus fact table (`--facts`)
+
+```bash
+npx @uwmd/batch deals --out batch-output --facts
+```
+
+Adds `uwmd-facts.jsonl` — one JSON object per line, one line per JSON fact in
+every parseable deal — plus `uwmd-facts-manifest.json` with the counts and the
+skip list. Each row is a normative `block_values` row (UW CSV Bundle spec §3,
+produced by `@uwmd/core`'s `flattenEnvelopeBlockValues`, never re-implemented
+here) prefixed with the deal's identity: `path`, `deal_id`, `asset_class`,
+`semantic_digest`, and `valid`. Deals that parse but fail validation are
+included with `valid: false`; files that cannot produce an envelope are listed
+in the manifest's `deals_skipped` — the fact table never silently drops a deal.
+
+The JSONL loads directly into DuckDB (`read_json('batch-output/uwmd-facts.jsonl')`),
+Snowflake, ClickHouse, or Pandas. The durable key for a fact is
+`(semantic_digest, block_ref, scope, pointer)`. See the
+[data-lake guide](../../docs/DATA_LAKE.md) for the full pipeline.

package/bin/uwmd-batch.mjs

@@ -1,6 +1,23 @@
 #!/usr/bin/env node
-import { indexUWMDDirectory, writeUWMDCollectionIndex } from '../dist/index.js';
-const [input, ...args] = process.argv.slice(2);
-const out = args[0] === '--out' ? args[1] : undefined;
-if (!input || !out) { console.error('Usage: uwmd-batch <input-directory> --out <output-directory>'); process.exitCode = 2; }
-else { const index = await indexUWMDDirectory(input); const files = await writeUWMDCollectionIndex(index, out); console.log(JSON.stringify({ ...index, outputs: files }, null, 2)); if (index.invalid_deals > 0) process.exitCode = 1; }
+import { buildUWMDFactTable, indexUWMDDirectory, writeUWMDCollectionIndex, writeUWMDFactTable } from '../dist/index.js';
+const argv = process.argv.slice(2);
+const facts = argv.includes('--facts');
+const args = argv.filter((arg) => arg !== '--facts');
+const [input, ...rest] = args;
+const out = rest[0] === '--out' ? rest[1] : undefined;
+if (!input || !out) {
+  console.error('Usage: uwmd-batch <input-directory> --out <output-directory> [--facts]');
+  process.exitCode = 2;
+} else {
+  const index = await indexUWMDDirectory(input);
+  const files = await writeUWMDCollectionIndex(index, out);
+  const outputs = { ...files };
+  if (facts) {
+    const table = await buildUWMDFactTable(input);
+    const factFiles = await writeUWMDFactTable(table, out);
+    outputs.facts_jsonl = factFiles.jsonl;
+    outputs.facts_manifest = factFiles.manifest;
+  }
+  console.log(JSON.stringify({ ...index, outputs }, null, 2));
+  if (index.invalid_deals > 0) process.exitCode = 1;
+}

package/dist/index.d.ts

@@ -1,4 +1,6 @@
+import { type UWBlockValueRow } from '@uwmd/core';
 export declare const UWMD_BATCH_INDEX_VERSION: "0.2";
+export declare const UWMD_FACT_TABLE_VERSION: "0.1";
 export type MetricComparison = 'gt' | 'gte' | 'lt' | 'lte' | 'eq';
 export interface BatchMetricFilter {
     metric: string;
@@ -77,3 +79,47 @@ export declare function summarizeUWMDCollection(index: UWMDCollectionIndex): UWM
  * retained: they sort with the highest error count instead of disappearing.
  */
 export declare function projectUnderwritingQueue(index: UWMDCollectionIndex, filters?: BatchDealFilters): UnderwritingQueueProjection;
+/**
+ * One row of the corpus-level fact table: a normative `block_values` row
+ * (UW CSV Bundle spec §3, via `flattenEnvelopeBlockValues`) prefixed with the
+ * deal's identity — path, `deal_id`, `asset_class`, semantic digest, and
+ * validation verdict. The durable key for a fact is
+ * `(semantic_digest, block_ref, scope, pointer)`; `path` and `deal_id` are
+ * conveniences and `block_ref` alone is not durable across encodings.
+ */
+export interface UWMDFactRow extends UWBlockValueRow {
+    path: string;
+    deal_id: string | null;
+    asset_class: string | null;
+    semantic_digest: string;
+    valid: boolean;
+}
+export interface UWMDFactTable {
+    fact_table_version: typeof UWMD_FACT_TABLE_VERSION;
+    files_scanned: number;
+    deals_included: number;
+    /** Files that could not be parsed into an envelope, with the refusal reason. */
+    deals_skipped: Array<{
+        path: string;
+        error: string;
+    }>;
+    rows: UWMDFactRow[];
+}
+/**
+ * Walks a directory of deal files and returns every JSON fact of every
+ * parseable deal as one flat table. Deals that parse but fail validation are
+ * INCLUDED with `valid: false` (mirroring the collection index, where invalid
+ * candidates stay visible); files that cannot produce an envelope at all are
+ * listed in `deals_skipped` — a fact table never silently drops a deal.
+ * Ordering is deterministic: files sorted by path, rows in envelope order.
+ */
+export declare function buildUWMDFactTable(inputDirectory: string): Promise<UWMDFactTable>;
+/**
+ * Writes the fact table as `uwmd-facts.jsonl` (one JSON object per line — the
+ * shape DuckDB's `read_json`, Snowflake, and ClickHouse ingest directly) plus
+ * `uwmd-facts-manifest.json` carrying the counts and skip list.
+ */
+export declare function writeUWMDFactTable(table: UWMDFactTable, outputDirectory: string): Promise<{
+    jsonl: string;
+    manifest: string;
+}>;

package/dist/index.js

@@ -1,7 +1,8 @@
 import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
 import { relative, resolve, sep } from 'node:path';
-import { computeEnvelopeDigest, parseUWFile, toUWEnvelope, validateUWFile } from '@uwmd/core';
+import { computeEnvelopeDigest, flattenEnvelopeBlockValues, parseUWFile, toUWEnvelope, validateUWFile, } from '@uwmd/core';
 export const UWMD_BATCH_INDEX_VERSION = '0.2';
+export const UWMD_FACT_TABLE_VERSION = '0.1';
 export class BatchError extends Error {
     code;
     constructor(code, message) {
@@ -69,13 +70,73 @@ export function projectUnderwritingQueue(index, filters = {}) {
         deals,
     };
 }
+/**
+ * Walks a directory of deal files and returns every JSON fact of every
+ * parseable deal as one flat table. Deals that parse but fail validation are
+ * INCLUDED with `valid: false` (mirroring the collection index, where invalid
+ * candidates stay visible); files that cannot produce an envelope at all are
+ * listed in `deals_skipped` — a fact table never silently drops a deal.
+ * Ordering is deterministic: files sorted by path, rows in envelope order.
+ */
+export async function buildUWMDFactTable(inputDirectory) {
+    const root = resolve(inputDirectory);
+    const files = (await discoverUWMD(root)).sort();
+    const rows = [];
+    const skipped = [];
+    let included = 0;
+    for (const file of files) {
+        const path = relative(root, file).split(sep).join('/');
+        try {
+            const parsed = parseUWFile(await readFile(file, 'utf8'));
+            if (!hasUWEnvelope(parsed.frontmatter)) {
+                throw new BatchError('MISSING_UW_ENVELOPE', 'File does not contain the required UW Markdown frontmatter envelope.');
+            }
+            const envelope = toUWEnvelope(parsed);
+            const digest = await computeEnvelopeDigest(envelope);
+            const validation = validateUWFile(parsed);
+            const valid = !validation.issues.some((issue) => issue.severity === 'error');
+            const frontmatter = parsed.frontmatter;
+            const deal_id = stringOrNull(frontmatter.deal_id);
+            const asset_class = stringOrNull(frontmatter.asset_class);
+            for (const fact of flattenEnvelopeBlockValues(envelope)) {
+                rows.push({ path, deal_id, asset_class, semantic_digest: digest, valid, ...fact });
+            }
+            included += 1;
+        }
+        catch (error) {
+            skipped.push({ path, error: error instanceof Error ? error.message : String(error) });
+        }
+    }
+    return {
+        fact_table_version: UWMD_FACT_TABLE_VERSION,
+        files_scanned: files.length,
+        deals_included: included,
+        deals_skipped: skipped,
+        rows,
+    };
+}
+/**
+ * Writes the fact table as `uwmd-facts.jsonl` (one JSON object per line — the
+ * shape DuckDB's `read_json`, Snowflake, and ClickHouse ingest directly) plus
+ * `uwmd-facts-manifest.json` carrying the counts and skip list.
+ */
+export async function writeUWMDFactTable(table, outputDirectory) {
+    await mkdir(outputDirectory, { recursive: true });
+    const jsonl = resolve(outputDirectory, 'uwmd-facts.jsonl');
+    const manifest = resolve(outputDirectory, 'uwmd-facts-manifest.json');
+    const lines = table.rows.map((row) => JSON.stringify(row));
+    await writeFile(jsonl, lines.length === 0 ? '' : `${lines.join('\n')}\n`, 'utf8');
+    const { rows: _rows, ...head } = table;
+    await writeFile(manifest, `${JSON.stringify({ ...head, row_count: table.rows.length }, null, 2)}\n`, 'utf8');
+    return { jsonl, manifest };
+}
 async function discoverUWMD(directory) {
     const entries = await readdir(directory, { withFileTypes: true });
     const found = await Promise.all(entries.map(async (entry) => {
         const path = resolve(directory, entry.name);
         if (entry.isDirectory())
             return discoverUWMD(path);
-        return entry.isFile() && entry.name.endsWith('.uw.md') ? [path] : [];
+        return entry.isFile() && (entry.name.endsWith('.uw.md') || entry.name.endsWith('.uwx.md')) ? [path] : [];
     }));
     return found.flat();
 }

package/package.json

@@ -1,7 +1,12 @@
 {
   "name": "@uwmd/batch",
-  "version": "0.2.0",
+  "version": "0.8.0",
   "description": "Deterministic local batch indexing for UW Markdown deal files.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/UWMD-OSP/UW-Markdown.git",
+    "directory": "packages/uwmd-batch"
+  },
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -13,7 +18,7 @@
     "test": "vitest run",
     "typecheck:tests": "tsc -p tsconfig.test.json"
   },
-  "dependencies": { "@uwmd/core": "1.3.0" },
+  "dependencies": { "@uwmd/core": "2.3.0" },
   "devDependencies": { "@types/node": "^20.0.0", "typescript": "^5.4.0", "vitest": "^3.2.6" },
   "engines": { "node": ">=18.0.0" },
   "license": "MIT"