@lexbuild/fr 1.14.0 → 1.15.0

package/dist/index.js

@@ -77,8 +77,8 @@ var FR_INLINE_ELEMENTS = /* @__PURE__ */ new Set([
 var FR_EMPHASIS_MAP = {
   "01": "bold",
   "02": "italic",
-  "03": "bold",
-  // bold italic in print — treat as bold for Markdown
+  "03": "italic",
+  // bold italic in print — FR uses T="03" for case names, citations, and publication titles which render as italic
   "04": "italic",
   // italic in headings
   "05": "italic",
@@ -289,12 +289,18 @@ var FrASTBuilder = class {
       return;
     }
     if (name === FR_FTREF_ELEMENT) {
-      const node = {
-        type: "inline",
-        inlineType: "footnoteRef",
-        idref: attrs["ID"]
-      };
-      this.stack.push({ kind: "inline", elementName: name, node, textBuffer: "" });
+      const parentFrame = this.stack[this.stack.length - 1];
+      if (parentFrame?.kind === "content" && parentFrame.node?.type === "content") {
+        const contentNode = parentFrame.node;
+        for (let i = contentNode.children.length - 1; i >= 0; i--) {
+          const child = contentNode.children[i];
+          if (child?.type === "inline" && child.inlineType === "sup") {
+            child.inlineType = "footnoteRef";
+            break;
+          }
+        }
+      }
+      this.ignoredContainerDepth = 1;
       return;
     }
     if (FR_NOTE_ELEMENTS.has(name)) {
@@ -414,25 +420,29 @@ var FrASTBuilder = class {
     }
     if (frame.kind === "content" && frame.node?.type === "content") {
       const contentNode = frame.node;
-      if (text) {
+      const normalized = text.replace(/\s+/g, " ");
+      if (normalized && normalized !== " ") {
         contentNode.children.push({
           type: "inline",
           inlineType: "text",
-          text
+          text: normalized
         });
       }
       return;
     }
     if (frame.kind === "inline" && frame.node?.type === "inline") {
       const inlineNode = frame.node;
+      const normalized = text.replace(/\s+/g, " ");
       if (inlineNode.children) {
-        inlineNode.children.push({
-          type: "inline",
-          inlineType: "text",
-          text
-        });
+        if (normalized && normalized !== " ") {
+          inlineNode.children.push({
+            type: "inline",
+            inlineType: "text",
+            text: normalized
+          });
+        }
       } else {
-        inlineNode.text = (inlineNode.text ?? "") + text;
+        inlineNode.text = (inlineNode.text ?? "") + normalized;
       }
       return;
     }
@@ -590,7 +600,8 @@ var FrASTBuilder = class {
     } else if (elementName === "B") {
       inlineType = "bold";
     } else if (elementName === "SU") {
-      inlineType = "sup";
+      const insideFootnote = this.findFrame("note") !== void 0;
+      inlineType = insideFootnote ? "footnoteRef" : "sup";
     } else if (elementName === "FR") {
       inlineType = "text";
     } else if (elementName === "E") {
@@ -887,9 +898,26 @@ var FrASTBuilder = class {
     const frame = this.popFrame(FR_FRDOC_ELEMENT);
     if (!frame || frame.kind !== "frdoc") return;
     const text = frame.textBuffer.trim();
-    const match = /FR\s+Doc\.\s+([\d-]+)/i.exec(text);
-    if (match) {
-      this.currentDocMeta.documentNumber = match[1];
+    const docMatch = /FR\s+Doc\.\s+([\w-]+)/i.exec(text);
+    if (docMatch) {
+      this.currentDocMeta.documentNumber = docMatch[1];
+    }
+    const dateMatch = /Filed\s+(\d{1,2})-(\d{1,2})-(\d{2})\b/.exec(text);
+    if (dateMatch) {
+      const [, mmStr, ddStr, yyStr] = dateMatch;
+      const mm = parseInt(mmStr ?? "0", 10);
+      const dd = parseInt(ddStr ?? "0", 10);
+      const yy = parseInt(yyStr ?? "0", 10);
+      const fullYear = yy < 50 ? 2e3 + yy : 1900 + yy;
+      const filed = new Date(fullYear, mm - 1, dd);
+      if (filed.getMonth() !== mm - 1 || filed.getDate() !== dd) {
+        return;
+      }
+      filed.setDate(filed.getDate() + 1);
+      const pubYear = filed.getFullYear();
+      const pubMonth = String(filed.getMonth() + 1).padStart(2, "0");
+      const pubDay = String(filed.getDate()).padStart(2, "0");
+      this.currentDocMeta.publicationDate = `${pubYear}-${pubMonth}-${pubDay}`;
     }
   }
   // ── Private helpers: Stack navigation ──
@@ -958,7 +986,7 @@ function normalizeDocumentType(apiType) {
 function buildFrFrontmatter(node, _context, xmlMeta, jsonMeta) {
   const documentNumber = jsonMeta?.document_number ?? xmlMeta.documentNumber ?? "";
   const subject = jsonMeta?.title ?? xmlMeta.subject ?? node.heading ?? "";
-  const publicationDate = jsonMeta?.publication_date ?? "";
+  const publicationDate = jsonMeta?.publication_date ?? xmlMeta.publicationDate ?? "";
   const documentType = jsonMeta ? normalizeDocumentType(jsonMeta.type) : xmlMeta.documentTypeNormalized;
   let agencies;
   if (jsonMeta?.agencies && jsonMeta.agencies.length > 0) {
@@ -1056,64 +1084,30 @@ import {
 var FR_DOC_TYPE_SET = new Set(FR_DOCUMENT_TYPE_KEYS);
 async function convertFrDocuments(options) {
   const xmlFiles = await discoverXmlFiles(options.input, options.from, options.to);
-  const files = [];
+  let documentsConverted = 0;
   let totalTokenEstimate = 0;
   let peakMemoryBytes = 0;
   const linkResolver = createLinkResolver();
-  const parsedFiles = /* @__PURE__ */ new Map();
+  let filesProcessed = 0;
   for (const xmlPath of xmlFiles) {
+    let collected;
     try {
-      const collected = await parseXmlFile(xmlPath);
-      parsedFiles.set(xmlPath, collected);
+      collected = await parseXmlFile(xmlPath);
     } catch (err) {
       console.warn(
         `Warning: Failed to parse ${xmlPath}: ${err instanceof Error ? err.message : String(err)}. Skipping.`
       );
+      continue;
     }
-  }
-  for (const [, collected] of parsedFiles) {
     for (const doc of collected) {
       if (options.types && options.types.length > 0) {
         if (!FR_DOC_TYPE_SET.has(doc.xmlMeta.documentType) || !options.types.includes(doc.xmlMeta.documentType)) {
           continue;
         }
       }
-      if (doc.node.identifier) {
-        const outputPath = buildFrOutputPath(
-          doc.documentNumber,
-          doc.publicationDate,
-          options.output
-        );
-        linkResolver.register(doc.node.identifier, outputPath);
-      }
-    }
-  }
-  if (options.dryRun) {
-    let count = 0;
-    for (const [, collected] of parsedFiles) {
-      for (const doc of collected) {
-        if (options.types && options.types.length > 0) {
-          if (!FR_DOC_TYPE_SET.has(doc.xmlMeta.documentType) || !options.types.includes(doc.xmlMeta.documentType)) {
-            continue;
-          }
-        }
-        count++;
-      }
-    }
-    return {
-      documentsConverted: count,
-      files: [],
-      totalTokenEstimate: 0,
-      peakMemoryBytes: 0,
-      dryRun: true
-    };
-  }
-  for (const [, collected] of parsedFiles) {
-    for (const doc of collected) {
-      if (options.types && options.types.length > 0) {
-        if (!FR_DOC_TYPE_SET.has(doc.xmlMeta.documentType) || !options.types.includes(doc.xmlMeta.documentType)) {
-          continue;
-        }
+      if (options.dryRun) {
+        documentsConverted++;
+        continue;
       }
       const outputPath = buildFrOutputPath(
         doc.documentNumber,
@@ -1128,21 +1122,28 @@ async function convertFrDocuments(options) {
       });
       await mkdir(dirname(outputPath), { recursive: true });
       await writeFile(outputPath, markdown, "utf-8");
-      files.push(outputPath);
-      const tokenEstimate = Math.round(markdown.length / 4);
-      totalTokenEstimate += tokenEstimate;
+      documentsConverted++;
+      totalTokenEstimate += Math.round(markdown.length / 4);
       const mem = process.memoryUsage().rss;
       if (mem > peakMemoryBytes) {
         peakMemoryBytes = mem;
       }
     }
+    filesProcessed++;
+    options.onProgress?.({
+      documentsConverted,
+      filesProcessed,
+      totalFiles: xmlFiles.length,
+      currentFile: xmlPath
+    });
   }
   return {
-    documentsConverted: files.length,
-    files,
+    documentsConverted,
+    files: [],
+    // Don't accumulate 750k+ file paths in memory
     totalTokenEstimate,
     peakMemoryBytes,
-    dryRun: false
+    dryRun: options.dryRun
   };
 }
 async function parseXmlFile(xmlPath) {
@@ -1241,9 +1242,13 @@ async function walkDir(dir, results) {
   }
 }
 function inferDateFromPath(filePath) {
-  const match = /(\d{4})\/(\d{2})\/[^/]+\.xml$/.exec(filePath);
-  if (match) {
-    return `${match[1]}-${match[2]}-01`;
+  const bulkMatch = /FR-(\d{4})-(\d{2})-(\d{2})\.xml$/.exec(filePath);
+  if (bulkMatch) {
+    return `${bulkMatch[1]}-${bulkMatch[2]}-${bulkMatch[3]}`;
+  }
+  const perDocMatch = /(\d{4})\/(\d{2})\/[^/]+\.xml$/.exec(filePath);
+  if (perDocMatch) {
+    return `${perDocMatch[1]}-${perDocMatch[2]}-01`;
   }
   return "";
 }
@@ -1256,7 +1261,7 @@ import { pipeline } from "stream/promises";
 import { Readable } from "stream";
 var FR_API_BASE = "https://www.federalregister.gov/api/v1";
 var PER_PAGE = 200;
-var DEFAULT_FETCH_DELAY_MS = 100;
+var DEFAULT_CONCURRENCY = 10;
 var MAX_RETRIES = 2;
 var RETRY_BASE_DELAY_MS = 2e3;
 var API_FIELDS = [
@@ -1299,7 +1304,7 @@ function buildFrApiListUrl(from, to, page, types) {
 }
 async function downloadFrDocuments(options) {
   const to = options.to ?? (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
-  const fetchDelay = options.fetchDelayMs ?? DEFAULT_FETCH_DELAY_MS;
+  const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
   const files = [];
   const failed = [];
   let totalBytes = 0;
@@ -1308,6 +1313,7 @@ async function downloadFrDocuments(options) {
   const chunks = buildMonthChunks(options.from, to);
   for (const chunk of chunks) {
     if (options.limit !== void 0 && files.length >= options.limit) break;
+    const chunkDocs = [];
     let page = 1;
     let hasMore = true;
     while (hasMore) {
@@ -1319,39 +1325,37 @@ async function downloadFrDocuments(options) {
           `Unexpected API response for ${listUrl}: missing or invalid 'count' field. The FederalRegister.gov API may have changed its response format.`
         );
       }
-      if (page === 1 && totalDocumentsFound === 0) {
-        totalDocumentsFound = data.count;
+      if (page === 1) {
+        totalDocumentsFound += data.count;
       }
       const results = data.results ?? [];
       for (const doc of results) {
-        if (options.limit !== void 0 && files.length >= options.limit) {
-          hasMore = false;
-          break;
-        }
-        options.onProgress?.({
-          documentsDownloaded: files.length,
-          totalDocuments: totalDocumentsFound,
-          currentDocument: doc.document_number,
-          currentChunk: `${chunk.from.slice(0, 7)}`
-        });
         if (!doc.full_text_xml_url) {
           skipped++;
           continue;
         }
-        try {
-          const result = await downloadSingleDocument(doc, options.output, fetchDelay);
-          files.push(result);
-          totalBytes += result.size;
-        } catch (err) {
-          failed.push({
-            documentNumber: doc.document_number,
-            error: err instanceof Error ? err.message : String(err)
-          });
-        }
+        chunkDocs.push(doc);
       }
-      hasMore = hasMore && page < (data.total_pages ?? 0);
+      hasMore = page < (data.total_pages ?? 0);
       page++;
     }
+    const remaining = options.limit !== void 0 ? options.limit - files.length : chunkDocs.length;
+    const docsToDownload = chunkDocs.slice(0, remaining);
+    const chunkLabel = chunk.from.slice(0, 7);
+    await downloadPool(docsToDownload, concurrency, options.output, (doc, result, error) => {
+      if (result) {
+        files.push(result);
+        totalBytes += result.size;
+      } else if (error) {
+        failed.push({ documentNumber: doc.document_number, error });
+      }
+      options.onProgress?.({
+        documentsDownloaded: files.length,
+        totalDocuments: totalDocumentsFound,
+        currentDocument: doc.document_number,
+        currentChunk: chunkLabel
+      });
+    });
   }
   return {
     documentsDownloaded: files.length,
@@ -1371,9 +1375,27 @@ async function downloadSingleFrDocument(documentNumber, output) {
       `Invalid API response for document ${documentNumber}: missing document_number or publication_date`
     );
   }
-  return downloadSingleDocument(doc, output, 0);
+  return downloadSingleDocument(doc, output);
 }
-async function downloadSingleDocument(doc, outputDir, fetchDelay) {
+async function downloadPool(docs, concurrency, outputDir, onComplete) {
+  let nextIndex = 0;
+  async function worker() {
+    while (nextIndex < docs.length) {
+      const i = nextIndex++;
+      const doc = docs[i];
+      if (!doc) break;
+      try {
+        const result = await downloadSingleDocument(doc, outputDir);
+        onComplete(doc, result, null);
+      } catch (err) {
+        onComplete(doc, null, err instanceof Error ? err.message : String(err));
+      }
+    }
+  }
+  const workerCount = Math.min(concurrency, docs.length);
+  await Promise.all(Array.from({ length: workerCount }, () => worker()));
+}
+async function downloadSingleDocument(doc, outputDir) {
   if (!doc.document_number || !doc.publication_date) {
     throw new Error(
       `Invalid document in API response: missing document_number or publication_date`
@@ -1389,9 +1411,6 @@ async function downloadSingleDocument(doc, outputDir, fetchDelay) {
   await mkdir2(dirname2(xmlPath), { recursive: true });
   const jsonContent = JSON.stringify(doc, null, 2);
   await fsWriteFile(jsonPath, jsonContent, "utf-8");
-  if (fetchDelay > 0) {
-    await sleep(fetchDelay);
-  }
   const xmlResponse = await fetchWithRetry(doc.full_text_xml_url);
   if (!xmlResponse.body) {
     throw new Error(`No response body for ${doc.document_number} XML`);
@@ -1466,6 +1485,138 @@ async function fetchWithRetry(url, attempt = 0) {
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+
+// src/govinfo-downloader.ts
+import { createWriteStream as createWriteStream2 } from "fs";
+import { mkdir as mkdir3, stat as stat3 } from "fs/promises";
+import { dirname as dirname3, join as join3 } from "path";
+import { pipeline as pipeline2 } from "stream/promises";
+import { Readable as Readable2 } from "stream";
+var GOVINFO_BASE = "https://www.govinfo.gov/content/pkg";
+var DEFAULT_CONCURRENCY2 = 10;
+var MAX_RETRIES2 = 2;
+var RETRY_BASE_DELAY_MS2 = 2e3;
+function buildGovinfoFrUrl(date) {
+  return `${GOVINFO_BASE}/FR-${date}/xml/FR-${date}.xml`;
+}
+function buildGovinfoBulkPath(date, outputDir) {
+  const year = date.slice(0, 4);
+  return join3(outputDir, "bulk", year, `FR-${date}.xml`);
+}
+async function downloadFrBulk(options) {
+  const to = options.to ?? (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+  const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY2;
+  const dates = generateDateRange(options.from, to);
+  const files = [];
+  let totalBytes = 0;
+  let skipped = 0;
+  let failed = 0;
+  let nextIndex = 0;
+  async function worker() {
+    while (nextIndex < dates.length) {
+      const i = nextIndex++;
+      const date = dates[i];
+      if (!date) break;
+      options.onProgress?.({
+        downloaded: files.length,
+        totalDays: dates.length,
+        skipped,
+        failed,
+        currentDate: date
+      });
+      const url = buildGovinfoFrUrl(date);
+      const filePath = buildGovinfoBulkPath(date, options.output);
+      try {
+        const result = await downloadSingleDay(url, filePath, date);
+        if (result) {
+          files.push(result);
+          totalBytes += result.size;
+        } else {
+          skipped++;
+        }
+      } catch (err) {
+        console.warn(`Warning: Failed to download ${date}: ${err instanceof Error ? err.message : String(err)}`);
+        failed++;
+      }
+    }
+  }
+  const workerCount = Math.min(concurrency, dates.length);
+  await Promise.all(Array.from({ length: workerCount }, () => worker()));
+  options.onProgress?.({
+    downloaded: files.length,
+    totalDays: dates.length,
+    skipped,
+    failed,
+    currentDate: "done"
+  });
+  return {
+    filesDownloaded: files.length,
+    files,
+    totalBytes,
+    dateRange: { from: options.from, to },
+    skipped,
+    failed
+  };
+}
+async function downloadSingleDay(url, filePath, date) {
+  const response = await fetchWithRetry2(url);
+  if (response.status === 404) {
+    return null;
+  }
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} for ${url}`);
+  }
+  if (!response.body) {
+    throw new Error(`No response body for ${url}`);
+  }
+  await mkdir3(dirname3(filePath), { recursive: true });
+  const dest = createWriteStream2(filePath);
+  await pipeline2(Readable2.fromWeb(response.body), dest);
+  const fileStat = await stat3(filePath);
+  return {
+    path: filePath,
+    date,
+    size: Number(fileStat.size)
+  };
+}
+function generateDateRange(from, to) {
+  const dates = [];
+  const current = /* @__PURE__ */ new Date(from + "T12:00:00Z");
+  const end = /* @__PURE__ */ new Date(to + "T12:00:00Z");
+  while (current <= end) {
+    dates.push(current.toISOString().slice(0, 10));
+    current.setUTCDate(current.getUTCDate() + 1);
+  }
+  return dates;
+}
+async function fetchWithRetry2(url, attempt = 0) {
+  let response;
+  try {
+    response = await fetch(url);
+  } catch (err) {
+    if (attempt < MAX_RETRIES2) {
+      const delay = RETRY_BASE_DELAY_MS2 * Math.pow(2, attempt);
+      await sleep2(delay);
+      return fetchWithRetry2(url, attempt + 1);
+    }
+    throw new Error(
+      `Network error after ${MAX_RETRIES2 + 1} attempts for ${url}: ${err instanceof Error ? err.message : String(err)}`,
+      { cause: err }
+    );
+  }
+  if (response.ok || response.status === 404) return response;
+  if ((response.status === 429 || response.status === 503 || response.status === 504) && attempt < MAX_RETRIES2) {
+    const retryAfter = response.headers.get("Retry-After");
+    const parsedRetry = retryAfter ? parseInt(retryAfter, 10) : NaN;
+    const delay = !isNaN(parsedRetry) && parsedRetry > 0 ? parsedRetry * 1e3 : RETRY_BASE_DELAY_MS2 * Math.pow(2, attempt);
+    await sleep2(delay);
+    return fetchWithRetry2(url, attempt + 1);
+  }
+  throw new Error(`HTTP ${response.status}: ${response.statusText} for ${url}`);
+}
+function sleep2(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
 export {
   FR_BLOCK_ELEMENTS,
   FR_CONTENT_ELEMENTS,
@@ -1493,9 +1644,12 @@ export {
   buildFrDownloadXmlPath,
   buildFrFrontmatter,
   buildFrOutputPath,
+  buildGovinfoBulkPath,
+  buildGovinfoFrUrl,
   buildMonthDir,
   buildYearDir,
   convertFrDocuments,
+  downloadFrBulk,
   downloadFrDocuments,
   downloadSingleFrDocument
 };