@terraleiloa/opportunity-extraction 0.1.0

package/README.md

@@ -0,0 +1,20 @@
+# @humandata/opportunity-extraction
+
+Reusable pipeline for:
+
+1. Discovering instruction document URLs from an opportunity page (LLM + regex fallback, Grants.gov Simpler API optional).
+2. Fetching instruction text via injected `DocumentFetcher`.
+3. Chunk analysis + synthesis to produce `applicationSections` (same contract as `LLM_Instruction.md` / document-upload API).
+
+## Public API
+
+- `extractOpportunitySections(input, deps)` — main orchestrator.
+- `findApplicationInstructionUrls`, `getGrantsGovInstructionUrlsFromApi` — lower-level discovery.
+- `parseSectionsFromAnalysisResponse`, `mapToAppSections` — parse Phase-2 JSON.
+- `createOpenAiChatLlmClient(openai)` — OpenAI v4 adapter.
+
+See [examples/second-app/README.md](./examples/second-app/README.md).
+
+## Tests
+
+From repo root: `pnpm test:opportunity-extraction` (after install).

package/dist/index.cjs

@@ -0,0 +1,612 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  createOpenAiChatLlmClient: () => createOpenAiChatLlmClient,
+  defaultFetchHtml: () => defaultFetchHtml,
+  defaultSplitIntoChunks: () => defaultSplitIntoChunks,
+  extractGrantSectionsFromCombinedText: () => extractGrantSectionsFromCombinedText,
+  extractOpportunityIdFromUrl: () => extractOpportunityIdFromUrl,
+  extractOpportunitySections: () => extractOpportunitySections,
+  extractTextFromHtml: () => extractTextFromHtml,
+  findApplicationInstructionUrls: () => findApplicationInstructionUrls,
+  getGrantsGovInstructionUrlsFromApi: () => getGrantsGovInstructionUrlsFromApi,
+  getSourceSpecificGuidance: () => getSourceSpecificGuidance,
+  getUrlDiscoveryPrompt: () => getUrlDiscoveryPrompt,
+  mapToAppSections: () => mapToAppSections,
+  parseSectionsFromAnalysisResponse: () => parseSectionsFromAnalysisResponse,
+  resolveSourceType: () => resolveSourceType,
+  synthesizeSectionsFromDocumentText: () => synthesizeSectionsFromDocumentText,
+  validateAndResolveUrl: () => validateAndResolveUrl
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/adapters/resolveSourceType.ts
+function resolveSourceType(pageUrl, hint) {
+  if (hint) return hint;
+  const url = pageUrl.toLowerCase();
+  if (url.includes("grants.gov")) return "grantsgov";
+  if (url.includes("sam.gov")) return "samgov";
+  if (url.includes("calgrants") || url.includes("california")) return "calgrants";
+  return "bespoke";
+}
+
+// src/discovery/html.ts
+function resolveUrl(baseUrl, relativeUrl) {
+  try {
+    return new URL(relativeUrl, baseUrl).href;
+  } catch {
+    return relativeUrl;
+  }
+}
+function extractTextFromHtml(html) {
+  const cleanedHtml = html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "");
+  const textMatches = cleanedHtml.match(/<[^>]+>|([^<]+)/g) || [];
+  const textParts = [];
+  for (const match of textMatches) {
+    if (!match.startsWith("<")) {
+      const trimmed = match.trim();
+      if (trimmed.length > 0) {
+        textParts.push(trimmed);
+      }
+    }
+  }
+  return textParts.join(" ").replace(/\s+/g, " ").trim();
+}
+function validateAndResolveUrl(url, baseUrl) {
+  try {
+    let resolved = url;
+    if (!url.startsWith("http://") && !url.startsWith("https://")) {
+      resolved = resolveUrl(baseUrl, url);
+    }
+    const urlObj = new URL(resolved);
+    const protocol = urlObj.protocol.toLowerCase();
+    if (protocol === "javascript:" || protocol === "data:" || protocol === "file:") {
+      return null;
+    }
+    if (protocol !== "http:" && protocol !== "https:") {
+      return null;
+    }
+    return resolved;
+  } catch {
+    return null;
+  }
+}
+
+// src/discovery/sourceGuidance.ts
+function getSourceSpecificGuidance(sourceType) {
+  const sourceGuidance = {
+    grantsgov: `**GRANTS.GOV SPECIFIC INSTRUCTIONS:**
+
+- Look for the "View Opportunity" link or opportunity synopsis
+- Find the full solicitation/FOA document (usually a PDF)
+- Check for required application guides or instructions
+- Note if submission is through Grants.gov Workspace or another system
+- Extract CFDA/ALN numbers if present
+- Look for synergy descriptions at grants.gov and also simpler.grants.gov (the newer site)`,
+    calgrants: `**CALGRANTS SPECIFIC INSTRUCTIONS:**
+
+- Look for the grant URL or application portal link
+- Find downloadable guidelines or instructions (often PDF)
+- Check for California-specific registration requirements
+- Note the application submission portal or email requirements
+- Extract any state-specific compliance requirements`,
+    samgov: `**SAM.GOV SPECIFIC INSTRUCTIONS:**
+
+- Look for the full solicitation or RFP documents
+- Check for amendments or modifications
+- Find Section L (Instructions) and Section M (Evaluation) if present
+- Note contract vehicle type and set-aside status
+- Extract NAICS code requirements and size standards`,
+    bespoke: `**CUSTOM OPPORTUNITY INSTRUCTIONS:**
+
+- Follow the provided URL to find application instructions
+- Look for any embedded or linked application guidelines
+- Extract contact information for questions
+- Note any unique submission requirements`
+  };
+  return sourceGuidance[sourceType ?? "bespoke"] ?? sourceGuidance.bespoke;
+}
+
+// src/discovery/findInstructionUrls.ts
+function stripJsonCodeFences(text) {
+  let s = text.trim();
+  if (s.includes("```json")) {
+    const match = s.match(/```json\n?([\s\S]*?)\n?```/);
+    if (match?.[1]) return match[1].trim();
+  }
+  if (s.includes("```")) {
+    const match = s.match(/```\n?([\s\S]*?)\n?```/);
+    if (match?.[1]) return match[1].trim();
+  }
+  return s;
+}
+function getUrlDiscoveryPrompt(pageUrl, sourceType) {
+  const sourceSpecific = getSourceSpecificGuidance(sourceType);
+  return `You are analyzing a government grant/contract opportunity page. Your task is to:
+
+1. **Extract all unique URLs** found on this page (including links in the HTML, buttons, data attributes, etc.)
+2. **Identify which URL(s) contain the full application instructions** - these are typically:
+   - Full announcement pages (often at files.simpler.grants.gov, grants.nih.gov, or agency-specific sites)
+   - PDF documents with "instructions", "guidance", "solicitation", "FOA", or "RFA" in the name
+   - Application guide pages
+   - Program summary pages
+
+3. **Rank URLs by likelihood** of containing complete instructions (most likely first, return top 1-2)
+
+Return your response as a JSON object with this structure:
+{
+  "instructionUrls": [
+    {
+      "url": "most likely instruction URL",
+      "confidence": "high|medium|low",
+      "reason": "why this URL likely contains instructions"
+    },
+    ...
+  ],
+  "uniqueUrls": ["url1", "url2", ...]
+}
+
+**URL TO ANALYZE: ${pageUrl}**
+
+${sourceSpecific}
+
+Extract URLs from the following HTML content:`;
+}
+var SYSTEM_MESSAGE_URL_DISCOVERY = 'You are a web scraping assistant that extracts URLs from HTML content and identifies which URLs are most likely to contain application instructions. Return only valid JSON with the structure: { "instructionUrls": [...], "uniqueUrls": [...] }.';
+function fallbackExtractInstructionUrls(html, baseUrl, maxCandidates) {
+  const urlRegex = /https?:\/\/[^\s"'<>)]+/gi;
+  const matches = html.match(urlRegex) || [];
+  const uniqueUrls = Array.from(new Set(matches));
+  const filtered = uniqueUrls.filter((u) => /(instruction|guide|solicitation|foa|rfa|announcement|full)/i.test(u)).slice(0, maxCandidates);
+  const resolved = [];
+  for (const u of filtered) {
+    const r = validateAndResolveUrl(u, baseUrl);
+    if (r) resolved.push(r);
+  }
+  return resolved;
+}
+var CONFIDENCE_ORDER = { high: 3, medium: 2, low: 1 };
+async function findApplicationInstructionUrls(options) {
+  const { pageUrl, html, sourceType, llmCaller, maxCandidates = 2 } = options;
+  const textContent = extractTextFromHtml(html);
+  const prompt = getUrlDiscoveryPrompt(pageUrl, sourceType);
+  const userContent = `${prompt}
+
+HTML Content:
+${textContent.substring(0, 5e4)}`;
+  if (llmCaller) {
+    try {
+      const response = await llmCaller([
+        { role: "system", content: SYSTEM_MESSAGE_URL_DISCOVERY },
+        { role: "user", content: userContent }
+      ]);
+      const jsonString = stripJsonCodeFences(response);
+      const parsed = JSON.parse(jsonString);
+      const candidates = parsed?.instructionUrls ?? [];
+      const sorted = [...candidates].sort((a, b) => {
+        const orderA = CONFIDENCE_ORDER[String(a.confidence).toLowerCase()] ?? 1;
+        const orderB = CONFIDENCE_ORDER[String(b.confidence).toLowerCase()] ?? 1;
+        return orderB - orderA;
+      });
+      const toTry = sorted.slice(0, maxCandidates);
+      const resolved = [];
+      const seen = /* @__PURE__ */ new Set();
+      for (const item of toTry) {
+        const url = item?.url;
+        if (!url || typeof url !== "string") continue;
+        const r = validateAndResolveUrl(url, pageUrl);
+        if (r && !seen.has(r)) {
+          seen.add(r);
+          resolved.push(r);
+        }
+      }
+      if (resolved.length > 0) return resolved;
+    } catch {
+    }
+  }
+  return fallbackExtractInstructionUrls(html, pageUrl, maxCandidates);
+}
+
+// src/discovery/grantsGovApi.ts
+async function getGrantsGovInstructionUrlsFromApi(opportunityId, apiKey) {
+  const url = `https://api.simpler.grants.gov/v1/opportunities/${opportunityId}`;
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), 3e4);
+  try {
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        "X-API-Key": apiKey,
+        "Content-Type": "application/json"
+      },
+      signal: controller.signal
+    });
+    clearTimeout(timeoutId);
+    if (!response.ok) {
+      if (response.status === 404) return [];
+      if (response.status === 401 || response.status === 403) {
+        throw new Error("Unauthorized: Invalid API key or insufficient permissions");
+      }
+      if (response.status === 429) {
+        throw new Error("Rate limit exceeded: Too many requests");
+      }
+      throw new Error(`Simpler API error: ${response.status} ${response.statusText}`);
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    if (!contentType.includes("application/json")) {
+      throw new Error("Simpler API returned non-JSON response");
+    }
+    const data = await response.json();
+    let attachments = [];
+    if (Array.isArray(data.attachments)) {
+      attachments = data.attachments;
+    } else if (data.data && Array.isArray(data.data.attachments)) {
+      attachments = data.data.attachments;
+    } else if (data.package_details) {
+      const packages = Array.isArray(data.package_details) ? data.package_details : [data.package_details];
+      const mostRecent = packages[packages.length - 1];
+      if (Array.isArray(mostRecent.attachments)) {
+        attachments = mostRecent.attachments;
+      } else if (Array.isArray(mostRecent.instruction_files)) {
+        attachments = mostRecent.instruction_files;
+      }
+    }
+    const instructionUrls = [];
+    for (const att of attachments) {
+      const downloadUrl = att.download_path ?? att.download_url ?? att.file_url ?? att.url ?? att.href;
+      if (!downloadUrl || typeof downloadUrl !== "string") continue;
+      const filename = String(att.file_name ?? att.filename ?? att.name ?? "").toLowerCase();
+      const type = String(att.type ?? att.contentType ?? att.content_type ?? "").toLowerCase();
+      const isPdf = type.includes("pdf") || filename.endsWith(".pdf");
+      const isHtml = type.includes("html") || filename.endsWith(".html") || filename.endsWith(".htm");
+      if (!isPdf && !isHtml) continue;
+      const isInstructionRelated = filename.includes("instruction") || filename.includes("guide") || filename.includes("application") || filename.includes("package") || filename.includes("solicitation") || filename.includes("foa") || filename.includes("rfa") || filename.includes("announcement") || att.isInstruction === true || att.category === "instruction";
+      if (isInstructionRelated) instructionUrls.push(downloadUrl);
+    }
+    if (instructionUrls.length === 0) {
+      const additionalInfoUrl = data.additional_info_url ?? data.additionalInfoUrl ?? data.data?.additional_info_url ?? data.data?.additionalInfoUrl;
+      if (additionalInfoUrl) instructionUrls.push(additionalInfoUrl);
+    }
+    const instructionUrl = data.instruction_url ?? data.instructionUrl ?? data.data?.instruction_url ?? data.data?.instructionUrl;
+    if (instructionUrl && !instructionUrls.includes(instructionUrl)) instructionUrls.push(instructionUrl);
+    const pdfUrl = data.pdf_url ?? data.pdfUrl ?? data.data?.pdf_url ?? data.data?.pdfUrl;
+    if (pdfUrl && !instructionUrls.includes(pdfUrl)) instructionUrls.push(pdfUrl);
+    return instructionUrls;
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+
+// src/discovery/opportunityId.ts
+function extractOpportunityIdFromUrl(url) {
+  try {
+    const detailMatch = url.match(/grants\.gov\/search-results-detail\/(\d+)/i);
+    if (detailMatch?.[1]) return detailMatch[1];
+    const simplerMatch = url.match(/simpler\.grants\.gov\/opportunity\/([a-f0-9-]+)/i);
+    if (simplerMatch?.[1]) return simplerMatch[1];
+    const oppIdMatch = url.match(/oppId=(\d+)/i);
+    if (oppIdMatch?.[1]) return oppIdMatch[1];
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+// src/section-extraction/parseSections.ts
+function parseSectionsFromAnalysisResponse(rawResponse) {
+  if (!rawResponse || typeof rawResponse !== "string") {
+    return [];
+  }
+  let jsonString = rawResponse.trim();
+  if (jsonString.includes("```json")) {
+    const match = jsonString.match(/```json\n?([\s\S]*?)\n?```/);
+    if (match?.[1]) {
+      jsonString = match[1].trim();
+    }
+  } else if (jsonString.includes("```")) {
+    const match = jsonString.match(/```\n?([\s\S]*?)\n?```/);
+    if (match?.[1]) {
+      jsonString = match[1].trim();
+    }
+  }
+  try {
+    const parsed = JSON.parse(jsonString);
+    if (!Array.isArray(parsed.applicationSections)) {
+      return [];
+    }
+    return parsed.applicationSections.filter(
+      (s) => typeof s === "object" && s !== null && typeof s.name === "string" && typeof s.required === "boolean" && Array.isArray(s.requirements)
+    );
+  } catch {
+    return [];
+  }
+}
+function mapToAppSections(sections) {
+  return sections.map((section) => ({
+    id: section.name.toLowerCase().replace(/\s+/g, "-"),
+    title: section.name,
+    description: section.description ?? "",
+    requirements: Array.isArray(section.requirements) ? section.requirements : [],
+    wordLimit: section.pageLimit ?? "Not specified",
+    required: section.required ?? true
+  }));
+}
+
+// src/section-extraction/synthesizeFromChunks.ts
+var PHASE1_SYSTEM = "You are a specialized AI writing assistant for creating Request for Proposals (RFPs) for federal, state, and local government agencies. Your primary function is to help government procurement and grant professionals draft clear, compliant, and effective solicitation documents for grants and contracts. Extract the eligibility section, method of application, websites to register at, links to application guides, required items for application and key information from this document chunk, and return a comprehensive summary of it. Focus on the specific content in this chunk and note if it contains eligibility section, method of application, websites to register at, links to application guides, required items for application or other key information.";
+var PHASE2_SYSTEM = 'You are a specialized AI writing assistant for creating Request for Proposals (RFPs) for federal, state, and local government agencies. Your primary function is to help government procurement and grant professionals draft clear, compliant, and effective solicitation documents for grants and contracts. You have been given analyses of different chunks of a document. Please synthesize these analyses into one comprehensive, coherent summary that covers the entire document. Focus on eligibility criteria, requirements, and key information across all chunks. Additionally, identify and extract all required grant application sections from this document. Return your response as a JSON object with the following structure: { "analysis": "comprehensive summary", "eligibilitySection": "extracted eligibility requirements", "methodOfApplication": "how to apply", "websitesToRegister": ["list of websites"], "applicationGuides": ["list of guides"], "requiredItems": ["list of required items"], "keyInformation": "other important information", "applicationSections": [{"name": "section name", "required": true/false, "pageLimit": "page limit if specified", "description": "what should be included", "requirements": ["list of specific requirements"]}] }';
+var DEFAULT_CHUNK_SIZE = 12e3;
+async function defaultSplitIntoChunks(text) {
+  if (text.length <= DEFAULT_CHUNK_SIZE) return [text];
+  const chunks = [];
+  for (let i = 0; i < text.length; i += DEFAULT_CHUNK_SIZE) {
+    chunks.push(text.slice(i, i + DEFAULT_CHUNK_SIZE));
+  }
+  return chunks;
+}
+async function synthesizeSectionsFromDocumentText(options) {
+  const { llm, combinedDocumentText, splitIntoChunks, maxChunksPerRequest = 3 } = options;
+  const splitter = splitIntoChunks ?? defaultSplitIntoChunks;
+  const chunks = await splitter(combinedDocumentText);
+  let combinedAnalysis = "";
+  for (let i = 0; i < chunks.length; i += maxChunksPerRequest) {
+    const currentChunks = chunks.slice(i, i + maxChunksPerRequest);
+    const chunkText = currentChunks.join("\n\n--- CHUNK SEPARATOR ---\n\n");
+    const chunkAnalysis = await llm([
+      { role: "system", content: PHASE1_SYSTEM },
+      {
+        role: "user",
+        content: `Please analyze this document chunk and extract the eligibility section, method of application, websites to register at, links to application guides, required items for application and key information. Provide a comprehensive summary.
+
+Document chunk content:
+${chunkText}`
+      }
+    ]);
+    combinedAnalysis += `
+
+--- CHUNK ${i + 1}-${Math.min(i + maxChunksPerRequest, chunks.length)} ANALYSIS ---
+
+${chunkAnalysis}`;
+  }
+  const rawPhase2 = await llm([
+    { role: "system", content: PHASE2_SYSTEM },
+    {
+      role: "user",
+      content: `Please synthesize the following chunk analyses into one comprehensive summary of the entire document:
+
+${combinedAnalysis}`
+    }
+  ]);
+  const artifacts = parseStructuredArtifacts(rawPhase2);
+  return { rawPhase2, artifacts };
+}
+function parseStructuredArtifacts(raw) {
+  let jsonString = raw.trim();
+  if (jsonString.includes("```json")) {
+    const m = jsonString.match(/```json\n([\s\S]*?)\n```/);
+    if (m?.[1]) jsonString = m[1].trim();
+  } else if (jsonString.includes("```")) {
+    const m = jsonString.match(/```\n([\s\S]*?)\n```/);
+    if (m?.[1]) jsonString = m[1].trim();
+  }
+  try {
+    const parsed = JSON.parse(jsonString);
+    return {
+      analysis: parsed.analysis,
+      eligibilitySection: parsed.eligibilitySection,
+      methodOfApplication: parsed.methodOfApplication,
+      websitesToRegister: Array.isArray(parsed.websitesToRegister) ? parsed.websitesToRegister : [],
+      applicationGuides: Array.isArray(parsed.applicationGuides) ? parsed.applicationGuides : [],
+      requiredItems: Array.isArray(parsed.requiredItems) ? parsed.requiredItems : [],
+      keyInformation: parsed.keyInformation
+    };
+  } catch {
+    return {};
+  }
+}
+async function extractGrantSectionsFromCombinedText(options) {
+  const { rawPhase2, artifacts } = await synthesizeSectionsFromDocumentText(options);
+  const grantSections = parseSectionsFromAnalysisResponse(rawPhase2);
+  return { rawPhase2, artifacts, grantSections };
+}
+
+// src/pipeline/defaultFetchHtml.ts
+async function defaultFetchHtml(url) {
+  const response = await fetch(url, {
+    headers: {
+      "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    },
+    redirect: "follow"
+  });
+  if (!response.ok) {
+    throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`);
+  }
+  return response.text();
+}
+
+// src/pipeline/extractOpportunitySections.ts
+async function extractOpportunitySections(input, deps) {
+  const warnings = [];
+  const instructionUrlsTried = [];
+  const pageUrl = input.pageUrl.trim();
+  const sourceType = resolveSourceType(pageUrl, input.sourceType);
+  const maxUrls = deps.maxInstructionUrls ?? 2;
+  const fetchHtml = deps.fetchHtml ?? defaultFetchHtml;
+  let candidateUrls = [];
+  if (sourceType === "grantsgov" && deps.grantsGovApiKey) {
+    const id = extractOpportunityIdFromUrl(pageUrl);
+    if (id) {
+      try {
+        candidateUrls = await getGrantsGovInstructionUrlsFromApi(id, deps.grantsGovApiKey);
+        candidateUrls = candidateUrls.slice(0, maxUrls);
+        for (const u of candidateUrls) {
+          instructionUrlsTried.push({ url: u, fetched: false });
+        }
+      } catch (e) {
+        warnings.push(`Simpler API: ${e instanceof Error ? e.message : String(e)}`);
+      }
+    }
+  }
+  if (candidateUrls.length === 0) {
+    let html;
+    try {
+      html = input.html ?? await fetchHtml(pageUrl);
+    } catch (e) {
+      return {
+        success: false,
+        sections: [],
+        artifacts: {},
+        instructionUrlsTried,
+        warnings,
+        error: e instanceof Error ? e.message : String(e)
+      };
+    }
+    const discovered = await findApplicationInstructionUrls({
+      pageUrl,
+      html,
+      sourceType,
+      llmCaller: deps.llm,
+      maxCandidates: maxUrls
+    });
+    candidateUrls = discovered;
+    for (const u of discovered) {
+      instructionUrlsTried.push({ url: u, fetched: false });
+    }
+  }
+  const parts = [];
+  for (const u of candidateUrls) {
+    try {
+      const doc = await deps.fetchDocument(u);
+      if (doc) {
+        const idx = instructionUrlsTried.findIndex((x) => x.url === u);
+        if (idx >= 0) {
+          instructionUrlsTried[idx] = { ...instructionUrlsTried[idx], fetched: true };
+        }
+        parts.push(
+          `
+
+--- Content from ${doc.finalUrl}${doc.filename ? ` (${doc.filename})` : ""} ---
+
+${doc.text.substring(0, 5e4)}`
+        );
+      } else {
+        warnings.push(`Failed to fetch document: ${u}`);
+      }
+    } catch (e) {
+      warnings.push(`Error fetching ${u}: ${e instanceof Error ? e.message : String(e)}`);
+    }
+  }
+  if (parts.length === 0 && input.html) {
+    const t = extractTextFromHtml(input.html);
+    if (t.length > 100) {
+      parts.push(`
+
+--- Opportunity page content ---
+
+${t.substring(0, 5e4)}`);
+      warnings.push("Used opportunity page HTML only; no instruction documents could be fetched.");
+    }
+  }
+  if (parts.length === 0 && !input.html) {
+    try {
+      const html = await fetchHtml(pageUrl);
+      const t = extractTextFromHtml(html);
+      if (t.length > 100) {
+        parts.push(`
+
+--- Opportunity page content ---
+
+${t.substring(0, 5e4)}`);
+        warnings.push("Used fetched opportunity page only; instruction documents could not be loaded.");
+      }
+    } catch (e) {
+      return {
+        success: false,
+        sections: [],
+        artifacts: {},
+        instructionUrlsTried,
+        warnings,
+        error: e instanceof Error ? e.message : String(e)
+      };
+    }
+  }
+  if (parts.length === 0) {
+    return {
+      success: false,
+      sections: [],
+      artifacts: {},
+      instructionUrlsTried,
+      warnings: [...warnings, "No instruction text available for synthesis."],
+      error: "No instruction text available for synthesis."
+    };
+  }
+  const combined = parts.join("\n\n");
+  const { grantSections, artifacts, rawPhase2 } = await extractGrantSectionsFromCombinedText({
+    llm: deps.llm,
+    combinedDocumentText: combined,
+    splitIntoChunks: deps.splitIntoChunks
+  });
+  const sections = mapToAppSections(grantSections);
+  return {
+    success: sections.length > 0 || Boolean(artifacts.analysis),
+    sections,
+    artifacts: {
+      ...artifacts,
+      analysis: artifacts.analysis ?? rawPhase2
+    },
+    instructionUrlsTried,
+    warnings
+  };
+}
+
+// src/llm/openaiChat.ts
+function createOpenAiChatLlmClient(openai, options) {
+  const model = options?.model ?? "gpt-4o";
+  const maxTokens = options?.maxTokens ?? 4e3;
+  return async (messages) => {
+    const completion = await openai.chat.completions.create({
+      model,
+      messages,
+      max_tokens: maxTokens
+    });
+    return completion.choices[0]?.message?.content ?? "";
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  createOpenAiChatLlmClient,
+  defaultFetchHtml,
+  defaultSplitIntoChunks,
+  extractGrantSectionsFromCombinedText,
+  extractOpportunityIdFromUrl,
+  extractOpportunitySections,
+  extractTextFromHtml,
+  findApplicationInstructionUrls,
+  getGrantsGovInstructionUrlsFromApi,
+  getSourceSpecificGuidance,
+  getUrlDiscoveryPrompt,
+  mapToAppSections,
+  parseSectionsFromAnalysisResponse,
+  resolveSourceType,
+  synthesizeSectionsFromDocumentText,
+  validateAndResolveUrl
+});