@terraleiloa/opportunity-extraction 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,213 @@
+/**
+ * Source-agnostic contracts for opportunity instruction discovery and section extraction.
+ */
+type SourceType = "grantsgov" | "calgrants" | "samgov" | "bespoke";
+interface LLMMessage {
+    role: "system" | "user" | "assistant";
+    content: string;
+}
+/** Generic chat completion; plug in OpenAI, Anthropic, etc. */
+type LLMClient = (messages: LLMMessage[]) => Promise<string>;
+interface FetchedDocument {
+    /** Original request URL */
+    url: string;
+    /** Final URL after redirects */
+    finalUrl: string;
+    /** Extracted plain text (HTML stripped, PDF parsed) */
+    text: string;
+    mimeType: string;
+    filename?: string;
+}
+type DocumentFetcher = (url: string) => Promise<FetchedDocument | null>;
+/**
+ * Fetch raw HTML for a page (needed for link discovery).
+ * If omitted, the pipeline uses a built-in fetch with a browser-like User-Agent.
+ */
+type HtmlFetcher = (url: string) => Promise<string>;
+interface OpportunityInput {
+    pageUrl: string;
+    sourceType?: SourceType;
+    /** When set, skips fetching the opportunity page */
+    html?: string;
+}
+interface OpportunityExtractionDeps {
+    llm: LLMClient;
+    /** Fetch and parse instruction documents (PDF/HTML → text). */
+    fetchDocument: DocumentFetcher;
+    /**
+     * Optional: fetch raw HTML for the opportunity listing page.
+     * Used when `html` is not provided on the input.
+     */
+    fetchHtml?: HtmlFetcher;
+    /** Grants.gov Simpler API key (optional; enables API-first discovery). */
+    grantsGovApiKey?: string;
+    maxInstructionUrls?: number;
+    /** Override default chunking for Phase 1 section synthesis. */
+    splitIntoChunks?: (text: string) => Promise<string[]>;
+}
+interface NormalizedGrantSection {
+    id: string;
+    title: string;
+    description: string;
+    requirements: string[];
+    wordLimit: string | number;
+    required: boolean;
+}
+interface InstructionUrlAttempt {
+    url: string;
+    confidence?: string;
+    reason?: string;
+    fetched?: boolean;
+}
+interface SectionExtractionArtifacts {
+    analysis?: string;
+    eligibilitySection?: string;
+    methodOfApplication?: string;
+    websitesToRegister?: string[];
+    applicationGuides?: string[];
+    requiredItems?: string[];
+    keyInformation?: string;
+}
+interface SectionExtractionResult {
+    success: boolean;
+    sections: NormalizedGrantSection[];
+    artifacts: SectionExtractionArtifacts;
+    instructionUrlsTried: InstructionUrlAttempt[];
+    warnings: string[];
+    error?: string;
+}
+
+/**
+ * Resolves source type from URL when not explicitly provided (matches handler registry behavior).
+ */
+declare function resolveSourceType(pageUrl: string, hint?: SourceType): SourceType;
+
+/**
+ * End-to-end: discover instruction URLs, fetch text, run chunk+synthesis, return normalized sections.
+ */
+declare function extractOpportunitySections(input: OpportunityInput, deps: OpportunityExtractionDeps): Promise<SectionExtractionResult>;
+
+/**
+ * Browser-like fetch for raw HTML (link discovery).
+ */
+declare function defaultFetchHtml(url: string): Promise<string>;
+
+/**
+ * Extracts text content from HTML (strips script/style, normalizes spaces).
+ */
+declare function extractTextFromHtml(html: string): string;
+/**
+ * Validates and resolves a URL against a base. Returns null for invalid or non-http(s) URLs.
+ */
+declare function validateAndResolveUrl(url: string, baseUrl: string): string | null;
+
+/**
+ * Extracts opportunity ID from various grants.gov URL formats.
+ */
+declare function extractOpportunityIdFromUrl(url: string): string | null;
+
+declare function getSourceSpecificGuidance(sourceType?: SourceType): string;
+
+interface InstructionUrlCandidate {
+    url: string;
+    confidence: "high" | "medium" | "low";
+    reason: string;
+}
+interface FindInstructionUrlsOptions {
+    pageUrl: string;
+    html: string;
+    sourceType?: SourceType;
+    llmCaller?: (messages: Array<{
+        role: "system" | "user" | "assistant";
+        content: string;
+    }>) => Promise<string>;
+    maxCandidates?: number;
+}
+declare function getUrlDiscoveryPrompt(pageUrl: string, sourceType?: SourceType): string;
+/**
+ * Finds URLs that are most likely to contain full application instructions from an opportunity page.
+ */
+declare function findApplicationInstructionUrls(options: FindInstructionUrlsOptions): Promise<string[]>;
+
+/**
+ * Grants.gov Simpler API – instruction URL discovery without scraping the listing page.
+ */
+declare function getGrantsGovInstructionUrlsFromApi(opportunityId: string, apiKey: string): Promise<string[]>;
+
+interface GrantApplicationSection {
+    name: string;
+    required: boolean;
+    pageLimit?: string;
+    description: string;
+    requirements: string[];
+}
+interface MappedAppSection {
+    id: string;
+    title: string;
+    description: string;
+    requirements: string[];
+    wordLimit: string | number;
+    required: boolean;
+}
+/**
+ * Parses Phase 2 synthesis LLM response and returns applicationSections.
+ */
+declare function parseSectionsFromAnalysisResponse(rawResponse: string): GrantApplicationSection[];
+declare function mapToAppSections(sections: GrantApplicationSection[]): MappedAppSection[];
+
+/**
+ * Simple fixed-size chunking when no custom splitter is provided.
+ */
+declare function defaultSplitIntoChunks(text: string): Promise<string[]>;
+interface SynthesizeOptions {
+    llm: LLMClient;
+    combinedDocumentText: string;
+    splitIntoChunks?: (text: string) => Promise<string[]>;
+    maxChunksPerRequest?: number;
+}
+/**
+ * Phase 1 + Phase 2 synthesis (matches document-upload route behavior).
+ */
+declare function synthesizeSectionsFromDocumentText(options: SynthesizeOptions): Promise<{
+    rawPhase2: string;
+    artifacts: SectionExtractionArtifacts;
+}>;
+/**
+ * Full path: chunk analyses → Phase 2 → parsed sections + artifacts.
+ */
+declare function extractGrantSectionsFromCombinedText(options: SynthesizeOptions): Promise<{
+    rawPhase2: string;
+    artifacts: SectionExtractionArtifacts;
+    grantSections: GrantApplicationSection[];
+}>;
+
+type ChatCompletionsCreate = (args: {
+    model: string;
+    messages: unknown[];
+    max_tokens?: number;
+    response_format?: {
+        type: "json_object";
+    };
+}) => Promise<{
+    choices: Array<{
+        message?: {
+            content?: string | null;
+        };
+    }>;
+}>;
+/**
+ * Adapts an OpenAI client (v4) to {@link LLMClient}.
+ * Use with `import OpenAI from "openai"` in the host application.
+ */
+declare function createOpenAiChatLlmClient(openai: {
+    chat: {
+        completions: {
+            create: ChatCompletionsCreate;
+        };
+    };
+}, options?: {
+    model?: string;
+    maxTokens?: number;
+}): LLMClient;
+
+export { type DocumentFetcher, type FetchedDocument, type FindInstructionUrlsOptions, type GrantApplicationSection, type HtmlFetcher, type InstructionUrlAttempt, type InstructionUrlCandidate, type LLMClient, type LLMMessage, type MappedAppSection, type NormalizedGrantSection, type OpportunityExtractionDeps, type OpportunityInput, type SectionExtractionArtifacts, type SectionExtractionResult, type SourceType, createOpenAiChatLlmClient, defaultFetchHtml, defaultSplitIntoChunks, extractGrantSectionsFromCombinedText, extractOpportunityIdFromUrl, extractOpportunitySections, extractTextFromHtml, findApplicationInstructionUrls, getGrantsGovInstructionUrlsFromApi, getSourceSpecificGuidance, getUrlDiscoveryPrompt, mapToAppSections, parseSectionsFromAnalysisResponse, resolveSourceType, synthesizeSectionsFromDocumentText, validateAndResolveUrl };

package/dist/index.d.ts

@@ -0,0 +1,213 @@
+/**
+ * Source-agnostic contracts for opportunity instruction discovery and section extraction.
+ */
+type SourceType = "grantsgov" | "calgrants" | "samgov" | "bespoke";
+interface LLMMessage {
+    role: "system" | "user" | "assistant";
+    content: string;
+}
+/** Generic chat completion; plug in OpenAI, Anthropic, etc. */
+type LLMClient = (messages: LLMMessage[]) => Promise<string>;
+interface FetchedDocument {
+    /** Original request URL */
+    url: string;
+    /** Final URL after redirects */
+    finalUrl: string;
+    /** Extracted plain text (HTML stripped, PDF parsed) */
+    text: string;
+    mimeType: string;
+    filename?: string;
+}
+type DocumentFetcher = (url: string) => Promise<FetchedDocument | null>;
+/**
+ * Fetch raw HTML for a page (needed for link discovery).
+ * If omitted, the pipeline uses a built-in fetch with a browser-like User-Agent.
+ */
+type HtmlFetcher = (url: string) => Promise<string>;
+interface OpportunityInput {
+    pageUrl: string;
+    sourceType?: SourceType;
+    /** When set, skips fetching the opportunity page */
+    html?: string;
+}
+interface OpportunityExtractionDeps {
+    llm: LLMClient;
+    /** Fetch and parse instruction documents (PDF/HTML → text). */
+    fetchDocument: DocumentFetcher;
+    /**
+     * Optional: fetch raw HTML for the opportunity listing page.
+     * Used when `html` is not provided on the input.
+     */
+    fetchHtml?: HtmlFetcher;
+    /** Grants.gov Simpler API key (optional; enables API-first discovery). */
+    grantsGovApiKey?: string;
+    maxInstructionUrls?: number;
+    /** Override default chunking for Phase 1 section synthesis. */
+    splitIntoChunks?: (text: string) => Promise<string[]>;
+}
+interface NormalizedGrantSection {
+    id: string;
+    title: string;
+    description: string;
+    requirements: string[];
+    wordLimit: string | number;
+    required: boolean;
+}
+interface InstructionUrlAttempt {
+    url: string;
+    confidence?: string;
+    reason?: string;
+    fetched?: boolean;
+}
+interface SectionExtractionArtifacts {
+    analysis?: string;
+    eligibilitySection?: string;
+    methodOfApplication?: string;
+    websitesToRegister?: string[];
+    applicationGuides?: string[];
+    requiredItems?: string[];
+    keyInformation?: string;
+}
+interface SectionExtractionResult {
+    success: boolean;
+    sections: NormalizedGrantSection[];
+    artifacts: SectionExtractionArtifacts;
+    instructionUrlsTried: InstructionUrlAttempt[];
+    warnings: string[];
+    error?: string;
+}
+
+/**
+ * Resolves source type from URL when not explicitly provided (matches handler registry behavior).
+ */
+declare function resolveSourceType(pageUrl: string, hint?: SourceType): SourceType;
+
+/**
+ * End-to-end: discover instruction URLs, fetch text, run chunk+synthesis, return normalized sections.
+ */
+declare function extractOpportunitySections(input: OpportunityInput, deps: OpportunityExtractionDeps): Promise<SectionExtractionResult>;
+
+/**
+ * Browser-like fetch for raw HTML (link discovery).
+ */
+declare function defaultFetchHtml(url: string): Promise<string>;
+
+/**
+ * Extracts text content from HTML (strips script/style, normalizes spaces).
+ */
+declare function extractTextFromHtml(html: string): string;
+/**
+ * Validates and resolves a URL against a base. Returns null for invalid or non-http(s) URLs.
+ */
+declare function validateAndResolveUrl(url: string, baseUrl: string): string | null;
+
+/**
+ * Extracts opportunity ID from various grants.gov URL formats.
+ */
+declare function extractOpportunityIdFromUrl(url: string): string | null;
+
+declare function getSourceSpecificGuidance(sourceType?: SourceType): string;
+
+interface InstructionUrlCandidate {
+    url: string;
+    confidence: "high" | "medium" | "low";
+    reason: string;
+}
+interface FindInstructionUrlsOptions {
+    pageUrl: string;
+    html: string;
+    sourceType?: SourceType;
+    llmCaller?: (messages: Array<{
+        role: "system" | "user" | "assistant";
+        content: string;
+    }>) => Promise<string>;
+    maxCandidates?: number;
+}
+declare function getUrlDiscoveryPrompt(pageUrl: string, sourceType?: SourceType): string;
+/**
+ * Finds URLs that are most likely to contain full application instructions from an opportunity page.
+ */
+declare function findApplicationInstructionUrls(options: FindInstructionUrlsOptions): Promise<string[]>;
+
+/**
+ * Grants.gov Simpler API – instruction URL discovery without scraping the listing page.
+ */
+declare function getGrantsGovInstructionUrlsFromApi(opportunityId: string, apiKey: string): Promise<string[]>;
+
+interface GrantApplicationSection {
+    name: string;
+    required: boolean;
+    pageLimit?: string;
+    description: string;
+    requirements: string[];
+}
+interface MappedAppSection {
+    id: string;
+    title: string;
+    description: string;
+    requirements: string[];
+    wordLimit: string | number;
+    required: boolean;
+}
+/**
+ * Parses Phase 2 synthesis LLM response and returns applicationSections.
+ */
+declare function parseSectionsFromAnalysisResponse(rawResponse: string): GrantApplicationSection[];
+declare function mapToAppSections(sections: GrantApplicationSection[]): MappedAppSection[];
+
+/**
+ * Simple fixed-size chunking when no custom splitter is provided.
+ */
+declare function defaultSplitIntoChunks(text: string): Promise<string[]>;
+interface SynthesizeOptions {
+    llm: LLMClient;
+    combinedDocumentText: string;
+    splitIntoChunks?: (text: string) => Promise<string[]>;
+    maxChunksPerRequest?: number;
+}
+/**
+ * Phase 1 + Phase 2 synthesis (matches document-upload route behavior).
+ */
+declare function synthesizeSectionsFromDocumentText(options: SynthesizeOptions): Promise<{
+    rawPhase2: string;
+    artifacts: SectionExtractionArtifacts;
+}>;
+/**
+ * Full path: chunk analyses → Phase 2 → parsed sections + artifacts.
+ */
+declare function extractGrantSectionsFromCombinedText(options: SynthesizeOptions): Promise<{
+    rawPhase2: string;
+    artifacts: SectionExtractionArtifacts;
+    grantSections: GrantApplicationSection[];
+}>;
+
+type ChatCompletionsCreate = (args: {
+    model: string;
+    messages: unknown[];
+    max_tokens?: number;
+    response_format?: {
+        type: "json_object";
+    };
+}) => Promise<{
+    choices: Array<{
+        message?: {
+            content?: string | null;
+        };
+    }>;
+}>;
+/**
+ * Adapts an OpenAI client (v4) to {@link LLMClient}.
+ * Use with `import OpenAI from "openai"` in the host application.
+ */
+declare function createOpenAiChatLlmClient(openai: {
+    chat: {
+        completions: {
+            create: ChatCompletionsCreate;
+        };
+    };
+}, options?: {
+    model?: string;
+    maxTokens?: number;
+}): LLMClient;
+
+export { type DocumentFetcher, type FetchedDocument, type FindInstructionUrlsOptions, type GrantApplicationSection, type HtmlFetcher, type InstructionUrlAttempt, type InstructionUrlCandidate, type LLMClient, type LLMMessage, type MappedAppSection, type NormalizedGrantSection, type OpportunityExtractionDeps, type OpportunityInput, type SectionExtractionArtifacts, type SectionExtractionResult, type SourceType, createOpenAiChatLlmClient, defaultFetchHtml, defaultSplitIntoChunks, extractGrantSectionsFromCombinedText, extractOpportunityIdFromUrl, extractOpportunitySections, extractTextFromHtml, findApplicationInstructionUrls, getGrantsGovInstructionUrlsFromApi, getSourceSpecificGuidance, getUrlDiscoveryPrompt, mapToAppSections, parseSectionsFromAnalysisResponse, resolveSourceType, synthesizeSectionsFromDocumentText, validateAndResolveUrl };