@bpinternal/site-scout 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,211 @@
+import { z } from '@bpinternal/zui';
+
+/** A source a URL was discovered from. Trust/recall differ per source. */
+type DiscoverySource = 'llms' | 'sitemap' | 'robots' | 'discoverUrls' | 'search';
+/** Per-URL metadata gathered during discovery (sparse; depends on source). */
+type UrlMeta = {
+    sources: DiscoverySource[];
+    lastmod?: string;
+    priority?: number;
+    changefreq?: string;
+    title?: string;
+    searchRank?: number;
+};
+/**
+ * A node in the prioritized tree. Each node corresponds to ONE path segment
+ * (the root is the host). A node may be:
+ *   - an internal "folder" (has children; `url` is the canonical folder URL if
+ *     that path itself resolves), and/or
+ *   - a "leaf" page (`url` set, usually no children).
+ * Folders carry `count` (# of leaf URLs anywhere beneath them) and an aggregate
+ * `score` so select() can reason about cardinality (10K products vs 670 docs)
+ * WITHOUT walking every leaf.
+ */
+type TreeNode = {
+    /** Path segment label for this node ('' for root; e.g. 'docs', 'api'). */
+    segment: string;
+    /** Full URL when this node is itself a real page (leaf or folder-with-page). */
+    url?: string;
+    /** Per-URL metadata, present when `url` is set. */
+    meta?: UrlMeta;
+    /** Child nodes keyed by their segment, sorted by score desc at read time. */
+    children: TreeNode[];
+    /** Deterministic priority score (aggregate for folders; own for leaves). */
+    score: number;
+    /** Number of leaf URLs at-or-below this node (own page counts as 1). */
+    count: number;
+    /** Human-readable signals contributing to `score` (explainable assertions). */
+    reasons?: string[];
+};
+/** One thing discovery did — surfaced via the onActivity callback + collected. */
+type DiscoveryStep = {
+    kind: 'robots' | 'sitemap' | 'sitemap-index' | 'llms' | 'discoverUrls' | 'search' | 'redirect' | 'site-type' | 'note';
+    detail: string;
+    /** URLs/items yielded by this step, when meaningful. */
+    count?: number;
+};
+type DetectedSiteType = {
+    type: 'shopify' | 'unknown';
+    excludePatterns: string[];
+    includeHints: string[];
+};
+/** What discover() returns — the contract handed to select(). */
+/** Why the pipeline stopped / how it resolved. */
+type StopReason = 'ok' | 'unsupported_site' | 'no_sources' | 'time_limit_reached';
+type MasterMap = {
+    /** Root of the prioritized tree (segment === ''). */
+    tree: TreeNode;
+    /** Total distinct leaf URLs in the tree. */
+    total: number;
+    /** Ordered log of what discovery did (also streamed via onActivity). */
+    steps: DiscoveryStep[];
+    /** True when discovery was aborted/timed-out and the tree is partial. */
+    aborted: boolean;
+    siteType: DetectedSiteType['type'];
+    /** Set when the site was refused by the guardrail (no discovery happened). */
+    unsupported?: boolean;
+};
+type DiscoverOptions = {
+    /** Topics/use-case hints — seed the site: search (find help/docs subdomains). */
+    topics?: string[];
+    /** Abort discovery; the tree built so far is returned (aborted: true). */
+    signal?: AbortSignal;
+    /** Soft wall-clock budget (ms) — discovery self-aborts at the deadline. */
+    budgetMs?: number;
+    /** Max distinct URLs to retain in the tree. */
+    maxUrls?: number;
+    /** Activity callback for live logging (searches, sitemaps, counts). */
+    onActivity?: (step: DiscoveryStep) => void;
+    /** `Date.parse`-able "now" for deterministic recency scoring in tests. */
+    now?: number;
+};
+type SelectOptions = {
+    /** Max URLs to return (default 100). */
+    limit?: number;
+    /**
+     * Optional natural-language refinement of what to prioritize for this use
+     * case (e.g. "focus on developer docs and API reference"), passed to the LLM
+     * pick call as extra guidance.
+     */
+    prompt?: string;
+    /** Company context passed to the LLM pick call. */
+    context?: {
+        company?: string;
+        website?: string;
+        overview?: string;
+    };
+};
+type Selection = {
+    /** Prioritized URLs to scrape, best first, ≤ limit. */
+    urls: string[];
+    /** Distinct useful URLs before the limit was applied. */
+    useful: number;
+};
+/** Tunable weights for the deterministic scorer. Defaults target a support KB. */
+type ScoreWeights = {
+    sectionWeights: Record<string, number>;
+    defaultSectionWeight: number;
+    depthPenalty: number;
+    depthWaivedSections: string[];
+    recencyBonus: number;
+    priorityBonus: number;
+    multiSourceBonus: number;
+    llmsBonus: number;
+    queryParamPenalty: number;
+};
+/** A discovered candidate page: a URL plus whatever metadata discovery gathered. */
+type Candidate = {
+    url: string;
+    meta: UrlMeta;
+};
+/** A flat scored URL — produced by score.ts, consumed by the tree builder. */
+type ScoredUrl = {
+    url: string;
+    score: number;
+    reasons: string[];
+    section: string;
+    meta: UrlMeta;
+};
+
+/** Fetch a URL's body as text, SSRF-guarded against `baseHost`. Null on any failure. */
+declare function fetchText(url: string, baseHost: string, timeoutMs?: number): Promise<string | null>;
+/**
+ * Fetch + parse a JSON endpoint, SSRF-guarded against `baseHost`. Tolerates
+ * stores that serve JSON with a non-JSON content-type (e.g. `text/javascript`
+ * on Shopify's `/products.json`) by parsing the raw text. Null on any failure
+ * (network, non-2xx, blocked, unparseable).
+ */
+declare function fetchJson<T = unknown>(url: string, baseHost: string, timeoutMs?: number): Promise<T | null>;
+/** The origin (scheme + host) of a base URL, tolerating a bare host. */
+declare function siteOrigin(baseUrl: string): string | null;
+/** Lower-cased hostname of a URL, tolerating a bare host. */
+declare function hostOf(url: string): string | null;
+
+type SearchResult = {
+    name: string;
+    url: string;
+    snippet?: string;
+};
+type DiscoverDeps = {
+    fetchText: typeof fetchText;
+    fetchJson: typeof fetchJson;
+    discoverUrls: (args: {
+        url: string;
+        count: number;
+        onlyHttps: boolean;
+    }) => Promise<{
+        urls: string[];
+        stopReason: string;
+    }>;
+    webSearch: (args: {
+        query: string;
+        includeSites?: string[];
+        count: number;
+    }) => Promise<{
+        results: SearchResult[];
+    }>;
+};
+declare function discover(website: string, opts: DiscoverOptions | undefined, deps: DiscoverDeps): Promise<MasterMap>;
+
+/**
+ * The one LLM touchpoint, host-provided. Mirrors zai's `extract` shape:
+ * feed text + a schema + instructions, get validated structured output back.
+ */
+type ExtractFn = <S extends z.ZodType>(input: string, schema: S, opts: {
+    instructions: string;
+}) => Promise<z.infer<S>>;
+/**
+ * Optional record/replay cache around side-effecting calls (the analogue of
+ * viber's cognitive cache). Defaults to passthrough — production hosts don't
+ * need to provide it.
+ */
+type CacheFn = <T>(kind: string, keyParts: unknown, fn: () => Promise<T>) => Promise<T>;
+type SelectDeps = {
+    extract: ExtractFn;
+    cache?: CacheFn;
+};
+
+declare function select(map: MasterMap, opts: SelectOptions | undefined, deps: SelectDeps): Promise<Selection>;
+
+declare function cachedDeps(group: string, real: SiteScoutDeps): SiteScoutDeps;
+
+/**
+ * Everything the pipeline needs from its host: discovery I/O (fetch, URL
+ * discovery, web search) plus the one LLM touchpoint and its optional cache.
+ */
+type SiteScoutDeps = DiscoverDeps & SelectDeps;
+type FindUrlsInput = {
+    website: string;
+    limit?: number;
+} & Pick<SelectOptions, 'prompt' | 'context'> & Pick<DiscoverOptions, 'topics' | 'signal' | 'budgetMs' | 'onActivity'>;
+/** Convenience: guard → discover → select in one call. */
+declare function findKnowledgeUrls(input: FindUrlsInput, deps: SiteScoutDeps, extra?: {
+    now?: number;
+}): Promise<Selection & {
+    discovered: number;
+    aborted: boolean;
+    siteType: string;
+    stopReason: StopReason;
+}>;
+
+export { type CacheFn, type Candidate, type DetectedSiteType, type DiscoverDeps, type DiscoverOptions, type DiscoverySource, type DiscoveryStep, type ExtractFn, type FindUrlsInput, type MasterMap, type ScoreWeights, type ScoredUrl, type SearchResult, type SelectDeps, type SelectOptions, type Selection, type SiteScoutDeps, type StopReason, type TreeNode, type UrlMeta, cachedDeps, discover, fetchJson, fetchText, findKnowledgeUrls, hostOf, select, siteOrigin };