@autorag/librarian 2.5.1 → 2.5.2

package/dist/web/search/providers/startpage.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Startpage proxies Google's index behind a privacy frontend and serves fully
+ * server-rendered result pages — no JS challenge on the happy path. Its bot
+ * defense keys on requests that skip the homepage handshake: the search form
+ * carries a session token (`sc`) plus sibling hidden inputs, and posting the
+ * form with a stale/absent token 302s to the `/en/errors/` CAPTCHA shell.
+ * The robust flow is therefore the same dance a real browser performs: GET
+ * the homepage, lift the form's hidden inputs, POST them back with the query.
+ *
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT)
+ * `packages/coding-agent/src/web/search/providers/startpage.ts`.
+ */
+import type { SearchProviderId, SearchResponse } from "../types.js";
+import type { SearchParams } from "./base.js";
+import { SearchProvider } from "./base.js";
+/** Execute a Startpage web search via the homepage-token form flow. */
+export declare function searchStartpage(params: SearchParams): Promise<SearchResponse>;
+/** Search provider for Startpage (no API key required). */
+export declare class StartpageProvider extends SearchProvider {
+    readonly id: SearchProviderId;
+    readonly label = "Startpage";
+    isAvailable(): boolean;
+    isExplicitlyAvailable(): boolean;
+    search(params: SearchParams): Promise<SearchResponse>;
+}

package/dist/web/search/providers/utils.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Shared helpers for web search providers.
+ *
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT) `web/search/providers/utils.ts`.
+ */
+import { SearchProviderError, type SearchProviderId, type SearchSource } from "../types.js";
+/**
+ * The 60-second default tolerates legitimate slow LLM-mediated responses
+ * while bounding stalls when the runtime's `AbortSignal` fails to propagate.
+ */
+export declare const SEARCH_HARD_TIMEOUT_MS: number;
+/**
+ * Compose a caller-supplied {@link AbortSignal} with a hard timeout so an
+ * outbound `fetch()` is guaranteed to settle within `ms` even when the
+ * runtime fails to propagate cancellation to the underlying transport.
+ */
+export declare function withHardTimeout(signal: AbortSignal | undefined, ms?: number): AbortSignal;
+/**
+ * Map a provider's raw source list to the unified SearchSource shape,
+ * clamped to the requested result count and annotated with ageSeconds.
+ */
+export declare function toSearchSources(sources: ReadonlyArray<{
+    title: string;
+    url: string;
+    snippet?: string;
+    publishedDate?: string;
+}>, numResults: number): SearchSource[];
+export declare function classifyProviderHttpError(provider: SearchProviderId, status: number, body: string): SearchProviderError | null;
+/**
+ * Collapse runs of whitespace in a loosely-typed provider field, returning
+ * `undefined` for missing/non-string/blank values. Shared so tab/newline
+ * folding cannot drift between providers.
+ */
+export declare function normalizeSearchText(value: unknown): string | undefined;
+/**
+ * Read a provider response body up to a byte cap, truncating or throwing when
+ * the limit is exceeded. Shared so streaming-cap fixes land in one place.
+ */
+export declare function readLimitedText(response: Response, provider: SearchProviderId, maxBytes: number, truncate?: boolean): Promise<string>;

package/dist/web/search/providers/xai.d.ts

@@ -0,0 +1,9 @@
+import type { SearchResponse } from "../types.js";
+import type { SearchAvailabilityContext, SearchParams } from "./base.js";
+import { SearchProvider } from "./base.js";
+export declare class XaiProvider extends SearchProvider {
+    readonly id: "xai";
+    readonly label = "xAI";
+    isAvailable(context?: SearchAvailabilityContext): boolean;
+    search(params: SearchParams): Promise<SearchResponse>;
+}

package/dist/web/search/query.d.ts

@@ -0,0 +1,193 @@
+/**
+ * Structured web-search query parsing.
+ *
+ * Agents habitually embed Google-style directives in search queries —
+ * `site:`, `before:`/`after:`, `inurl:`, `filetype:`, quoted phrases, `OR`
+ * groups, `-exclusions` — regardless of whether the backing engine parses
+ * them. This module turns a raw query into a {@link StructuredQuery} so each
+ * provider can:
+ *
+ * 1. map constraints onto native API parameters where they exist (Tavily
+ *    `include_domains`, Exa date bounds, …),
+ * 2. rebuild a query string containing only the syntax the target engine
+ *    understands ({@link formatQuery}), and
+ * 3. post-filter returned sources leniently ({@link applyQueryConstraints}):
+ *    a constraint dimension that would eliminate every result is dropped and
+ *    reported rather than returning nothing.
+ *
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT)
+ * `packages/coding-agent/src/web/search/query.ts`.
+ */
+import type { SearchSource } from "./types.js";
+/** One free-text token of the query (everything that is not a recognized directive). */
+export interface QueryTerm {
+    /** Term text without quotes or operator prefixes. */
+    text: string;
+    /** Quoted exact phrase (`"like this"`) or verbatim-required (`+term`). */
+    phrase?: boolean;
+    /** Excluded via `-term` or `NOT term`. */
+    negated?: boolean;
+    /**
+     * OR-group id. Terms sharing an id are alternatives (`a OR b`); terms
+     * without a group are implicitly AND-ed. Groups are always contiguous
+     * runs in {@link StructuredQuery.terms}.
+     */
+    group?: number;
+}
+/**
+ * A raw query decomposed into free text plus every recognized constraint.
+ *
+ * All list fields are always present (possibly empty) so consumers can map
+ * over them without null checks. Values are stored as typed by the user
+ * except for normalization noted per field.
+ */
+export interface StructuredQuery {
+    /** Original query string, verbatim. */
+    raw: string;
+    /**
+     * Free-text remainder with all recognized directives removed; phrases
+     * stay quoted, exclusions keep `-`, OR groups keep `OR`. Empty when the
+     * query was directives only — use {@link formatQuery} for a never-empty
+     * engine query.
+     */
+    text: string;
+    /** Ordered free-text terms (phrases, exclusions, OR groups). */
+    terms: QueryTerm[];
+    /** `site:`/`domain:`/`host:` includes — any-of. Lowercased, scheme stripped, may carry a path (`github.com/anthropics`). */
+    sites: string[];
+    /** `-site:` exclusions, same normalization as {@link sites}. */
+    excludedSites: string[];
+    /** `inurl:`/`url:`/`allinurl:` substrings — all must appear in the URL. */
+    inUrl: string[];
+    /** `-inurl:` substrings — none may appear in the URL. */
+    excludedInUrl: string[];
+    /** `intitle:`/`title:`/`allintitle:` substrings — all must appear in the title. */
+    inTitle: string[];
+    /** `-intitle:` substrings — none may appear in the title. */
+    excludedInTitle: string[];
+    /** `intext:`/`inbody:`/`inanchor:`/`allintext:` body substrings. Not post-filterable (snippets are partial); query-building only. */
+    inText: string[];
+    /** `-intext:` body exclusions. Query-building only. */
+    excludedInText: string[];
+    /** `filetype:`/`ext:` extensions — any-of. Lowercased, no leading dot. */
+    filetypes: string[];
+    /** `-filetype:`/`-ext:` extensions — none may match. */
+    excludedFiletypes: string[];
+    /** Inclusive lower publish-date bound from `after:`/`since:`, ISO `YYYY-MM-DD`. */
+    after?: string;
+    /** Exclusive upper publish-date bound from `before:`/`until:`, ISO `YYYY-MM-DD`. */
+    before?: string;
+    /** Language code from `lang:`/`language:`, lowercased (e.g. `en`, `en-us`). */
+    lang?: string;
+    /** True when any directive or boolean operator was recognized. */
+    hasDirectives: boolean;
+    /** True when any post-filterable constraint is set (sites, url/title terms, filetypes, date bounds). */
+    hasConstraints: boolean;
+}
+/**
+ * Query-syntax capabilities of a target engine, used by {@link formatQuery}
+ * to decide which parsed features are re-emitted as query text. Everything
+ * defaults to `false`: the zero-value produces plain keywords suitable for
+ * natural-language APIs.
+ */
+export interface QuerySyntax {
+    /** Emit `"quoted phrases"`. */
+    phrases?: boolean;
+    /** Emit `-term` exclusions (negated terms are dropped otherwise). */
+    negation?: boolean;
+    /** Emit `OR` between alternatives (groups are flattened to keywords otherwise). */
+    or?: boolean;
+    /** Emit `site:`/`-site:`. */
+    site?: boolean;
+    /** Emit `inurl:`/`-inurl:`. */
+    inUrl?: boolean;
+    /** Emit `intitle:`/`-intitle:`. */
+    inTitle?: boolean;
+    /** Emit `intext:`/`-intext:`. */
+    inText?: boolean;
+    /** Emit `filetype:`/`-filetype:`. */
+    filetype?: boolean;
+    /** Emit `before:`/`after:` ISO date bounds. */
+    dateRange?: boolean;
+}
+/** Full Google-style syntax: engines that parse the classic operator set (Google, Startpage, Ecosia, Brave, Kagi, Mojeek, SearXNG…). */
+export declare const GOOGLE_QUERY_SYNTAX: QuerySyntax;
+/** Result of {@link applyQueryConstraints}. */
+export interface ConstraintFilterResult {
+    /** Sources surviving the lenient filter — never empty when the input was non-empty. */
+    sources: SearchSource[];
+    /**
+     * Directive renderings (`site:arxiv.org`, `before:2024-01-01`, …) of the
+     * constraint dimensions that matched zero sources and were therefore
+     * relaxed instead of enforced.
+     */
+    dropped: string[];
+}
+/**
+ * Parse a `before:`/`after:` value into ISO `YYYY-MM-DD`.
+ * Accepts `YYYY`, `YYYY-MM`, `YYYY-MM-DD` (also `/` and `.` separators) and
+ * `MM/DD/YYYY` (day-first assumed when the first field exceeds 12).
+ * Bare years/months resolve to the first day of the period, matching
+ * Google's `after:2024` ≙ `after:2024-01-01` semantics.
+ */
+export declare function parseDateValue(value: string): string | undefined;
+/**
+ * Parse a raw query into a {@link StructuredQuery}.
+ *
+ * Lenient by construction: unknown `name:value` tokens (URLs, `C:\paths`,
+ * `TS2345:`, jargon) stay in the free text verbatim, and a directive with an
+ * unparseable value (`before:someday`) degrades to a plain term instead of
+ * being dropped.
+ */
+export declare function parseSearchQuery(raw: string): StructuredQuery;
+/**
+ * Rebuild a query string for an engine with the given {@link QuerySyntax}.
+ *
+ * Constraints whose syntax the engine lacks are omitted (the caller maps
+ * them onto API parameters or relies on {@link applyQueryConstraints}).
+ * Never returns an empty string for a non-empty input: a directives-only
+ * query falls back to the constraint values as keywords, then to `raw` — an
+ * engine searching *something* beats an empty-query error.
+ */
+export declare function formatQuery(q: StructuredQuery, syntax?: QuerySyntax): string;
+/**
+ * Build the engine query for a credential-free HTML engine (Google,
+ * Startpage, DuckDuckGo, Ecosia, Mojeek, SearXNG, and the Public Web
+ * fan-out over them).
+ *
+ * Canonicalizes directives via {@link formatQuery} with the engine's
+ * {@link QuerySyntax} (default: full Google syntax), after demoting the
+ * operators that zero-match across the scraper set: engines only match
+ * `site:` against a bare domain (a path yields zero results everywhere),
+ * and DuckDuckGo ignores `inurl:` entirely — so either operator silently
+ * empties the result set. The raw URL as a plain term matches fine, so
+ * bare-domain `site:` filters are kept while path-carrying `site:` and all
+ * `inurl:` values become plain keywords; the demotion is structural (before
+ * formatting), so OR-grouped and quoted directives are covered. Negated
+ * forms (`-site:`, `-inurl:`) pass through untouched — demoting them would
+ * invert an exclusion into a search term; the pipeline post-filter
+ * ({@link applyQueryConstraints}) enforces every demoted or unsupported
+ * constraint on the returned sources. Directive-free queries pass through
+ * byte-identical.
+ */
+export declare function formatScraperQuery(query: string, parsedQuery?: StructuredQuery, syntax?: QuerySyntax): string;
+/**
+ * `site:` matcher: exact host or subdomain of `site`; when `site` carries a
+ * path (`github.com/anthropics`), the URL path must start with it.
+ */
+export declare function matchesSite(url: string, site: string): boolean;
+/**
+ * Strict per-source constraint check: every filterable dimension of `q` must
+ * pass. Sources without a resolvable date pass date bounds (a missing date
+ * is not proof of violation). For custom provider flows; the standard path
+ * is {@link applyQueryConstraints}.
+ */
+export declare function matchesQueryConstraints(source: SearchSource, q: StructuredQuery): boolean;
+/**
+ * Lenient post-filter: applies each constraint dimension of `q` in turn,
+ * skipping (and reporting) any dimension that would eliminate every
+ * remaining source. Guarantees a non-empty result for a non-empty input, so
+ * a mis-scoped directive degrades to unfiltered results plus a note instead
+ * of a dead search.
+ */
+export declare function applyQueryConstraints(sources: readonly SearchSource[], q: StructuredQuery): ConstraintFilterResult;

package/dist/web/search/types.d.ts

@@ -0,0 +1,198 @@
+/**
+ * Web Search Types
+ *
+ * Unified types for web search responses across supported providers.
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT) `web/search/types.ts`,
+ * trimmed to a credential-free-only surface: model-native providers that
+ * reuse the agent's own model credentials (gemini, anthropic, codex, xai —
+ * see `model-auth.ts`), the always-keyless routes (perplexity anonymous ask,
+ * parallel MCP), the explicitly-advanced SearXNG endpoint, and the
+ * credential-free scraped engines. oh-my-pi's vendor-key providers (brave,
+ * tavily, exa, jina, kagi, kimi, zai, tinyfish, synthetic, ollama,
+ * firecrawl) are intentionally absent: a default AutoRAG install must never
+ * require a user-issued search key or signup.
+ */
+export declare const SEARCH_PROVIDER_OPTIONS: readonly [{
+    readonly value: "auto";
+    readonly label: "Auto";
+    readonly description: "Automatically uses the first configured web-search provider";
+}, {
+    readonly value: "gemini";
+    readonly label: "Gemini";
+    readonly description: "Google Search grounding via the agent's Gemini model credentials (GEMINI_API_KEY/GOOGLE_API_KEY)";
+}, {
+    readonly value: "anthropic";
+    readonly label: "Anthropic";
+    readonly description: "Claude's built-in web_search via the agent's Anthropic model credentials (ANTHROPIC_API_KEY)";
+}, {
+    readonly value: "codex";
+    readonly label: "OpenAI";
+    readonly description: "OpenAI Responses web_search via the agent's OpenAI model credentials (OPENAI_API_KEY)";
+}, {
+    readonly value: "xai";
+    readonly label: "xAI";
+    readonly description: "xAI Responses web_search via the agent's xAI model credentials (XAI_API_KEY)";
+}, {
+    readonly value: "perplexity";
+    readonly label: "Perplexity";
+    readonly description: "Anonymous Perplexity ask endpoint; no key or signup required";
+}, {
+    readonly value: "parallel";
+    readonly label: "Parallel";
+    readonly description: "Parallel's keyless web search MCP endpoint; no key or signup required";
+}, {
+    readonly value: "searxng";
+    readonly label: "SearXNG";
+    readonly description: "Advanced: requires a self-hosted SEARXNG_ENDPOINT; never part of the default experience";
+}, {
+    readonly value: "startpage";
+    readonly label: "Startpage";
+    readonly description: "Credential-free scrape of Startpage (Google-backed) results; may be bot-challenged";
+}, {
+    readonly value: "duckduckgo";
+    readonly label: "DuckDuckGo";
+    readonly description: "Credential-free best-effort fallback; may be bot-challenged on datacenter/shared-egress IPs";
+}, {
+    readonly value: "ecosia";
+    readonly label: "Ecosia";
+    readonly description: "Credential-free scrape of Ecosia (Google-backed) results";
+}, {
+    readonly value: "google";
+    readonly label: "Google";
+    readonly description: "Credential-free fallback; slower and may be bot-challenged";
+}, {
+    readonly value: "mojeek";
+    readonly label: "Mojeek";
+    readonly description: "Credential-free scrape of Mojeek's independent index";
+}, {
+    readonly value: "public";
+    readonly label: "Public Web";
+    readonly description: "Queries every credential-free engine in parallel and consolidates deduplicated results";
+}];
+/** Default hard timeout for each web-search provider transport. */
+export declare const DEFAULT_WEB_SEARCH_TIMEOUT_SECONDS = 60;
+/** Maximum configurable hard timeout for each web-search provider transport. */
+export declare const MAX_WEB_SEARCH_TIMEOUT_SECONDS = 300;
+/** Supported web search providers (every option except `auto`). */
+export type SearchProviderId = Exclude<(typeof SEARCH_PROVIDER_OPTIONS)[number]["value"], "auto">;
+/**
+ * Auto-resolution priority order. Derived from {@link SEARCH_PROVIDER_OPTIONS}
+ * (minus `auto`) so any dropdown/setting and `resolveProviderChain()` share
+ * one source of truth and never drift apart: credential-free and
+ * model-credential-reusing providers first, then the scraped engines, with
+ * the public fan-out last (explicit selection only).
+ */
+export declare const SEARCH_PROVIDER_ORDER: readonly SearchProviderId[];
+/** Concrete provider choices (no `auto` sentinel) — for list-valued settings like order/exclude. */
+export declare const SEARCH_PROVIDER_CHOICES: ({
+    readonly value: "gemini";
+    readonly label: "Gemini";
+    readonly description: "Google Search grounding via the agent's Gemini model credentials (GEMINI_API_KEY/GOOGLE_API_KEY)";
+} | {
+    readonly value: "anthropic";
+    readonly label: "Anthropic";
+    readonly description: "Claude's built-in web_search via the agent's Anthropic model credentials (ANTHROPIC_API_KEY)";
+} | {
+    readonly value: "codex";
+    readonly label: "OpenAI";
+    readonly description: "OpenAI Responses web_search via the agent's OpenAI model credentials (OPENAI_API_KEY)";
+} | {
+    readonly value: "xai";
+    readonly label: "xAI";
+    readonly description: "xAI Responses web_search via the agent's xAI model credentials (XAI_API_KEY)";
+} | {
+    readonly value: "perplexity";
+    readonly label: "Perplexity";
+    readonly description: "Anonymous Perplexity ask endpoint; no key or signup required";
+} | {
+    readonly value: "parallel";
+    readonly label: "Parallel";
+    readonly description: "Parallel's keyless web search MCP endpoint; no key or signup required";
+} | {
+    readonly value: "searxng";
+    readonly label: "SearXNG";
+    readonly description: "Advanced: requires a self-hosted SEARXNG_ENDPOINT; never part of the default experience";
+} | {
+    readonly value: "startpage";
+    readonly label: "Startpage";
+    readonly description: "Credential-free scrape of Startpage (Google-backed) results; may be bot-challenged";
+} | {
+    readonly value: "duckduckgo";
+    readonly label: "DuckDuckGo";
+    readonly description: "Credential-free best-effort fallback; may be bot-challenged on datacenter/shared-egress IPs";
+} | {
+    readonly value: "ecosia";
+    readonly label: "Ecosia";
+    readonly description: "Credential-free scrape of Ecosia (Google-backed) results";
+} | {
+    readonly value: "google";
+    readonly label: "Google";
+    readonly description: "Credential-free fallback; slower and may be bot-challenged";
+} | {
+    readonly value: "mojeek";
+    readonly label: "Mojeek";
+    readonly description: "Credential-free scrape of Mojeek's independent index";
+} | {
+    readonly value: "public";
+    readonly label: "Public Web";
+    readonly description: "Queries every credential-free engine in parallel and consolidates deduplicated results";
+})[];
+export declare const SEARCH_PROVIDER_PREFERENCES: readonly ["auto", ...SearchProviderId[]];
+/** Display labels, derived from {@link SEARCH_PROVIDER_OPTIONS}. */
+export declare const SEARCH_PROVIDER_LABELS: Record<SearchProviderId, string>;
+export declare function isSearchProviderId(value: string): value is SearchProviderId;
+export declare function isSearchProviderPreference(value: string): value is SearchProviderId | "auto";
+/** Source returned by search (all providers) */
+export interface SearchSource {
+    title: string;
+    url: string;
+    snippet?: string;
+    /** ISO date string or relative ("2d ago") */
+    publishedDate?: string;
+    /** Age in seconds for consistent formatting */
+    ageSeconds?: number;
+    author?: string;
+}
+/** Citation with text reference (LLM-mediated providers) */
+export interface SearchCitation {
+    url: string;
+    title: string;
+    citedText?: string;
+}
+/** Usage metrics */
+export interface SearchUsage {
+    inputTokens?: number;
+    outputTokens?: number;
+    /** Anthropic: number of web search requests made */
+    searchRequests?: number;
+    /** Perplexity: combined token count */
+    totalTokens?: number;
+}
+/** Unified response across providers */
+export interface SearchResponse {
+    provider: SearchProviderId | "none";
+    /** Synthesized answer text (LLM-mediated providers) */
+    answer?: string;
+    /** Search result sources */
+    sources: SearchSource[];
+    /** Text citations with context */
+    citations?: SearchCitation[];
+    /** Intermediate search queries */
+    searchQueries?: string[];
+    /** Follow-up question suggestions (provider-dependent) */
+    relatedQuestions?: string[];
+    /** Token usage metrics */
+    usage?: SearchUsage;
+    /** Model used */
+    model?: string;
+    /** Request ID for debugging */
+    requestId?: string;
+    /** Authentication mode used by the provider (e.g. oauth, api-key) */
+    authMode?: string;
+}
+/** Provider-specific error with optional HTTP status */
+export declare class SearchProviderError extends Error {
+    readonly provider: SearchProviderId;
+    readonly status?: number;
+    constructor(provider: SearchProviderId, message: string, status?: number);
+}

package/dist/web/search/utils.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Small shared helpers for web search providers.
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT) `web/search/utils.ts`.
+ */
+/** Calculate age in seconds from an ISO date string. Returns undefined on invalid input. */
+export declare function dateToAgeSeconds(dateStr: string | null | undefined): number | undefined;
+/** Clamp a result count to [1, maxVal], returning defaultVal when value is absent or NaN. */
+export declare function clampNumResults(value: number | undefined, defaultVal: number, maxVal: number): number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.5.1",
+  "version": "2.5.2",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "MIT",
   "repository": {
@@ -60,6 +60,7 @@
     "iconv-lite": "^0.7.3",
     "jszip": "^3.10.1",
     "mailparser": "^3.9.20",
+    "puppeteer-core": "^25.11.0",
     "slides-grab": "^1.5.1",
     "smol-toml": "1.8.0",
     "tesseract.js": "^7.0.0",

package/skills/autorag/SKILL.md

@@ -100,6 +100,7 @@ autorag feedback <sessionId> --useful 1,3 --not-useful 2 --json
 ## Maintenance
 
 ```bash
+autorag setup --format json
 autorag status --json
 autorag health --json
 autorag refresh --json
@@ -138,7 +139,7 @@ Never target source documents. `memory inspect` is read-only and path-opaque.
 - Prefer `--json --debug` when another agent consumes search output or will
   call `autorag feedback`.
 - Do not invent CLI commands. `autorag --help` is the command list of record,
-  including the shipped `gateway`, `models`, `serve`, `p2p`, `tui`, and `lite`
+  including the shipped `setup`, `gateway`, `models`, `serve`, `p2p`, `tui`, and `lite`
   commands. `autorag <command> --help` prints that command's own flags, and
   `autorag --version` prints the installed version. See
   `docs/embedding-runtime.md` for the local embedding runtime