@inflexa-ai/harness 0.15.1 → 0.17.0

package/dist/lib/rate-limit.d.ts

@@ -0,0 +1,35 @@
+export interface RateLimitConfig {
+    readonly maxConcurrency: number;
+    readonly requestsPerSecond: number;
+    /**
+     * Retry policy for the source's HTTP layer. Retries re-enter the limiter as
+     * fresh attempts, so they are paced like any other request; the limiter
+     * itself does not read these two fields.
+     */
+    readonly maxRetries: number;
+    readonly maxRetryDelayMs: number;
+}
+export interface RateLimitRuntime {
+    readonly now?: () => number;
+    readonly sleep?: (milliseconds: number, signal?: AbortSignal) => Promise<void>;
+}
+export declare function sleepWithSignal(milliseconds: number, signal?: AbortSignal): Promise<void>;
+export declare class BoundedRateLimiter {
+    private readonly config;
+    private readonly semaphore;
+    private readonly now;
+    private readonly sleep;
+    private reservationTail;
+    private nextStartAt;
+    constructor(config: RateLimitConfig, runtime?: RateLimitRuntime);
+    private reserveStart;
+    run<T>(operation: () => Promise<T>, signal?: AbortSignal): Promise<T>;
+}
+/**
+ * Admission gate for a source's HTTP layer (`SourceHttpOptions.schedule`).
+ *
+ * It gates the network call alone: the caller arms its request timeout inside
+ * the admitted operation, so a request held behind pacing or a concurrency
+ * ceiling cannot be reported as a timeout before it ever reaches the network.
+ */
+export declare function createRateLimitSchedule(config: RateLimitConfig, runtime?: RateLimitRuntime): <T>(operation: () => Promise<T>, signal?: AbortSignal) => Promise<T>;

package/dist/lib/rate-limit.js

@@ -0,0 +1,121 @@
+function abortReason(signal) {
+    return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
+}
+export function sleepWithSignal(milliseconds, signal) {
+    if (milliseconds <= 0)
+        return Promise.resolve();
+    if (signal?.aborted)
+        return Promise.reject(abortReason(signal));
+    return new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, milliseconds);
+        const onAbort = () => {
+            clearTimeout(timeout);
+            reject(abortReason(signal));
+        };
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}
+class Semaphore {
+    maximum;
+    active = 0;
+    waiters = [];
+    constructor(maximum) {
+        this.maximum = maximum;
+    }
+    async acquire(signal) {
+        if (signal?.aborted)
+            throw abortReason(signal);
+        if (this.active < this.maximum) {
+            this.active += 1;
+            return;
+        }
+        await new Promise((resolve, reject) => {
+            const waiter = { resolve, reject, ...(signal === undefined ? {} : { signal }) };
+            if (signal !== undefined) {
+                waiter.onAbort = () => {
+                    const index = this.waiters.indexOf(waiter);
+                    if (index >= 0)
+                        this.waiters.splice(index, 1);
+                    reject(abortReason(signal));
+                };
+                signal.addEventListener("abort", waiter.onAbort, { once: true });
+            }
+            this.waiters.push(waiter);
+        });
+    }
+    release() {
+        const waiter = this.waiters.shift();
+        if (waiter === undefined) {
+            this.active -= 1;
+            return;
+        }
+        if (waiter.signal !== undefined && waiter.onAbort !== undefined)
+            waiter.signal.removeEventListener("abort", waiter.onAbort);
+        waiter.resolve();
+    }
+}
+export class BoundedRateLimiter {
+    config;
+    semaphore;
+    now;
+    sleep;
+    reservationTail = Promise.resolve();
+    nextStartAt = 0;
+    constructor(config, runtime = {}) {
+        this.config = config;
+        if (!Number.isInteger(config.maxConcurrency) || config.maxConcurrency < 1)
+            throw new Error("maxConcurrency must be a positive integer");
+        if (!(config.requestsPerSecond > 0))
+            throw new Error("requestsPerSecond must be positive");
+        this.semaphore = new Semaphore(config.maxConcurrency);
+        this.now = runtime.now ?? Date.now;
+        this.sleep = runtime.sleep ?? sleepWithSignal;
+    }
+    async reserveStart(signal) {
+        let release;
+        const predecessor = this.reservationTail;
+        this.reservationTail = new Promise((resolve) => {
+            release = resolve;
+        });
+        if (signal?.aborted) {
+            release();
+            throw abortReason(signal);
+        }
+        await predecessor;
+        try {
+            if (signal?.aborted)
+                throw abortReason(signal);
+            const startAt = Math.max(this.now(), this.nextStartAt);
+            this.nextStartAt = startAt + 1000 / this.config.requestsPerSecond;
+            return startAt;
+        }
+        finally {
+            release();
+        }
+    }
+    async run(operation, signal) {
+        const startAt = await this.reserveStart(signal);
+        await this.sleep(Math.max(0, startAt - this.now()), signal);
+        await this.semaphore.acquire(signal);
+        try {
+            return await operation();
+        }
+        finally {
+            this.semaphore.release();
+        }
+    }
+}
+/**
+ * Admission gate for a source's HTTP layer (`SourceHttpOptions.schedule`).
+ *
+ * It gates the network call alone: the caller arms its request timeout inside
+ * the admitted operation, so a request held behind pacing or a concurrency
+ * ceiling cannot be reported as a timeout before it ever reaches the network.
+ */
+export function createRateLimitSchedule(config, runtime = {}) {
+    const limiter = new BoundedRateLimiter(config, runtime);
+    return (operation, signal) => limiter.run(operation, signal);
+}

package/dist/literature/sources/arxiv.d.ts

@@ -0,0 +1,22 @@
+import { type SourceHttpOptions, type SourceHttpResult } from "./http.js";
+export interface ArxivPaper {
+    id: string;
+    title: string;
+    abstract: string;
+    authors: string[];
+    published: string;
+    categories: string[];
+    url: string;
+    pdfUrl?: string;
+}
+export interface ArxivSearchInput {
+    readonly query: string;
+    readonly categories?: readonly string[];
+    readonly limit: number;
+}
+export interface ArxivSource {
+    search(input: ArxivSearchInput, signal?: AbortSignal): Promise<SourceHttpResult<ArxivPaper[]>>;
+    lookupExact(id: string, limit: number, signal?: AbortSignal): Promise<SourceHttpResult<ArxivPaper[]>>;
+}
+export declare function parseArxivAtom(xml: string): ArxivPaper[];
+export declare function createArxivSource(options?: SourceHttpOptions): ArxivSource;

package/dist/literature/sources/arxiv.js

@@ -0,0 +1,103 @@
+import { requestText } from "./http.js";
+const BASE_URL = "https://export.arxiv.org/api/query";
+const stripCdata = (value) => value.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1");
+const decodeXmlEntities = (value) => value
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#x([0-9a-fA-F]+);/g, (_, hexadecimal) => String.fromCodePoint(parseInt(hexadecimal, 16)))
+    .replace(/&#(\d+);/g, (_, decimal) => String.fromCodePoint(parseInt(decimal, 10)))
+    .replace(/&amp;/g, "&");
+const collapseWhitespace = (value) => decodeXmlEntities(stripCdata(value)).replace(/\s+/g, " ").trim();
+function extractEntries(xml) {
+    const expression = /<entry\b[^>]*>([\s\S]*?)<\/entry>/g;
+    const entries = [];
+    let match;
+    while ((match = expression.exec(xml)) !== null)
+        entries.push(match[1]);
+    return entries;
+}
+function firstMatch(entry, tag) {
+    return entry.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`))?.[1];
+}
+function allMatches(entry, tag) {
+    const expression = new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, "g");
+    const values = [];
+    let match;
+    while ((match = expression.exec(entry)) !== null)
+        values.push(match[1]);
+    return values;
+}
+function linkAttribute(entry, predicate) {
+    const expression = /<link\b([^>]*)\/>/g;
+    let match;
+    while ((match = expression.exec(entry)) !== null) {
+        const attributes = match[1];
+        if (predicate(attributes))
+            return attributes.match(/href="([^"]+)"/)?.[1];
+    }
+    return undefined;
+}
+function categoryTerms(entry) {
+    const expression = /<category\b([^>]*)\/>/g;
+    const terms = [];
+    let match;
+    while ((match = expression.exec(entry)) !== null) {
+        const term = match[1].match(/term="([^"]+)"/)?.[1];
+        if (term !== undefined)
+            terms.push(term);
+    }
+    return terms;
+}
+export function parseArxivAtom(xml) {
+    return extractEntries(xml).flatMap((entry) => {
+        const rawId = firstMatch(entry, "id");
+        const title = firstMatch(entry, "title");
+        if (rawId === undefined || title === undefined)
+            return [];
+        const idMatch = rawId.match(/abs\/(.+)$/);
+        const authors = allMatches(entry, "author")
+            .map((author) => firstMatch(author, "name"))
+            .filter((name) => name !== undefined)
+            .map(collapseWhitespace);
+        const htmlUrl = linkAttribute(entry, (attributes) => /rel="alternate"/.test(attributes) && /type="text\/html"/.test(attributes));
+        const pdfUrl = linkAttribute(entry, (attributes) => /title="pdf"/.test(attributes));
+        return [
+            {
+                id: idMatch?.[1] ?? collapseWhitespace(rawId),
+                title: collapseWhitespace(title),
+                abstract: collapseWhitespace(firstMatch(entry, "summary") ?? ""),
+                authors,
+                published: collapseWhitespace(firstMatch(entry, "published") ?? ""),
+                categories: categoryTerms(entry),
+                url: htmlUrl ?? collapseWhitespace(rawId),
+                ...(pdfUrl === undefined ? {} : { pdfUrl }),
+            },
+        ];
+    });
+}
+export function createArxivSource(options = {}) {
+    async function fetchPapers(params, signal) {
+        const result = await requestText(`${BASE_URL}?${params}`, options, signal);
+        if (result.status !== "ok")
+            return result;
+        if (!/<feed\b/.test(result.value))
+            return { status: "unavailable", detail: "response schema mismatch: missing Atom feed" };
+        return { status: "ok", value: parseArxivAtom(result.value) };
+    }
+    return {
+        search: (input, signal) => {
+            const categoryQuery = input.categories !== undefined && input.categories.length > 0
+                ? ` AND (${input.categories.map((category) => `cat:${category}`).join(" OR ")})`
+                : "";
+            return fetchPapers(new URLSearchParams({
+                search_query: `all:${input.query}${categoryQuery}`,
+                max_results: String(input.limit),
+                sortBy: "relevance",
+                sortOrder: "descending",
+            }), signal);
+        },
+        lookupExact: (id, limit, signal) => fetchPapers(new URLSearchParams({ id_list: id, max_results: String(limit) }), signal),
+    };
+}

package/dist/literature/sources/http.d.ts

@@ -0,0 +1,28 @@
+import { z } from "zod";
+export interface SourceHttpOptions {
+    readonly fetch?: typeof globalThis.fetch;
+    /**
+     * Admission gate wrapped around each network attempt — a rate limiter, a
+     * concurrency semaphore, or both. The request timeout is armed *inside* it,
+     * so time an attempt spends queueing is never charged against the timeout.
+     */
+    readonly schedule?: <T>(operation: () => Promise<T>, signal?: AbortSignal) => Promise<T>;
+    readonly timeoutMs?: number;
+    readonly maxRetries?: number;
+    readonly retryDelayMs?: number;
+    /** Ceiling on any single retry wait, including one a server asks for via `Retry-After`. */
+    readonly maxRetryDelayMs?: number;
+    readonly sleep?: (milliseconds: number, signal?: AbortSignal) => Promise<void>;
+}
+export type SourceHttpResult<T> = {
+    readonly status: "ok";
+    readonly value: T;
+} | {
+    readonly status: "no_data";
+    readonly detail: string;
+} | {
+    readonly status: "unavailable";
+    readonly detail: string;
+};
+export declare function requestJson<S extends z.ZodType>(url: string, schema: S, options: SourceHttpOptions, signal?: AbortSignal, init?: RequestInit): Promise<SourceHttpResult<z.infer<S>>>;
+export declare function requestText(url: string, options: SourceHttpOptions, signal?: AbortSignal, init?: RequestInit): Promise<SourceHttpResult<string>>;

package/dist/literature/sources/http.js

@@ -0,0 +1,97 @@
+const RETRYABLE_STATUSES = new Set([429, 503]);
+function cancellationReason(signal) {
+    return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
+}
+async function delay(ms, signal) {
+    if (signal?.aborted)
+        throw cancellationReason(signal);
+    await new Promise((resolve, reject) => {
+        const onAbort = () => {
+            clearTimeout(timeout);
+            reject(cancellationReason(signal));
+        };
+        const timeout = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}
+function retryAfterMilliseconds(response) {
+    const value = response.headers.get("retry-after")?.trim();
+    if (!value)
+        return undefined;
+    const seconds = Number(value);
+    if (Number.isFinite(seconds) && seconds >= 0)
+        return seconds * 1000;
+    const date = Date.parse(value);
+    return Number.isFinite(date) ? Math.max(0, date - Date.now()) : undefined;
+}
+async function request(url, options, signal, init) {
+    if (signal?.aborted)
+        throw cancellationReason(signal);
+    const timeoutMs = options.timeoutMs ?? 15_000;
+    const maxRetries = options.maxRetries ?? 0;
+    const retryDelayMs = options.retryDelayMs ?? 1_000;
+    const maxRetryDelayMs = options.maxRetryDelayMs ?? Number.POSITIVE_INFINITY;
+    const schedule = options.schedule ?? ((operation) => operation());
+    const sleep = options.sleep ?? delay;
+    const fetcher = options.fetch ?? globalThis.fetch;
+    const backoff = (attempt) => Math.min(maxRetryDelayMs, retryDelayMs * 2 ** attempt);
+    for (let attempt = 0;; attempt += 1) {
+        let timeout;
+        try {
+            const response = await schedule(async () => {
+                timeout = AbortSignal.timeout(timeoutMs);
+                const combined = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
+                return await fetcher(url, { ...init, signal: combined });
+            }, signal);
+            if (RETRYABLE_STATUSES.has(response.status) && attempt < maxRetries) {
+                await sleep(Math.min(maxRetryDelayMs, retryAfterMilliseconds(response) ?? backoff(attempt)), signal);
+                continue;
+            }
+            if (response.status === 404 || response.status === 400 || response.status === 406) {
+                return { status: "no_data", detail: `HTTP ${response.status}` };
+            }
+            if (!response.ok)
+                return { status: "unavailable", detail: `HTTP ${response.status}` };
+            return { status: "ok", value: response };
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw cancellationReason(signal);
+            if (timeout?.aborted)
+                return { status: "unavailable", detail: `request timed out after ${timeoutMs}ms` };
+            if (attempt < maxRetries) {
+                await sleep(backoff(attempt), signal);
+                continue;
+            }
+            return { status: "unavailable", detail: error instanceof Error ? error.message : String(error) };
+        }
+    }
+}
+export async function requestJson(url, schema, options, signal, init = {}) {
+    const response = await request(url, options, signal, init);
+    if (response.status !== "ok")
+        return response;
+    try {
+        const parsed = schema.safeParse(await response.value.json());
+        if (!parsed.success)
+            return { status: "unavailable", detail: `response schema mismatch: ${parsed.error.issues[0]?.message ?? "unknown"}` };
+        return { status: "ok", value: parsed.data };
+    }
+    catch (error) {
+        return { status: "unavailable", detail: `invalid JSON response: ${error instanceof Error ? error.message : String(error)}` };
+    }
+}
+export async function requestText(url, options, signal, init = {}) {
+    const response = await request(url, options, signal, init);
+    if (response.status !== "ok")
+        return response;
+    try {
+        return { status: "ok", value: await response.value.text() };
+    }
+    catch (error) {
+        return { status: "unavailable", detail: `invalid text response: ${error instanceof Error ? error.message : String(error)}` };
+    }
+}

package/dist/literature/sources/pubmed.d.ts

@@ -0,0 +1,68 @@
+import { type SourceHttpOptions, type SourceHttpResult } from "./http.js";
+export declare const NCBI_BASE = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils";
+export declare const NCBI_IDCONV = "https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0";
+export interface PubmedSourceOptions extends SourceHttpOptions {
+    readonly apiKey?: string;
+}
+export interface PubmedSearchOptions {
+    readonly maxResults?: number;
+    readonly sort?: "relevance" | "date";
+    readonly dateRange?: {
+        readonly from: string;
+        readonly to: string;
+    };
+}
+export interface PubmedCitationFields {
+    readonly venue?: string;
+    readonly year?: string | number;
+    readonly volume?: string;
+    readonly firstPage?: string;
+    readonly firstAuthor?: string;
+    readonly key: string;
+}
+export interface PubMedSummary {
+    pmid: string;
+    title: string;
+    journal: string;
+    year: string;
+    authors: string;
+}
+export interface ArticleDetail {
+    pmid: string;
+    title: string;
+    abstract: string;
+    authors: string[];
+    journal: string;
+    year: string;
+    doi: string;
+    meshTerms: string[];
+    pmcId: string | null;
+}
+export interface ArticleSection {
+    heading: string;
+    text: string;
+}
+export interface FullTextResult {
+    fullText: string;
+    sections: ArticleSection[];
+}
+export interface PubmedSource {
+    search(query: string, options?: PubmedSearchOptions, signal?: AbortSignal): Promise<SourceHttpResult<{
+        totalFound: number;
+        results: PubMedSummary[];
+    }>>;
+    searchIds(query: string, maxResults: number, signal?: AbortSignal): Promise<SourceHttpResult<string[]>>;
+    matchCitation(fields: PubmedCitationFields, signal?: AbortSignal): Promise<SourceHttpResult<string[]>>;
+    fetchArticles(pmids: readonly string[], signal?: AbortSignal): Promise<SourceHttpResult<ArticleDetail[]>>;
+    articleDetails(pmids: readonly string[], signal?: AbortSignal): Promise<SourceHttpResult<{
+        articles: ArticleDetail[];
+        notFound: string[];
+    }>>;
+    fetchFullText(pmcId: string, signal?: AbortSignal): Promise<SourceHttpResult<FullTextResult | null>>;
+}
+export declare function ncbiUrl(apiKey: string | undefined, base: string, params: Record<string, string | number | undefined>): string;
+export declare function parseEsummary(xml: string): PubMedSummary[];
+export declare function parseEfetch(xml: string): ArticleDetail[];
+export declare function parseIdConvResponse(data: unknown): Map<string, string>;
+export declare function parsePmcFullText(xml: string): FullTextResult | null;
+export declare function createPubmedSource(options?: PubmedSourceOptions): PubmedSource;