@oomol-lab/open-connector 1.9.1 → 1.9.3

package/src/providers/tinyfish/data-actions.js

@@ -0,0 +1,186 @@
+import { readSchemaProperties } from "../../core/json-schema.js";
+import { s } from "../../core/json-schema.js";
+import { fetchInput, fetchOutput, searchInput, searchOutput, usageInput, usageOutput } from "./data-schemas.js";
+import { monitorCreateInput, monitorId, monitorOutput, monitorUpdateInput } from "./monitor-schemas.js";
+import { researchId, researchInput, researchListInput, researchOutput, researchStatus } from "./research-schemas.js";
+const researchIdInput = s.requiredObject("Research run identifier.", { research_run_id: researchId });
+const monitorIdInput = s.requiredObject("Monitor identifier.", { id: monitorId });
+const usageQueryKeys = Object.keys(readSchemaProperties(usageInput));
+export const dataEndpoints = [
+    {
+        name: "search",
+        description: "Search web pages, news or research papers with optional content fetching.",
+        operationType: "read",
+        origin: "search",
+        method: "GET",
+        path: "/",
+        queryKeys: Object.keys(readSchemaProperties(searchInput)),
+        inputSchema: searchInput,
+        outputSchema: searchOutput,
+    },
+    {
+        name: "list_search_usage",
+        description: "List paginated search usage records and recorded results.",
+        operationType: "read",
+        origin: "search",
+        method: "GET",
+        path: "/usage",
+        queryKeys: usageQueryKeys,
+        inputSchema: usageInput,
+        outputSchema: usageOutput,
+    },
+    {
+        name: "fetch_urls",
+        description: "Fetch up to ten URLs with content extraction, selectors and conditional request support; inspect per-URL errors.",
+        operationType: "read",
+        origin: "fetchApi",
+        method: "POST",
+        path: "/",
+        inputSchema: fetchInput,
+        outputSchema: fetchOutput,
+    },
+    {
+        name: "list_fetch_usage",
+        description: "List paginated fetch usage records.",
+        operationType: "read",
+        origin: "fetchApi",
+        method: "GET",
+        path: "/usage",
+        queryKeys: usageQueryKeys,
+        inputSchema: usageInput,
+        outputSchema: usageOutput,
+    },
+    {
+        name: "start_research",
+        description: "Start deep or maximum-depth research and retrieve its reports through get_research_run.",
+        operationType: "write",
+        origin: "main",
+        method: "POST",
+        path: "/v1/automation/run-research-async",
+        inputSchema: researchInput,
+        outputSchema: s.looseObject("Research submission.", {
+            research_run_id: researchId,
+            error: s.nullable(s.looseObject("Run creation error; null on success.")),
+        }),
+    },
+    {
+        name: "list_research_runs",
+        description: "List research runs with query, state, timestamp and cursor filters.",
+        operationType: "read",
+        origin: "main",
+        method: "GET",
+        path: "/v1/research-run",
+        queryKeys: Object.keys(readSchemaProperties(researchListInput)),
+        inputSchema: researchListInput,
+        outputSchema: s.looseObject("Research run page.", {
+            data: s.array("Research runs.", researchOutput),
+            pagination: s.looseObject("Pagination metadata.", {
+                total: s.integer("Total matching runs."),
+                next_cursor: s.nullable(s.string("Next-page cursor.")),
+                has_more: s.boolean("Whether more results exist."),
+            }),
+        }),
+    },
+    {
+        name: "get_research_run",
+        description: "Get research progress, final reports, sources and terminal outcome.",
+        operationType: "read",
+        origin: "main",
+        method: "GET",
+        path: "/v1/research-run/{research_run_id}",
+        pathKeys: ["research_run_id"],
+        inputSchema: researchIdInput,
+        outputSchema: researchOutput,
+    },
+    {
+        name: "cancel_research_run",
+        description: "Cancel a research run and return whether its state changed.",
+        operationType: "destructive",
+        origin: "main",
+        method: "POST",
+        path: "/v1/research-run/{research_run_id}/cancel",
+        pathKeys: ["research_run_id"],
+        inputSchema: researchIdInput,
+        outputSchema: s.looseObject("Cancellation outcome.", {
+            research_run_id: researchId,
+            status: researchStatus,
+            cancelled: s.boolean("Whether this request cancelled the run."),
+        }),
+    },
+    {
+        name: "create_monitor",
+        description: "Create a Page or Topic Monitor with a schedule and baseline run; requires a personal API key.",
+        operationType: "write",
+        origin: "main",
+        method: "POST",
+        path: "/v1/monitors",
+        inputSchema: monitorCreateInput,
+        outputSchema: s.looseObject("Created monitor and baseline.", {
+            monitor: monitorOutput,
+            run: s.looseObject("Baseline page or topic run."),
+        }),
+    },
+    {
+        name: "list_monitors",
+        description: "List Page and Topic Monitors owned by the personal API key.",
+        operationType: "read",
+        origin: "main",
+        method: "GET",
+        path: "/v1/monitors",
+        inputSchema: s.requiredObject("No parameters.", {}),
+        outputSchema: s.looseObject("Monitor list.", {
+            monitors: s.array("Owned monitors.", monitorOutput),
+        }),
+    },
+    {
+        name: "get_monitor",
+        description: "Get a Page or Topic Monitor and its configuration.",
+        operationType: "read",
+        origin: "main",
+        method: "GET",
+        path: "/v1/monitors/{id}",
+        pathKeys: ["id"],
+        inputSchema: monitorIdInput,
+        outputSchema: monitorOutput,
+    },
+    {
+        name: "update_monitor",
+        description: "Pause or resume a Monitor, or update its schedule and optional metadata; these update modes cannot be combined.",
+        operationType: "destructive",
+        origin: "main",
+        method: "PATCH",
+        path: "/v1/monitors/{id}",
+        pathKeys: ["id"],
+        inputSchema: monitorUpdateInput,
+        outputSchema: monitorOutput,
+    },
+    {
+        name: "delete_monitor",
+        description: "Delete a Monitor and stop its scheduled runs.",
+        operationType: "destructive",
+        origin: "main",
+        method: "DELETE",
+        path: "/v1/monitors/{id}",
+        pathKeys: ["id"],
+        inputSchema: monitorIdInput,
+        outputSchema: s.looseObject("Deletion result.", {
+            deleted: s.literal(true, {
+                description: "Whether deletion succeeded.",
+            }),
+            id: monitorId,
+        }),
+    },
+    {
+        name: "run_monitor",
+        description: "Run a Monitor immediately and return its Page or Topic results.",
+        operationType: "write",
+        origin: "main",
+        method: "POST",
+        path: "/v1/monitors/{id}/runs",
+        pathKeys: ["id"],
+        inputSchema: monitorIdInput,
+        outputSchema: s.looseObject("Monitor run result.", {
+            run: s.looseObject("Page or Topic run, including change detection results."),
+        }),
+    },
+];

package/src/providers/tinyfish/data-schemas.d.ts

@@ -0,0 +1,7 @@
+import type { JsonSchema } from "../../core/types.ts";
+export declare const usageInput: JsonSchema;
+export declare const usageOutput: JsonSchema;
+export declare const fetchInput: JsonSchema;
+export declare const fetchOutput: JsonSchema;
+export declare const searchInput: JsonSchema;
+export declare const searchOutput: JsonSchema;

package/src/providers/tinyfish/data-schemas.js

@@ -0,0 +1,103 @@
+import { s } from "../../core/json-schema.js";
+export const usageInput = s.object("Usage filters.", {
+    start_after: s.string("Start timestamp, exclusive.", { format: "date-time" }),
+    end_before: s.string("End timestamp, exclusive.", { format: "date-time" }),
+    status: s.stringEnum("Usage outcome.", ["completed", "failed"]),
+    limit: s.integer("Records per page; defaults to 100.", { minimum: 1, maximum: 1000 }),
+    page: s.integer("Page number, starting at 1.", { minimum: 1 }),
+}, { required: [] });
+export const usageOutput = s.looseObject("Paginated usage records.", {
+    items: s.array("Usage records.", s.looseObject("Usage record.")),
+    total: s.integer("Total records."),
+    limit: s.integer("Page size."),
+    page: s.integer("Current page."),
+    total_pages: s.integer("Total pages."),
+    has_more: s.boolean("Whether another page exists."),
+});
+const fetchOptions = {
+    purpose: s.string("Underlying task used to tailor extraction.", {
+        minLength: 1,
+        maxLength: 2000,
+    }),
+    format: s.stringEnum("Extracted content format; defaults to markdown.", ["markdown", "html", "json"]),
+    links: s.boolean("Extract outbound links."),
+    image_links: s.boolean("Extract image URLs."),
+    iframe_links: s.boolean("Extract iframe URLs; browser-rendered frames require browser rendering."),
+    page_metadata: s.boolean("Include canonical URL, favicon, robots, Open Graph, Twitter and other page metadata."),
+    ttl: s.integer("Cache freshness tolerance in seconds; 0 prefers live fetch, but origin caching may extend tolerance.", { minimum: 0 }),
+    per_url_timeout_ms: s.integer("Independent timeout budget per URL in milliseconds.", {
+        minimum: 1,
+        maximum: 110000,
+    }),
+    if_none_match: s.string("ETag to replay; valid only with one URL.", { minLength: 1 }),
+    if_modified_since: s.string("Last-Modified validator to replay; valid only with one URL.", {
+        minLength: 1,
+    }),
+    include_etag_and_last_modified: s.boolean("Include validators and conditional not_modified results."),
+    include_selectors: s.array("Extract only matching CSS regions; total misses return selector_not_matched with retry hints.", s.string("CSS selector.", { minLength: 1, maxLength: 1000 }), { minItems: 1, maxItems: 20 }),
+    exclude_selectors: s.array("Remove matching CSS regions before extraction; direct PDF/CSV returns selector_unsupported.", s.string("CSS selector.", { minLength: 1, maxLength: 1000 }), { minItems: 1, maxItems: 20 }),
+};
+export const fetchInput = {
+    ...s.object("Fetch one or more URLs independently.", {
+        urls: s.array("URLs fetched by TinyFish.", s.string("Public URL.", { format: "uri" }), {
+            minItems: 1,
+            maxItems: 10,
+        }),
+        ...fetchOptions,
+    }, { required: ["urls"] }),
+    anyOf: [
+        { not: { anyOf: [{ required: ["if_none_match"] }, { required: ["if_modified_since"] }] } },
+        { properties: { urls: { maxItems: 1 } } },
+    ],
+};
+export const fetchOutput = s.looseObject("Fetch results with independent per-URL errors.", {
+    results: s.array("Successfully fetched URLs including conditional results.", s.looseObject("Fetched page.", {
+        url: s.string("Requested URL."),
+        final_url: s.nullable(s.string("Final URL.")),
+        title: s.nullable(s.string("Page title.")),
+        text: s.unknown("Extracted markdown, HTML or structured JSON; may be null."),
+        format: s.stringEnum("Content format.", ["markdown", "html", "json"]),
+        not_modified: s.boolean("True when the origin returned 304."),
+        etag: s.string("ETag validator."),
+        last_modified: s.string("Last-Modified validator."),
+        unmatched_selectors: s.stringArray("Selectors that did not match while others succeeded."),
+    })),
+    errors: s.array("Failures for individual URLs.", s.looseObject("Fetch failure.", {
+        url: s.string("Failed URL."),
+        error: s.string("Provider error code."),
+        status: s.integer("Origin HTTP status when applicable."),
+        unmatched_selectors: s.stringArray("Selectors with no match."),
+        candidate_selectors: s.stringArray("Suggested selectors for a retry."),
+    })),
+    request_id: s.string("Correlation ID, present for failed URLs."),
+});
+export const searchInput = s.object("Search query and filters.", {
+    query: s.string("Search query.", { minLength: 1, maxLength: 2000 }),
+    purpose: s.string("Underlying task used to rank results.", { minLength: 1, maxLength: 2000 }),
+    location: s.string("Country code; inferred from language when omitted."),
+    language: s.string("Language code; inferred from location when omitted."),
+    include_domains: s.string("Comma-separated domains to include."),
+    exclude_domains: s.string("Comma-separated domains to exclude."),
+    domain_type: s.stringEnum("Search source pool.", ["web", "news", "research_paper"]),
+    after_date: s.string("Return results after YYYY-MM-DD."),
+    before_date: s.string("Return results before YYYY-MM-DD."),
+    recency_minutes: s.integer("Freshness window in minutes.", { minimum: 1, maximum: 5256000 }),
+    pub_year_min: s.integer("Minimum research-paper publication year. Only supported for domain_type=research_paper.", {
+        minimum: 0,
+        maximum: 9999,
+    }),
+    pub_year_max: s.integer("Maximum research-paper publication year. Only supported for domain_type=research_paper.", {
+        minimum: 0,
+        maximum: 9999,
+    }),
+    page: s.nullable(s.integer("Page index, starting at 0.", { minimum: 0, maximum: 10 })),
+    include_thumbnail: s.stringEnum("Include available thumbnails.", ["true", "false"]),
+    fetch: s.looseObject("Fetch configuration; encoded as JSON in the query string, maximum 256 encoded characters."),
+}, { required: ["query"] });
+export const searchOutput = s.looseObject("Search results.", {
+    query: s.string("Executed query."),
+    results: s.array("Ranked results with optional fetched content and fetch_error.", s.looseObject("Search result.")),
+    total_results: s.integer("Total results."),
+    page: s.integer("Current zero-based page."),
+    request_id: s.string("Correlation ID when any fetch failed."),
+});

package/src/providers/tinyfish/endpoints.d.ts

@@ -0,0 +1,14 @@
+import type { ActionDefinition, JsonSchema } from "../../core/types.ts";
+export interface TinyfishEndpoint {
+    name: string;
+    description: string;
+    operationType: ActionDefinition["operationType"];
+    origin: "main" | "search" | "fetchApi" | "browser";
+    path: string;
+    method: "GET" | "POST" | "PATCH" | "DELETE";
+    pathKeys?: string[];
+    queryKeys?: string[];
+    inputSchema: JsonSchema;
+    outputSchema: JsonSchema;
+    responseType?: "screenshot" | "html";
+}

package/src/providers/tinyfish/endpoints.js

@@ -0,0 +1 @@
+export {};

package/src/providers/tinyfish/executors.d.ts

@@ -0,0 +1,4 @@
+import type { CredentialValidators, ProviderExecutors, ProviderProxyExecutor } from "../../core/types.ts";
+export declare const credentialValidators: CredentialValidators;
+export declare const executors: ProviderExecutors;
+export declare const proxy: ProviderProxyExecutor;

package/src/providers/tinyfish/executors.js

@@ -0,0 +1,133 @@
+import { optionalRecord, optionalString } from "../../core/cast.js";
+import { storeResponseInTransit } from "../../core/request.js";
+import { requiredResponseRecord, isAbortLikeError, runProviderRequest, ProviderRequestError, providerInputError, providerResponseError, defineApiKeyProviderExecutors, defineProviderProxy, } from "../provider-runtime.js";
+import { tinyfishActions } from "./actions.js";
+import { tinyfishEndpointByName } from "./actions.js";
+const origins = {
+    main: "https://agent.tinyfish.ai",
+    search: "https://api.search.tinyfish.ai",
+    fetchApi: "https://api.fetch.tinyfish.ai",
+    browser: "https://api.browser.tinyfish.ai",
+};
+async function request(endpoint, input, apiKey, fetcher, validating = false, transitFiles, parentSignal) {
+    const body = { ...input };
+    let path = endpoint.path;
+    for (const key of endpoint.pathKeys ?? []) {
+        const value = String(body[key]);
+        if (value === "." || value === "..")
+            throw providerInputError(`Invalid ${key}`);
+        path = path.replace(`{${key}}`, encodeURIComponent(value));
+        delete body[key];
+    }
+    const url = new URL(path, origins[endpoint.origin]);
+    for (const key of endpoint.queryKeys ?? []) {
+        const value = body[key];
+        if (endpoint.name === "search" && key === "fetch" && JSON.stringify(value)?.length > 256)
+            throw providerInputError("Search fetch configuration must encode to at most 256 characters");
+        if (value !== undefined && value !== null)
+            url.searchParams.set(key, typeof value === "object" ? JSON.stringify(value) : String(value));
+        delete body[key];
+    }
+    const headers = new Headers({
+        "X-API-Key": apiKey,
+        accept: endpoint.responseType === "screenshot"
+            ? "image/jpeg"
+            : endpoint.responseType === "html"
+                ? "text/html"
+                : "application/json",
+    });
+    if (endpoint.name === "top_up_wallet") {
+        const amount = Number(body.amount);
+        if (amount < 10 || amount > 500)
+            throw providerInputError("Wallet top-up amount must be between 10 and 500 USD");
+        const paymentAuthorization = optionalString(body.paymentAuthorization);
+        if (paymentAuthorization)
+            headers.set("Payment-Authorization", paymentAuthorization);
+        delete body.paymentAuthorization;
+    }
+    const hasBody = endpoint.method === "POST" || endpoint.method === "PATCH";
+    if (hasBody)
+        headers.set("content-type", "application/json");
+    if (endpoint.responseType && !transitFiles)
+        throw new ProviderRequestError(500, "TinyFish snapshots require file transit and execution context", undefined, "provider_error");
+    const timeoutMs = endpoint.origin === "fetchApi" && endpoint.method === "POST" ? 150000 : undefined;
+    return runProviderRequest({ label: "TinyFish", timeoutMs, signal: parentSignal }, async (signal) => {
+        const response = await fetcher(url, {
+            method: endpoint.method,
+            headers,
+            body: hasBody ? JSON.stringify(body) : undefined,
+            signal,
+        });
+        if (response.ok && endpoint.responseType) {
+            const mimeType = endpoint.responseType === "screenshot" ? "image/jpeg" : "text/html";
+            const file = await storeResponseInTransit(response, transitFiles, {
+                name: endpoint.responseType === "screenshot" ? "tinyfish-screenshot.jpg" : "tinyfish-snapshot.html",
+                mimeType,
+                fieldName: "TinyFish snapshot",
+                createError: providerResponseError,
+                signal,
+            });
+            return { url: file.downloadUrl, mimeType };
+        }
+        if (response.status === 204 && response.ok)
+            return { success: true };
+        const payload = await response.json().catch((error) => {
+            if (isAbortLikeError(error))
+                throw error;
+            return undefined;
+        });
+        if (endpoint.name === "top_up_wallet" && response.status === 402 && response.headers.has("www-authenticate")) {
+            return {
+                status: "payment_required",
+                challenge: response.headers.get("www-authenticate"),
+                body: payload ?? null,
+            };
+        }
+        if (!response.ok) {
+            const object = optionalRecord(payload);
+            const error = optionalRecord(object?.error);
+            const code = optionalString(error?.code);
+            const message = optionalString(error?.message) ??
+                optionalString(object?.message) ??
+                optionalString(object?.detail) ??
+                optionalString(object?.title);
+            const hint = optionalString(object?.hint) ?? optionalString(error?.hint);
+            const invalidKey = code === "INVALID_API_KEY" || code === "MISSING_API_KEY";
+            throw new ProviderRequestError(invalidKey && validating ? 400 : response.status, `TinyFish request failed (${response.status})${code ? `: ${code}` : ""}${message ? `: ${message}` : ""}${hint ? `: ${hint}` : ""}`, undefined, invalidKey ? (validating ? "invalid_input" : "authorization_failed") : "provider_error");
+        }
+        const result = requiredResponseRecord(payload, "TinyFish response");
+        const taskIdFields = { submit_run: "run_id", start_research: "research_run_id" };
+        const taskIdField = taskIdFields[endpoint.name];
+        if (taskIdField && !optionalString(result[taskIdField]))
+            throw providerResponseError("TinyFish task creation failed: missing task ID");
+        if (endpoint.name === "submit_run_batch" && !Array.isArray(result.run_ids)) {
+            const error = optionalRecord(result.error);
+            throw providerResponseError(`TinyFish batch creation failed: ${optionalString(error?.message) ?? "missing run IDs"}`);
+        }
+        return result;
+    });
+}
+async function execute(actionName, values, context) {
+    const { apiKey, fetcher } = context;
+    const endpoint = tinyfishEndpointByName.get(actionName);
+    if (!endpoint)
+        throw providerInputError("Unknown TinyFish action");
+    return request(endpoint, values, apiKey, fetcher, false, context.transitFiles, context.signal);
+}
+export const credentialValidators = {
+    async apiKey(input, { fetcher, signal }) {
+        const endpoint = [...tinyfishEndpointByName.values()].find((entry) => entry.origin === "search" && entry.path === "/usage");
+        await request(endpoint, { limit: 1 }, input.apiKey, fetcher, true, undefined, signal);
+        return { profile: { displayName: "TinyFish API Key" }, grantedScopes: [], metadata: {} };
+    },
+};
+export const executors = defineApiKeyProviderExecutors("tinyfish", Object.fromEntries(tinyfishActions.map((action) => [action.name, (values, context) => execute(action.name, values, context)])));
+export const proxy = defineProviderProxy({
+    service: "tinyfish",
+    baseUrl: origins.main,
+    allowedOrigins: Object.values(origins),
+    auth: { type: "api_key_header", name: "X-API-Key" },
+    customizeRequest({ headers }) {
+        headers.set("accept", "application/json");
+    },
+});

package/src/providers/tinyfish/management-actions.d.ts

@@ -0,0 +1,2 @@
+import type { TinyfishEndpoint } from "./endpoints.ts";
+export declare const managementEndpoints: TinyfishEndpoint[];