@oomol-lab/open-connector 1.9.1 → 1.9.3

package/src/providers/scraperapi/runtime.js

@@ -1,6 +1,6 @@
 import { compactObject, optionalBoolean, optionalNumber, optionalRecord, optionalRawString, optionalString, } from "../../core/cast.js";
 import { assertPublicHttpUrl, readBoundedResponseBytes } from "../../core/request.js";
-import { createProviderTimeout, isAbortLikeError, providerUserAgent, ProviderRequestError, } from "../provider-runtime.js";
+import { createProviderTimeout, isAbortLikeError, providerUserAgent, ProviderRequestError, providerInputError, } from "../provider-runtime.js";
 export const scraperapiApiBaseUrl = "https://api.scraperapi.com/";
 const scraperapiAccountUrl = "https://api.scraperapi.com/account";
 const scraperapiRequestTimeoutMs = 80_000;
@@ -102,7 +102,7 @@ async function requestScraperapiUrl(url, query, init, context, phase) {
 function buildScraperapiUrl(url, query, apiKey) {
     const requestUrl = assertPublicHttpUrl(url, {
         fieldName: "url",
-        createError: (message) => new ProviderRequestError(400, message),
+        createError: providerInputError,
     });
     requestUrl.searchParams.set("api_key", apiKey);
     for (const [key, value] of Object.entries(query)) {
@@ -190,13 +190,13 @@ function createScraperapiError(status, body, phase) {
         return new ProviderRequestError(429, message);
     }
     if (phase === "validate" && status >= 400 && status < 500) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     if (phase === "execute" && status === 401) {
         return new ProviderRequestError(401, message);
     }
     if (phase === "execute" && status === 400) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     return new ProviderRequestError(status || 500, message);
 }

package/src/providers/scrapfly/executors.js

@@ -1,5 +1,5 @@
 import { compactObject, optionalBoolean, optionalInteger, optionalRawString, optionalRecord, optionalString, requiredString, } from "../../core/cast.js";
-import { defineApiKeyProviderExecutors, defineProviderProxy, ProviderRequestError, providerUserAgent, requiredInputString, } from "../provider-runtime.js";
+import { defineApiKeyProviderExecutors, defineProviderProxy, ProviderRequestError, providerUserAgent, requiredInputString, providerInputError, } from "../provider-runtime.js";
 const service = "scrapfly";
 const scrapflyApiBaseUrl = "https://api.scrapfly.io";
 const scrapflyScrapeApiUrl = `${scrapflyApiBaseUrl}/scrape`;
@@ -203,7 +203,7 @@ function createScrapflyError(status, body, headers, phase) {
         return new ProviderRequestError(429, message);
     }
     if (phase === "validate" && status >= 400 && status < 500) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     return new ProviderRequestError(status || 500, message);
 }

package/src/providers/scrapingant/executors.js

@@ -1,9 +1,8 @@
 import { compactObject, optionalBoolean, optionalInteger, optionalRawString, optionalRecord, optionalString, requiredString, } from "../../core/cast.js";
-import { defineApiKeyProviderExecutors, defineProviderProxy, ProviderRequestError, providerUserAgent, requiredInputString, } from "../provider-runtime.js";
+import { defineApiKeyProviderExecutors, defineProviderProxy, ProviderRequestError, providerUserAgent, requiredInputString, providerInputError, defaultProviderRequestTimeoutMs, } from "../provider-runtime.js";
 const service = "scrapingant";
 const scrapingantApiOrigin = "https://api.scrapingant.com";
 const scrapingantApiBaseUrl = `${scrapingantApiOrigin}/v2`;
-const scrapingantDefaultTimeoutMs = 30_000;
 export const scrapingantActionHandlers = {
     scrape_with_extended_json_output(input, context) {
         return requestScrapingantExtended(input, context);
@@ -138,15 +137,15 @@ function buildScrapingantBody(input) {
 function buildConnectorTimeoutMs(input) {
     const timeoutSeconds = optionalInteger(input.timeout);
     return timeoutSeconds === undefined
-        ? scrapingantDefaultTimeoutMs
-        : Math.max(scrapingantDefaultTimeoutMs, timeoutSeconds * 1000 + 10_000);
+        ? defaultProviderRequestTimeoutMs
+        : Math.max(defaultProviderRequestTimeoutMs, timeoutSeconds * 1000 + 10_000);
 }
 function resolveScrapingantMethod(input) {
     return optionalString(input.method) ?? "GET";
 }
 async function requestScrapingantJson(input) {
     const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? scrapingantDefaultTimeoutMs);
+    const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? defaultProviderRequestTimeoutMs);
     const signal = mergeAbortSignals(controller.signal, input.context.signal);
     try {
         const response = await input.context.fetcher(buildScrapingantUrl(input.path, input.query), {
@@ -291,13 +290,13 @@ function createScrapingantError(status, payload, phase) {
         return new ProviderRequestError(429, message);
     }
     if (phase === "validate" && status >= 400 && status < 500) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     if (phase === "execute" && status === 403) {
         return new ProviderRequestError(403, message);
     }
     if (phase === "execute" && (status === 400 || status === 404 || status === 405 || status === 422)) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     return new ProviderRequestError(status || 500, message);
 }

package/src/providers/scrapingbee/executors.js

@@ -1,5 +1,5 @@
 import { booleanString, compactObject, optionalRecord, requiredString } from "../../core/cast.js";
-import { defineApiKeyProviderExecutors, defineProviderProxy, ProviderRequestError } from "../provider-runtime.js";
+import { defineApiKeyProviderExecutors, defineProviderProxy, ProviderRequestError, providerInputError, providerResponseError, } from "../provider-runtime.js";
 const service = "scrapingbee";
 const scrapingbeeApiOrigin = "https://app.scrapingbee.com";
 const scrapingbeeApiBasePath = "/api/v1/";
@@ -140,7 +140,7 @@ function parseScrapingbeeUsage(payload) {
 function createScrapingbeeError(status, body, phase) {
     const message = extractErrorMessage(body) ?? `ScrapingBee request failed with status ${status}`;
     if (phase === "validate" && [400, 401, 403].includes(status)) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     return new ProviderRequestError(status || 500, message);
 }
@@ -168,7 +168,7 @@ function parseScrapingbeeJson(body, invalidMessage) {
         return JSON.parse(body);
     }
     catch {
-        throw new ProviderRequestError(502, invalidMessage);
+        throw providerResponseError(invalidMessage);
     }
 }
 function readRequiredUrl(value) {

package/src/providers/scrapingdog/executors.js

@@ -169,7 +169,7 @@ function buildScrapingdogUrl(path, query, apiKey) {
 function createScrapingdogError(status, body, phase) {
     const message = extractScrapingdogMessage(body) ?? `Scrapingdog request failed with ${status}`;
     if (phase === "validate" && [400, 401, 403].includes(status)) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     if (status === 403) {
         return new ProviderRequestError(429, message);

package/src/providers/screenshot_fyi/executors.js

@@ -1,9 +1,8 @@
 import { compactObject, optionalBoolean, optionalInteger, optionalRecord, optionalString } from "../../core/cast.js";
-import { defineApiKeyProviderExecutors, defineProviderProxy, providerUserAgent, ProviderRequestError, } from "../provider-runtime.js";
+import { defineApiKeyProviderExecutors, defineProviderProxy, providerUserAgent, ProviderRequestError, providerInputError, defaultProviderRequestTimeoutMs, } from "../provider-runtime.js";
 const service = "screenshot_fyi";
 const screenshotFyiApiBaseUrl = "https://www.screenshot.fyi";
 const screenshotFyiCapturePath = "/api/take";
-const screenshotFyiDefaultRequestTimeoutMs = 30_000;
 const screenshotFyiValidationUrl = "https://example.com";
 export const screenshotFyiActionHandlers = {
     async take_screenshot(input, context) {
@@ -68,7 +67,7 @@ function buildTakeScreenshotQuery(input) {
     });
 }
 async function requestScreenshotFyiJson(input) {
-    const timeout = createTimeoutSignal(input.context.signal, screenshotFyiDefaultRequestTimeoutMs);
+    const timeout = createTimeoutSignal(input.context.signal, defaultProviderRequestTimeoutMs);
     try {
         const response = await input.context.fetcher(buildScreenshotFyiUrl(input.apiKey, input.query), {
             method: "GET",
@@ -131,7 +130,7 @@ function parseScreenshotFyiCapturePayload(payload) {
 function createScreenshotFyiError(response, payload, phase) {
     const message = extractScreenshotFyiErrorMessage(payload) ?? `screenshot.fyi request failed with status ${response.status}`;
     if (phase === "validate" && response.status >= 400 && response.status < 500) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     return new ProviderRequestError(response.status || 500, message);
 }

package/src/providers/screenshotbase/runtime.js

@@ -1,5 +1,5 @@
 import { optionalBoolean, optionalInteger, optionalNumber, optionalRecord, optionalString } from "../../core/cast.js";
-import { createProviderTimeout, ProviderRequestError, providerUserAgent, requiredResponseRecord, } from "../provider-runtime.js";
+import { createProviderTimeout, ProviderRequestError, providerUserAgent, requiredResponseRecord, providerInputError, } from "../provider-runtime.js";
 export const screenshotbaseApiBaseUrl = "https://api.screenshotbase.com";
 const screenshotbaseStatusPath = "/v1/status";
 const screenshotbaseTakePath = "/v1/take";
@@ -144,13 +144,13 @@ function createScreenshotbaseError(status, payload, phase) {
         return new ProviderRequestError(429, message);
     }
     if (phase === "validate" && (status === 401 || status === 403)) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     if (phase === "execute" && (status === 401 || status === 403)) {
         return new ProviderRequestError(401, message);
     }
     if (status === 400 || status === 422) {
-        return new ProviderRequestError(400, message);
+        return providerInputError(message);
     }
     if (status === 408) {
         return new ProviderRequestError(504, message);

package/src/providers/search1api/actions.d.ts

@@ -0,0 +1,2 @@
+import type { ActionDefinition } from "../../core/types.ts";
+export declare const search1apiActions: ActionDefinition[];

package/src/providers/search1api/actions.js

@@ -0,0 +1,39 @@
+import { s } from "../../core/json-schema.js";
+import { defineProviderAction } from "../../core/provider-definition.js";
+import { search1apiInputs, search1apiOutputs } from "./schemas.js";
+const actionLifecycles = {
+    deepcrawl: { startActionId: "search1api.deepcrawl", statusActionId: "search1api.get_deepcrawl" },
+};
+const descriptions = {
+    search: "Search the web with engine, site, language, time, pagination, image and full-content options.",
+    search_batch: "Run multiple web searches in one batch, preserving each success, failure and credit cost.",
+    news: "Search recent news with source, site, language, time, image and full-content options.",
+    news_batch: "Run multiple news searches in one batch, preserving each success, failure and credit cost.",
+    ask: "Select search engines and rank relevant sources from a natural-language query. Returns sources, not a generated answer; partial engine failures are preserved.",
+    crawl: "Read a public webpage as clean content, with optional fallback retrieval.",
+    crawl_batch: "Read multiple public webpages in one batch, preserving individual results and failures.",
+    sitemap: "Discover page URLs from a site's sitemap or all available links.",
+    trending: "Retrieve trending topics from the requested source.",
+    extract: "Extract structured data from a webpage using a prompt and response schema. This upstream API is in beta.",
+    deepcrawl: "Start an asynchronous whole-site crawl. Use get_deepcrawl to wait for the archive download URL. This upstream API is in beta.",
+    get_deepcrawl: "Read the state and archive download URL of a deepcrawl task.",
+    screenshot: "Capture a public webpage or element as PNG, JPEG or WebP and return a temporary file URL.",
+    usage: "Read the authenticated account's remaining credits without consuming credits.",
+    health: "Check public API health. This does not validate credentials.",
+    feedback: "Submit product feedback. Do not include credentials, personal data or private content; do not automatically repeat the same report.",
+};
+export const search1apiActions = Object.entries(descriptions).map(([name, description]) => defineProviderAction("search1api", {
+    name,
+    description,
+    operationType: name === "deepcrawl" || name === "feedback" ? "write" : "read",
+    requiredScopes: [],
+    inputSchema: search1apiInputs[name],
+    outputSchema: name === "screenshot"
+        ? s.requiredObject("Captured screenshot file.", {
+            url: s.string("Temporary download URL for the screenshot."),
+            mimeType: s.string("Screenshot MIME type."),
+            requestId: s.nullable(s.string("Upstream request identifier, when available.")),
+        })
+        : search1apiOutputs[name],
+    asyncLifecycle: actionLifecycles[name],
+}));

package/src/providers/search1api/executors.d.ts

@@ -0,0 +1,4 @@
+import type { CredentialValidators, ProviderExecutors, ProviderProxyExecutor } from "../../core/types.ts";
+export declare const credentialValidators: CredentialValidators;
+export declare const executors: ProviderExecutors;
+export declare const proxy: ProviderProxyExecutor;

package/src/providers/search1api/executors.js

@@ -0,0 +1,72 @@
+import { optionalString } from "../../core/cast.js";
+import { storeResponseInTransit } from "../../core/request.js";
+import { runProviderRequest, ProviderRequestError, providerResponseError, defineBearerProviderExecutors, defineProviderProxy, } from "../provider-runtime.js";
+import { search1apiActions } from "./actions.js";
+import { readSearch1apiJson, requestSearch1api, search1apiBaseUrl } from "./runtime.js";
+async function execute(actionName, values, context) {
+    const { accessToken: token, fetcher, transitFiles } = context;
+    const data = values;
+    if (actionName === "screenshot") {
+        if (!transitFiles)
+            throw new ProviderRequestError(503, "Search1API screenshots require configured file transit and execution identity", undefined, "provider_error");
+        return runProviderRequest({ label: "Search1API screenshot", signal: context.signal, timeoutMs: 60000 }, async (signal) => {
+            const response = await fetcher(`${search1apiBaseUrl}/screenshot`, {
+                method: "POST",
+                headers: {
+                    authorization: `Bearer ${token}`,
+                    "content-type": "application/json",
+                    accept: "image/png, image/jpeg, image/webp",
+                },
+                body: JSON.stringify(data),
+                signal,
+            });
+            if (!response.ok)
+                await readSearch1apiJson(response);
+            const mimeType = response.headers.get("content-type")?.split(";")[0]?.trim();
+            if (!mimeType || !["image/png", "image/jpeg", "image/webp"].includes(mimeType) || !response.body) {
+                await response.body?.cancel();
+                throw providerResponseError("Search1API returned an invalid screenshot response");
+            }
+            const extension = mimeType.slice("image/".length);
+            const result = await storeResponseInTransit(response, transitFiles, {
+                name: `search1api-screenshot.${extension}`,
+                mimeType,
+                fieldName: "Search1API screenshot",
+                createError: providerResponseError,
+                signal,
+            });
+            return {
+                url: result.downloadUrl,
+                mimeType,
+                requestId: response.headers.get("x-request-id"),
+            };
+        });
+    }
+    const body = await requestSearch1api(token, actionName, data, fetcher, context.signal);
+    if (actionName === "deepcrawl")
+        if (!optionalString(body.taskId))
+            throw providerResponseError("Search1API task ID is missing");
+    if (actionName === "get_deepcrawl")
+        if (!optionalString(body.status))
+            throw providerResponseError("Search1API task status is missing");
+    return body;
+}
+export const credentialValidators = {
+    async apiKey(input, { fetcher, signal }) {
+        await requestSearch1api(input.apiKey, "usage", {}, fetcher, signal);
+        return {
+            profile: { displayName: "Search1API API Key" },
+            grantedScopes: [],
+            metadata: { apiBaseUrl: search1apiBaseUrl },
+        };
+    },
+};
+export const executors = defineBearerProviderExecutors("search1api", Object.fromEntries(search1apiActions.map((action) => [action.name, (values, context) => execute(action.name, values, context)])));
+export const proxy = defineProviderProxy({
+    service: "search1api",
+    baseUrl: search1apiBaseUrl,
+    auth: { type: "bearer" },
+    customizeRequest({ headers }) {
+        headers.set("accept", "application/json");
+    },
+});

package/src/providers/search1api/runtime.d.ts

@@ -0,0 +1,3 @@
+export declare const search1apiBaseUrl = "https://api.search1api.com";
+export declare function readSearch1apiJson(response: Response, crawlBatch?: boolean): Promise<Record<string, unknown>>;
+export declare function requestSearch1api(token: string, name: string, input: Record<string, unknown>, fetcher: typeof fetch, parentSignal?: AbortSignal): Promise<Record<string, unknown>>;

package/src/providers/search1api/runtime.js

@@ -0,0 +1,44 @@
+import { optionalRecord, optionalString } from "../../core/cast.js";
+import { requiredResponseRecord, isAbortLikeError, runProviderRequest, ProviderRequestError, providerResponseError, withRetryAfterSeconds, } from "../provider-runtime.js";
+export const search1apiBaseUrl = "https://api.search1api.com";
+export async function readSearch1apiJson(response, crawlBatch = false) {
+    const value = await response.json().catch((error) => {
+        if (isAbortLikeError(error))
+            throw error;
+        return undefined;
+    });
+    const body = optionalRecord(value);
+    if (!response.ok || body?.ok === false) {
+        const detail = optionalString(body?.message) ?? optionalString(body?.detail) ?? optionalString(body?.error_description);
+        const code = optionalString(body?.error) ?? optionalString(body?.title);
+        throw new ProviderRequestError(response.ok ? 502 : response.status, `Search1API request failed: ${[code, detail].filter(Boolean).join(": ") || `HTTP ${response.status}`}`, withRetryAfterSeconds(response), response.status === 429 ? "rate_limited" : "provider_error");
+    }
+    if (crawlBatch) {
+        if (!Array.isArray(value))
+            throw providerResponseError("Search1API batch crawl response must be an array");
+        return { results: value };
+    }
+    return requiredResponseRecord(value, "Search1API response");
+}
+export function requestSearch1api(token, name, input, fetcher, parentSignal) {
+    return runProviderRequest({ label: "Search1API", signal: parentSignal, timeoutMs: name === "ask" ? 45000 : undefined }, async (signal) => {
+        const batch = name.endsWith("_batch");
+        const action = batch ? name.slice(0, -6) : name;
+        const path = action === "get_deepcrawl" ? `/deepcrawl/status/${encodeURIComponent(String(input.taskId))}` : `/${action}`;
+        const url = new URL(path, search1apiBaseUrl);
+        const method = ["health", "usage", "get_deepcrawl"].includes(action) ? "GET" : "POST";
+        if (action === "usage" && input.period !== undefined)
+            url.searchParams.set("period", String(input.period));
+        const response = await fetcher(url, {
+            method,
+            headers: {
+                accept: "application/json",
+                "content-type": "application/json",
+                ...(action === "health" ? {} : { authorization: `Bearer ${token}` }),
+            },
+            body: method === "POST" ? JSON.stringify(batch ? input.requests : input) : undefined,
+            signal,
+        });
+        return readSearch1apiJson(response, name === "crawl_batch");
+    });
+}

package/src/providers/search1api/schemas.d.ts

@@ -0,0 +1,3 @@
+import type { JsonSchema } from "../../core/types.ts";
+export declare const search1apiInputs: Record<string, JsonSchema>;
+export declare const search1apiOutputs: Record<string, JsonSchema>;