@oomol-lab/open-connector 1.9.1 → 1.9.2

package/src/providers/aisa/executors.js

@@ -1,5 +1,9 @@
 import { optionalRecord, optionalString } from "../../core/cast.js";
+import { readSchemaProperties } from "../../core/json-schema.js";
+import { randomUUIDv7 } from "../../core/uuid-v7.js";
+import { requiredResponseRecord } from "../provider-runtime.js";
 import { defineApiKeyProviderExecutors, defineProviderProxy, providerInputError, ProviderRequestError, runProviderRequest, } from "../provider-runtime.js";
+import { aisaFirecrawlFamilies } from "./firecrawl-actions.js";
 const service = "aisa";
 const aisaApiOrigin = "https://api.aisa.one";
 function dataForSeoRoute(path, body) {
@@ -50,6 +54,148 @@ const similarwebRowsQuery = {
     trafficSource: "traffic_source",
 };
 const aisaActionRoutes = {
+    ...Object.fromEntries(aisaFirecrawlFamilies.flatMap((family) => {
+        const base = `/apis/v1/firecrawl/${family.path}`;
+        return [
+            [
+                `submit_firecrawl_${family.name}`,
+                {
+                    path: base,
+                    method: "POST",
+                    query: {},
+                    asyncJob: "submit",
+                    body: Object.fromEntries(Object.keys(readSchemaProperties(family.inputSchema))
+                        .filter((key) => key !== "maxPriceUsd")
+                        .map((key) => [key, key])),
+                },
+            ],
+            [
+                `get_firecrawl_${family.name}`,
+                {
+                    path: `${base}/{taskId}`,
+                    query: {},
+                    pathParams: { taskId: "taskId" },
+                    asyncJob: "get",
+                },
+            ],
+            [
+                `cancel_firecrawl_${family.name}`,
+                {
+                    path: `${base}/{taskId}/cancel`,
+                    method: "POST",
+                    query: {},
+                    pathParams: { taskId: "taskId" },
+                    asyncJob: "cancel",
+                },
+            ],
+        ];
+    })),
+    search_exa: {
+        path: "/apis/v1/exa/search",
+        method: "POST",
+        query: {},
+        timeoutMs: 180000,
+        body: {
+            query: "query",
+            stream: "stream",
+            type: "type",
+            numResults: "numResults",
+            category: "category",
+            includeDomains: "includeDomains",
+            excludeDomains: "excludeDomains",
+            startPublishedDate: "startPublishedDate",
+            endPublishedDate: "endPublishedDate",
+            contents: "contents",
+            outputSchema: "outputSchema",
+            systemPrompt: "systemPrompt",
+        },
+    },
+    get_exa_contents: {
+        path: "/apis/v1/exa/contents",
+        method: "POST",
+        query: {},
+        timeoutMs: 180000,
+        bodyDefaults: { text: true },
+        body: {
+            urls: "ids",
+            stream: "stream",
+            text: "text",
+            highlights: "highlights",
+            summary: "summary",
+            livecrawl: "livecrawl",
+            subpages: "subpages",
+        },
+    },
+    answer_exa: {
+        path: "/apis/v1/exa/answer",
+        method: "POST",
+        query: {},
+        timeoutMs: 180000,
+        body: {
+            query: "query",
+            type: "type",
+            stream: "stream",
+            text: "text",
+            outputSchema: "outputSchema",
+        },
+    },
+    scrape_firecrawl: {
+        path: "/apis/v1/firecrawl/scrape",
+        method: "POST",
+        query: {},
+        timeoutMs: 180000,
+        body: { url: "url" },
+        bodyDefaults: { proxy: "basic", formats: ["markdown"], onlyCleanContent: false },
+    },
+    map_firecrawl: {
+        path: "/apis/v1/firecrawl/map",
+        method: "POST",
+        query: {},
+        timeoutMs: 180000,
+        body: { url: "url", limit: "limit" },
+    },
+    crawl_tavily: {
+        path: "/apis/v1/tavily/crawl",
+        method: "POST",
+        query: {},
+        timeoutMs: 180000,
+        body: {
+            url: "url",
+            limit: "limit",
+            maxDepth: "max_depth",
+            maxBreadth: "max_breadth",
+            instructions: "instructions",
+            selectPaths: "select_paths",
+            excludePaths: "exclude_paths",
+            selectDomains: "select_domains",
+            excludeDomains: "exclude_domains",
+            allowExternal: "allow_external",
+            chunksPerSource: "chunks_per_source",
+            extractDepth: "extract_depth",
+            format: "format",
+            includeFavicon: "include_favicon",
+            includeImages: "include_images",
+            includeUsage: "include_usage",
+            timeout: "timeout",
+        },
+    },
+    extract_tavily: {
+        path: "/apis/v1/tavily/extract",
+        method: "POST",
+        query: {},
+        timeoutMs: 90000,
+        body: {
+            urls: "urls",
+            query: "query",
+            chunksPerSource: "chunks_per_source",
+            extractDepth: "extract_depth",
+            format: "format",
+            includeFavicon: "include_favicon",
+            includeImages: "include_images",
+            includeUsage: "include_usage",
+            timeout: "timeout",
+        },
+    },
     get_credits_balance: { path: "/v1/credits/balance", query: {} },
     get_usage: {
         path: "/v1/usage",
@@ -799,9 +945,12 @@ function buildAisaBody(input, route) {
     if (!route.body) {
         return undefined;
     }
-    const body = Object.fromEntries(Object.entries(route.body)
-        .filter(([inputName]) => input[inputName] !== undefined)
-        .map(([inputName, bodyName]) => [bodyName, input[inputName]]));
+    const body = {
+        ...route.bodyDefaults,
+        ...Object.fromEntries(Object.entries(route.body)
+            .filter(([inputName]) => input[inputName] !== undefined)
+            .map(([inputName, bodyName]) => [bodyName, input[inputName]])),
+    };
     const normalized = route.snakeCaseBody ? toSnakeCaseJson(body, route.bodyKeyAliases) : body;
     return route.bodyArray ? [normalized] : normalized;
 }
@@ -866,7 +1015,29 @@ function buildAisaQuery(input, mapping, commaSeparated, joined, fixed, omitDefau
 }
 const handlers = Object.fromEntries(Object.entries(aisaActionRoutes).map(([name, route]) => [
     name,
-    (input, context) => requestAisa(buildAisaPath(route.path, input, route.pathParams), buildAisaQuery(input, route.query, route.commaSeparated ?? [], route.joined ?? {}, route.fixedQuery ?? {}, route.omitQueryDefaults ?? {}), context, route.method ?? "GET", buildAisaBody(input, route), "execute", route.response),
+    async (input, context) => {
+        const normalized = normalizeFirecrawlInput(name, input);
+        const headers = {};
+        if (route.asyncJob === "submit") {
+            headers["Idempotency-Key"] = randomUUIDv7();
+            const maxPriceUsd = optionalString(normalized.maxPriceUsd);
+            if (maxPriceUsd !== undefined)
+                headers["X-AISA-Max-Price-USD"] = maxPriceUsd;
+        }
+        const payload = await requestAisa(buildAisaPath(route.path, normalized, route.pathParams), buildAisaQuery(normalized, route.query, route.commaSeparated ?? [], route.joined ?? {}, route.fixedQuery ?? {}, route.omitQueryDefaults ?? {}), context, route.method ?? "GET", buildAisaBody(normalized, route), "execute", route.response, route.timeoutMs, headers);
+        if (!route.asyncJob)
+            return payload;
+        const job = requiredResponseRecord(payload, "AIsa job");
+        const id = optionalString(job.id);
+        if (!id || (route.asyncJob !== "submit" && !optionalString(job.status)))
+            throw new ProviderRequestError(502, "AIsa returned an invalid job handle or state");
+        if (route.asyncJob !== "submit" && id !== normalized.taskId)
+            throw new ProviderRequestError(502, "AIsa returned a different job");
+        if (route.asyncJob === "submit")
+            return { taskId: id };
+        const { id: _id, object: _object, ...result } = job;
+        return { ...result, taskId: id };
+    },
 ]));
 export const executors = defineApiKeyProviderExecutors(service, handlers, {
     skipDnsValidation: true,
@@ -893,11 +1064,12 @@ export const credentialValidators = {
         };
     },
 };
-async function requestAisa(path, query, context, method, body, phase = "execute", responseFormat) {
-    return runProviderRequest({ signal: context.signal, label: "AIsa" }, async (signal) => {
+async function requestAisa(path, query, context, method, body, phase = "execute", responseFormat, timeoutMs, extraHeaders) {
+    return runProviderRequest({ signal: context.signal, label: "AIsa", timeoutMs }, async (signal) => {
         const url = new URL(path, aisaApiOrigin);
         url.search = query.toString();
         const headers = new Headers({
+            ...extraHeaders,
             accept: responseFormat ? "text/plain, application/json;q=0.9" : "application/json",
             authorization: `Bearer ${context.apiKey}`,
         });
@@ -1046,3 +1218,63 @@ function createAisaError(response, payload, phase) {
 function isCredentialErrorCode(code) {
     return code === "missing_api_key" || code === "invalid_api_key" || code === "revoked_api_key";
 }
+function normalizeFirecrawlInput(name, input) {
+    if (!name.includes("firecrawl"))
+        return input;
+    const normalized = { ...input };
+    const submission = name.startsWith("submit_");
+    const validateUrl = (value) => {
+        const text = String(value).trim();
+        const url = URL.parse(text);
+        if (!url ||
+            url.protocol !== "https:" ||
+            !url.hostname ||
+            ((submission || name === "scrape_firecrawl") && url.pathname.toLowerCase().endsWith(".pdf")) ||
+            (submission && (url.username || url.password || url.hash)))
+            throw providerInputError("Firecrawl requires an HTTPS URL without credentials, fragment or PDF suffix");
+        return text;
+    };
+    if (input.url !== undefined)
+        normalized.url = validateUrl(input.url);
+    if (!submission)
+        return normalized;
+    if (Array.isArray(input.urls)) {
+        const urls = input.urls.map(validateUrl);
+        const unique = urls.map((value) => {
+            while (value.endsWith("/"))
+                value = value.slice(0, -1);
+            return value;
+        });
+        if (new Set(unique).size !== urls.length)
+            throw providerInputError("URLs must be unique after trimming whitespace and trailing slashes");
+        normalized.urls = urls;
+    }
+    if (input.maxPriceUsd !== undefined) {
+        const amount = String(input.maxPriceUsd).trim();
+        if (!amount || !Number.isFinite(Number(amount)) || Number(amount) < 0)
+            throw providerInputError("Expected a non-negative USD amount");
+        normalized.maxPriceUsd = amount;
+    }
+    const normalizeSelectors = (value) => {
+        const output = { ...value };
+        for (const [field, limit] of [
+            ["includeTags", 128],
+            ["excludeTags", 128],
+            ["includePaths", 256],
+            ["excludePaths", 256],
+        ]) {
+            if (!Array.isArray(value[field]))
+                continue;
+            output[field] = value[field].map((entry) => {
+                const text = String(entry).trim();
+                if (!text || Buffer.byteLength(text) > limit || text.includes("\r") || text.includes("\n"))
+                    throw providerInputError(`${field} entries must fit ${limit} UTF-8 bytes and contain no newlines`);
+                return text;
+            });
+        }
+        return output;
+    };
+    if (optionalRecord(input.scrapeOptions))
+        normalized.scrapeOptions = normalizeSelectors(optionalRecord(input.scrapeOptions));
+    return normalizeSelectors(normalized);
+}

package/src/providers/aisa/firecrawl-actions.d.ts

@@ -0,0 +1,9 @@
+import type { ActionDefinition, JsonSchema } from "../../core/types.ts";
+interface FirecrawlFamily {
+    name: string;
+    path: string;
+    inputSchema: JsonSchema;
+}
+export declare const aisaFirecrawlFamilies: FirecrawlFamily[];
+export declare const aisaFirecrawlActions: readonly ActionDefinition[];
+export {};

package/src/providers/aisa/firecrawl-actions.js

@@ -0,0 +1,115 @@
+import { s } from "../../core/json-schema.js";
+import { defineProviderAction } from "../../core/provider-definition.js";
+const actionLifecycles = {
+    submit_firecrawl_crawl: { startActionId: "aisa.submit_firecrawl_crawl", statusActionId: "aisa.get_firecrawl_crawl" },
+    submit_firecrawl_batch_scrape: {
+        startActionId: "aisa.submit_firecrawl_batch_scrape",
+        statusActionId: "aisa.get_firecrawl_batch_scrape",
+    },
+};
+const url = s.string("HTTPS URL without credentials, fragment or PDF suffix.", {
+    format: "uri",
+});
+const tags = (description) => s.nullable(s.array(description, s.nonEmptyString("An HTML tag selector, up to 128 UTF-8 bytes."), {
+    maxItems: 50,
+}));
+const scrapeProperties = {
+    includeTags: tags("Only include matching HTML tags."),
+    excludeTags: tags("Exclude matching HTML tags."),
+    onlyMainContent: s.nullable(s.boolean("Extract only the main page content.")),
+    maxAge: s.nullable(s.integer("Maximum cached content age in milliseconds.", { minimum: 0, maximum: 31536000000 })),
+    minAge: s.nullable(s.integer("Minimum cached content age in milliseconds.", { minimum: 0, maximum: 31536000000 })),
+    timeout: s.nullable(s.integer("Per-page scrape timeout in milliseconds.", { minimum: 1000, maximum: 300000 })),
+};
+const submissionProperties = {
+    maxPriceUsd: s.string("Optional AIsa admission authorization in USD; not a guarantee of the final charge.", {
+        minLength: 1,
+    }),
+};
+const paths = (description) => s.nullable(s.array(description, s.nonEmptyString("A path pattern, up to 256 UTF-8 bytes."), {
+    maxItems: 20,
+}));
+const concurrency = s.nullable(s.integer("Maximum concurrent page requests.", { minimum: 1, maximum: 20 }));
+const crawlInput = s.object("Submit an asynchronous website crawl through AIsa.", {
+    ...submissionProperties,
+    url,
+    limit: s.integer("Maximum pages to crawl.", { minimum: 1, maximum: 1000 }),
+    allowExternalLinks: s.nullable(s.boolean("Allow links to external websites.")),
+    allowSubdomains: s.nullable(s.boolean("Allow crawling subdomains.")),
+    crawlEntireDomain: s.nullable(s.boolean("Crawl across the entire domain.")),
+    delay: s.nullable(s.number("Delay between requests in seconds.", { minimum: 0, maximum: 30 })),
+    excludePaths: paths("Path patterns to exclude from crawling."),
+    includePaths: paths("Path patterns to include in crawling."),
+    ignoreQueryParameters: s.nullable(s.boolean("Ignore query parameters when identifying pages.")),
+    maxConcurrency: concurrency,
+    maxDiscoveryDepth: s.nullable(s.integer("Maximum link discovery depth.", { minimum: 0, maximum: 10 })),
+    scrapeOptions: s.nullable(s.object("Options applied to each crawled page.", scrapeProperties, { required: [] })),
+    sitemap: s.nullable(s.stringEnum("How to use the website sitemap.", ["skip", "include", "only"])),
+}, { required: ["url", "limit"] });
+const batchInput = s.object("Submit an asynchronous batch of known page URLs through AIsa.", {
+    ...submissionProperties,
+    ...scrapeProperties,
+    maxConcurrency: concurrency,
+    urls: s.array("Distinct page URLs; duplicates after trimming trailing slashes are not allowed.", url, {
+        minItems: 1,
+        maxItems: 1000,
+        uniqueItems: true,
+    }),
+}, { required: ["urls"] });
+export const aisaFirecrawlFamilies = [
+    { name: "crawl", path: "crawl", inputSchema: crawlInput },
+    { name: "batch_scrape", path: "batch-scrape", inputSchema: batchInput },
+];
+const taskId = s.string("Task ID returned by submission; use it with the same connection.", {
+    minLength: 1,
+});
+const jobOutput = s.looseObject("The job state, output and authoritative final pricing when available.", {
+    taskId,
+    status: s.string("Job state: queued, running, completed, failed or cancelled."),
+    output: s.unknown("Upstream crawl or batch scrape output; may be unavailable before completion or after expiry."),
+    outputExpired: s.nullable(s.boolean("Whether the upstream output has expired.")),
+    error: s.nullable(s.looseObject("Upstream job error details.")),
+    pricing: s.nullable(s.looseObject("AIsa pricing information, in micro-USD.", {
+        authorizedMicrosUSD: s.nullable(s.integer("Initially authorized amount; not the final charge.")),
+        finalMicrosUSD: s.nullable(s.integer("Final customer charge; null means unknown, zero means no charge.")),
+        currency: s.nullable(s.string("Pricing currency.")),
+        billingMode: s.nullable(s.string("AIsa billing mode.")),
+    })),
+});
+export const aisaFirecrawlActions = aisaFirecrawlFamilies.flatMap((family) => [
+    defineProviderAction("aisa", {
+        name: `submit_firecrawl_${family.name}`,
+        operationType: "write",
+        requiredScopes: [],
+        description: family.name === "crawl"
+            ? "Start a background website crawl through AIsa Firecrawl. Poll the returned task ID for results."
+            : "Start a background batch scrape of known URLs through AIsa Firecrawl. Poll the returned task ID for results.",
+        inputSchema: family.inputSchema,
+        outputSchema: s.object("The submitted task handle.", { taskId }, {
+            optional: [],
+        }),
+        asyncLifecycle: actionLifecycles[`submit_firecrawl_${family.name}`],
+    }),
+    defineProviderAction("aisa", {
+        name: `get_firecrawl_${family.name}`,
+        operationType: "read",
+        requiredScopes: [],
+        description: "Get a Firecrawl task state and results. Check outputExpired and job errors even when the task has finished.",
+        inputSchema: s.object("Identify the task to query.", { taskId }, {
+            optional: [],
+        }),
+        outputSchema: jobOutput,
+        asyncLifecycle: actionLifecycles[`get_firecrawl_${family.name}`],
+    }),
+    defineProviderAction("aisa", {
+        name: `cancel_firecrawl_${family.name}`,
+        operationType: "destructive",
+        requiredScopes: [],
+        description: "Request cancellation of a Firecrawl task. Work already performed may still be charged; poll the task for final state and pricing.",
+        inputSchema: s.object("Identify the task to cancel.", { taskId }, {
+            optional: [],
+        }),
+        outputSchema: jobOutput,
+        asyncLifecycle: actionLifecycles[`cancel_firecrawl_${family.name}`],
+    }),
+]);

package/src/providers/akta/actions.d.ts

@@ -0,0 +1,9 @@
+import type { ActionDefinition } from "../../core/types.ts";
+export declare const aktaActions: ActionDefinition[];
+interface AktaRoute {
+    method: string;
+    path: string;
+    timeoutMs?: number;
+}
+export declare const aktaRoutes: Record<string, AktaRoute>;
+export {};