@usagekit/providers 0.8.0 → 0.9.1

package/dist/dataforseo/extractors.js

@@ -1,7 +1,9 @@
-import { formatQuantity } from "@usagekit/core";
+import { addQuantity, formatQuantity } from "@usagekit/core";
+import { descriptor } from "./descriptor.js";
+import { historyWindowOptions } from "./history-window.js";
 import { listEstimate } from "../prices.js";
 import { pointer } from "../pointer.js";
-import { moneyFromDollars, requestsMeasured } from "../receipts.js";
+import { decimalQuantity, moneyFromDollars, requestsMeasured } from "../receipts.js";
 const tasksOf = (body) => {
     const tasks = pointer(body, "/tasks");
     return Array.isArray(tasks) ? tasks : [];
@@ -34,23 +36,287 @@ export const priceKeys = {
     ...Object.fromEntries(["summary", "history", "backlinks"].map((n) => [`backlinks.live.${n}`, `backlinks.${n}.live`])),
 };
 const withPriority = new Set(["serp.google.organic.task_post"]);
+const itemCountFields = {
+    "dataforseo_labs.google.keyword_overview.live": "keywords",
+    "dataforseo_labs.google.keyword_ideas.live": "limit",
+    "dataforseo_labs.google.keyword_suggestions.live": "limit",
+    "dataforseo_labs.google.related_keywords.live": "limit",
+    "dataforseo_labs.google.relevant_pages.live": "limit",
+    "backlinks.backlinks.live": "limit",
+    "dataforseo_labs.google.ranked_keywords.live": "limit",
+    "dataforseo_labs.google.domain_rank_overview.live": "limit",
+};
 function timestamp(text) {
     if (typeof text !== "string")
         return undefined;
     const m = /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) \+00:00$/.exec(text);
     return m ? new Date(`${m[1]}T${m[2]}Z`).toISOString() : undefined;
 }
+const nonNegativeInteger = (v) => typeof v === "number" && Number.isSafeInteger(v) && v >= 0;
+/**
+ * The single task of a present billable body, with documented JSON types for every pricing
+ * field. Ambiguous body shapes (text, bare objects, empty or nested arrays) are InvalidInput.
+ * Unlisted keys, including alternate spellings, are separately marked unpriceable.
+ * Undefined fields count as omitted, as JSON serialization drops them. A request without a
+ * body carries no task and keeps the defaults.
+ */
+function billableTask(body) {
+    if (body === undefined)
+        return undefined;
+    if (Array.isArray(body) && body.length > 1)
+        throw new Error("InvalidInput: multiple DataForSEO tasks");
+    const task = Array.isArray(body) ? body[0] : undefined;
+    if (!task || typeof task !== "object" || Array.isArray(task))
+        throw new Error("InvalidInput: DataForSEO body must be an array with one task object");
+    const t = task;
+    for (const field of ["limit", "depth"])
+        if (t[field] !== undefined && !nonNegativeInteger(t[field]))
+            throw new Error(`InvalidInput: DataForSEO ${field}`);
+    if (t.priority !== undefined && t.priority !== 1 && t.priority !== 2)
+        throw new Error("InvalidInput: DataForSEO priority");
+    if (t.keywords !== undefined &&
+        (!Array.isArray(t.keywords) || t.keywords.some((keyword) => typeof keyword !== "string")))
+        throw new Error("InvalidInput: DataForSEO keywords");
+    for (const field of ["include_clickstream_data", "include_seed_keyword"])
+        if (t[field] !== undefined && typeof t[field] !== "boolean")
+            throw new Error(`InvalidInput: DataForSEO ${field}`);
+    return t;
+}
+const isSerp = (id) => id.startsWith("serp.google.organic.");
+function validateSerpDepth(depth) {
+    if (depth !== undefined &&
+        (!/^\d+$/.test(String(depth)) ||
+            BigInt(String(depth)) < 1n ||
+            BigInt(String(depth)) > BigInt(Number.MAX_SAFE_INTEGER)))
+        throw new Error("InvalidInput: DataForSEO SERP depth must be a positive safe integer");
+}
+// "Contains" is intentional, including embedded forms such as android: (id:).
+// Each distinct documented substring contributes a factor of five.
+// https://docs.dataforseo.com/v3/serp/google/organic/live/advanced/
+// https://docs.dataforseo.com/v3/serp/google/organic/task_post/
+// https://dataforseo.com/pricing/serp/google-organic-serp-api
+const searchOperators = [
+    "allinanchor:",
+    "allintext:",
+    "allintitle:",
+    "allinurl:",
+    "cache:",
+    "define:",
+    "definition:",
+    "filetype:",
+    "id:",
+    "inanchor:",
+    "info:",
+    "intext:",
+    "intitle:",
+    "inurl:",
+    "link:",
+    "site:",
+];
+function operatorCount(task) {
+    const values = [
+        ...(typeof task.keyword === "string" ? [task.keyword, decodeKeyword(task.keyword)] : []),
+        ...(typeof task.search_param === "string"
+            ? [task.search_param, decodeSearchParam(task.search_param)]
+            : []),
+    ];
+    const normalized = values
+        .flatMap((v) => [
+        v.normalize("NFKC").toLowerCase(),
+        v.normalize("NFKC").toUpperCase().toLowerCase(),
+        v
+            .normalize("NFKD")
+            .replace(/[\p{M}\p{Default_Ignorable_Code_Point}]/gu, "")
+            .toLowerCase(),
+    ])
+        .map((v) => v.replaceAll("-site:", ""));
+    // -site: is explicitly free. Other operator matches are conservative upper bounds.
+    // https://dataforseo.com/help-center/rules-and-limitations-of-keyword-and-keywords-fields-in-dataforseo-apis
+    return searchOperators.filter((op) => normalized.some((value) => value.includes(op))).length;
+}
+/** Keywords are text, not query strings: preserve literal percent signs. Decode each
+ * valid escape run leniently; invalid UTF-8 bytes become replacement characters. */
+function decodeKeyword(value) {
+    value = value.normalize("NFKC");
+    for (let i = 0; i < 16; i++) {
+        const decoded = value
+            .replace(/(?:%[0-9a-f]{2})+/gi, (run) => {
+            try {
+                return decodeURIComponent(run);
+            }
+            catch {
+                return new TextDecoder().decode(Uint8Array.from(run.match(/[0-9a-f]{2}/gi), (hex) => parseInt(hex, 16)));
+            }
+        })
+            .normalize("NFKC");
+        if (decoded === value)
+            return decoded;
+        value = decoded;
+    }
+    throw new Error("InvalidInput: DataForSEO keyword encoding depth");
+}
+function validateSerpOptions(options) {
+    validateSerpDepth(options.depth);
+    for (const [field, max] of [
+        ["operatorCount", searchOperators.length],
+        ["clicks", 4],
+    ]) {
+        const value = options[field];
+        if (value !== undefined && (!/^\d+$/.test(value) || BigInt(value) > BigInt(max)))
+            throw new Error(`InvalidInput: DataForSEO SERP ${field}`);
+    }
+}
+function decodeSearchParam(value) {
+    value = value.normalize("NFKC");
+    // Decode nested query escapes before checking operators. Fail closed when malformed
+    // or still changing after the bound rather than passing an unchecked encoded operator.
+    for (let i = 0; i < 16; i++) {
+        let decoded;
+        try {
+            decoded = decodeURIComponent(value.replace(/\+/g, " ")).normalize("NFKC");
+        }
+        catch {
+            throw new Error("InvalidInput: DataForSEO malformed search_param escape");
+        }
+        if (decoded === value)
+            return decoded;
+        value = decoded;
+    }
+    throw new Error("InvalidInput: DataForSEO search_param encoding depth");
+}
+function validateSerpTask(task) {
+    validateSerpDepth(task.depth);
+    for (const field of ["calculate_rectangles", "load_async_ai_overview"])
+        if (task[field] !== undefined && typeof task[field] !== "boolean")
+            throw new Error(`InvalidInput: DataForSEO ${field}`);
+    for (const [field, max] of [
+        ["people_also_ask_click_depth", 4],
+        ["max_crawl_pages", 100],
+    ])
+        if (task[field] !== undefined &&
+            (!nonNegativeInteger(task[field]) || Number(task[field]) < 1 || Number(task[field]) > max))
+            throw new Error(`InvalidInput: DataForSEO ${field}`);
+    if (task.stop_crawl_on_match !== undefined && !Array.isArray(task.stop_crawl_on_match))
+        throw new Error("InvalidInput: DataForSEO stop_crawl_on_match");
+    for (const field of ["keyword", "search_param"])
+        if (task[field] !== undefined && typeof task[field] !== "string")
+            throw new Error(`InvalidInput: DataForSEO ${field}`);
+}
 export const extractors = {
-    estimate: listEstimate,
-    /** priority 1 is normal (the documented default when absent), 2 is high; anything else is unreadable. */
+    canEstimate: (options) => options.pricingUnknown !== "true",
+    // https://dataforseo.com/pricing/serp/google-organic-serp-api
+    // https://dataforseo.com/pricing/dataforseo-labs/dataforseo-google-api
+    requiresListPriceFloor: (options) => Number(options.operatorCount ?? "0") > 0 ||
+        options.rectangles === "true" ||
+        options.asyncOverview === "true" ||
+        Number(options.clicks ?? "0") > 0 ||
+        options.crawlPages !== undefined ||
+        options.clickstream === "true",
+    estimate: (input) => {
+        if (isSerp(input.operation.id))
+            validateSerpOptions(input.options);
+        if (!extractors.canEstimate(input.options))
+            return [];
+        const estimates = listEstimate(input);
+        if (!isSerp(input.operation.id))
+            return estimates;
+        // Bound every page and operator combination, including add-ons. Field docs give
+        // $0.002 live / $0.0006 queued; the pricing table gives one base (higher for high priority).
+        // https://docs.dataforseo.com/v3/serp/google/organic/live/advanced/
+        // https://docs.dataforseo.com/v3/serp/google/organic/task_post/
+        // https://dataforseo.com/pricing/serp/google-organic-serp-api
+        // https://dataforseo.com/help-center/serp-api-cost-explained
+        const pages = (BigInt(input.options.depth ?? "10") + 9n) / 10n;
+        const operators = BigInt(input.options.operatorCount ?? "0");
+        const factor = 5n ** operators;
+        const addOns = Number(input.options.rectangles === "true") + Number(input.options.asyncOverview === "true");
+        const extra = decimalQuantity(input.operation.id.endsWith("task_post")
+            ? input.options.priority === "high"
+                ? "0.12"
+                : "0.06"
+            : "0.2", "cents");
+        const clicks = decimalQuantity("0.015", "cents");
+        return estimates.map((q) => {
+            if (q.unit !== "cents")
+                return q;
+            const total = addQuantity(addQuantity(q, { ...extra, value: extra.value * BigInt(addOns) * pages }), { ...clicks, value: clicks.value * BigInt(input.options.clicks ?? "0") * pages });
+            return decimalQuantity(formatQuantity({ ...total, value: total.value * factor }), "cents");
+        });
+    },
+    /** Request dimensions and documented defaults, separate from observed response quantities. */
     options: (operation, request) => {
-        if (!operation.optionKeys?.includes("priority"))
-            return {};
-        const first = Array.isArray(request.body) ? request.body[0] : null;
+        const first = operation.billable
+            ? billableTask(request.body)
+            : Array.isArray(request.body)
+                ? request.body[0]
+                : null;
         if (!first || typeof first !== "object")
             return {};
-        const priority = first.priority ?? 1;
-        return priority === 1 ? { priority: "normal" } : priority === 2 ? { priority: "high" } : {};
+        const task = first, options = {};
+        const keys = operation.optionKeys ?? [];
+        Object.assign(options, historyWindowOptions(operation.id, task));
+        if (operation.billable) {
+            const fields = operation.requestPricing;
+            const unlisted = !fields ||
+                Object.keys(task).some((key) => task[key] !== undefined &&
+                    !Object.hasOwn(fields.priceFields, key) &&
+                    !fields.neutralFields.includes(key));
+            if (unlisted ||
+                task.limit === 0 ||
+                Number(task.limit) > 1000 ||
+                Object.entries(fields?.priceFields ?? {}).some(([field, rule]) => rule.unpricedWhen === "present"
+                    ? task[field] !== undefined
+                    : rule.unpricedWhen === "true" && task[field] === true))
+                options.pricingUnknown = "true";
+        }
+        if (operation.billable && isSerp(operation.id))
+            validateSerpTask(task);
+        if (keys.includes("priority")) {
+            const priority = task.priority ?? 1;
+            if (priority === 1)
+                options.priority = "normal";
+            else if (priority === 2)
+                options.priority = "high";
+        }
+        if (keys.includes("depth") && task.depth !== undefined)
+            options.depth = String(task.depth);
+        if (operation.billable && isSerp(operation.id)) {
+            if (task.max_crawl_pages !== undefined) {
+                options.crawlPages = String(task.max_crawl_pages);
+                options.depth = String(Math.max(Number(task.depth ?? 10), Number(task.max_crawl_pages) * 10));
+            }
+            const count = operatorCount(task);
+            if (count)
+                options.operatorCount = String(count);
+            if (task.calculate_rectangles === true)
+                options.rectangles = "true";
+            if (task.load_async_ai_overview === true)
+                options.asyncOverview = "true";
+            if (task.people_also_ask_click_depth !== undefined)
+                options.clicks = String(task.people_also_ask_click_depth);
+        }
+        if (keys.includes("itemCount")) {
+            const field = itemCountFields[operation.id];
+            const count = field === "keywords"
+                ? Array.isArray(task.keywords)
+                    ? task.keywords.length
+                    : undefined
+                : field === "limit"
+                    ? // Only omission uses the documented default; zero marks the request unknown.
+                        task.limit === undefined
+                            ? operation.requestPricing?.priceFields.limit?.defaultCount
+                            : task.limit
+                    : // Summary returns one row; explicit history dates were bounded above.
+                        // https://docs.dataforseo.com/v3/backlinks/summary/live/
+                        operation.id === "backlinks.summary.live"
+                            ? 1
+                            : undefined;
+            if (count !== undefined)
+                options.itemCount = String(count);
+        }
+        if (keys.includes("clickstream") && task.include_clickstream_data !== undefined)
+            options.clickstream = task.include_clickstream_data === false ? "false" : "true";
+        return options;
     },
     /**
      * Cost from the envelope `cost` (USD, every task of the call). A task status of 40000 or
@@ -115,12 +381,15 @@ export const extractors = {
                             continue;
                         if (!withPriority.has(operation) && level !== "normal")
                             continue;
+                        const calculation = descriptor.prices.find((r) => r.operation === operation && (!r.option || r.option.priority === level))?.calculation;
                         rows.push({
                             validFrom: checkedAt,
                             operation,
                             ...(withPriority.has(operation) ? { option: { priority: level } } : {}),
                             unit: "cents",
                             perUnit: formatQuantity(centsQuantity(entry.cost)),
+                            // The endpoint refreshes base prices; retain static formula metadata.
+                            ...(calculation ? { calculation } : {}),
                             source: priceListSource,
                             checkedAt,
                         });

package/dist/dataforseo/history-window.d.ts

@@ -0,0 +1,5 @@
+/** Both history APIs return at most one item per month. Include partial boundary months.
+ * https://docs.dataforseo.com/v3/backlinks-history-live/
+ * https://docs.dataforseo.com/v3/dataforseo_labs-google-historical_rank_overview-live/
+ */
+export declare function historyWindowOptions(operation: string, task: Readonly<Record<string, unknown>>): Record<string, string>;

package/dist/dataforseo/history-window.js

@@ -0,0 +1,36 @@
+const historyOperations = new Set([
+    "backlinks.history.live",
+    "dataforseo_labs.google.historical_rank_overview.live",
+]);
+function validDate(value) {
+    if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(value))
+        return false;
+    const date = new Date(`${value}T00:00:00Z`);
+    return Number.isFinite(date.getTime()) && date.toISOString().slice(0, 10) === value;
+}
+/** Both history APIs return at most one item per month. Include partial boundary months.
+ * https://docs.dataforseo.com/v3/backlinks-history-live/
+ * https://docs.dataforseo.com/v3/dataforseo_labs-google-historical_rank_overview-live/
+ */
+export function historyWindowOptions(operation, task) {
+    if (!historyOperations.has(operation))
+        return {};
+    const from = task.date_from;
+    const to = task.date_to;
+    const options = {};
+    if (validDate(from))
+        options.dateFrom = from;
+    else if (from !== undefined)
+        options.pricingUnknown = "true";
+    if (validDate(to))
+        options.dateTo = to;
+    else if (to !== undefined)
+        options.pricingUnknown = "true";
+    if (!validDate(from) || !validDate(to))
+        return options;
+    if (from > to)
+        return { ...options, pricingUnknown: "true" };
+    const monthIndex = (date) => Number(date.slice(0, 4)) * 12 + Number(date.slice(5, 7));
+    options.itemCount = String(monthIndex(to) - monthIndex(from) + 1);
+    return options;
+}

package/dist/dataforseo/request-pricing.d.ts

@@ -0,0 +1,2 @@
+import type { ProviderOperation } from "@usagekit/core";
+export declare const requestPricing: Readonly<Record<string, NonNullable<ProviderOperation["requestPricing"]>>>;