@usagekit/providers 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.d.ts +8 -14
- package/dist/catalog.js +102 -17
- package/dist/dataforseo/descriptor.d.ts +2 -2
- package/dist/dataforseo/descriptor.js +47 -13
- package/dist/dataforseo/extractors.js +277 -9
- package/dist/dataforseo/request-pricing.d.ts +2 -0
- package/dist/dataforseo/request-pricing.js +401 -0
- package/dist/execute.d.ts +9 -7
- package/dist/execute.js +5 -11
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -1
- package/dist/priced-request.d.ts +49 -0
- package/dist/priced-request.js +234 -0
- package/dist/prices.d.ts +3 -1
- package/dist/prices.js +59 -2
- package/dist/serpapi/descriptor.js +16 -9
- package/dist/testing/example.js +1 -0
- package/dist/types.d.ts +4 -0
- package/package.json +2 -2
package/dist/catalog.d.ts
CHANGED
|
@@ -1,21 +1,10 @@
|
|
|
1
|
-
import type { BalanceSnapshot, Budget, EstimateSource, OperationId, ProviderDescriptor, ProviderId, ProviderOperation, ProviderPlan, Quantity } from "@usagekit/core";
|
|
1
|
+
import type { BalanceSnapshot, Budget, EstimateSource, ConnectionPolicy, OperationId, ProviderDescriptor, ProviderId, ProviderOperation, ProviderPlan, Quantity } from "@usagekit/core";
|
|
2
2
|
import type { Extractors, ProviderModule, ProviderRequest, ProviderResponse, ReceiptDraft } from "./types.js";
|
|
3
3
|
import type { BudgetTarget } from "./budget.js";
|
|
4
4
|
export type { EstimateSource };
|
|
5
5
|
/** Connection data the catalog needs; the catalog itself stays storage-free. */
|
|
6
|
-
export type CatalogConnection = {
|
|
6
|
+
export type CatalogConnection = Omit<ConnectionPolicy, "tracking"> & {
|
|
7
7
|
id: string;
|
|
8
|
-
provider: ProviderId;
|
|
9
|
-
plan?: string;
|
|
10
|
-
/** Per-call manual price for (connection, operation). */
|
|
11
|
-
manualPrices?: Readonly<Record<OperationId, Quantity>>;
|
|
12
|
-
/** Median of this connection's settled receipts for the operation, with its sample size. */
|
|
13
|
-
measured?: (operation: OperationId) => {
|
|
14
|
-
quantity: Quantity;
|
|
15
|
-
samples: number;
|
|
16
|
-
} | undefined;
|
|
17
|
-
/** The plan allowance is exhausted: only overage rows apply. */
|
|
18
|
-
overage?: boolean;
|
|
19
8
|
};
|
|
20
9
|
export type CatalogEstimate = {
|
|
21
10
|
/** Always includes one request for a billable operation; empty for a free operation. */
|
|
@@ -23,6 +12,8 @@ export type CatalogEstimate = {
|
|
|
23
12
|
source: EstimateSource;
|
|
24
13
|
/** "<provider>:<validFrom>" of the list row, when the source is list. */
|
|
25
14
|
priceVersion?: string;
|
|
15
|
+
checkedAt?: string;
|
|
16
|
+
sampleSize?: number;
|
|
26
17
|
};
|
|
27
18
|
export type CatalogMatch = {
|
|
28
19
|
operation: ProviderOperation;
|
|
@@ -45,7 +36,10 @@ export declare function createCatalog({ providers, enabled, measuredMinSamples,
|
|
|
45
36
|
}): {
|
|
46
37
|
providers: () => ProviderDescriptor[];
|
|
47
38
|
match(providerId: ProviderId, request: ProviderRequest): CatalogMatch;
|
|
48
|
-
|
|
39
|
+
canEstimate(providerId: ProviderId, options: Readonly<Record<string, string>>): boolean;
|
|
40
|
+
/** BYOK caller estimates also retain any documented surcharge floor. */
|
|
41
|
+
boundEstimate(providerId: ProviderId, operationId: OperationId, options: Readonly<Record<string, string>>, quantities: readonly Quantity[]): Quantity[];
|
|
42
|
+
/** Resolution order: request safety, manual, measured with enough samples, list row, unknown. */
|
|
49
43
|
estimate(connection: CatalogConnection, operationId: OperationId, options: Readonly<Record<string, string>>): CatalogEstimate;
|
|
50
44
|
extract(providerId: ProviderId, operationId: OperationId | "unknown", request: ProviderRequest, response: ProviderResponse, body: unknown): ReceiptDraft;
|
|
51
45
|
/** Price list and billing export parsers of an enabled descriptor, when it has them. */
|
package/dist/catalog.js
CHANGED
|
@@ -1,11 +1,23 @@
|
|
|
1
|
-
import { validateDescriptor } from "@usagekit/core";
|
|
1
|
+
import { compareQuantity, validateDescriptor } from "@usagekit/core";
|
|
2
2
|
import { matchOperation } from "./match.js";
|
|
3
|
-
import { selectPriceRow } from "./prices.js";
|
|
3
|
+
import { calculatePrice, selectPriceRow } from "./prices.js";
|
|
4
4
|
import { pointer } from "./pointer.js";
|
|
5
5
|
import { decimalQuantity, requestsMeasured } from "./receipts.js";
|
|
6
6
|
import { proposePlanBudget } from "./budget.js";
|
|
7
7
|
const oneRequest = () => ({ value: 1n, scale: 0, unit: "requests" });
|
|
8
8
|
const withRequest = (q) => q.some((x) => x.unit === "requests") ? q : [...q, oneRequest()];
|
|
9
|
+
/** Compare exact quantities at their own scales; never round a reservation down. */
|
|
10
|
+
function withFloor(quantities, floor) {
|
|
11
|
+
const result = [...quantities];
|
|
12
|
+
for (const bound of floor) {
|
|
13
|
+
const index = result.findIndex((q) => q.unit === bound.unit);
|
|
14
|
+
if (index < 0)
|
|
15
|
+
result.push(bound);
|
|
16
|
+
else if (compareQuantity(result[index], bound) < 0)
|
|
17
|
+
result[index] = bound;
|
|
18
|
+
}
|
|
19
|
+
return result;
|
|
20
|
+
}
|
|
9
21
|
function isoTimestamp(value) {
|
|
10
22
|
if (typeof value !== "string" || !value)
|
|
11
23
|
return undefined;
|
|
@@ -72,26 +84,60 @@ export function createCatalog({ providers, enabled, measuredMinSamples = 5, now
|
|
|
72
84
|
providers: () => [...enabled].map((id) => modules.get(id).descriptor),
|
|
73
85
|
match(providerId, request) {
|
|
74
86
|
const module = active(providerId);
|
|
75
|
-
|
|
76
|
-
if (!module || !operation)
|
|
87
|
+
if (!module)
|
|
77
88
|
return { operation: "unknown" };
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
89
|
+
const url = new URL(request.url, module.descriptor.upstream);
|
|
90
|
+
if (module.descriptor.pricingInputs.includes("query")) {
|
|
91
|
+
const keys = new Set();
|
|
92
|
+
for (const key of url.searchParams.keys()) {
|
|
93
|
+
if (keys.has(key))
|
|
94
|
+
throw new Error("InvalidInput: duplicate query key");
|
|
95
|
+
keys.add(key);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
else
|
|
99
|
+
url.search = "";
|
|
100
|
+
// Pricing sees only declared inputs. Receipts and transport retain the full request.
|
|
101
|
+
const { body, ...rest } = request;
|
|
102
|
+
const pricingRequest = {
|
|
103
|
+
...rest,
|
|
104
|
+
url: url.href,
|
|
105
|
+
...(module.descriptor.pricingInputs.includes("body") && body !== undefined ? { body } : {}),
|
|
106
|
+
};
|
|
107
|
+
const operation = matchOperation(module.descriptor, pricingRequest);
|
|
108
|
+
if (!operation)
|
|
109
|
+
return { operation: "unknown" };
|
|
110
|
+
return { operation, options: module.extractors.options(operation, pricingRequest) };
|
|
111
|
+
},
|
|
112
|
+
canEstimate(providerId, options) {
|
|
113
|
+
return require(providerId).extractors.canEstimate?.(options) ?? true;
|
|
81
114
|
},
|
|
82
|
-
/**
|
|
115
|
+
/** BYOK caller estimates also retain any documented surcharge floor. */
|
|
116
|
+
boundEstimate(providerId, operationId, options, quantities) {
|
|
117
|
+
const module = require(providerId);
|
|
118
|
+
const operation = module.descriptor.operations.find((o) => o.id === operationId);
|
|
119
|
+
if (!operation || !module.extractors.requiresListPriceFloor?.(options))
|
|
120
|
+
return [...quantities];
|
|
121
|
+
const listed = module.extractors.estimate({
|
|
122
|
+
operation,
|
|
123
|
+
options,
|
|
124
|
+
prices: validPrices(module.descriptor),
|
|
125
|
+
});
|
|
126
|
+
return withFloor(quantities, listed);
|
|
127
|
+
},
|
|
128
|
+
/** Resolution order: request safety, manual, measured with enough samples, list row, unknown. */
|
|
83
129
|
estimate(connection, operationId, options) {
|
|
84
130
|
const module = active(connection.provider), operation = module?.descriptor.operations.find((o) => o.id === operationId);
|
|
85
131
|
if (!module || !operation)
|
|
86
132
|
return { quantities: [oneRequest()], source: "unknown" };
|
|
87
133
|
if (!operation.billable)
|
|
88
134
|
return { quantities: [], source: "list" };
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
if (
|
|
94
|
-
return { quantities:
|
|
135
|
+
// Provider prices estimate BYOK upstream cost only. A host must supply customer prices
|
|
136
|
+
// explicitly for platform funding; neither manual nor list prices price credits.
|
|
137
|
+
if (connection.fundingSource !== "byok")
|
|
138
|
+
return { quantities: [oneRequest()], source: "unknown" };
|
|
139
|
+
if (module.extractors.canEstimate?.(options) === false)
|
|
140
|
+
return { quantities: [oneRequest()], source: "unknown" };
|
|
95
141
|
const input = {
|
|
96
142
|
operation,
|
|
97
143
|
options,
|
|
@@ -99,14 +145,53 @@ export function createCatalog({ providers, enabled, measuredMinSamples = 5, now
|
|
|
99
145
|
...(connection.plan !== undefined ? { plan: connection.plan } : {}),
|
|
100
146
|
...(connection.overage ? { overage: true } : {}),
|
|
101
147
|
};
|
|
102
|
-
const listed = module.extractors.estimate(input);
|
|
148
|
+
const listed = module.extractors.estimate(input), row = selectPriceRow(input);
|
|
149
|
+
const manual = connection.manualPrices?.[operationId];
|
|
150
|
+
const measured = manual ? undefined : connection.measured?.(operationId);
|
|
151
|
+
const override = manual ??
|
|
152
|
+
(measured &&
|
|
153
|
+
Number.isSafeInteger(measured.samples) &&
|
|
154
|
+
measured.samples >= measuredMinSamples
|
|
155
|
+
? measured.quantity
|
|
156
|
+
: undefined);
|
|
157
|
+
if (override) {
|
|
158
|
+
const basis = manual ? connection.manualPriceBasis?.[operationId] : measured?.basis;
|
|
159
|
+
if (basis === "item" && (!row || row.calculation?.kind !== "base_plus_items"))
|
|
160
|
+
return {
|
|
161
|
+
quantities: withRequest(listed.filter((q) => q.unit !== override.unit)),
|
|
162
|
+
source: "unknown",
|
|
163
|
+
};
|
|
164
|
+
const quantity = basis === "item" ? calculatePrice(row, options, override) : override;
|
|
165
|
+
if (!quantity)
|
|
166
|
+
return {
|
|
167
|
+
quantities: withRequest(listed.filter((q) => q.unit !== override.unit)),
|
|
168
|
+
source: "unknown",
|
|
169
|
+
};
|
|
170
|
+
return {
|
|
171
|
+
quantities: withRequest(withFloor([
|
|
172
|
+
quantity,
|
|
173
|
+
...listed.filter((q) => q.unit !== quantity.unit && q.unit !== "requests"),
|
|
174
|
+
], module.extractors.requiresListPriceFloor?.(options) ? listed : [])),
|
|
175
|
+
source: manual ? "manual" : "measured",
|
|
176
|
+
...(basis === "item" && row
|
|
177
|
+
? { priceVersion: `${module.descriptor.id}:${row.validFrom}`, checkedAt: row.checkedAt }
|
|
178
|
+
: {}),
|
|
179
|
+
...(!manual && measured
|
|
180
|
+
? {
|
|
181
|
+
sampleSize: measured.samples,
|
|
182
|
+
...(measured.checkedAt ? { checkedAt: measured.checkedAt } : {}),
|
|
183
|
+
}
|
|
184
|
+
: {}),
|
|
185
|
+
};
|
|
186
|
+
}
|
|
103
187
|
if (!listed.length)
|
|
104
188
|
return { quantities: [oneRequest()], source: "unknown" };
|
|
105
|
-
const row = selectPriceRow(input);
|
|
106
189
|
return {
|
|
107
190
|
quantities: withRequest(listed),
|
|
108
191
|
source: "list",
|
|
109
|
-
...(row
|
|
192
|
+
...(row
|
|
193
|
+
? { priceVersion: `${module.descriptor.id}:${row.validFrom}`, checkedAt: row.checkedAt }
|
|
194
|
+
: {}),
|
|
110
195
|
};
|
|
111
196
|
},
|
|
112
197
|
extract(providerId, operationId, request, response, body) {
|
|
@@ -4,7 +4,7 @@ export declare const labsPricing = "https://dataforseo.com/pricing/dataforseo-la
|
|
|
4
4
|
export declare const backlinksPricing = "https://dataforseo.com/pricing/backlinks/backlinks";
|
|
5
5
|
/**
|
|
6
6
|
* Provider A: cost per call in every response, prepaid money balance, basic auth. List prices
|
|
7
|
-
*
|
|
8
|
-
*
|
|
7
|
+
* include request formulas in cents. Actual settlement continues to use response evidence.
|
|
8
|
+
* Formulas preserve host characterization values; the existing check date is not refreshed.
|
|
9
9
|
*/
|
|
10
10
|
export declare const descriptor: ProviderDescriptor;
|
|
@@ -1,14 +1,17 @@
|
|
|
1
|
+
import { requestPricing } from "./request-pricing.js";
|
|
1
2
|
const docs = "https://docs.dataforseo.com/v3/";
|
|
2
3
|
export const serpPricing = "https://dataforseo.com/pricing/serp/google-organic-serp-api";
|
|
3
4
|
export const labsPricing = "https://dataforseo.com/pricing/dataforseo-labs/dataforseo-google-api";
|
|
4
5
|
export const backlinksPricing = "https://dataforseo.com/pricing/backlinks/backlinks";
|
|
5
6
|
const checked = "2026-09-25";
|
|
6
|
-
const live = (id, label, path) => ({
|
|
7
|
+
const live = (id, label, path, optionKeys) => ({
|
|
7
8
|
id,
|
|
8
9
|
label,
|
|
9
10
|
billable: true,
|
|
11
|
+
requestPricing: requestPricing[id],
|
|
10
12
|
match: [{ method: "POST", path }],
|
|
11
13
|
costEvidence: "response",
|
|
14
|
+
...(optionKeys ? { optionKeys } : {}),
|
|
12
15
|
});
|
|
13
16
|
const free = (id, label, method, path) => ({
|
|
14
17
|
id,
|
|
@@ -32,34 +35,64 @@ const backlinks = [
|
|
|
32
35
|
["history", "Backlinks history"],
|
|
33
36
|
["backlinks", "Backlinks live"],
|
|
34
37
|
];
|
|
35
|
-
const row = (operation, perUnit, source, option) => ({
|
|
38
|
+
const row = (operation, perUnit, source, option, calculation) => ({
|
|
36
39
|
validFrom: checked,
|
|
37
40
|
operation,
|
|
38
41
|
...(option ? { option } : {}),
|
|
39
42
|
unit: "cents",
|
|
40
43
|
perUnit,
|
|
44
|
+
...(calculation ? { calculation } : {}),
|
|
41
45
|
source,
|
|
42
46
|
checkedAt: checked,
|
|
43
47
|
});
|
|
48
|
+
// Bill each SERP of up to 10 results at the full base: base * ceil(depth / 10).
|
|
49
|
+
// https://docs.dataforseo.com/v3/serp/google/organic/live/advanced/
|
|
50
|
+
// https://docs.dataforseo.com/v3/serp/google/organic/task_post/
|
|
51
|
+
// The pricing page multiplies for each 10 results; the cost formula charges a full page 2.
|
|
52
|
+
// https://dataforseo.com/pricing/serp/google-organic-serp-api
|
|
53
|
+
// https://dataforseo.com/help-center/serp-api-cost-explained
|
|
54
|
+
// The conflicting 25% later-page discount can only lower the charge, so do not apply it.
|
|
55
|
+
// Unreturned results are refunded: full base per page is a conservative upper bound
|
|
56
|
+
// that never under-reserves; omitted depth still means one page.
|
|
57
|
+
// https://dataforseo.com/help-center/above-100-results-with-serp-api
|
|
58
|
+
const pages = {
|
|
59
|
+
kind: "additional_pages",
|
|
60
|
+
depthOption: "depth",
|
|
61
|
+
pageSize: 10,
|
|
62
|
+
additionalPageFactor: "1",
|
|
63
|
+
};
|
|
64
|
+
const items = (perItem, clickstream = true) => ({
|
|
65
|
+
kind: "base_plus_items",
|
|
66
|
+
itemCountOption: "itemCount",
|
|
67
|
+
perItem,
|
|
68
|
+
...(clickstream ? { multiplierOption: { key: "clickstream", value: "true", factor: "2" } } : {}),
|
|
69
|
+
});
|
|
44
70
|
/**
|
|
45
71
|
* Provider A: cost per call in every response, prepaid money balance, basic auth. List prices
|
|
46
|
-
*
|
|
47
|
-
*
|
|
72
|
+
* include request formulas in cents. Actual settlement continues to use response evidence.
|
|
73
|
+
* Formulas preserve host characterization values; the existing check date is not refreshed.
|
|
48
74
|
*/
|
|
49
75
|
export const descriptor = {
|
|
50
76
|
id: "dataforseo",
|
|
51
77
|
label: "DataForSEO",
|
|
52
78
|
upstream: "https://api.dataforseo.com",
|
|
79
|
+
pricingInputs: ["body"],
|
|
53
80
|
auth: { kind: "basic" },
|
|
54
81
|
stripRequestHeaders: ["authorization"],
|
|
55
82
|
billing: { unit: "cents", cycle: "prepaid" },
|
|
56
83
|
plans: [{ id: "prepaid", label: "Prepaid balance", cycle: "prepaid", allowanceMode: "hard" }],
|
|
57
84
|
prices: [
|
|
58
|
-
row("serp.google.organic.live.advanced", "0.2", serpPricing),
|
|
59
|
-
row("serp.google.organic.task_post", "0.06", serpPricing, { priority: "normal" }),
|
|
60
|
-
row("serp.google.organic.task_post", "0.12", serpPricing, { priority: "high" }),
|
|
61
|
-
...labs.map(([name]) => row(`dataforseo_labs.google.${name}.live`, name === "historical_rank_overview" ? "12" : "1.2", labsPricing
|
|
62
|
-
|
|
85
|
+
row("serp.google.organic.live.advanced", "0.2", serpPricing, undefined, pages),
|
|
86
|
+
row("serp.google.organic.task_post", "0.06", serpPricing, { priority: "normal" }, pages),
|
|
87
|
+
row("serp.google.organic.task_post", "0.12", serpPricing, { priority: "high" }, pages),
|
|
88
|
+
...labs.map(([name]) => row(`dataforseo_labs.google.${name}.live`, name === "historical_rank_overview" ? "12" : "1.2", labsPricing, undefined,
|
|
89
|
+
// Per-request plus per-item, including ranked_keywords/domain_rank_overview.
|
|
90
|
+
// https://dataforseo.com/pricing/dataforseo-labs/dataforseo-google-api
|
|
91
|
+
items(name === "historical_rank_overview" ? "0.12" : "0.012"))),
|
|
92
|
+
...backlinks.map(([name]) => row(`backlinks.${name}.live`, "2.4", backlinksPricing, undefined,
|
|
93
|
+
// Every returned row is billed, including summary and history.
|
|
94
|
+
// https://dataforseo.com/pricing/backlinks/backlinks
|
|
95
|
+
items("0.0036", false))),
|
|
63
96
|
],
|
|
64
97
|
priceList: {
|
|
65
98
|
kind: "endpoint",
|
|
@@ -68,19 +101,20 @@ export const descriptor = {
|
|
|
68
101
|
mapping: "provider-specific",
|
|
69
102
|
},
|
|
70
103
|
operations: [
|
|
71
|
-
live("serp.google.organic.live.advanced", "Google organic SERP, live advanced", "/v3/serp/google/organic/live/advanced"),
|
|
104
|
+
live("serp.google.organic.live.advanced", "Google organic SERP, live advanced", "/v3/serp/google/organic/live/advanced", ["depth"]),
|
|
72
105
|
{
|
|
73
106
|
id: "serp.google.organic.task_post",
|
|
74
107
|
label: "Google organic SERP, task post",
|
|
75
108
|
billable: true,
|
|
76
109
|
match: [{ method: "POST", path: "/v3/serp/google/organic/task_post" }],
|
|
77
|
-
optionKeys: ["priority"],
|
|
110
|
+
optionKeys: ["priority", "depth"],
|
|
78
111
|
costEvidence: "deferred",
|
|
112
|
+
requestPricing: requestPricing["serp.google.organic.task_post"],
|
|
79
113
|
},
|
|
80
114
|
free("serp.google.organic.tasks_ready", "Google organic SERP, tasks ready", "GET", "/v3/serp/google/organic/tasks_ready"),
|
|
81
115
|
free("serp.google.organic.task_get.advanced", "Google organic SERP, task get advanced", "GET", "/v3/serp/google/organic/task_get/advanced/:id"),
|
|
82
|
-
...labs.map(([name, label]) => live(`dataforseo_labs.google.${name}.live`, label, `/v3/dataforseo_labs/google/${name}/live
|
|
83
|
-
...backlinks.map(([name, label]) => live(`backlinks.${name}.live`, label, `/v3/backlinks/${name}/live
|
|
116
|
+
...labs.map(([name, label]) => live(`dataforseo_labs.google.${name}.live`, label, `/v3/dataforseo_labs/google/${name}/live`, ["itemCount", "clickstream"])),
|
|
117
|
+
...backlinks.map(([name, label]) => live(`backlinks.${name}.live`, label, `/v3/backlinks/${name}/live`, ["itemCount"])),
|
|
84
118
|
free("appendix.user_data", "User data (balance, limits, prices)", "GET", "/v3/appendix/user_data"),
|
|
85
119
|
free("dataforseo_labs.status", "Labs status", "GET", "/v3/dataforseo_labs/status"),
|
|
86
120
|
],
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { formatQuantity } from "@usagekit/core";
|
|
1
|
+
import { addQuantity, formatQuantity } from "@usagekit/core";
|
|
2
|
+
import { descriptor } from "./descriptor.js";
|
|
2
3
|
import { listEstimate } from "../prices.js";
|
|
3
4
|
import { pointer } from "../pointer.js";
|
|
4
|
-
import { moneyFromDollars, requestsMeasured } from "../receipts.js";
|
|
5
|
+
import { decimalQuantity, moneyFromDollars, requestsMeasured } from "../receipts.js";
|
|
5
6
|
const tasksOf = (body) => {
|
|
6
7
|
const tasks = pointer(body, "/tasks");
|
|
7
8
|
return Array.isArray(tasks) ? tasks : [];
|
|
@@ -34,23 +35,287 @@ export const priceKeys = {
|
|
|
34
35
|
...Object.fromEntries(["summary", "history", "backlinks"].map((n) => [`backlinks.live.${n}`, `backlinks.${n}.live`])),
|
|
35
36
|
};
|
|
36
37
|
const withPriority = new Set(["serp.google.organic.task_post"]);
|
|
38
|
+
const itemCountFields = {
|
|
39
|
+
"dataforseo_labs.google.keyword_overview.live": "keywords",
|
|
40
|
+
"dataforseo_labs.google.keyword_ideas.live": "limit",
|
|
41
|
+
"dataforseo_labs.google.keyword_suggestions.live": "limit",
|
|
42
|
+
"dataforseo_labs.google.related_keywords.live": "limit",
|
|
43
|
+
"dataforseo_labs.google.relevant_pages.live": "limit",
|
|
44
|
+
"backlinks.backlinks.live": "limit",
|
|
45
|
+
"dataforseo_labs.google.ranked_keywords.live": "limit",
|
|
46
|
+
"dataforseo_labs.google.domain_rank_overview.live": "limit",
|
|
47
|
+
};
|
|
37
48
|
function timestamp(text) {
|
|
38
49
|
if (typeof text !== "string")
|
|
39
50
|
return undefined;
|
|
40
51
|
const m = /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) \+00:00$/.exec(text);
|
|
41
52
|
return m ? new Date(`${m[1]}T${m[2]}Z`).toISOString() : undefined;
|
|
42
53
|
}
|
|
54
|
+
const nonNegativeInteger = (v) => typeof v === "number" && Number.isSafeInteger(v) && v >= 0;
|
|
55
|
+
/**
|
|
56
|
+
* The single task of a present billable body, with documented JSON types for every pricing
|
|
57
|
+
* field. Ambiguous body shapes (text, bare objects, empty or nested arrays) are InvalidInput.
|
|
58
|
+
* Unlisted keys, including alternate spellings, are separately marked unpriceable.
|
|
59
|
+
* Undefined fields count as omitted, as JSON serialization drops them. A request without a
|
|
60
|
+
* body carries no task and keeps the defaults.
|
|
61
|
+
*/
|
|
62
|
+
function billableTask(body) {
|
|
63
|
+
if (body === undefined)
|
|
64
|
+
return undefined;
|
|
65
|
+
if (Array.isArray(body) && body.length > 1)
|
|
66
|
+
throw new Error("InvalidInput: multiple DataForSEO tasks");
|
|
67
|
+
const task = Array.isArray(body) ? body[0] : undefined;
|
|
68
|
+
if (!task || typeof task !== "object" || Array.isArray(task))
|
|
69
|
+
throw new Error("InvalidInput: DataForSEO body must be an array with one task object");
|
|
70
|
+
const t = task;
|
|
71
|
+
for (const field of ["limit", "depth"])
|
|
72
|
+
if (t[field] !== undefined && !nonNegativeInteger(t[field]))
|
|
73
|
+
throw new Error(`InvalidInput: DataForSEO ${field}`);
|
|
74
|
+
if (t.priority !== undefined && t.priority !== 1 && t.priority !== 2)
|
|
75
|
+
throw new Error("InvalidInput: DataForSEO priority");
|
|
76
|
+
if (t.keywords !== undefined &&
|
|
77
|
+
(!Array.isArray(t.keywords) || t.keywords.some((keyword) => typeof keyword !== "string")))
|
|
78
|
+
throw new Error("InvalidInput: DataForSEO keywords");
|
|
79
|
+
for (const field of ["include_clickstream_data", "include_seed_keyword"])
|
|
80
|
+
if (t[field] !== undefined && typeof t[field] !== "boolean")
|
|
81
|
+
throw new Error(`InvalidInput: DataForSEO ${field}`);
|
|
82
|
+
return t;
|
|
83
|
+
}
|
|
84
|
+
const isSerp = (id) => id.startsWith("serp.google.organic.");
|
|
85
|
+
function validateSerpDepth(depth) {
|
|
86
|
+
if (depth !== undefined &&
|
|
87
|
+
(!/^\d+$/.test(String(depth)) ||
|
|
88
|
+
BigInt(String(depth)) < 1n ||
|
|
89
|
+
BigInt(String(depth)) > BigInt(Number.MAX_SAFE_INTEGER)))
|
|
90
|
+
throw new Error("InvalidInput: DataForSEO SERP depth must be a positive safe integer");
|
|
91
|
+
}
|
|
92
|
+
// "Contains" is intentional, including embedded forms such as android: (id:).
|
|
93
|
+
// Each distinct documented substring contributes a factor of five.
|
|
94
|
+
// https://docs.dataforseo.com/v3/serp/google/organic/live/advanced/
|
|
95
|
+
// https://docs.dataforseo.com/v3/serp/google/organic/task_post/
|
|
96
|
+
// https://dataforseo.com/pricing/serp/google-organic-serp-api
|
|
97
|
+
const searchOperators = [
|
|
98
|
+
"allinanchor:",
|
|
99
|
+
"allintext:",
|
|
100
|
+
"allintitle:",
|
|
101
|
+
"allinurl:",
|
|
102
|
+
"cache:",
|
|
103
|
+
"define:",
|
|
104
|
+
"definition:",
|
|
105
|
+
"filetype:",
|
|
106
|
+
"id:",
|
|
107
|
+
"inanchor:",
|
|
108
|
+
"info:",
|
|
109
|
+
"intext:",
|
|
110
|
+
"intitle:",
|
|
111
|
+
"inurl:",
|
|
112
|
+
"link:",
|
|
113
|
+
"site:",
|
|
114
|
+
];
|
|
115
|
+
function operatorCount(task) {
|
|
116
|
+
const values = [
|
|
117
|
+
...(typeof task.keyword === "string" ? [task.keyword, decodeKeyword(task.keyword)] : []),
|
|
118
|
+
...(typeof task.search_param === "string"
|
|
119
|
+
? [task.search_param, decodeSearchParam(task.search_param)]
|
|
120
|
+
: []),
|
|
121
|
+
];
|
|
122
|
+
const normalized = values
|
|
123
|
+
.flatMap((v) => [
|
|
124
|
+
v.normalize("NFKC").toLowerCase(),
|
|
125
|
+
v.normalize("NFKC").toUpperCase().toLowerCase(),
|
|
126
|
+
v
|
|
127
|
+
.normalize("NFKD")
|
|
128
|
+
.replace(/[\p{M}\p{Default_Ignorable_Code_Point}]/gu, "")
|
|
129
|
+
.toLowerCase(),
|
|
130
|
+
])
|
|
131
|
+
.map((v) => v.replaceAll("-site:", ""));
|
|
132
|
+
// -site: is explicitly free. Other operator matches are conservative upper bounds.
|
|
133
|
+
// https://dataforseo.com/help-center/rules-and-limitations-of-keyword-and-keywords-fields-in-dataforseo-apis
|
|
134
|
+
return searchOperators.filter((op) => normalized.some((value) => value.includes(op))).length;
|
|
135
|
+
}
|
|
136
|
+
/** Keywords are text, not query strings: preserve literal percent signs. Decode each
|
|
137
|
+
* valid escape run leniently; invalid UTF-8 bytes become replacement characters. */
|
|
138
|
+
function decodeKeyword(value) {
|
|
139
|
+
value = value.normalize("NFKC");
|
|
140
|
+
for (let i = 0; i < 16; i++) {
|
|
141
|
+
const decoded = value
|
|
142
|
+
.replace(/(?:%[0-9a-f]{2})+/gi, (run) => {
|
|
143
|
+
try {
|
|
144
|
+
return decodeURIComponent(run);
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return new TextDecoder().decode(Uint8Array.from(run.match(/[0-9a-f]{2}/gi), (hex) => parseInt(hex, 16)));
|
|
148
|
+
}
|
|
149
|
+
})
|
|
150
|
+
.normalize("NFKC");
|
|
151
|
+
if (decoded === value)
|
|
152
|
+
return decoded;
|
|
153
|
+
value = decoded;
|
|
154
|
+
}
|
|
155
|
+
throw new Error("InvalidInput: DataForSEO keyword encoding depth");
|
|
156
|
+
}
|
|
157
|
+
function validateSerpOptions(options) {
|
|
158
|
+
validateSerpDepth(options.depth);
|
|
159
|
+
for (const [field, max] of [
|
|
160
|
+
["operatorCount", searchOperators.length],
|
|
161
|
+
["clicks", 4],
|
|
162
|
+
]) {
|
|
163
|
+
const value = options[field];
|
|
164
|
+
if (value !== undefined && (!/^\d+$/.test(value) || BigInt(value) > BigInt(max)))
|
|
165
|
+
throw new Error(`InvalidInput: DataForSEO SERP ${field}`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
function decodeSearchParam(value) {
|
|
169
|
+
value = value.normalize("NFKC");
|
|
170
|
+
// Decode nested query escapes before checking operators. Fail closed when malformed
|
|
171
|
+
// or still changing after the bound rather than passing an unchecked encoded operator.
|
|
172
|
+
for (let i = 0; i < 16; i++) {
|
|
173
|
+
let decoded;
|
|
174
|
+
try {
|
|
175
|
+
decoded = decodeURIComponent(value.replace(/\+/g, " ")).normalize("NFKC");
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
throw new Error("InvalidInput: DataForSEO malformed search_param escape");
|
|
179
|
+
}
|
|
180
|
+
if (decoded === value)
|
|
181
|
+
return decoded;
|
|
182
|
+
value = decoded;
|
|
183
|
+
}
|
|
184
|
+
throw new Error("InvalidInput: DataForSEO search_param encoding depth");
|
|
185
|
+
}
|
|
186
|
+
function validateSerpTask(task) {
|
|
187
|
+
validateSerpDepth(task.depth);
|
|
188
|
+
for (const field of ["calculate_rectangles", "load_async_ai_overview"])
|
|
189
|
+
if (task[field] !== undefined && typeof task[field] !== "boolean")
|
|
190
|
+
throw new Error(`InvalidInput: DataForSEO ${field}`);
|
|
191
|
+
for (const [field, max] of [
|
|
192
|
+
["people_also_ask_click_depth", 4],
|
|
193
|
+
["max_crawl_pages", 100],
|
|
194
|
+
])
|
|
195
|
+
if (task[field] !== undefined &&
|
|
196
|
+
(!nonNegativeInteger(task[field]) || Number(task[field]) < 1 || Number(task[field]) > max))
|
|
197
|
+
throw new Error(`InvalidInput: DataForSEO ${field}`);
|
|
198
|
+
if (task.stop_crawl_on_match !== undefined && !Array.isArray(task.stop_crawl_on_match))
|
|
199
|
+
throw new Error("InvalidInput: DataForSEO stop_crawl_on_match");
|
|
200
|
+
for (const field of ["keyword", "search_param"])
|
|
201
|
+
if (task[field] !== undefined && typeof task[field] !== "string")
|
|
202
|
+
throw new Error(`InvalidInput: DataForSEO ${field}`);
|
|
203
|
+
}
|
|
43
204
|
export const extractors = {
|
|
44
|
-
|
|
45
|
-
|
|
205
|
+
canEstimate: (options) => options.pricingUnknown !== "true",
|
|
206
|
+
// https://dataforseo.com/pricing/serp/google-organic-serp-api
|
|
207
|
+
// https://dataforseo.com/pricing/dataforseo-labs/dataforseo-google-api
|
|
208
|
+
requiresListPriceFloor: (options) => Number(options.operatorCount ?? "0") > 0 ||
|
|
209
|
+
options.rectangles === "true" ||
|
|
210
|
+
options.asyncOverview === "true" ||
|
|
211
|
+
Number(options.clicks ?? "0") > 0 ||
|
|
212
|
+
options.crawlPages !== undefined ||
|
|
213
|
+
options.clickstream === "true",
|
|
214
|
+
estimate: (input) => {
|
|
215
|
+
if (isSerp(input.operation.id))
|
|
216
|
+
validateSerpOptions(input.options);
|
|
217
|
+
if (!extractors.canEstimate(input.options))
|
|
218
|
+
return [];
|
|
219
|
+
const estimates = listEstimate(input);
|
|
220
|
+
if (!isSerp(input.operation.id))
|
|
221
|
+
return estimates;
|
|
222
|
+
// Bound every page and operator combination, including add-ons. Field docs give
|
|
223
|
+
// $0.002 live / $0.0006 queued; the pricing table gives one base (higher for high priority).
|
|
224
|
+
// https://docs.dataforseo.com/v3/serp/google/organic/live/advanced/
|
|
225
|
+
// https://docs.dataforseo.com/v3/serp/google/organic/task_post/
|
|
226
|
+
// https://dataforseo.com/pricing/serp/google-organic-serp-api
|
|
227
|
+
// https://dataforseo.com/help-center/serp-api-cost-explained
|
|
228
|
+
const pages = (BigInt(input.options.depth ?? "10") + 9n) / 10n;
|
|
229
|
+
const operators = BigInt(input.options.operatorCount ?? "0");
|
|
230
|
+
const factor = 5n ** operators;
|
|
231
|
+
const addOns = Number(input.options.rectangles === "true") + Number(input.options.asyncOverview === "true");
|
|
232
|
+
const extra = decimalQuantity(input.operation.id.endsWith("task_post")
|
|
233
|
+
? input.options.priority === "high"
|
|
234
|
+
? "0.12"
|
|
235
|
+
: "0.06"
|
|
236
|
+
: "0.2", "cents");
|
|
237
|
+
const clicks = decimalQuantity("0.015", "cents");
|
|
238
|
+
return estimates.map((q) => {
|
|
239
|
+
if (q.unit !== "cents")
|
|
240
|
+
return q;
|
|
241
|
+
const total = addQuantity(addQuantity(q, { ...extra, value: extra.value * BigInt(addOns) * pages }), { ...clicks, value: clicks.value * BigInt(input.options.clicks ?? "0") * pages });
|
|
242
|
+
return decimalQuantity(formatQuantity({ ...total, value: total.value * factor }), "cents");
|
|
243
|
+
});
|
|
244
|
+
},
|
|
245
|
+
/** Request dimensions and documented defaults, separate from observed response quantities. */
|
|
46
246
|
options: (operation, request) => {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
247
|
+
const first = operation.billable
|
|
248
|
+
? billableTask(request.body)
|
|
249
|
+
: Array.isArray(request.body)
|
|
250
|
+
? request.body[0]
|
|
251
|
+
: null;
|
|
50
252
|
if (!first || typeof first !== "object")
|
|
51
253
|
return {};
|
|
52
|
-
const
|
|
53
|
-
|
|
254
|
+
const task = first, options = {};
|
|
255
|
+
const keys = operation.optionKeys ?? [];
|
|
256
|
+
if (operation.billable) {
|
|
257
|
+
const fields = operation.requestPricing;
|
|
258
|
+
const unlisted = !fields ||
|
|
259
|
+
Object.keys(task).some((key) => task[key] !== undefined &&
|
|
260
|
+
!Object.hasOwn(fields.priceFields, key) &&
|
|
261
|
+
!fields.neutralFields.includes(key));
|
|
262
|
+
if (unlisted ||
|
|
263
|
+
task.limit === 0 ||
|
|
264
|
+
Number(task.limit) > 1000 ||
|
|
265
|
+
Object.entries(fields?.priceFields ?? {}).some(([field, rule]) => rule.unpricedWhen === "present"
|
|
266
|
+
? task[field] !== undefined
|
|
267
|
+
: rule.unpricedWhen === "true" && task[field] === true))
|
|
268
|
+
options.pricingUnknown = "true";
|
|
269
|
+
}
|
|
270
|
+
if (operation.billable && isSerp(operation.id))
|
|
271
|
+
validateSerpTask(task);
|
|
272
|
+
if (keys.includes("priority")) {
|
|
273
|
+
const priority = task.priority ?? 1;
|
|
274
|
+
if (priority === 1)
|
|
275
|
+
options.priority = "normal";
|
|
276
|
+
else if (priority === 2)
|
|
277
|
+
options.priority = "high";
|
|
278
|
+
}
|
|
279
|
+
if (keys.includes("depth") && task.depth !== undefined)
|
|
280
|
+
options.depth = String(task.depth);
|
|
281
|
+
if (operation.billable && isSerp(operation.id)) {
|
|
282
|
+
if (task.max_crawl_pages !== undefined) {
|
|
283
|
+
options.crawlPages = String(task.max_crawl_pages);
|
|
284
|
+
options.depth = String(Math.max(Number(task.depth ?? 10), Number(task.max_crawl_pages) * 10));
|
|
285
|
+
}
|
|
286
|
+
const count = operatorCount(task);
|
|
287
|
+
if (count)
|
|
288
|
+
options.operatorCount = String(count);
|
|
289
|
+
if (task.calculate_rectangles === true)
|
|
290
|
+
options.rectangles = "true";
|
|
291
|
+
if (task.load_async_ai_overview === true)
|
|
292
|
+
options.asyncOverview = "true";
|
|
293
|
+
if (task.people_also_ask_click_depth !== undefined)
|
|
294
|
+
options.clicks = String(task.people_also_ask_click_depth);
|
|
295
|
+
}
|
|
296
|
+
if (keys.includes("itemCount")) {
|
|
297
|
+
const field = itemCountFields[operation.id];
|
|
298
|
+
const count = field === "keywords"
|
|
299
|
+
? Array.isArray(task.keywords)
|
|
300
|
+
? task.keywords.length
|
|
301
|
+
: undefined
|
|
302
|
+
: field === "limit"
|
|
303
|
+
? // Only omission uses the documented default; zero marks the request unknown.
|
|
304
|
+
task.limit === undefined
|
|
305
|
+
? operation.requestPricing?.priceFields.limit?.defaultCount
|
|
306
|
+
: task.limit
|
|
307
|
+
: // Summary returns one row. History has no documented request count.
|
|
308
|
+
// https://docs.dataforseo.com/v3/backlinks/summary/live/
|
|
309
|
+
// https://docs.dataforseo.com/v3/backlinks/history/live/
|
|
310
|
+
operation.id === "backlinks.summary.live"
|
|
311
|
+
? 1
|
|
312
|
+
: undefined;
|
|
313
|
+
if (count !== undefined)
|
|
314
|
+
options.itemCount = String(count);
|
|
315
|
+
}
|
|
316
|
+
if (keys.includes("clickstream") && task.include_clickstream_data !== undefined)
|
|
317
|
+
options.clickstream = task.include_clickstream_data === false ? "false" : "true";
|
|
318
|
+
return options;
|
|
54
319
|
},
|
|
55
320
|
/**
|
|
56
321
|
* Cost from the envelope `cost` (USD, every task of the call). A task status of 40000 or
|
|
@@ -115,12 +380,15 @@ export const extractors = {
|
|
|
115
380
|
continue;
|
|
116
381
|
if (!withPriority.has(operation) && level !== "normal")
|
|
117
382
|
continue;
|
|
383
|
+
const calculation = descriptor.prices.find((r) => r.operation === operation && (!r.option || r.option.priority === level))?.calculation;
|
|
118
384
|
rows.push({
|
|
119
385
|
validFrom: checkedAt,
|
|
120
386
|
operation,
|
|
121
387
|
...(withPriority.has(operation) ? { option: { priority: level } } : {}),
|
|
122
388
|
unit: "cents",
|
|
123
389
|
perUnit: formatQuantity(centsQuantity(entry.cost)),
|
|
390
|
+
// The endpoint refreshes base prices; retain static formula metadata.
|
|
391
|
+
...(calculation ? { calculation } : {}),
|
|
124
392
|
source: priceListSource,
|
|
125
393
|
checkedAt,
|
|
126
394
|
});
|