mcp-scraper 0.88.2 → 0.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -15
- package/README.md +6 -5
- package/THIRD_PARTY_NOTICES.html +203 -0
- package/dist/analytics-repository-2BMT5JNE.js +1 -0
- package/dist/bin/api-server.js +2 -41
- package/dist/bin/mcp-scraper-cli.js +39 -756
- package/dist/bin/mcp-scraper-core.js +1 -60
- package/dist/bin/mcp-scraper-install.js +2 -25
- package/dist/bin/mcp-stdio-server.js +1 -19
- package/dist/bin/paa-harvest.js +1 -41
- package/dist/chunk-3GP5CYZX.js +1 -0
- package/dist/chunk-4AI7DOS7.js +59 -0
- package/dist/chunk-4FROKQJN.js +1 -0
- package/dist/chunk-7YGVI5J4.js +21710 -0
- package/dist/chunk-CCYWSJNG.js +16 -0
- package/dist/chunk-CFI6CXIV.js +182 -0
- package/dist/chunk-E2WRWV3A.js +1 -0
- package/dist/chunk-GMWKPIYX.js +1172 -0
- package/dist/chunk-HDPYG3XV.js +102 -0
- package/dist/chunk-HE45FFBU.js +1 -0
- package/dist/chunk-HUV2WTRW.js +1 -0
- package/dist/chunk-KJQXUZ4Y.js +4 -0
- package/dist/chunk-L4CGLFPU.js +4 -0
- package/dist/chunk-M22MM4N4.js +84 -0
- package/dist/chunk-MASR22K4.js +73 -0
- package/dist/chunk-MZN4U5BL.js +1 -0
- package/dist/chunk-PUHFVA7P.js +1280 -0
- package/dist/chunk-QPWPR5XG.js +10 -0
- package/dist/chunk-TMB56NCA.js +1 -0
- package/dist/chunk-TXENITMS.js +20 -0
- package/dist/chunk-W2BVJ7S2.js +13 -0
- package/dist/chunk-WO3N5FH2.js +5 -0
- package/dist/chunk-WSCGYRWA.js +2595 -0
- package/dist/chunk-X54CQLK2.js +1 -0
- package/dist/chunk-XLWNEVUZ.js +27 -0
- package/dist/chunk-XPZVJIZ2.js +100 -0
- package/dist/chunk-YQZGZBB4.js +1 -0
- package/dist/chunk-Z2QGQJS2.js +1 -0
- package/dist/db-F2MX63GI.js +1 -0
- package/dist/extract-bundle-SNUIHM3J.js +26 -0
- package/dist/gmail-service-BZ3H75XC.js +1 -0
- package/dist/index.cjs +21750 -6045
- package/dist/index.d.cts +14 -14
- package/dist/index.d.ts +14 -14
- package/dist/index.js +18 -315
- package/dist/lead-list-enrichment-repository-S2H3U7T7.js +1 -0
- package/dist/location-data-repository-OTWHWMV6.js +1 -0
- package/dist/server-RFR2A5UJ.js +7303 -0
- package/dist/site-extract-repository-SE776XDC.js +1 -0
- package/dist/worker-XUDSM3AL.js +1 -0
- package/package.json +17 -124
- package/dist/analytics-repository-GGJJCVVP.js +0 -194
- package/dist/chunk-4QMUF6XM.js +0 -1013
- package/dist/chunk-6HAV7LCE.js +0 -265
- package/dist/chunk-ABF2CGOZ.js +0 -113
- package/dist/chunk-C5Z4OFKW.js +0 -404
- package/dist/chunk-DNM65UCK.js +0 -299
- package/dist/chunk-EQGTEHLZ.js +0 -592
- package/dist/chunk-F5GQJWZU.js +0 -732
- package/dist/chunk-GGZEC22A.js +0 -215
- package/dist/chunk-GXBZXWXB.js +0 -184
- package/dist/chunk-IHXAXYIS.js +0 -843
- package/dist/chunk-K3Z5AQYE.js +0 -683
- package/dist/chunk-K45K75OF.js +0 -6
- package/dist/chunk-LFW2FRPJ.js +0 -224
- package/dist/chunk-MZDNZQWT.js +0 -2078
- package/dist/chunk-NVUKO5NN.js +0 -256
- package/dist/chunk-OM7HVEJ3.js +0 -26
- package/dist/chunk-OPQIGAFB.js +0 -286
- package/dist/chunk-OZJMVCDK.js +0 -16
- package/dist/chunk-P7FWOMU7.js +0 -505
- package/dist/chunk-PGJQDMC2.js +0 -383
- package/dist/chunk-PKZS6SHW.js +0 -33139
- package/dist/chunk-RJ7JVYKU.js +0 -68
- package/dist/chunk-S24LFPL7.js +0 -5262
- package/dist/chunk-T3MZISOF.js +0 -240
- package/dist/chunk-UZPTGUDV.js +0 -1915
- package/dist/chunk-X623GTBV.js +0 -8290
- package/dist/chunk-YXNDOQXN.js +0 -4018
- package/dist/db-Z34LPZNR.js +0 -284
- package/dist/extract-bundle-565SBZCR.js +0 -1003
- package/dist/gmail-service-E6ALS7JG.js +0 -25
- package/dist/lead-list-enrichment-repository-36RPVV6N.js +0 -67
- package/dist/location-data-repository-WPRG62GE.js +0 -34
- package/dist/server-SQZ3A7SY.js +0 -86606
- package/dist/site-extract-repository-VYFZASPU.js +0 -69
- package/dist/worker-LDCAULWL.js +0 -146
package/dist/chunk-NVUKO5NN.js
DELETED
|
@@ -1,256 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
isVendorUnavailableError
|
|
3
|
-
} from "./chunk-OZJMVCDK.js";
|
|
4
|
-
import {
|
|
5
|
-
CaptchaError,
|
|
6
|
-
LocationMismatchError,
|
|
7
|
-
PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
8
|
-
RequestAbortedError,
|
|
9
|
-
attachHarvestAttemptProviderSession,
|
|
10
|
-
buildPublicErrorEnvelope,
|
|
11
|
-
finishHarvestAttempt,
|
|
12
|
-
publicErrorMessage,
|
|
13
|
-
startHarvestAttempt
|
|
14
|
-
} from "./chunk-YXNDOQXN.js";
|
|
15
|
-
|
|
16
|
-
// src/api/harvest-problems.ts
|
|
17
|
-
function errorMessage(err) {
|
|
18
|
-
return err instanceof Error ? err.message : String(err);
|
|
19
|
-
}
|
|
20
|
-
function looksLikeTimeout(err, message) {
|
|
21
|
-
if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return true;
|
|
22
|
-
return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message);
|
|
23
|
-
}
|
|
24
|
-
function looksLikeCaptcha(message) {
|
|
25
|
-
return /captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(message);
|
|
26
|
-
}
|
|
27
|
-
function looksLikeProxyTunnelFailure(message) {
|
|
28
|
-
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
29
|
-
}
|
|
30
|
-
function looksLikeProxyUnavailable(message) {
|
|
31
|
-
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
32
|
-
}
|
|
33
|
-
function looksLikeInternalServiceUnavailable(message) {
|
|
34
|
-
return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
|
|
35
|
-
}
|
|
36
|
-
function looksLikeBrowserSessionInterrupted(message) {
|
|
37
|
-
return /browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message);
|
|
38
|
-
}
|
|
39
|
-
function classifyHarvestProblem(err) {
|
|
40
|
-
const message = errorMessage(err);
|
|
41
|
-
if (err instanceof RequestAbortedError) {
|
|
42
|
-
return {
|
|
43
|
-
error_code: "request_aborted",
|
|
44
|
-
error_type: "request_aborted",
|
|
45
|
-
message: publicErrorMessage("request_aborted"),
|
|
46
|
-
retryable: true,
|
|
47
|
-
httpStatus: 408,
|
|
48
|
-
terminalStatus: "cancelled"
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
if (err instanceof CaptchaError || looksLikeCaptcha(message)) {
|
|
52
|
-
return {
|
|
53
|
-
error_code: "captcha_exhausted",
|
|
54
|
-
error_type: "captcha",
|
|
55
|
-
message: publicErrorMessage("captcha_exhausted"),
|
|
56
|
-
retryable: true,
|
|
57
|
-
httpStatus: 503,
|
|
58
|
-
terminalStatus: "failed"
|
|
59
|
-
};
|
|
60
|
-
}
|
|
61
|
-
if (isVendorUnavailableError(err, message)) {
|
|
62
|
-
return {
|
|
63
|
-
error_code: "vendor_unavailable",
|
|
64
|
-
error_type: "service_unavailable",
|
|
65
|
-
message: publicErrorMessage("vendor_unavailable"),
|
|
66
|
-
retryable: true,
|
|
67
|
-
httpStatus: 503,
|
|
68
|
-
terminalStatus: "failed"
|
|
69
|
-
};
|
|
70
|
-
}
|
|
71
|
-
if (err instanceof LocationMismatchError) {
|
|
72
|
-
return {
|
|
73
|
-
error_code: "location_mismatch",
|
|
74
|
-
error_type: "location_mismatch",
|
|
75
|
-
message: publicErrorMessage("location_mismatch"),
|
|
76
|
-
retryable: true,
|
|
77
|
-
httpStatus: 503,
|
|
78
|
-
terminalStatus: "failed"
|
|
79
|
-
};
|
|
80
|
-
}
|
|
81
|
-
if (looksLikeProxyTunnelFailure(message)) {
|
|
82
|
-
return {
|
|
83
|
-
error_code: "proxy_tunnel_failed",
|
|
84
|
-
error_type: "connection",
|
|
85
|
-
message: publicErrorMessage("proxy_tunnel_failed"),
|
|
86
|
-
retryable: true,
|
|
87
|
-
httpStatus: 503,
|
|
88
|
-
terminalStatus: "failed"
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
if (looksLikeProxyUnavailable(message)) {
|
|
92
|
-
return {
|
|
93
|
-
error_code: "proxy_unavailable",
|
|
94
|
-
error_type: "connection",
|
|
95
|
-
message: publicErrorMessage("proxy_unavailable"),
|
|
96
|
-
retryable: true,
|
|
97
|
-
httpStatus: 503,
|
|
98
|
-
terminalStatus: "failed"
|
|
99
|
-
};
|
|
100
|
-
}
|
|
101
|
-
if (looksLikeBrowserSessionInterrupted(message)) {
|
|
102
|
-
return {
|
|
103
|
-
error_code: "browser_session_interrupted",
|
|
104
|
-
error_type: "extraction",
|
|
105
|
-
message: publicErrorMessage("browser_session_interrupted"),
|
|
106
|
-
retryable: true,
|
|
107
|
-
httpStatus: 503,
|
|
108
|
-
terminalStatus: "failed"
|
|
109
|
-
};
|
|
110
|
-
}
|
|
111
|
-
if (looksLikeTimeout(err, message)) {
|
|
112
|
-
return {
|
|
113
|
-
error_code: "harvest_timeout",
|
|
114
|
-
error_type: "timeout",
|
|
115
|
-
message: publicErrorMessage("harvest_timeout"),
|
|
116
|
-
retryable: true,
|
|
117
|
-
httpStatus: 504,
|
|
118
|
-
terminalStatus: "failed"
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
if (looksLikeInternalServiceUnavailable(message)) {
|
|
122
|
-
return {
|
|
123
|
-
error_code: "service_unavailable",
|
|
124
|
-
error_type: "service_unavailable",
|
|
125
|
-
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
126
|
-
retryable: true,
|
|
127
|
-
httpStatus: 503,
|
|
128
|
-
terminalStatus: "failed"
|
|
129
|
-
};
|
|
130
|
-
}
|
|
131
|
-
return {
|
|
132
|
-
error_code: "extraction_failed",
|
|
133
|
-
error_type: "extraction",
|
|
134
|
-
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
135
|
-
retryable: false,
|
|
136
|
-
httpStatus: 500,
|
|
137
|
-
terminalStatus: "failed"
|
|
138
|
-
};
|
|
139
|
-
}
|
|
140
|
-
function serializeHarvestProblem(problem) {
|
|
141
|
-
return JSON.stringify({
|
|
142
|
-
error_code: problem.error_code,
|
|
143
|
-
error_type: problem.error_type,
|
|
144
|
-
message: problem.message,
|
|
145
|
-
retryable: problem.retryable
|
|
146
|
-
});
|
|
147
|
-
}
|
|
148
|
-
function harvestProblemResponse(problem, context = {}) {
|
|
149
|
-
const envelope = buildPublicErrorEnvelope({
|
|
150
|
-
errorCode: problem.error_code,
|
|
151
|
-
errorType: problem.error_type,
|
|
152
|
-
retryable: problem.retryable,
|
|
153
|
-
retryAfterSeconds: context.retryAfterSeconds,
|
|
154
|
-
chargeStatus: context.chargeStatus,
|
|
155
|
-
details: context.details
|
|
156
|
-
});
|
|
157
|
-
return {
|
|
158
|
-
error: envelope.message,
|
|
159
|
-
...envelope
|
|
160
|
-
};
|
|
161
|
-
}
|
|
162
|
-
function harvestProblemEnvelope(problem, context = {}) {
|
|
163
|
-
const { error: _legacyAlias, ...envelope } = harvestProblemResponse(problem, context);
|
|
164
|
-
return envelope;
|
|
165
|
-
}
|
|
166
|
-
function publicErrorBoundary(err, context = {}) {
|
|
167
|
-
const problem = classifyHarvestProblem(err);
|
|
168
|
-
return {
|
|
169
|
-
body: harvestProblemResponse(problem, context),
|
|
170
|
-
status: problem.httpStatus
|
|
171
|
-
};
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
// src/api/harvest-attempt-events.ts
|
|
175
|
-
function createHarvestAttemptRecorder(jobId, userId) {
|
|
176
|
-
return async (event) => {
|
|
177
|
-
if (event.type === "started") {
|
|
178
|
-
await startHarvestAttempt({
|
|
179
|
-
jobId,
|
|
180
|
-
userId,
|
|
181
|
-
attemptNumber: event.attemptNumber,
|
|
182
|
-
maxAttempts: event.maxAttempts,
|
|
183
|
-
query: event.query,
|
|
184
|
-
location: event.location,
|
|
185
|
-
maxQuestions: event.maxQuestions,
|
|
186
|
-
startedAt: event.startedAt
|
|
187
|
-
});
|
|
188
|
-
return;
|
|
189
|
-
}
|
|
190
|
-
if (event.type === "provider_session_started") {
|
|
191
|
-
await attachHarvestAttemptProviderSession({
|
|
192
|
-
jobId,
|
|
193
|
-
attemptNumber: event.attemptNumber,
|
|
194
|
-
browserProvider: event.provider,
|
|
195
|
-
providerSessionId: event.providerSessionId,
|
|
196
|
-
observedAt: event.observedAt
|
|
197
|
-
});
|
|
198
|
-
return;
|
|
199
|
-
}
|
|
200
|
-
await finishHarvestAttempt({
|
|
201
|
-
jobId,
|
|
202
|
-
attemptNumber: event.attemptNumber,
|
|
203
|
-
outcome: event.outcome,
|
|
204
|
-
kernelSessionId: event.kernelSessionId,
|
|
205
|
-
questionCount: event.questionCount,
|
|
206
|
-
durationMs: event.durationMs,
|
|
207
|
-
error: event.error,
|
|
208
|
-
willRetry: event.willRetry,
|
|
209
|
-
kernelDeleteStarted: event.cleanup.kernelDeleteStarted,
|
|
210
|
-
kernelDeleteSucceeded: event.cleanup.kernelDeleteSucceeded,
|
|
211
|
-
kernelDeleteError: event.cleanup.kernelDeleteError,
|
|
212
|
-
browserCloseSucceeded: event.cleanup.browserCloseSucceeded,
|
|
213
|
-
browserCloseError: event.cleanup.browserCloseError,
|
|
214
|
-
browserProvider: event.cleanup.provider ?? null,
|
|
215
|
-
providerSessionId: event.cleanup.providerSessionId ?? null,
|
|
216
|
-
providerLookupStatus: event.cleanup.providerSessionId ? "pending" : "not_applicable",
|
|
217
|
-
providerErrorCode: event.cleanup.providerDisconnectObserved ? "browser_session_interrupted" : null,
|
|
218
|
-
providerErrorMessage: event.cleanup.providerDisconnectMessage ?? null,
|
|
219
|
-
providerReasonSource: event.cleanup.providerDisconnectObserved ? "client_runtime" : null,
|
|
220
|
-
providerDisconnectObserved: event.cleanup.providerDisconnectObserved ?? false,
|
|
221
|
-
providerDisconnectMessage: event.cleanup.providerDisconnectMessage ?? null,
|
|
222
|
-
debug: event.debug,
|
|
223
|
-
completedAt: event.completedAt
|
|
224
|
-
});
|
|
225
|
-
};
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
// src/api/webhook.ts
|
|
229
|
-
async function deliverWebhook(url, payload, retries = 3) {
|
|
230
|
-
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
231
|
-
try {
|
|
232
|
-
const res = await fetch(url, {
|
|
233
|
-
method: "POST",
|
|
234
|
-
headers: { "content-type": "application/json" },
|
|
235
|
-
body: JSON.stringify(payload),
|
|
236
|
-
signal: AbortSignal.timeout(1e4)
|
|
237
|
-
});
|
|
238
|
-
if (res.ok) return;
|
|
239
|
-
console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
|
|
240
|
-
} catch (err) {
|
|
241
|
-
console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
|
|
242
|
-
}
|
|
243
|
-
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
|
|
244
|
-
}
|
|
245
|
-
console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
export {
|
|
249
|
-
classifyHarvestProblem,
|
|
250
|
-
serializeHarvestProblem,
|
|
251
|
-
harvestProblemResponse,
|
|
252
|
-
harvestProblemEnvelope,
|
|
253
|
-
publicErrorBoundary,
|
|
254
|
-
createHarvestAttemptRecorder,
|
|
255
|
-
deliverWebhook
|
|
256
|
-
};
|
package/dist/chunk-OM7HVEJ3.js
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
// src/lib/browser-service-env.ts
|
|
2
|
-
function browserServiceApiKey() {
|
|
3
|
-
const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim();
|
|
4
|
-
return value || void 0;
|
|
5
|
-
}
|
|
6
|
-
function browserServiceProxyId() {
|
|
7
|
-
return void 0;
|
|
8
|
-
}
|
|
9
|
-
function browserServiceProfileName() {
|
|
10
|
-
const value = (process.env.BROWSER_AGENT_PROFILE_NAME ?? process.env.BROWSER_SERVICE_PROFILE_NAME ?? process.env.KERNEL_BROWSER_PROFILE_NAME ?? process.env.KERNEL_PROFILE_NAME)?.trim();
|
|
11
|
-
return value || void 0;
|
|
12
|
-
}
|
|
13
|
-
function browserServiceProfileSaveChanges() {
|
|
14
|
-
const value = (process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ?? process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();
|
|
15
|
-
if (!value) return void 0;
|
|
16
|
-
if (["1", "true", "yes", "on"].includes(value)) return true;
|
|
17
|
-
if (["0", "false", "no", "off"].includes(value)) return false;
|
|
18
|
-
return void 0;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export {
|
|
22
|
-
browserServiceApiKey,
|
|
23
|
-
browserServiceProxyId,
|
|
24
|
-
browserServiceProfileName,
|
|
25
|
-
browserServiceProfileSaveChanges
|
|
26
|
-
};
|
package/dist/chunk-OPQIGAFB.js
DELETED
|
@@ -1,286 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
getDb
|
|
3
|
-
} from "./chunk-YXNDOQXN.js";
|
|
4
|
-
|
|
5
|
-
// src/api/cost-context.ts
|
|
6
|
-
import { AsyncLocalStorage } from "async_hooks";
|
|
7
|
-
var HEADLESS_HARVEST_TEST_ENV = "MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST";
|
|
8
|
-
var storage = new AsyncLocalStorage();
|
|
9
|
-
function runWithCostContext(ctx, fn) {
|
|
10
|
-
return storage.run(ctx, fn);
|
|
11
|
-
}
|
|
12
|
-
function currentCostContext() {
|
|
13
|
-
return storage.getStore();
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
// src/api/cost-telemetry.ts
|
|
17
|
-
import { randomUUID } from "crypto";
|
|
18
|
-
|
|
19
|
-
// src/api/cost-rates.ts
|
|
20
|
-
var KERNEL_HEADLESS_USD_PER_SEC = 166667e-10;
|
|
21
|
-
var KERNEL_HEADFUL_USD_PER_SEC = 1333336e-10;
|
|
22
|
-
var HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "fb_search", "fb_ad", "instagram"]);
|
|
23
|
-
var FAL_WIZPER_USD_PER_COMPUTE_SEC = 111e-5;
|
|
24
|
-
var FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR = 4;
|
|
25
|
-
var DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 4e-4;
|
|
26
|
-
function envRate(name, fallback) {
|
|
27
|
-
const raw = process.env[name]?.trim();
|
|
28
|
-
if (!raw) return fallback;
|
|
29
|
-
const parsed = Number(raw);
|
|
30
|
-
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
|
31
|
-
}
|
|
32
|
-
var NANGO_USD_PER_CONNECTION_MONTH = envRate("NANGO_USD_PER_CONNECTION_MONTH", 1);
|
|
33
|
-
var NANGO_USD_PER_FUNCTION_RUN = envRate("NANGO_USD_PER_FUNCTION_RUN", 1e-4);
|
|
34
|
-
var NANGO_USD_PER_PROXY_REQUEST = envRate("NANGO_USD_PER_PROXY_REQUEST", 1e-4);
|
|
35
|
-
var NANGO_USD_PER_COMPUTE_SEC = envRate("NANGO_USD_PER_COMPUTE_SEC", 2e-4);
|
|
36
|
-
var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB", 8);
|
|
37
|
-
function kernelCostUsd(ms, headful) {
|
|
38
|
-
const sec = Math.max(0, ms) / 1e3;
|
|
39
|
-
return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
|
|
40
|
-
}
|
|
41
|
-
function vendorCostUsd(vendor, units) {
|
|
42
|
-
if (vendor === "fal_wizper") {
|
|
43
|
-
const estimatedComputeSec = Math.max(0, units) / FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR;
|
|
44
|
-
return estimatedComputeSec * FAL_WIZPER_USD_PER_COMPUTE_SEC;
|
|
45
|
-
}
|
|
46
|
-
if (vendor === "deepinfra_qwen") return Math.max(0, units) / 1e3 * DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS;
|
|
47
|
-
if (vendor === "openrouter") return Math.max(0, units);
|
|
48
|
-
if (vendor === "mcp_memory_video") return Math.max(0, units);
|
|
49
|
-
if (vendor === "mcp_memory_ai") return Math.max(0, units);
|
|
50
|
-
if (vendor === "nango_connection") return Math.max(0, units) * NANGO_USD_PER_CONNECTION_MONTH;
|
|
51
|
-
if (vendor === "nango_function_run") return Math.max(0, units) * NANGO_USD_PER_FUNCTION_RUN;
|
|
52
|
-
if (vendor === "nango_proxy_request") return Math.max(0, units) * NANGO_USD_PER_PROXY_REQUEST;
|
|
53
|
-
if (vendor === "nango_compute") return Math.max(0, units) * NANGO_USD_PER_COMPUTE_SEC;
|
|
54
|
-
if (vendor === "brightdata_browser_api") {
|
|
55
|
-
return Math.max(0, units) / 1e9 * BRIGHTDATA_BROWSER_USD_PER_GB;
|
|
56
|
-
}
|
|
57
|
-
return 0;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
// src/api/cost-telemetry.ts
|
|
61
|
-
var _ready = false;
|
|
62
|
-
var _migrationPromise = null;
|
|
63
|
-
async function migrateCostTelemetry() {
|
|
64
|
-
if (_ready) return;
|
|
65
|
-
if (_migrationPromise) return _migrationPromise;
|
|
66
|
-
_migrationPromise = runCostTelemetryMigration().finally(() => {
|
|
67
|
-
_migrationPromise = null;
|
|
68
|
-
});
|
|
69
|
-
return _migrationPromise;
|
|
70
|
-
}
|
|
71
|
-
async function runCostTelemetryMigration() {
|
|
72
|
-
const db = getDb();
|
|
73
|
-
const existing = await db.execute(`
|
|
74
|
-
SELECT
|
|
75
|
-
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
76
|
-
AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
77
|
-
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key', 'provider_duration_ms', 'provider_captcha', 'provider_status')) = 5
|
|
78
|
-
AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
|
|
79
|
-
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
80
|
-
AS ready
|
|
81
|
-
`);
|
|
82
|
-
if (Number(existing.rows[0]?.ready ?? 0) === 1) {
|
|
83
|
-
_ready = true;
|
|
84
|
-
return;
|
|
85
|
-
}
|
|
86
|
-
await db.execute(`
|
|
87
|
-
CREATE TABLE IF NOT EXISTS kernel_session_log (
|
|
88
|
-
id TEXT PRIMARY KEY,
|
|
89
|
-
kernel_session_id TEXT,
|
|
90
|
-
op TEXT,
|
|
91
|
-
source TEXT NOT NULL,
|
|
92
|
-
probe_run_id TEXT,
|
|
93
|
-
user_id INTEGER,
|
|
94
|
-
stealth INTEGER,
|
|
95
|
-
headless_sent INTEGER,
|
|
96
|
-
proxy_used INTEGER,
|
|
97
|
-
fallback INTEGER NOT NULL DEFAULT 0,
|
|
98
|
-
opened_at TEXT,
|
|
99
|
-
closed_at TEXT,
|
|
100
|
-
duration_ms INTEGER NOT NULL DEFAULT 0,
|
|
101
|
-
est_cost_usd_headless REAL NOT NULL DEFAULT 0,
|
|
102
|
-
est_cost_usd_headful REAL NOT NULL DEFAULT 0,
|
|
103
|
-
error TEXT,
|
|
104
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
105
|
-
)
|
|
106
|
-
`);
|
|
107
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_op ON kernel_session_log(op)`);
|
|
108
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_probe ON kernel_session_log(probe_run_id)`);
|
|
109
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_created ON kernel_session_log(created_at)`);
|
|
110
|
-
try {
|
|
111
|
-
await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_source TEXT`);
|
|
112
|
-
} catch {
|
|
113
|
-
}
|
|
114
|
-
try {
|
|
115
|
-
await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_type TEXT`);
|
|
116
|
-
} catch {
|
|
117
|
-
}
|
|
118
|
-
try {
|
|
119
|
-
await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN method TEXT`);
|
|
120
|
-
} catch {
|
|
121
|
-
}
|
|
122
|
-
await db.execute(`
|
|
123
|
-
CREATE TABLE IF NOT EXISTS vendor_usage_log (
|
|
124
|
-
id TEXT PRIMARY KEY,
|
|
125
|
-
op TEXT,
|
|
126
|
-
probe_run_id TEXT,
|
|
127
|
-
user_id INTEGER,
|
|
128
|
-
vendor TEXT NOT NULL,
|
|
129
|
-
model TEXT,
|
|
130
|
-
units REAL NOT NULL DEFAULT 0,
|
|
131
|
-
unit_type TEXT,
|
|
132
|
-
est_cost_usd REAL NOT NULL DEFAULT 0,
|
|
133
|
-
error TEXT,
|
|
134
|
-
source_key TEXT,
|
|
135
|
-
provider_duration_ms INTEGER,
|
|
136
|
-
provider_captcha INTEGER,
|
|
137
|
-
provider_status TEXT,
|
|
138
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
139
|
-
)
|
|
140
|
-
`);
|
|
141
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_op ON vendor_usage_log(op)`);
|
|
142
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_probe ON vendor_usage_log(probe_run_id)`);
|
|
143
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_created ON vendor_usage_log(created_at)`);
|
|
144
|
-
try {
|
|
145
|
-
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN method TEXT`);
|
|
146
|
-
} catch {
|
|
147
|
-
}
|
|
148
|
-
try {
|
|
149
|
-
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN source_key TEXT`);
|
|
150
|
-
} catch {
|
|
151
|
-
}
|
|
152
|
-
try {
|
|
153
|
-
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_duration_ms INTEGER`);
|
|
154
|
-
} catch {
|
|
155
|
-
}
|
|
156
|
-
try {
|
|
157
|
-
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_captcha INTEGER`);
|
|
158
|
-
} catch {
|
|
159
|
-
}
|
|
160
|
-
try {
|
|
161
|
-
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_status TEXT`);
|
|
162
|
-
} catch {
|
|
163
|
-
}
|
|
164
|
-
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS vendor_usage_log_vendor_source_key ON vendor_usage_log(vendor, source_key) WHERE source_key IS NOT NULL`);
|
|
165
|
-
await db.execute(`
|
|
166
|
-
CREATE TABLE IF NOT EXISTS cost_probe_runs (
|
|
167
|
-
id TEXT PRIMARY KEY,
|
|
168
|
-
tool TEXT NOT NULL,
|
|
169
|
-
target TEXT,
|
|
170
|
-
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
171
|
-
wall_ms INTEGER NOT NULL DEFAULT 0,
|
|
172
|
-
success INTEGER NOT NULL DEFAULT 0,
|
|
173
|
-
error TEXT,
|
|
174
|
-
http_only INTEGER NOT NULL DEFAULT 0,
|
|
175
|
-
kernel_used INTEGER NOT NULL DEFAULT 0,
|
|
176
|
-
kernel_sessions INTEGER NOT NULL DEFAULT 0,
|
|
177
|
-
kernel_seconds_total REAL NOT NULL DEFAULT 0,
|
|
178
|
-
kernel_tier_observed TEXT,
|
|
179
|
-
est_kernel_cost_usd_headless REAL NOT NULL DEFAULT 0,
|
|
180
|
-
est_kernel_cost_usd_headful REAL NOT NULL DEFAULT 0,
|
|
181
|
-
vendor_cost_usd REAL NOT NULL DEFAULT 0,
|
|
182
|
-
charged_credits REAL NOT NULL DEFAULT 0,
|
|
183
|
-
margin_usd_headless_starter REAL NOT NULL DEFAULT 0,
|
|
184
|
-
margin_usd_headful_starter REAL NOT NULL DEFAULT 0,
|
|
185
|
-
notes TEXT
|
|
186
|
-
)
|
|
187
|
-
`);
|
|
188
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)`);
|
|
189
|
-
try {
|
|
190
|
-
await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN units REAL`);
|
|
191
|
-
} catch {
|
|
192
|
-
}
|
|
193
|
-
try {
|
|
194
|
-
await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT`);
|
|
195
|
-
} catch {
|
|
196
|
-
}
|
|
197
|
-
try {
|
|
198
|
-
await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT`);
|
|
199
|
-
} catch {
|
|
200
|
-
}
|
|
201
|
-
_ready = true;
|
|
202
|
-
}
|
|
203
|
-
async function recordKernelSession(r) {
|
|
204
|
-
try {
|
|
205
|
-
await migrateCostTelemetry();
|
|
206
|
-
const ctx = currentCostContext();
|
|
207
|
-
const durationMs = Math.max(0, r.closedAtMs - r.openedAtMs);
|
|
208
|
-
const db = getDb();
|
|
209
|
-
await db.execute({
|
|
210
|
-
sql: `INSERT INTO kernel_session_log
|
|
211
|
-
(id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)
|
|
212
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
213
|
-
args: [
|
|
214
|
-
randomUUID(),
|
|
215
|
-
r.kernelSessionId ?? null,
|
|
216
|
-
ctx?.op ?? null,
|
|
217
|
-
r.source,
|
|
218
|
-
ctx?.probeRunId ?? null,
|
|
219
|
-
ctx?.userId ?? null,
|
|
220
|
-
boolToInt(r.stealth),
|
|
221
|
-
boolToInt(r.headlessSent),
|
|
222
|
-
boolToInt(r.proxyUsed),
|
|
223
|
-
r.proxySource ?? null,
|
|
224
|
-
r.proxyType ?? null,
|
|
225
|
-
r.fallback ? 1 : 0,
|
|
226
|
-
new Date(r.openedAtMs).toISOString(),
|
|
227
|
-
new Date(r.closedAtMs).toISOString(),
|
|
228
|
-
durationMs,
|
|
229
|
-
kernelCostUsd(durationMs, false),
|
|
230
|
-
kernelCostUsd(durationMs, true),
|
|
231
|
-
r.error ?? null,
|
|
232
|
-
ctx?.subOp ?? null
|
|
233
|
-
]
|
|
234
|
-
});
|
|
235
|
-
} catch (err) {
|
|
236
|
-
console.warn("[cost-telemetry] recordKernelSession failed:", err instanceof Error ? err.message : String(err));
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
async function recordVendorUsage(r) {
|
|
240
|
-
try {
|
|
241
|
-
await migrateCostTelemetry();
|
|
242
|
-
const ctx = currentCostContext();
|
|
243
|
-
const db = getDb();
|
|
244
|
-
const result = await db.execute({
|
|
245
|
-
sql: `INSERT OR IGNORE INTO vendor_usage_log
|
|
246
|
-
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key, provider_duration_ms, provider_captcha, provider_status)
|
|
247
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
248
|
-
args: [
|
|
249
|
-
randomUUID(),
|
|
250
|
-
r.op ?? ctx?.op ?? null,
|
|
251
|
-
r.probeRunId ?? ctx?.probeRunId ?? null,
|
|
252
|
-
r.userId ?? ctx?.userId ?? null,
|
|
253
|
-
r.vendor,
|
|
254
|
-
r.model ?? null,
|
|
255
|
-
r.units,
|
|
256
|
-
r.unitType,
|
|
257
|
-
vendorCostUsd(r.vendor, r.units),
|
|
258
|
-
r.error ?? null,
|
|
259
|
-
r.method ?? ctx?.subOp ?? null,
|
|
260
|
-
r.sourceKey ?? null,
|
|
261
|
-
r.providerDurationMs ?? null,
|
|
262
|
-
boolToInt(r.providerCaptcha),
|
|
263
|
-
r.providerStatus ?? null
|
|
264
|
-
]
|
|
265
|
-
});
|
|
266
|
-
return result.rowsAffected === 1;
|
|
267
|
-
} catch (err) {
|
|
268
|
-
console.warn("[cost-telemetry] recordVendorUsage failed:", err instanceof Error ? err.message : String(err));
|
|
269
|
-
return false;
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
function boolToInt(v) {
|
|
273
|
-
if (v === null || v === void 0) return null;
|
|
274
|
-
return v ? 1 : 0;
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
export {
|
|
278
|
-
HEADLESS_HARVEST_TEST_ENV,
|
|
279
|
-
runWithCostContext,
|
|
280
|
-
currentCostContext,
|
|
281
|
-
HEADLESS_OPS,
|
|
282
|
-
vendorCostUsd,
|
|
283
|
-
migrateCostTelemetry,
|
|
284
|
-
recordKernelSession,
|
|
285
|
-
recordVendorUsage
|
|
286
|
-
};
|
package/dist/chunk-OZJMVCDK.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
// src/api/vendor-errors.ts
|
|
2
|
-
import {
|
|
3
|
-
AuthenticationError as KernelAuthenticationError,
|
|
4
|
-
APIConnectionError as KernelAPIConnectionError,
|
|
5
|
-
APIConnectionTimeoutError as KernelAPIConnectionTimeoutError,
|
|
6
|
-
InternalServerError as KernelInternalServerError,
|
|
7
|
-
PermissionDeniedError as KernelPermissionDeniedError
|
|
8
|
-
} from "@onkernel/sdk";
|
|
9
|
-
var BRIGHTDATA_NOT_CONFIGURED_MESSAGE = "SERP browser service is not configured";
|
|
10
|
-
function isVendorUnavailableError(err, message) {
|
|
11
|
-
return err instanceof KernelAuthenticationError || err instanceof KernelPermissionDeniedError || err instanceof KernelAPIConnectionError || err instanceof KernelAPIConnectionTimeoutError || err instanceof KernelInternalServerError || /^401\s+Invalid or disabled API key\s*$/i.test(message.trim()) || message === BRIGHTDATA_NOT_CONFIGURED_MESSAGE;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export {
|
|
15
|
-
isVendorUnavailableError
|
|
16
|
-
};
|