@dealcrawl/sdk 2.13.0 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +408 -1
- package/dist/index.d.mts +523 -2
- package/dist/index.d.ts +523 -2
- package/dist/index.js +233 -0
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +232 -1
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -2668,6 +2668,77 @@ var KeysResource = class {
|
|
|
2668
2668
|
}
|
|
2669
2669
|
};
|
|
2670
2670
|
|
|
2671
|
+
// src/resources/postprocessors.ts
|
|
2672
|
+
var PostprocessorsResource = class {
|
|
2673
|
+
constructor(ctx) {
|
|
2674
|
+
this.ctx = ctx;
|
|
2675
|
+
}
|
|
2676
|
+
/**
|
|
2677
|
+
* List all available postprocessors
|
|
2678
|
+
*
|
|
2679
|
+
* Returns information about each postprocessor including:
|
|
2680
|
+
* - Supported domains
|
|
2681
|
+
* - Extracted fields
|
|
2682
|
+
* - Description
|
|
2683
|
+
*
|
|
2684
|
+
* @example
|
|
2685
|
+
* ```ts
|
|
2686
|
+
* const result = await client.postprocessors.list();
|
|
2687
|
+
* console.log(result.postprocessors);
|
|
2688
|
+
* // [
|
|
2689
|
+
* // {
|
|
2690
|
+
* // name: "amazon",
|
|
2691
|
+
* // domains: ["amazon.com", "amazon.fr", ...],
|
|
2692
|
+
* // extractedFields: ["productId", "seller", "shipping", ...],
|
|
2693
|
+
* // description: "Amazon product page enrichment..."
|
|
2694
|
+
* // },
|
|
2695
|
+
* // ...
|
|
2696
|
+
* // ]
|
|
2697
|
+
* console.log(result.totalDomains); // 15
|
|
2698
|
+
* ```
|
|
2699
|
+
*/
|
|
2700
|
+
async list() {
|
|
2701
|
+
const result = await get(
|
|
2702
|
+
this.ctx,
|
|
2703
|
+
"/v1/postprocessors"
|
|
2704
|
+
);
|
|
2705
|
+
return result.data;
|
|
2706
|
+
}
|
|
2707
|
+
/**
|
|
2708
|
+
* Check which postprocessor applies to a given URL
|
|
2709
|
+
*
|
|
2710
|
+
* Useful for:
|
|
2711
|
+
* - Determining if a URL will be enriched before scraping
|
|
2712
|
+
* - Understanding what data will be extracted
|
|
2713
|
+
* - Filtering URLs by supported sites
|
|
2714
|
+
*
|
|
2715
|
+
* @example
|
|
2716
|
+
* ```ts
|
|
2717
|
+
* // Check Amazon URL
|
|
2718
|
+
* const check = await client.postprocessors.check({
|
|
2719
|
+
* url: "https://www.amazon.fr/dp/B09V3KXJPB"
|
|
2720
|
+
* });
|
|
2721
|
+
* console.log(check.hasPostprocessor); // true
|
|
2722
|
+
* console.log(check.postprocessor); // "amazon"
|
|
2723
|
+
* console.log(check.extractedFields); // ["productId", "seller", "shipping", ...]
|
|
2724
|
+
*
|
|
2725
|
+
* // Check unsupported URL
|
|
2726
|
+
* const noMatch = await client.postprocessors.check({
|
|
2727
|
+
* url: "https://example.com/product"
|
|
2728
|
+
* });
|
|
2729
|
+
* console.log(noMatch.hasPostprocessor); // false
|
|
2730
|
+
* console.log(noMatch.postprocessor); // null
|
|
2731
|
+
* ```
|
|
2732
|
+
*/
|
|
2733
|
+
async check(options) {
|
|
2734
|
+
const result = await get(
|
|
2735
|
+
this.ctx,
|
|
2736
|
+
`/v1/postprocessors/check?url=${encodeURIComponent(options.url)}`
|
|
2737
|
+
);
|
|
2738
|
+
return result.data;
|
|
2739
|
+
}
|
|
2740
|
+
};
|
|
2741
|
+
|
|
2671
2742
|
// src/resources/scrape.ts
|
|
2672
2743
|
var ScrapeResource = class {
|
|
2673
2744
|
constructor(ctx) {
|
|
@@ -3166,6 +3237,116 @@ var StatusResource = class {
|
|
|
3166
3237
|
}
|
|
3167
3238
|
};
|
|
3168
3239
|
|
|
3240
|
+
// src/resources/usage.ts
|
|
3241
|
+
var UsageResource = class {
|
|
3242
|
+
constructor(ctx) {
|
|
3243
|
+
this.ctx = ctx;
|
|
3244
|
+
}
|
|
3245
|
+
/**
|
|
3246
|
+
* Get current usage and quotas
|
|
3247
|
+
*
|
|
3248
|
+
* Returns the current billing period usage statistics including:
|
|
3249
|
+
* - Current usage counts for all operation types
|
|
3250
|
+
* - Quota limits based on tier
|
|
3251
|
+
* - Percentage used for each resource
|
|
3252
|
+
* - Time until quota reset
|
|
3253
|
+
*
|
|
3254
|
+
* @example
|
|
3255
|
+
* ```ts
|
|
3256
|
+
* const usage = await client.usage.current();
|
|
3257
|
+
* console.log(usage.tier); // "pro"
|
|
3258
|
+
* console.log(usage.usage.scrapes); // 150
|
|
3259
|
+
* console.log(usage.quotas.scrapes); // 10000
|
|
3260
|
+
* console.log(usage.percentUsed.scrapes); // 1.5
|
|
3261
|
+
* console.log(usage.billingPeriod.daysRemaining); // 15
|
|
3262
|
+
* ```
|
|
3263
|
+
*/
|
|
3264
|
+
async current() {
|
|
3265
|
+
const result = await get(this.ctx, "/v1/usage");
|
|
3266
|
+
return result.data;
|
|
3267
|
+
}
|
|
3268
|
+
/**
|
|
3269
|
+
* Get historical usage data
|
|
3270
|
+
*
|
|
3271
|
+
* Returns monthly usage snapshots for billing analysis.
|
|
3272
|
+
*
|
|
3273
|
+
* @example
|
|
3274
|
+
* ```ts
|
|
3275
|
+
* const history = await client.usage.history({ months: 6 });
|
|
3276
|
+
* history.data.forEach(period => {
|
|
3277
|
+
* console.log(`${period.periodStart}: ${period.usage.scrapes} scrapes`);
|
|
3278
|
+
* });
|
|
3279
|
+
* ```
|
|
3280
|
+
*/
|
|
3281
|
+
async history(options = {}) {
|
|
3282
|
+
const params = new URLSearchParams();
|
|
3283
|
+
if (options.months) {
|
|
3284
|
+
params.set("months", options.months.toString());
|
|
3285
|
+
}
|
|
3286
|
+
const query = params.toString();
|
|
3287
|
+
const result = await get(
|
|
3288
|
+
this.ctx,
|
|
3289
|
+
`/v1/usage/history${query ? `?${query}` : ""}`
|
|
3290
|
+
);
|
|
3291
|
+
return result.data;
|
|
3292
|
+
}
|
|
3293
|
+
/**
|
|
3294
|
+
* Get LLM token usage summary
|
|
3295
|
+
*
|
|
3296
|
+
* Returns aggregated token consumption and estimated costs.
|
|
3297
|
+
*
|
|
3298
|
+
* @example
|
|
3299
|
+
* ```ts
|
|
3300
|
+
* const tokens = await client.usage.tokens({ days: 30 });
|
|
3301
|
+
* console.log(tokens.totals.totalTokens); // 1234567
|
|
3302
|
+
* console.log(tokens.totals.estimatedCostUsd); // 12.34
|
|
3303
|
+
* tokens.breakdown.forEach(item => {
|
|
3304
|
+
* console.log(`${item.provider}/${item.model}: ${item.totalTokens} tokens`);
|
|
3305
|
+
* });
|
|
3306
|
+
* ```
|
|
3307
|
+
*/
|
|
3308
|
+
async tokens(options = {}) {
|
|
3309
|
+
const params = new URLSearchParams();
|
|
3310
|
+
if (options.days) {
|
|
3311
|
+
params.set("days", options.days.toString());
|
|
3312
|
+
}
|
|
3313
|
+
if (options.provider) {
|
|
3314
|
+
params.set("provider", options.provider);
|
|
3315
|
+
}
|
|
3316
|
+
const query = params.toString();
|
|
3317
|
+
const result = await get(
|
|
3318
|
+
this.ctx,
|
|
3319
|
+
`/v1/usage/tokens${query ? `?${query}` : ""}`
|
|
3320
|
+
);
|
|
3321
|
+
return result.data;
|
|
3322
|
+
}
|
|
3323
|
+
/**
|
|
3324
|
+
* Get daily token usage breakdown
|
|
3325
|
+
*
|
|
3326
|
+
* Returns day-by-day token consumption for charting.
|
|
3327
|
+
*
|
|
3328
|
+
* @example
|
|
3329
|
+
* ```ts
|
|
3330
|
+
* const daily = await client.usage.dailyTokens({ days: 7 });
|
|
3331
|
+
* daily.daily.forEach(day => {
|
|
3332
|
+
* console.log(`${day.date}: ${day.totalTokens} tokens ($${day.estimatedCostUsd})`);
|
|
3333
|
+
* });
|
|
3334
|
+
* ```
|
|
3335
|
+
*/
|
|
3336
|
+
async dailyTokens(options = {}) {
|
|
3337
|
+
const params = new URLSearchParams();
|
|
3338
|
+
if (options.days) {
|
|
3339
|
+
params.set("days", options.days.toString());
|
|
3340
|
+
}
|
|
3341
|
+
const query = params.toString();
|
|
3342
|
+
const result = await get(
|
|
3343
|
+
this.ctx,
|
|
3344
|
+
`/v1/usage/tokens/daily${query ? `?${query}` : ""}`
|
|
3345
|
+
);
|
|
3346
|
+
return result.data;
|
|
3347
|
+
}
|
|
3348
|
+
};
|
|
3349
|
+
|
|
3169
3350
|
// src/resources/webhooks.ts
|
|
3170
3351
|
var WebhooksResource = class {
|
|
3171
3352
|
constructor(ctx) {
|
|
@@ -3690,6 +3871,54 @@ var DealCrawl = class {
|
|
|
3690
3871
|
* ```
|
|
3691
3872
|
*/
|
|
3692
3873
|
events;
|
|
3874
|
+
/**
|
|
3875
|
+
* Postprocessors resource - Discover site-specific enrichment postprocessors
|
|
3876
|
+
*
|
|
3877
|
+
* Postprocessors automatically enrich scrape results for supported e-commerce sites
|
|
3878
|
+
* (Amazon, Fnac, eBay) with additional data like seller info, shipping, availability.
|
|
3879
|
+
*
|
|
3880
|
+
* @example
|
|
3881
|
+
* ```ts
|
|
3882
|
+
* // List all available postprocessors
|
|
3883
|
+
* const list = await client.postprocessors.list();
|
|
3884
|
+
* console.log(list.postprocessors);
|
|
3885
|
+
* // [{ name: "amazon", domains: ["amazon.com", ...], extractedFields: [...] }, ...]
|
|
3886
|
+
*
|
|
3887
|
+
* // Check if a URL will be enriched
|
|
3888
|
+
* const check = await client.postprocessors.check({
|
|
3889
|
+
* url: "https://www.amazon.fr/dp/B09V3KXJPB"
|
|
3890
|
+
* });
|
|
3891
|
+
* console.log(check.hasPostprocessor); // true
|
|
3892
|
+
* console.log(check.postprocessor); // "amazon"
|
|
3893
|
+
* console.log(check.extractedFields); // ["productId", "seller", ...]
|
|
3894
|
+
* ```
|
|
3895
|
+
*/
|
|
3896
|
+
postprocessors;
|
|
3897
|
+
/**
|
|
3898
|
+
* Usage resource - Monitor usage, quotas, and token consumption
|
|
3899
|
+
*
|
|
3900
|
+
* @example
|
|
3901
|
+
* ```ts
|
|
3902
|
+
* // Get current usage and quotas
|
|
3903
|
+
* const usage = await client.usage.current();
|
|
3904
|
+
* console.log(usage.tier); // "pro"
|
|
3905
|
+
* console.log(usage.usage.scrapes); // 150
|
|
3906
|
+
* console.log(usage.quotas.scrapes); // 10000
|
|
3907
|
+
* console.log(usage.percentUsed.scrapes); // 1.5
|
|
3908
|
+
*
|
|
3909
|
+
* // Get historical usage
|
|
3910
|
+
* const history = await client.usage.history({ months: 6 });
|
|
3911
|
+
*
|
|
3912
|
+
* // Get LLM token usage
|
|
3913
|
+
* const tokens = await client.usage.tokens({ days: 30 });
|
|
3914
|
+
* console.log(tokens.totals.totalTokens);
|
|
3915
|
+
* console.log(tokens.totals.estimatedCostUsd);
|
|
3916
|
+
*
|
|
3917
|
+
* // Get daily token breakdown
|
|
3918
|
+
* const daily = await client.usage.dailyTokens({ days: 7 });
|
|
3919
|
+
* ```
|
|
3920
|
+
*/
|
|
3921
|
+
usage;
|
|
3693
3922
|
// ============================================
|
|
3694
3923
|
// CONSTRUCTOR
|
|
3695
3924
|
// ============================================
|
|
@@ -3750,6 +3979,8 @@ var DealCrawl = class {
|
|
|
3750
3979
|
this.screenshots = new ScreenshotsResource(this.ctx);
|
|
3751
3980
|
this.auth = new AuthResource(this.ctx);
|
|
3752
3981
|
this.events = new EventsResource(this.ctx);
|
|
3982
|
+
this.postprocessors = new PostprocessorsResource(this.ctx);
|
|
3983
|
+
this.usage = new UsageResource(this.ctx);
|
|
3753
3984
|
}
|
|
3754
3985
|
// ============================================
|
|
3755
3986
|
// POLLING METHODS
|
|
@@ -3893,6 +4124,6 @@ var DealCrawl = class {
|
|
|
3893
4124
|
}
|
|
3894
4125
|
};
|
|
3895
4126
|
|
|
3896
|
-
export { AccountResource, AgentResource, AuthResource, ConvertResource, CrawlResource, DEFAULT_CONFIG, DataResource, DealCrawl, DealCrawlError, DorkResource, ERROR_CODES, ERROR_MESSAGES, EventsResource, ExtractResource, KeysResource, ScrapeResource, ScreenshotsResource, SearchResource, StatusResource, WebhooksResource, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|
|
4127
|
+
export { AccountResource, AgentResource, AuthResource, ConvertResource, CrawlResource, DEFAULT_CONFIG, DataResource, DealCrawl, DealCrawlError, DorkResource, ERROR_CODES, ERROR_MESSAGES, EventsResource, ExtractResource, KeysResource, PostprocessorsResource, ScrapeResource, ScreenshotsResource, SearchResource, StatusResource, UsageResource, WebhooksResource, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|
|
3897
4128
|
//# sourceMappingURL=index.mjs.map
|
|
3898
4129
|
//# sourceMappingURL=index.mjs.map
|