@dealcrawl/sdk 2.12.0 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +408 -1
- package/dist/index.d.mts +539 -5
- package/dist/index.d.ts +539 -5
- package/dist/index.js +233 -0
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +232 -1
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -1
package/dist/index.js
CHANGED
|
@@ -2672,6 +2672,77 @@ var KeysResource = class {
|
|
|
2672
2672
|
}
|
|
2673
2673
|
};
|
|
2674
2674
|
|
|
2675
|
+
// src/resources/postprocessors.ts
|
|
2676
|
+
var PostprocessorsResource = class {
|
|
2677
|
+
constructor(ctx) {
|
|
2678
|
+
this.ctx = ctx;
|
|
2679
|
+
}
|
|
2680
|
+
/**
|
|
2681
|
+
* List all available postprocessors
|
|
2682
|
+
*
|
|
2683
|
+
* Returns information about each postprocessor including:
|
|
2684
|
+
* - Supported domains
|
|
2685
|
+
* - Extracted fields
|
|
2686
|
+
* - Description
|
|
2687
|
+
*
|
|
2688
|
+
* @example
|
|
2689
|
+
* ```ts
|
|
2690
|
+
* const result = await client.postprocessors.list();
|
|
2691
|
+
* console.log(result.postprocessors);
|
|
2692
|
+
* // [
|
|
2693
|
+
* // {
|
|
2694
|
+
* // name: "amazon",
|
|
2695
|
+
* // domains: ["amazon.com", "amazon.fr", ...],
|
|
2696
|
+
* // extractedFields: ["productId", "seller", "shipping", ...],
|
|
2697
|
+
* // description: "Amazon product page enrichment..."
|
|
2698
|
+
* // },
|
|
2699
|
+
* // ...
|
|
2700
|
+
* // ]
|
|
2701
|
+
* console.log(result.totalDomains); // 15
|
|
2702
|
+
* ```
|
|
2703
|
+
*/
|
|
2704
|
+
async list() {
|
|
2705
|
+
const result = await get(
|
|
2706
|
+
this.ctx,
|
|
2707
|
+
"/v1/postprocessors"
|
|
2708
|
+
);
|
|
2709
|
+
return result.data;
|
|
2710
|
+
}
|
|
2711
|
+
/**
|
|
2712
|
+
* Check which postprocessor applies to a given URL
|
|
2713
|
+
*
|
|
2714
|
+
* Useful for:
|
|
2715
|
+
* - Determining if a URL will be enriched before scraping
|
|
2716
|
+
* - Understanding what data will be extracted
|
|
2717
|
+
* - Filtering URLs by supported sites
|
|
2718
|
+
*
|
|
2719
|
+
* @example
|
|
2720
|
+
* ```ts
|
|
2721
|
+
* // Check Amazon URL
|
|
2722
|
+
* const check = await client.postprocessors.check({
|
|
2723
|
+
* url: "https://www.amazon.fr/dp/B09V3KXJPB"
|
|
2724
|
+
* });
|
|
2725
|
+
* console.log(check.hasPostprocessor); // true
|
|
2726
|
+
* console.log(check.postprocessor); // "amazon"
|
|
2727
|
+
* console.log(check.extractedFields); // ["productId", "seller", "shipping", ...]
|
|
2728
|
+
*
|
|
2729
|
+
* // Check unsupported URL
|
|
2730
|
+
* const noMatch = await client.postprocessors.check({
|
|
2731
|
+
* url: "https://example.com/product"
|
|
2732
|
+
* });
|
|
2733
|
+
* console.log(noMatch.hasPostprocessor); // false
|
|
2734
|
+
* console.log(noMatch.postprocessor); // null
|
|
2735
|
+
* ```
|
|
2736
|
+
*/
|
|
2737
|
+
async check(options) {
|
|
2738
|
+
const result = await get(
|
|
2739
|
+
this.ctx,
|
|
2740
|
+
`/v1/postprocessors/check?url=${encodeURIComponent(options.url)}`
|
|
2741
|
+
);
|
|
2742
|
+
return result.data;
|
|
2743
|
+
}
|
|
2744
|
+
};
|
|
2745
|
+
|
|
2675
2746
|
// src/resources/scrape.ts
|
|
2676
2747
|
var ScrapeResource = class {
|
|
2677
2748
|
constructor(ctx) {
|
|
@@ -3170,6 +3241,116 @@ var StatusResource = class {
|
|
|
3170
3241
|
}
|
|
3171
3242
|
};
|
|
3172
3243
|
|
|
3244
|
+
// src/resources/usage.ts
|
|
3245
|
+
var UsageResource = class {
|
|
3246
|
+
constructor(ctx) {
|
|
3247
|
+
this.ctx = ctx;
|
|
3248
|
+
}
|
|
3249
|
+
/**
|
|
3250
|
+
* Get current usage and quotas
|
|
3251
|
+
*
|
|
3252
|
+
* Returns the current billing period usage statistics including:
|
|
3253
|
+
* - Current usage counts for all operation types
|
|
3254
|
+
* - Quota limits based on tier
|
|
3255
|
+
* - Percentage used for each resource
|
|
3256
|
+
* - Time until quota reset
|
|
3257
|
+
*
|
|
3258
|
+
* @example
|
|
3259
|
+
* ```ts
|
|
3260
|
+
* const usage = await client.usage.current();
|
|
3261
|
+
* console.log(usage.tier); // "pro"
|
|
3262
|
+
* console.log(usage.usage.scrapes); // 150
|
|
3263
|
+
* console.log(usage.quotas.scrapes); // 10000
|
|
3264
|
+
* console.log(usage.percentUsed.scrapes); // 1.5
|
|
3265
|
+
* console.log(usage.billingPeriod.daysRemaining); // 15
|
|
3266
|
+
* ```
|
|
3267
|
+
*/
|
|
3268
|
+
async current() {
|
|
3269
|
+
const result = await get(this.ctx, "/v1/usage");
|
|
3270
|
+
return result.data;
|
|
3271
|
+
}
|
|
3272
|
+
/**
|
|
3273
|
+
* Get historical usage data
|
|
3274
|
+
*
|
|
3275
|
+
* Returns monthly usage snapshots for billing analysis.
|
|
3276
|
+
*
|
|
3277
|
+
* @example
|
|
3278
|
+
* ```ts
|
|
3279
|
+
* const history = await client.usage.history({ months: 6 });
|
|
3280
|
+
* history.data.forEach(period => {
|
|
3281
|
+
* console.log(`${period.periodStart}: ${period.usage.scrapes} scrapes`);
|
|
3282
|
+
* });
|
|
3283
|
+
* ```
|
|
3284
|
+
*/
|
|
3285
|
+
async history(options = {}) {
|
|
3286
|
+
const params = new URLSearchParams();
|
|
3287
|
+
if (options.months) {
|
|
3288
|
+
params.set("months", options.months.toString());
|
|
3289
|
+
}
|
|
3290
|
+
const query = params.toString();
|
|
3291
|
+
const result = await get(
|
|
3292
|
+
this.ctx,
|
|
3293
|
+
`/v1/usage/history${query ? `?${query}` : ""}`
|
|
3294
|
+
);
|
|
3295
|
+
return result.data;
|
|
3296
|
+
}
|
|
3297
|
+
/**
|
|
3298
|
+
* Get LLM token usage summary
|
|
3299
|
+
*
|
|
3300
|
+
* Returns aggregated token consumption and estimated costs.
|
|
3301
|
+
*
|
|
3302
|
+
* @example
|
|
3303
|
+
* ```ts
|
|
3304
|
+
* const tokens = await client.usage.tokens({ days: 30 });
|
|
3305
|
+
* console.log(tokens.totals.totalTokens); // 1234567
|
|
3306
|
+
* console.log(tokens.totals.estimatedCostUsd); // 12.34
|
|
3307
|
+
* tokens.breakdown.forEach(item => {
|
|
3308
|
+
* console.log(`${item.provider}/${item.model}: ${item.totalTokens} tokens`);
|
|
3309
|
+
* });
|
|
3310
|
+
* ```
|
|
3311
|
+
*/
|
|
3312
|
+
async tokens(options = {}) {
|
|
3313
|
+
const params = new URLSearchParams();
|
|
3314
|
+
if (options.days) {
|
|
3315
|
+
params.set("days", options.days.toString());
|
|
3316
|
+
}
|
|
3317
|
+
if (options.provider) {
|
|
3318
|
+
params.set("provider", options.provider);
|
|
3319
|
+
}
|
|
3320
|
+
const query = params.toString();
|
|
3321
|
+
const result = await get(
|
|
3322
|
+
this.ctx,
|
|
3323
|
+
`/v1/usage/tokens${query ? `?${query}` : ""}`
|
|
3324
|
+
);
|
|
3325
|
+
return result.data;
|
|
3326
|
+
}
|
|
3327
|
+
/**
|
|
3328
|
+
* Get daily token usage breakdown
|
|
3329
|
+
*
|
|
3330
|
+
* Returns day-by-day token consumption for charting.
|
|
3331
|
+
*
|
|
3332
|
+
* @example
|
|
3333
|
+
* ```ts
|
|
3334
|
+
* const daily = await client.usage.dailyTokens({ days: 7 });
|
|
3335
|
+
* daily.daily.forEach(day => {
|
|
3336
|
+
* console.log(`${day.date}: ${day.totalTokens} tokens ($${day.estimatedCostUsd})`);
|
|
3337
|
+
* });
|
|
3338
|
+
* ```
|
|
3339
|
+
*/
|
|
3340
|
+
async dailyTokens(options = {}) {
|
|
3341
|
+
const params = new URLSearchParams();
|
|
3342
|
+
if (options.days) {
|
|
3343
|
+
params.set("days", options.days.toString());
|
|
3344
|
+
}
|
|
3345
|
+
const query = params.toString();
|
|
3346
|
+
const result = await get(
|
|
3347
|
+
this.ctx,
|
|
3348
|
+
`/v1/usage/tokens/daily${query ? `?${query}` : ""}`
|
|
3349
|
+
);
|
|
3350
|
+
return result.data;
|
|
3351
|
+
}
|
|
3352
|
+
};
|
|
3353
|
+
|
|
3173
3354
|
// src/resources/webhooks.ts
|
|
3174
3355
|
var WebhooksResource = class {
|
|
3175
3356
|
constructor(ctx) {
|
|
@@ -3694,6 +3875,54 @@ var DealCrawl = class {
|
|
|
3694
3875
|
* ```
|
|
3695
3876
|
*/
|
|
3696
3877
|
events;
|
|
3878
|
+
/**
|
|
3879
|
+
* Postprocessors resource - Discover site-specific enrichment postprocessors
|
|
3880
|
+
*
|
|
3881
|
+
* Postprocessors automatically enrich scrape results for supported e-commerce sites
|
|
3882
|
+
* (Amazon, Fnac, eBay) with additional data like seller info, shipping, availability.
|
|
3883
|
+
*
|
|
3884
|
+
* @example
|
|
3885
|
+
* ```ts
|
|
3886
|
+
* // List all available postprocessors
|
|
3887
|
+
* const list = await client.postprocessors.list();
|
|
3888
|
+
* console.log(list.postprocessors);
|
|
3889
|
+
* // [{ name: "amazon", domains: ["amazon.com", ...], extractedFields: [...] }, ...]
|
|
3890
|
+
*
|
|
3891
|
+
* // Check if a URL will be enriched
|
|
3892
|
+
* const check = await client.postprocessors.check({
|
|
3893
|
+
* url: "https://www.amazon.fr/dp/B09V3KXJPB"
|
|
3894
|
+
* });
|
|
3895
|
+
* console.log(check.hasPostprocessor); // true
|
|
3896
|
+
* console.log(check.postprocessor); // "amazon"
|
|
3897
|
+
* console.log(check.extractedFields); // ["productId", "seller", ...]
|
|
3898
|
+
* ```
|
|
3899
|
+
*/
|
|
3900
|
+
postprocessors;
|
|
3901
|
+
/**
|
|
3902
|
+
* Usage resource - Monitor usage, quotas, and token consumption
|
|
3903
|
+
*
|
|
3904
|
+
* @example
|
|
3905
|
+
* ```ts
|
|
3906
|
+
* // Get current usage and quotas
|
|
3907
|
+
* const usage = await client.usage.current();
|
|
3908
|
+
* console.log(usage.tier); // "pro"
|
|
3909
|
+
* console.log(usage.usage.scrapes); // 150
|
|
3910
|
+
* console.log(usage.quotas.scrapes); // 10000
|
|
3911
|
+
* console.log(usage.percentUsed.scrapes); // 1.5
|
|
3912
|
+
*
|
|
3913
|
+
* // Get historical usage
|
|
3914
|
+
* const history = await client.usage.history({ months: 6 });
|
|
3915
|
+
*
|
|
3916
|
+
* // Get LLM token usage
|
|
3917
|
+
* const tokens = await client.usage.tokens({ days: 30 });
|
|
3918
|
+
* console.log(tokens.totals.totalTokens);
|
|
3919
|
+
* console.log(tokens.totals.estimatedCostUsd);
|
|
3920
|
+
*
|
|
3921
|
+
* // Get daily token breakdown
|
|
3922
|
+
* const daily = await client.usage.dailyTokens({ days: 7 });
|
|
3923
|
+
* ```
|
|
3924
|
+
*/
|
|
3925
|
+
usage;
|
|
3697
3926
|
// ============================================
|
|
3698
3927
|
// CONSTRUCTOR
|
|
3699
3928
|
// ============================================
|
|
@@ -3754,6 +3983,8 @@ var DealCrawl = class {
|
|
|
3754
3983
|
this.screenshots = new ScreenshotsResource(this.ctx);
|
|
3755
3984
|
this.auth = new AuthResource(this.ctx);
|
|
3756
3985
|
this.events = new EventsResource(this.ctx);
|
|
3986
|
+
this.postprocessors = new PostprocessorsResource(this.ctx);
|
|
3987
|
+
this.usage = new UsageResource(this.ctx);
|
|
3757
3988
|
}
|
|
3758
3989
|
// ============================================
|
|
3759
3990
|
// POLLING METHODS
|
|
@@ -3912,10 +4143,12 @@ exports.ERROR_MESSAGES = ERROR_MESSAGES;
|
|
|
3912
4143
|
exports.EventsResource = EventsResource;
|
|
3913
4144
|
exports.ExtractResource = ExtractResource;
|
|
3914
4145
|
exports.KeysResource = KeysResource;
|
|
4146
|
+
exports.PostprocessorsResource = PostprocessorsResource;
|
|
3915
4147
|
exports.ScrapeResource = ScrapeResource;
|
|
3916
4148
|
exports.ScreenshotsResource = ScreenshotsResource;
|
|
3917
4149
|
exports.SearchResource = SearchResource;
|
|
3918
4150
|
exports.StatusResource = StatusResource;
|
|
4151
|
+
exports.UsageResource = UsageResource;
|
|
3919
4152
|
exports.WebhooksResource = WebhooksResource;
|
|
3920
4153
|
exports.default = DealCrawl;
|
|
3921
4154
|
exports.getErrorMessage = getErrorMessage;
|