@stack0/sdk 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cdn/index.js +0 -1
- package/dist/cdn/index.js.map +1 -1
- package/dist/cdn/index.mjs +0 -1
- package/dist/cdn/index.mjs.map +1 -1
- package/dist/extraction/index.d.mts +374 -0
- package/dist/extraction/index.d.ts +374 -0
- package/dist/extraction/index.js +455 -0
- package/dist/extraction/index.js.map +1 -0
- package/dist/extraction/index.mjs +453 -0
- package/dist/extraction/index.mjs.map +1 -0
- package/dist/index.d.mts +13 -4
- package/dist/index.d.ts +13 -4
- package/dist/index.js +739 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +738 -2
- package/dist/index.mjs.map +1 -1
- package/dist/screenshots/index.d.mts +377 -0
- package/dist/screenshots/index.d.ts +377 -0
- package/dist/screenshots/index.js +423 -0
- package/dist/screenshots/index.js.map +1 -0
- package/dist/screenshots/index.mjs +421 -0
- package/dist/screenshots/index.mjs.map +1 -0
- package/dist/shared-types-B0PyC7cF.d.mts +40 -0
- package/dist/shared-types-B0PyC7cF.d.ts +40 -0
- package/package.json +20 -1
package/dist/index.mjs
CHANGED
|
@@ -155,7 +155,6 @@ var CDN = class {
|
|
|
155
155
|
*/
|
|
156
156
|
async confirmUpload(assetId) {
|
|
157
157
|
const response = await this.http.post(`/cdn/upload/${assetId}/confirm`, {});
|
|
158
|
-
console.log("confirm upload response", response);
|
|
159
158
|
return this.convertAssetDates(response);
|
|
160
159
|
}
|
|
161
160
|
/**
|
|
@@ -384,6 +383,736 @@ var CDN = class {
|
|
|
384
383
|
}
|
|
385
384
|
};
|
|
386
385
|
|
|
386
|
+
// src/screenshots/client.ts
|
|
387
|
+
var Screenshots = class {
|
|
388
|
+
http;
|
|
389
|
+
constructor(config) {
|
|
390
|
+
this.http = new HttpClient(config);
|
|
391
|
+
}
|
|
392
|
+
// ==========================================================================
|
|
393
|
+
// SCREENSHOTS
|
|
394
|
+
// ==========================================================================
|
|
395
|
+
/**
|
|
396
|
+
* Capture a screenshot of a URL
|
|
397
|
+
*
|
|
398
|
+
* @example
|
|
399
|
+
* ```typescript
|
|
400
|
+
* const { id, status } = await screenshots.capture({
|
|
401
|
+
* url: 'https://example.com',
|
|
402
|
+
* format: 'png',
|
|
403
|
+
* fullPage: true,
|
|
404
|
+
* deviceType: 'desktop',
|
|
405
|
+
* });
|
|
406
|
+
*
|
|
407
|
+
* // Poll for completion
|
|
408
|
+
* const screenshot = await screenshots.get({ id });
|
|
409
|
+
* console.log(screenshot.imageUrl);
|
|
410
|
+
* ```
|
|
411
|
+
*/
|
|
412
|
+
async capture(request) {
|
|
413
|
+
return this.http.post("/webdata/screenshots", request);
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* Get a screenshot by ID
|
|
417
|
+
*
|
|
418
|
+
* @example
|
|
419
|
+
* ```typescript
|
|
420
|
+
* const screenshot = await screenshots.get({ id: 'screenshot-id' });
|
|
421
|
+
* if (screenshot.status === 'completed') {
|
|
422
|
+
* console.log(screenshot.imageUrl);
|
|
423
|
+
* }
|
|
424
|
+
* ```
|
|
425
|
+
*/
|
|
426
|
+
async get(request) {
|
|
427
|
+
const params = new URLSearchParams();
|
|
428
|
+
if (request.environment) params.set("environment", request.environment);
|
|
429
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
430
|
+
const query = params.toString();
|
|
431
|
+
const path = `/webdata/screenshots/${request.id}${query ? `?${query}` : ""}`;
|
|
432
|
+
const response = await this.http.get(path);
|
|
433
|
+
return this.convertDates(response);
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* List screenshots with pagination and filters
|
|
437
|
+
*
|
|
438
|
+
* @example
|
|
439
|
+
* ```typescript
|
|
440
|
+
* const { items, nextCursor } = await screenshots.list({
|
|
441
|
+
* status: 'completed',
|
|
442
|
+
* limit: 20,
|
|
443
|
+
* });
|
|
444
|
+
* ```
|
|
445
|
+
*/
|
|
446
|
+
async list(request = {}) {
|
|
447
|
+
const params = new URLSearchParams();
|
|
448
|
+
if (request.environment) params.set("environment", request.environment);
|
|
449
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
450
|
+
if (request.status) params.set("status", request.status);
|
|
451
|
+
if (request.url) params.set("url", request.url);
|
|
452
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
453
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
454
|
+
const query = params.toString();
|
|
455
|
+
const response = await this.http.get(
|
|
456
|
+
`/webdata/screenshots${query ? `?${query}` : ""}`
|
|
457
|
+
);
|
|
458
|
+
return {
|
|
459
|
+
...response,
|
|
460
|
+
items: response.items.map((item) => this.convertDates(item))
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Delete a screenshot
|
|
465
|
+
*
|
|
466
|
+
* @example
|
|
467
|
+
* ```typescript
|
|
468
|
+
* await screenshots.delete({ id: 'screenshot-id' });
|
|
469
|
+
* ```
|
|
470
|
+
*/
|
|
471
|
+
async delete(request) {
|
|
472
|
+
const params = new URLSearchParams();
|
|
473
|
+
if (request.environment) params.set("environment", request.environment);
|
|
474
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
475
|
+
const query = params.toString();
|
|
476
|
+
return this.http.delete(
|
|
477
|
+
`/webdata/screenshots/${request.id}${query ? `?${query}` : ""}`
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Capture a screenshot and wait for completion
|
|
482
|
+
*
|
|
483
|
+
* @example
|
|
484
|
+
* ```typescript
|
|
485
|
+
* const screenshot = await screenshots.captureAndWait({
|
|
486
|
+
* url: 'https://example.com',
|
|
487
|
+
* format: 'png',
|
|
488
|
+
* });
|
|
489
|
+
* console.log(screenshot.imageUrl);
|
|
490
|
+
* ```
|
|
491
|
+
*/
|
|
492
|
+
async captureAndWait(request, options = {}) {
|
|
493
|
+
const { pollInterval = 1e3, timeout = 6e4 } = options;
|
|
494
|
+
const startTime = Date.now();
|
|
495
|
+
const { id } = await this.capture(request);
|
|
496
|
+
while (Date.now() - startTime < timeout) {
|
|
497
|
+
const screenshot = await this.get({
|
|
498
|
+
id,
|
|
499
|
+
environment: request.environment,
|
|
500
|
+
projectId: request.projectId
|
|
501
|
+
});
|
|
502
|
+
if (screenshot.status === "completed" || screenshot.status === "failed") {
|
|
503
|
+
if (screenshot.status === "failed") {
|
|
504
|
+
throw new Error(screenshot.error || "Screenshot failed");
|
|
505
|
+
}
|
|
506
|
+
return screenshot;
|
|
507
|
+
}
|
|
508
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
509
|
+
}
|
|
510
|
+
throw new Error("Screenshot timed out");
|
|
511
|
+
}
|
|
512
|
+
// ==========================================================================
|
|
513
|
+
// BATCH JOBS
|
|
514
|
+
// ==========================================================================
|
|
515
|
+
/**
|
|
516
|
+
* Create a batch screenshot job for multiple URLs
|
|
517
|
+
*
|
|
518
|
+
* @example
|
|
519
|
+
* ```typescript
|
|
520
|
+
* const { id, totalUrls } = await screenshots.batch({
|
|
521
|
+
* urls: [
|
|
522
|
+
* 'https://example.com',
|
|
523
|
+
* 'https://example.org',
|
|
524
|
+
* ],
|
|
525
|
+
* config: { format: 'png', fullPage: true },
|
|
526
|
+
* });
|
|
527
|
+
*
|
|
528
|
+
* // Poll for completion
|
|
529
|
+
* const job = await screenshots.getBatchJob({ id });
|
|
530
|
+
* console.log(`Progress: ${job.processedUrls}/${job.totalUrls}`);
|
|
531
|
+
* ```
|
|
532
|
+
*/
|
|
533
|
+
async batch(request) {
|
|
534
|
+
return this.http.post("/webdata/batch/screenshots", request);
|
|
535
|
+
}
|
|
536
|
+
/**
|
|
537
|
+
* Get a batch job by ID
|
|
538
|
+
*/
|
|
539
|
+
async getBatchJob(request) {
|
|
540
|
+
const params = new URLSearchParams();
|
|
541
|
+
if (request.environment) params.set("environment", request.environment);
|
|
542
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
543
|
+
const query = params.toString();
|
|
544
|
+
const path = `/webdata/batch/${request.id}${query ? `?${query}` : ""}`;
|
|
545
|
+
const response = await this.http.get(path);
|
|
546
|
+
return this.convertBatchJobDates(response);
|
|
547
|
+
}
|
|
548
|
+
/**
|
|
549
|
+
* List batch jobs with pagination and filters
|
|
550
|
+
*/
|
|
551
|
+
async listBatchJobs(request = {}) {
|
|
552
|
+
const params = new URLSearchParams();
|
|
553
|
+
if (request.environment) params.set("environment", request.environment);
|
|
554
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
555
|
+
if (request.status) params.set("status", request.status);
|
|
556
|
+
params.set("type", "screenshot");
|
|
557
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
558
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
559
|
+
const query = params.toString();
|
|
560
|
+
const response = await this.http.get(
|
|
561
|
+
`/webdata/batch${query ? `?${query}` : ""}`
|
|
562
|
+
);
|
|
563
|
+
return {
|
|
564
|
+
...response,
|
|
565
|
+
items: response.items.map((item) => this.convertBatchJobDates(item))
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
/**
|
|
569
|
+
* Cancel a batch job
|
|
570
|
+
*/
|
|
571
|
+
async cancelBatchJob(request) {
|
|
572
|
+
const params = new URLSearchParams();
|
|
573
|
+
if (request.environment) params.set("environment", request.environment);
|
|
574
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
575
|
+
const query = params.toString();
|
|
576
|
+
return this.http.post(
|
|
577
|
+
`/webdata/batch/${request.id}/cancel${query ? `?${query}` : ""}`,
|
|
578
|
+
{}
|
|
579
|
+
);
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* Create a batch screenshot job and wait for completion
|
|
583
|
+
*/
|
|
584
|
+
async batchAndWait(request, options = {}) {
|
|
585
|
+
const { pollInterval = 2e3, timeout = 3e5 } = options;
|
|
586
|
+
const startTime = Date.now();
|
|
587
|
+
const { id } = await this.batch(request);
|
|
588
|
+
while (Date.now() - startTime < timeout) {
|
|
589
|
+
const job = await this.getBatchJob({
|
|
590
|
+
id,
|
|
591
|
+
environment: request.environment,
|
|
592
|
+
projectId: request.projectId
|
|
593
|
+
});
|
|
594
|
+
if (job.status === "completed" || job.status === "failed" || job.status === "cancelled") {
|
|
595
|
+
return job;
|
|
596
|
+
}
|
|
597
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
598
|
+
}
|
|
599
|
+
throw new Error("Batch job timed out");
|
|
600
|
+
}
|
|
601
|
+
// ==========================================================================
|
|
602
|
+
// SCHEDULES
|
|
603
|
+
// ==========================================================================
|
|
604
|
+
/**
|
|
605
|
+
* Create a scheduled screenshot job
|
|
606
|
+
*
|
|
607
|
+
* @example
|
|
608
|
+
* ```typescript
|
|
609
|
+
* const { id } = await screenshots.createSchedule({
|
|
610
|
+
* name: 'Daily homepage screenshot',
|
|
611
|
+
* url: 'https://example.com',
|
|
612
|
+
* frequency: 'daily',
|
|
613
|
+
* config: { format: 'png', fullPage: true },
|
|
614
|
+
* });
|
|
615
|
+
* ```
|
|
616
|
+
*/
|
|
617
|
+
async createSchedule(request) {
|
|
618
|
+
return this.http.post("/webdata/schedules", {
|
|
619
|
+
...request,
|
|
620
|
+
type: "screenshot"
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
/**
|
|
624
|
+
* Update a schedule
|
|
625
|
+
*/
|
|
626
|
+
async updateSchedule(request) {
|
|
627
|
+
const { id, environment, projectId, ...data } = request;
|
|
628
|
+
const params = new URLSearchParams();
|
|
629
|
+
if (environment) params.set("environment", environment);
|
|
630
|
+
if (projectId) params.set("projectId", projectId);
|
|
631
|
+
const query = params.toString();
|
|
632
|
+
return this.http.post(
|
|
633
|
+
`/webdata/schedules/${id}${query ? `?${query}` : ""}`,
|
|
634
|
+
data
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* Get a schedule by ID
|
|
639
|
+
*/
|
|
640
|
+
async getSchedule(request) {
|
|
641
|
+
const params = new URLSearchParams();
|
|
642
|
+
if (request.environment) params.set("environment", request.environment);
|
|
643
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
644
|
+
const query = params.toString();
|
|
645
|
+
const path = `/webdata/schedules/${request.id}${query ? `?${query}` : ""}`;
|
|
646
|
+
const response = await this.http.get(path);
|
|
647
|
+
return this.convertScheduleDates(response);
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* List schedules with pagination and filters
|
|
651
|
+
*/
|
|
652
|
+
async listSchedules(request = {}) {
|
|
653
|
+
const params = new URLSearchParams();
|
|
654
|
+
if (request.environment) params.set("environment", request.environment);
|
|
655
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
656
|
+
params.set("type", "screenshot");
|
|
657
|
+
if (request.isActive !== void 0) params.set("isActive", request.isActive.toString());
|
|
658
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
659
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
660
|
+
const query = params.toString();
|
|
661
|
+
const response = await this.http.get(
|
|
662
|
+
`/webdata/schedules${query ? `?${query}` : ""}`
|
|
663
|
+
);
|
|
664
|
+
return {
|
|
665
|
+
...response,
|
|
666
|
+
items: response.items.map((item) => this.convertScheduleDates(item))
|
|
667
|
+
};
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* Delete a schedule
|
|
671
|
+
*/
|
|
672
|
+
async deleteSchedule(request) {
|
|
673
|
+
const params = new URLSearchParams();
|
|
674
|
+
if (request.environment) params.set("environment", request.environment);
|
|
675
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
676
|
+
const query = params.toString();
|
|
677
|
+
return this.http.delete(
|
|
678
|
+
`/webdata/schedules/${request.id}${query ? `?${query}` : ""}`
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
/**
|
|
682
|
+
* Toggle a schedule on or off
|
|
683
|
+
*/
|
|
684
|
+
async toggleSchedule(request) {
|
|
685
|
+
const params = new URLSearchParams();
|
|
686
|
+
if (request.environment) params.set("environment", request.environment);
|
|
687
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
688
|
+
const query = params.toString();
|
|
689
|
+
return this.http.post(
|
|
690
|
+
`/webdata/schedules/${request.id}/toggle${query ? `?${query}` : ""}`,
|
|
691
|
+
{}
|
|
692
|
+
);
|
|
693
|
+
}
|
|
694
|
+
// ==========================================================================
|
|
695
|
+
// HELPERS
|
|
696
|
+
// ==========================================================================
|
|
697
|
+
convertDates(screenshot) {
|
|
698
|
+
if (typeof screenshot.createdAt === "string") {
|
|
699
|
+
screenshot.createdAt = new Date(screenshot.createdAt);
|
|
700
|
+
}
|
|
701
|
+
if (screenshot.completedAt && typeof screenshot.completedAt === "string") {
|
|
702
|
+
screenshot.completedAt = new Date(screenshot.completedAt);
|
|
703
|
+
}
|
|
704
|
+
return screenshot;
|
|
705
|
+
}
|
|
706
|
+
convertBatchJobDates(job) {
|
|
707
|
+
if (typeof job.createdAt === "string") {
|
|
708
|
+
job.createdAt = new Date(job.createdAt);
|
|
709
|
+
}
|
|
710
|
+
if (job.startedAt && typeof job.startedAt === "string") {
|
|
711
|
+
job.startedAt = new Date(job.startedAt);
|
|
712
|
+
}
|
|
713
|
+
if (job.completedAt && typeof job.completedAt === "string") {
|
|
714
|
+
job.completedAt = new Date(job.completedAt);
|
|
715
|
+
}
|
|
716
|
+
return job;
|
|
717
|
+
}
|
|
718
|
+
convertScheduleDates(schedule) {
|
|
719
|
+
if (typeof schedule.createdAt === "string") {
|
|
720
|
+
schedule.createdAt = new Date(schedule.createdAt);
|
|
721
|
+
}
|
|
722
|
+
if (typeof schedule.updatedAt === "string") {
|
|
723
|
+
schedule.updatedAt = new Date(schedule.updatedAt);
|
|
724
|
+
}
|
|
725
|
+
if (schedule.lastRunAt && typeof schedule.lastRunAt === "string") {
|
|
726
|
+
schedule.lastRunAt = new Date(schedule.lastRunAt);
|
|
727
|
+
}
|
|
728
|
+
if (schedule.nextRunAt && typeof schedule.nextRunAt === "string") {
|
|
729
|
+
schedule.nextRunAt = new Date(schedule.nextRunAt);
|
|
730
|
+
}
|
|
731
|
+
return schedule;
|
|
732
|
+
}
|
|
733
|
+
};
|
|
734
|
+
|
|
735
|
+
// src/extraction/client.ts
|
|
736
|
+
var Extraction = class {
|
|
737
|
+
http;
|
|
738
|
+
constructor(config) {
|
|
739
|
+
this.http = new HttpClient(config);
|
|
740
|
+
}
|
|
741
|
+
// ==========================================================================
|
|
742
|
+
// EXTRACTIONS
|
|
743
|
+
// ==========================================================================
|
|
744
|
+
/**
|
|
745
|
+
* Extract content from a URL
|
|
746
|
+
*
|
|
747
|
+
* @example
|
|
748
|
+
* ```typescript
|
|
749
|
+
* const { id, status } = await extraction.extract({
|
|
750
|
+
* url: 'https://example.com/article',
|
|
751
|
+
* mode: 'markdown',
|
|
752
|
+
* includeMetadata: true,
|
|
753
|
+
* });
|
|
754
|
+
*
|
|
755
|
+
* // Poll for completion
|
|
756
|
+
* const result = await extraction.get({ id });
|
|
757
|
+
* console.log(result.markdown);
|
|
758
|
+
* ```
|
|
759
|
+
*/
|
|
760
|
+
async extract(request) {
|
|
761
|
+
return this.http.post("/webdata/extractions", request);
|
|
762
|
+
}
|
|
763
|
+
/**
|
|
764
|
+
* Get an extraction by ID
|
|
765
|
+
*
|
|
766
|
+
* @example
|
|
767
|
+
* ```typescript
|
|
768
|
+
* const extraction = await extraction.get({ id: 'extraction-id' });
|
|
769
|
+
* if (extraction.status === 'completed') {
|
|
770
|
+
* console.log(extraction.markdown);
|
|
771
|
+
* console.log(extraction.pageMetadata);
|
|
772
|
+
* }
|
|
773
|
+
* ```
|
|
774
|
+
*/
|
|
775
|
+
async get(request) {
|
|
776
|
+
const params = new URLSearchParams();
|
|
777
|
+
if (request.environment) params.set("environment", request.environment);
|
|
778
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
779
|
+
const query = params.toString();
|
|
780
|
+
const path = `/webdata/extractions/${request.id}${query ? `?${query}` : ""}`;
|
|
781
|
+
const response = await this.http.get(path);
|
|
782
|
+
return this.convertDates(response);
|
|
783
|
+
}
|
|
784
|
+
/**
|
|
785
|
+
* List extractions with pagination and filters
|
|
786
|
+
*
|
|
787
|
+
* @example
|
|
788
|
+
* ```typescript
|
|
789
|
+
* const { items, nextCursor } = await extraction.list({
|
|
790
|
+
* status: 'completed',
|
|
791
|
+
* limit: 20,
|
|
792
|
+
* });
|
|
793
|
+
* ```
|
|
794
|
+
*/
|
|
795
|
+
async list(request = {}) {
|
|
796
|
+
const params = new URLSearchParams();
|
|
797
|
+
if (request.environment) params.set("environment", request.environment);
|
|
798
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
799
|
+
if (request.status) params.set("status", request.status);
|
|
800
|
+
if (request.url) params.set("url", request.url);
|
|
801
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
802
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
803
|
+
const query = params.toString();
|
|
804
|
+
const response = await this.http.get(
|
|
805
|
+
`/webdata/extractions${query ? `?${query}` : ""}`
|
|
806
|
+
);
|
|
807
|
+
return {
|
|
808
|
+
...response,
|
|
809
|
+
items: response.items.map((item) => this.convertDates(item))
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
/**
|
|
813
|
+
* Delete an extraction
|
|
814
|
+
*
|
|
815
|
+
* @example
|
|
816
|
+
* ```typescript
|
|
817
|
+
* await extraction.delete({ id: 'extraction-id' });
|
|
818
|
+
* ```
|
|
819
|
+
*/
|
|
820
|
+
async delete(request) {
|
|
821
|
+
const params = new URLSearchParams();
|
|
822
|
+
if (request.environment) params.set("environment", request.environment);
|
|
823
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
824
|
+
const query = params.toString();
|
|
825
|
+
return this.http.delete(
|
|
826
|
+
`/webdata/extractions/${request.id}${query ? `?${query}` : ""}`
|
|
827
|
+
);
|
|
828
|
+
}
|
|
829
|
+
/**
|
|
830
|
+
* Extract content and wait for completion
|
|
831
|
+
*
|
|
832
|
+
* @example
|
|
833
|
+
* ```typescript
|
|
834
|
+
* const extraction = await extraction.extractAndWait({
|
|
835
|
+
* url: 'https://example.com/article',
|
|
836
|
+
* mode: 'markdown',
|
|
837
|
+
* });
|
|
838
|
+
* console.log(extraction.markdown);
|
|
839
|
+
* ```
|
|
840
|
+
*/
|
|
841
|
+
async extractAndWait(request, options = {}) {
|
|
842
|
+
const { pollInterval = 1e3, timeout = 6e4 } = options;
|
|
843
|
+
const startTime = Date.now();
|
|
844
|
+
const { id } = await this.extract(request);
|
|
845
|
+
while (Date.now() - startTime < timeout) {
|
|
846
|
+
const extraction = await this.get({
|
|
847
|
+
id,
|
|
848
|
+
environment: request.environment,
|
|
849
|
+
projectId: request.projectId
|
|
850
|
+
});
|
|
851
|
+
if (extraction.status === "completed" || extraction.status === "failed") {
|
|
852
|
+
if (extraction.status === "failed") {
|
|
853
|
+
throw new Error(extraction.error || "Extraction failed");
|
|
854
|
+
}
|
|
855
|
+
return extraction;
|
|
856
|
+
}
|
|
857
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
858
|
+
}
|
|
859
|
+
throw new Error("Extraction timed out");
|
|
860
|
+
}
|
|
861
|
+
// ==========================================================================
|
|
862
|
+
// BATCH JOBS
|
|
863
|
+
// ==========================================================================
|
|
864
|
+
/**
|
|
865
|
+
* Create a batch extraction job for multiple URLs
|
|
866
|
+
*
|
|
867
|
+
* @example
|
|
868
|
+
* ```typescript
|
|
869
|
+
* const { id, totalUrls } = await extraction.batch({
|
|
870
|
+
* urls: [
|
|
871
|
+
* 'https://example.com/article1',
|
|
872
|
+
* 'https://example.com/article2',
|
|
873
|
+
* ],
|
|
874
|
+
* config: { mode: 'markdown' },
|
|
875
|
+
* });
|
|
876
|
+
* ```
|
|
877
|
+
*/
|
|
878
|
+
async batch(request) {
|
|
879
|
+
return this.http.post("/webdata/batch/extractions", request);
|
|
880
|
+
}
|
|
881
|
+
/**
|
|
882
|
+
* Get a batch job by ID
|
|
883
|
+
*/
|
|
884
|
+
async getBatchJob(request) {
|
|
885
|
+
const params = new URLSearchParams();
|
|
886
|
+
if (request.environment) params.set("environment", request.environment);
|
|
887
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
888
|
+
const query = params.toString();
|
|
889
|
+
const path = `/webdata/batch/${request.id}${query ? `?${query}` : ""}`;
|
|
890
|
+
const response = await this.http.get(path);
|
|
891
|
+
return this.convertBatchJobDates(response);
|
|
892
|
+
}
|
|
893
|
+
/**
|
|
894
|
+
* List batch jobs with pagination and filters
|
|
895
|
+
*/
|
|
896
|
+
async listBatchJobs(request = {}) {
|
|
897
|
+
const params = new URLSearchParams();
|
|
898
|
+
if (request.environment) params.set("environment", request.environment);
|
|
899
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
900
|
+
if (request.status) params.set("status", request.status);
|
|
901
|
+
params.set("type", "extraction");
|
|
902
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
903
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
904
|
+
const query = params.toString();
|
|
905
|
+
const response = await this.http.get(
|
|
906
|
+
`/webdata/batch${query ? `?${query}` : ""}`
|
|
907
|
+
);
|
|
908
|
+
return {
|
|
909
|
+
...response,
|
|
910
|
+
items: response.items.map((item) => this.convertBatchJobDates(item))
|
|
911
|
+
};
|
|
912
|
+
}
|
|
913
|
+
/**
|
|
914
|
+
* Cancel a batch job
|
|
915
|
+
*/
|
|
916
|
+
async cancelBatchJob(request) {
|
|
917
|
+
const params = new URLSearchParams();
|
|
918
|
+
if (request.environment) params.set("environment", request.environment);
|
|
919
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
920
|
+
const query = params.toString();
|
|
921
|
+
return this.http.post(
|
|
922
|
+
`/webdata/batch/${request.id}/cancel${query ? `?${query}` : ""}`,
|
|
923
|
+
{}
|
|
924
|
+
);
|
|
925
|
+
}
|
|
926
|
+
/**
|
|
927
|
+
* Create a batch extraction job and wait for completion
|
|
928
|
+
*/
|
|
929
|
+
async batchAndWait(request, options = {}) {
|
|
930
|
+
const { pollInterval = 2e3, timeout = 3e5 } = options;
|
|
931
|
+
const startTime = Date.now();
|
|
932
|
+
const { id } = await this.batch(request);
|
|
933
|
+
while (Date.now() - startTime < timeout) {
|
|
934
|
+
const job = await this.getBatchJob({
|
|
935
|
+
id,
|
|
936
|
+
environment: request.environment,
|
|
937
|
+
projectId: request.projectId
|
|
938
|
+
});
|
|
939
|
+
if (job.status === "completed" || job.status === "failed" || job.status === "cancelled") {
|
|
940
|
+
return job;
|
|
941
|
+
}
|
|
942
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
943
|
+
}
|
|
944
|
+
throw new Error("Batch job timed out");
|
|
945
|
+
}
|
|
946
|
+
// ==========================================================================
|
|
947
|
+
// SCHEDULES
|
|
948
|
+
// ==========================================================================
|
|
949
|
+
/**
|
|
950
|
+
* Create a scheduled extraction job
|
|
951
|
+
*
|
|
952
|
+
* @example
|
|
953
|
+
* ```typescript
|
|
954
|
+
* const { id } = await extraction.createSchedule({
|
|
955
|
+
* name: 'Daily price monitoring',
|
|
956
|
+
* url: 'https://competitor.com/pricing',
|
|
957
|
+
* frequency: 'daily',
|
|
958
|
+
* config: { mode: 'schema', schema: { ... } },
|
|
959
|
+
* });
|
|
960
|
+
* ```
|
|
961
|
+
*/
|
|
962
|
+
async createSchedule(request) {
|
|
963
|
+
return this.http.post("/webdata/schedules", {
|
|
964
|
+
...request,
|
|
965
|
+
type: "extraction"
|
|
966
|
+
});
|
|
967
|
+
}
|
|
968
|
+
/**
|
|
969
|
+
* Update a schedule
|
|
970
|
+
*/
|
|
971
|
+
async updateSchedule(request) {
|
|
972
|
+
const { id, environment, projectId, ...data } = request;
|
|
973
|
+
const params = new URLSearchParams();
|
|
974
|
+
if (environment) params.set("environment", environment);
|
|
975
|
+
if (projectId) params.set("projectId", projectId);
|
|
976
|
+
const query = params.toString();
|
|
977
|
+
return this.http.post(
|
|
978
|
+
`/webdata/schedules/${id}${query ? `?${query}` : ""}`,
|
|
979
|
+
data
|
|
980
|
+
);
|
|
981
|
+
}
|
|
982
|
+
/**
|
|
983
|
+
* Get a schedule by ID
|
|
984
|
+
*/
|
|
985
|
+
async getSchedule(request) {
|
|
986
|
+
const params = new URLSearchParams();
|
|
987
|
+
if (request.environment) params.set("environment", request.environment);
|
|
988
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
989
|
+
const query = params.toString();
|
|
990
|
+
const path = `/webdata/schedules/${request.id}${query ? `?${query}` : ""}`;
|
|
991
|
+
const response = await this.http.get(path);
|
|
992
|
+
return this.convertScheduleDates(response);
|
|
993
|
+
}
|
|
994
|
+
/**
|
|
995
|
+
* List schedules with pagination and filters
|
|
996
|
+
*/
|
|
997
|
+
async listSchedules(request = {}) {
|
|
998
|
+
const params = new URLSearchParams();
|
|
999
|
+
if (request.environment) params.set("environment", request.environment);
|
|
1000
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
1001
|
+
params.set("type", "extraction");
|
|
1002
|
+
if (request.isActive !== void 0) params.set("isActive", request.isActive.toString());
|
|
1003
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
1004
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
1005
|
+
const query = params.toString();
|
|
1006
|
+
const response = await this.http.get(
|
|
1007
|
+
`/webdata/schedules${query ? `?${query}` : ""}`
|
|
1008
|
+
);
|
|
1009
|
+
return {
|
|
1010
|
+
...response,
|
|
1011
|
+
items: response.items.map((item) => this.convertScheduleDates(item))
|
|
1012
|
+
};
|
|
1013
|
+
}
|
|
1014
|
+
/**
|
|
1015
|
+
* Delete a schedule
|
|
1016
|
+
*/
|
|
1017
|
+
async deleteSchedule(request) {
|
|
1018
|
+
const params = new URLSearchParams();
|
|
1019
|
+
if (request.environment) params.set("environment", request.environment);
|
|
1020
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
1021
|
+
const query = params.toString();
|
|
1022
|
+
return this.http.delete(
|
|
1023
|
+
`/webdata/schedules/${request.id}${query ? `?${query}` : ""}`
|
|
1024
|
+
);
|
|
1025
|
+
}
|
|
1026
|
+
/**
|
|
1027
|
+
* Toggle a schedule on or off
|
|
1028
|
+
*/
|
|
1029
|
+
async toggleSchedule(request) {
|
|
1030
|
+
const params = new URLSearchParams();
|
|
1031
|
+
if (request.environment) params.set("environment", request.environment);
|
|
1032
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
1033
|
+
const query = params.toString();
|
|
1034
|
+
return this.http.post(
|
|
1035
|
+
`/webdata/schedules/${request.id}/toggle${query ? `?${query}` : ""}`,
|
|
1036
|
+
{}
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
// ==========================================================================
|
|
1040
|
+
// USAGE
|
|
1041
|
+
// ==========================================================================
|
|
1042
|
+
/**
|
|
1043
|
+
* Get usage statistics
|
|
1044
|
+
*
|
|
1045
|
+
* @example
|
|
1046
|
+
* ```typescript
|
|
1047
|
+
* const usage = await extraction.getUsage({
|
|
1048
|
+
* periodStart: '2024-01-01T00:00:00Z',
|
|
1049
|
+
* periodEnd: '2024-01-31T23:59:59Z',
|
|
1050
|
+
* });
|
|
1051
|
+
* console.log(`Extractions: ${usage.extractionsTotal}`);
|
|
1052
|
+
* console.log(`Tokens used: ${usage.extractionTokensUsed}`);
|
|
1053
|
+
* ```
|
|
1054
|
+
*/
|
|
1055
|
+
async getUsage(request = {}) {
|
|
1056
|
+
const params = new URLSearchParams();
|
|
1057
|
+
if (request.environment) params.set("environment", request.environment);
|
|
1058
|
+
if (request.periodStart) params.set("periodStart", request.periodStart);
|
|
1059
|
+
if (request.periodEnd) params.set("periodEnd", request.periodEnd);
|
|
1060
|
+
const query = params.toString();
|
|
1061
|
+
const response = await this.http.get(
|
|
1062
|
+
`/webdata/usage${query ? `?${query}` : ""}`
|
|
1063
|
+
);
|
|
1064
|
+
return this.convertUsageDates(response);
|
|
1065
|
+
}
|
|
1066
|
+
// ==========================================================================
|
|
1067
|
+
// HELPERS
|
|
1068
|
+
// ==========================================================================
|
|
1069
|
+
convertDates(extraction) {
|
|
1070
|
+
if (typeof extraction.createdAt === "string") {
|
|
1071
|
+
extraction.createdAt = new Date(extraction.createdAt);
|
|
1072
|
+
}
|
|
1073
|
+
if (extraction.completedAt && typeof extraction.completedAt === "string") {
|
|
1074
|
+
extraction.completedAt = new Date(extraction.completedAt);
|
|
1075
|
+
}
|
|
1076
|
+
return extraction;
|
|
1077
|
+
}
|
|
1078
|
+
convertBatchJobDates(job) {
|
|
1079
|
+
if (typeof job.createdAt === "string") {
|
|
1080
|
+
job.createdAt = new Date(job.createdAt);
|
|
1081
|
+
}
|
|
1082
|
+
if (job.startedAt && typeof job.startedAt === "string") {
|
|
1083
|
+
job.startedAt = new Date(job.startedAt);
|
|
1084
|
+
}
|
|
1085
|
+
if (job.completedAt && typeof job.completedAt === "string") {
|
|
1086
|
+
job.completedAt = new Date(job.completedAt);
|
|
1087
|
+
}
|
|
1088
|
+
return job;
|
|
1089
|
+
}
|
|
1090
|
+
convertScheduleDates(schedule) {
|
|
1091
|
+
if (typeof schedule.createdAt === "string") {
|
|
1092
|
+
schedule.createdAt = new Date(schedule.createdAt);
|
|
1093
|
+
}
|
|
1094
|
+
if (typeof schedule.updatedAt === "string") {
|
|
1095
|
+
schedule.updatedAt = new Date(schedule.updatedAt);
|
|
1096
|
+
}
|
|
1097
|
+
if (schedule.lastRunAt && typeof schedule.lastRunAt === "string") {
|
|
1098
|
+
schedule.lastRunAt = new Date(schedule.lastRunAt);
|
|
1099
|
+
}
|
|
1100
|
+
if (schedule.nextRunAt && typeof schedule.nextRunAt === "string") {
|
|
1101
|
+
schedule.nextRunAt = new Date(schedule.nextRunAt);
|
|
1102
|
+
}
|
|
1103
|
+
return schedule;
|
|
1104
|
+
}
|
|
1105
|
+
convertUsageDates(usage) {
|
|
1106
|
+
if (typeof usage.periodStart === "string") {
|
|
1107
|
+
usage.periodStart = new Date(usage.periodStart);
|
|
1108
|
+
}
|
|
1109
|
+
if (typeof usage.periodEnd === "string") {
|
|
1110
|
+
usage.periodEnd = new Date(usage.periodEnd);
|
|
1111
|
+
}
|
|
1112
|
+
return usage;
|
|
1113
|
+
}
|
|
1114
|
+
};
|
|
1115
|
+
|
|
387
1116
|
// src/webdata/client.ts
|
|
388
1117
|
var Webdata = class {
|
|
389
1118
|
http;
|
|
@@ -988,6 +1717,11 @@ var Webdata = class {
|
|
|
988
1717
|
var Stack0 = class {
|
|
989
1718
|
mail;
|
|
990
1719
|
cdn;
|
|
1720
|
+
screenshots;
|
|
1721
|
+
extraction;
|
|
1722
|
+
/**
|
|
1723
|
+
* @deprecated Use `screenshots` and `extraction` instead. Will be removed in a future version.
|
|
1724
|
+
*/
|
|
991
1725
|
webdata;
|
|
992
1726
|
constructor(config) {
|
|
993
1727
|
const clientConfig = {
|
|
@@ -996,11 +1730,13 @@ var Stack0 = class {
|
|
|
996
1730
|
};
|
|
997
1731
|
this.mail = new Mail(clientConfig);
|
|
998
1732
|
this.cdn = new CDN(clientConfig);
|
|
1733
|
+
this.screenshots = new Screenshots(clientConfig);
|
|
1734
|
+
this.extraction = new Extraction(clientConfig);
|
|
999
1735
|
this.webdata = new Webdata(clientConfig);
|
|
1000
1736
|
}
|
|
1001
1737
|
};
|
|
1002
1738
|
var src_default = Stack0;
|
|
1003
1739
|
|
|
1004
|
-
export { CDN, Mail, Stack0, Webdata, src_default as default };
|
|
1740
|
+
export { CDN, Extraction, Mail, Screenshots, Stack0, Webdata, src_default as default };
|
|
1005
1741
|
//# sourceMappingURL=index.mjs.map
|
|
1006
1742
|
//# sourceMappingURL=index.mjs.map
|