@stack0/sdk 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extraction/index.d.mts +374 -0
- package/dist/extraction/index.d.ts +374 -0
- package/dist/extraction/index.js +455 -0
- package/dist/extraction/index.js.map +1 -0
- package/dist/extraction/index.mjs +453 -0
- package/dist/extraction/index.mjs.map +1 -0
- package/dist/index.d.mts +13 -4
- package/dist/index.d.ts +13 -4
- package/dist/index.js +739 -0
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +738 -1
- package/dist/index.mjs.map +1 -1
- package/dist/screenshots/index.d.mts +377 -0
- package/dist/screenshots/index.d.ts +377 -0
- package/dist/screenshots/index.js +423 -0
- package/dist/screenshots/index.js.map +1 -0
- package/dist/screenshots/index.mjs +421 -0
- package/dist/screenshots/index.mjs.map +1 -0
- package/dist/shared-types-B0PyC7cF.d.mts +40 -0
- package/dist/shared-types-B0PyC7cF.d.ts +40 -0
- package/package.json +20 -1
package/dist/index.js
CHANGED
|
@@ -387,6 +387,736 @@ var CDN = class {
|
|
|
387
387
|
}
|
|
388
388
|
};
|
|
389
389
|
|
|
390
|
+
// src/screenshots/client.ts
|
|
391
|
+
var Screenshots = class {
|
|
392
|
+
http;
|
|
393
|
+
constructor(config) {
|
|
394
|
+
this.http = new HttpClient(config);
|
|
395
|
+
}
|
|
396
|
+
// ==========================================================================
|
|
397
|
+
// SCREENSHOTS
|
|
398
|
+
// ==========================================================================
|
|
399
|
+
/**
|
|
400
|
+
* Capture a screenshot of a URL
|
|
401
|
+
*
|
|
402
|
+
* @example
|
|
403
|
+
* ```typescript
|
|
404
|
+
* const { id, status } = await screenshots.capture({
|
|
405
|
+
* url: 'https://example.com',
|
|
406
|
+
* format: 'png',
|
|
407
|
+
* fullPage: true,
|
|
408
|
+
* deviceType: 'desktop',
|
|
409
|
+
* });
|
|
410
|
+
*
|
|
411
|
+
* // Poll for completion
|
|
412
|
+
* const screenshot = await screenshots.get({ id });
|
|
413
|
+
* console.log(screenshot.imageUrl);
|
|
414
|
+
* ```
|
|
415
|
+
*/
|
|
416
|
+
async capture(request) {
|
|
417
|
+
return this.http.post("/webdata/screenshots", request);
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Get a screenshot by ID
|
|
421
|
+
*
|
|
422
|
+
* @example
|
|
423
|
+
* ```typescript
|
|
424
|
+
* const screenshot = await screenshots.get({ id: 'screenshot-id' });
|
|
425
|
+
* if (screenshot.status === 'completed') {
|
|
426
|
+
* console.log(screenshot.imageUrl);
|
|
427
|
+
* }
|
|
428
|
+
* ```
|
|
429
|
+
*/
|
|
430
|
+
async get(request) {
|
|
431
|
+
const params = new URLSearchParams();
|
|
432
|
+
if (request.environment) params.set("environment", request.environment);
|
|
433
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
434
|
+
const query = params.toString();
|
|
435
|
+
const path = `/webdata/screenshots/${request.id}${query ? `?${query}` : ""}`;
|
|
436
|
+
const response = await this.http.get(path);
|
|
437
|
+
return this.convertDates(response);
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* List screenshots with pagination and filters
|
|
441
|
+
*
|
|
442
|
+
* @example
|
|
443
|
+
* ```typescript
|
|
444
|
+
* const { items, nextCursor } = await screenshots.list({
|
|
445
|
+
* status: 'completed',
|
|
446
|
+
* limit: 20,
|
|
447
|
+
* });
|
|
448
|
+
* ```
|
|
449
|
+
*/
|
|
450
|
+
async list(request = {}) {
|
|
451
|
+
const params = new URLSearchParams();
|
|
452
|
+
if (request.environment) params.set("environment", request.environment);
|
|
453
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
454
|
+
if (request.status) params.set("status", request.status);
|
|
455
|
+
if (request.url) params.set("url", request.url);
|
|
456
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
457
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
458
|
+
const query = params.toString();
|
|
459
|
+
const response = await this.http.get(
|
|
460
|
+
`/webdata/screenshots${query ? `?${query}` : ""}`
|
|
461
|
+
);
|
|
462
|
+
return {
|
|
463
|
+
...response,
|
|
464
|
+
items: response.items.map((item) => this.convertDates(item))
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Delete a screenshot
|
|
469
|
+
*
|
|
470
|
+
* @example
|
|
471
|
+
* ```typescript
|
|
472
|
+
* await screenshots.delete({ id: 'screenshot-id' });
|
|
473
|
+
* ```
|
|
474
|
+
*/
|
|
475
|
+
async delete(request) {
|
|
476
|
+
const params = new URLSearchParams();
|
|
477
|
+
if (request.environment) params.set("environment", request.environment);
|
|
478
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
479
|
+
const query = params.toString();
|
|
480
|
+
return this.http.delete(
|
|
481
|
+
`/webdata/screenshots/${request.id}${query ? `?${query}` : ""}`
|
|
482
|
+
);
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* Capture a screenshot and wait for completion
|
|
486
|
+
*
|
|
487
|
+
* @example
|
|
488
|
+
* ```typescript
|
|
489
|
+
* const screenshot = await screenshots.captureAndWait({
|
|
490
|
+
* url: 'https://example.com',
|
|
491
|
+
* format: 'png',
|
|
492
|
+
* });
|
|
493
|
+
* console.log(screenshot.imageUrl);
|
|
494
|
+
* ```
|
|
495
|
+
*/
|
|
496
|
+
async captureAndWait(request, options = {}) {
|
|
497
|
+
const { pollInterval = 1e3, timeout = 6e4 } = options;
|
|
498
|
+
const startTime = Date.now();
|
|
499
|
+
const { id } = await this.capture(request);
|
|
500
|
+
while (Date.now() - startTime < timeout) {
|
|
501
|
+
const screenshot = await this.get({
|
|
502
|
+
id,
|
|
503
|
+
environment: request.environment,
|
|
504
|
+
projectId: request.projectId
|
|
505
|
+
});
|
|
506
|
+
if (screenshot.status === "completed" || screenshot.status === "failed") {
|
|
507
|
+
if (screenshot.status === "failed") {
|
|
508
|
+
throw new Error(screenshot.error || "Screenshot failed");
|
|
509
|
+
}
|
|
510
|
+
return screenshot;
|
|
511
|
+
}
|
|
512
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
513
|
+
}
|
|
514
|
+
throw new Error("Screenshot timed out");
|
|
515
|
+
}
|
|
516
|
+
// ==========================================================================
|
|
517
|
+
// BATCH JOBS
|
|
518
|
+
// ==========================================================================
|
|
519
|
+
/**
|
|
520
|
+
* Create a batch screenshot job for multiple URLs
|
|
521
|
+
*
|
|
522
|
+
* @example
|
|
523
|
+
* ```typescript
|
|
524
|
+
* const { id, totalUrls } = await screenshots.batch({
|
|
525
|
+
* urls: [
|
|
526
|
+
* 'https://example.com',
|
|
527
|
+
* 'https://example.org',
|
|
528
|
+
* ],
|
|
529
|
+
* config: { format: 'png', fullPage: true },
|
|
530
|
+
* });
|
|
531
|
+
*
|
|
532
|
+
* // Poll for completion
|
|
533
|
+
* const job = await screenshots.getBatchJob({ id });
|
|
534
|
+
* console.log(`Progress: ${job.processedUrls}/${job.totalUrls}`);
|
|
535
|
+
* ```
|
|
536
|
+
*/
|
|
537
|
+
async batch(request) {
|
|
538
|
+
return this.http.post("/webdata/batch/screenshots", request);
|
|
539
|
+
}
|
|
540
|
+
/**
|
|
541
|
+
* Get a batch job by ID
|
|
542
|
+
*/
|
|
543
|
+
async getBatchJob(request) {
|
|
544
|
+
const params = new URLSearchParams();
|
|
545
|
+
if (request.environment) params.set("environment", request.environment);
|
|
546
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
547
|
+
const query = params.toString();
|
|
548
|
+
const path = `/webdata/batch/${request.id}${query ? `?${query}` : ""}`;
|
|
549
|
+
const response = await this.http.get(path);
|
|
550
|
+
return this.convertBatchJobDates(response);
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* List batch jobs with pagination and filters
|
|
554
|
+
*/
|
|
555
|
+
async listBatchJobs(request = {}) {
|
|
556
|
+
const params = new URLSearchParams();
|
|
557
|
+
if (request.environment) params.set("environment", request.environment);
|
|
558
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
559
|
+
if (request.status) params.set("status", request.status);
|
|
560
|
+
params.set("type", "screenshot");
|
|
561
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
562
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
563
|
+
const query = params.toString();
|
|
564
|
+
const response = await this.http.get(
|
|
565
|
+
`/webdata/batch${query ? `?${query}` : ""}`
|
|
566
|
+
);
|
|
567
|
+
return {
|
|
568
|
+
...response,
|
|
569
|
+
items: response.items.map((item) => this.convertBatchJobDates(item))
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* Cancel a batch job
|
|
574
|
+
*/
|
|
575
|
+
async cancelBatchJob(request) {
|
|
576
|
+
const params = new URLSearchParams();
|
|
577
|
+
if (request.environment) params.set("environment", request.environment);
|
|
578
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
579
|
+
const query = params.toString();
|
|
580
|
+
return this.http.post(
|
|
581
|
+
`/webdata/batch/${request.id}/cancel${query ? `?${query}` : ""}`,
|
|
582
|
+
{}
|
|
583
|
+
);
|
|
584
|
+
}
|
|
585
|
+
/**
|
|
586
|
+
* Create a batch screenshot job and wait for completion
|
|
587
|
+
*/
|
|
588
|
+
async batchAndWait(request, options = {}) {
|
|
589
|
+
const { pollInterval = 2e3, timeout = 3e5 } = options;
|
|
590
|
+
const startTime = Date.now();
|
|
591
|
+
const { id } = await this.batch(request);
|
|
592
|
+
while (Date.now() - startTime < timeout) {
|
|
593
|
+
const job = await this.getBatchJob({
|
|
594
|
+
id,
|
|
595
|
+
environment: request.environment,
|
|
596
|
+
projectId: request.projectId
|
|
597
|
+
});
|
|
598
|
+
if (job.status === "completed" || job.status === "failed" || job.status === "cancelled") {
|
|
599
|
+
return job;
|
|
600
|
+
}
|
|
601
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
602
|
+
}
|
|
603
|
+
throw new Error("Batch job timed out");
|
|
604
|
+
}
|
|
605
|
+
// ==========================================================================
|
|
606
|
+
// SCHEDULES
|
|
607
|
+
// ==========================================================================
|
|
608
|
+
/**
|
|
609
|
+
* Create a scheduled screenshot job
|
|
610
|
+
*
|
|
611
|
+
* @example
|
|
612
|
+
* ```typescript
|
|
613
|
+
* const { id } = await screenshots.createSchedule({
|
|
614
|
+
* name: 'Daily homepage screenshot',
|
|
615
|
+
* url: 'https://example.com',
|
|
616
|
+
* frequency: 'daily',
|
|
617
|
+
* config: { format: 'png', fullPage: true },
|
|
618
|
+
* });
|
|
619
|
+
* ```
|
|
620
|
+
*/
|
|
621
|
+
async createSchedule(request) {
|
|
622
|
+
return this.http.post("/webdata/schedules", {
|
|
623
|
+
...request,
|
|
624
|
+
type: "screenshot"
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* Update a schedule
|
|
629
|
+
*/
|
|
630
|
+
async updateSchedule(request) {
|
|
631
|
+
const { id, environment, projectId, ...data } = request;
|
|
632
|
+
const params = new URLSearchParams();
|
|
633
|
+
if (environment) params.set("environment", environment);
|
|
634
|
+
if (projectId) params.set("projectId", projectId);
|
|
635
|
+
const query = params.toString();
|
|
636
|
+
return this.http.post(
|
|
637
|
+
`/webdata/schedules/${id}${query ? `?${query}` : ""}`,
|
|
638
|
+
data
|
|
639
|
+
);
|
|
640
|
+
}
|
|
641
|
+
/**
|
|
642
|
+
* Get a schedule by ID
|
|
643
|
+
*/
|
|
644
|
+
async getSchedule(request) {
|
|
645
|
+
const params = new URLSearchParams();
|
|
646
|
+
if (request.environment) params.set("environment", request.environment);
|
|
647
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
648
|
+
const query = params.toString();
|
|
649
|
+
const path = `/webdata/schedules/${request.id}${query ? `?${query}` : ""}`;
|
|
650
|
+
const response = await this.http.get(path);
|
|
651
|
+
return this.convertScheduleDates(response);
|
|
652
|
+
}
|
|
653
|
+
/**
|
|
654
|
+
* List schedules with pagination and filters
|
|
655
|
+
*/
|
|
656
|
+
async listSchedules(request = {}) {
|
|
657
|
+
const params = new URLSearchParams();
|
|
658
|
+
if (request.environment) params.set("environment", request.environment);
|
|
659
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
660
|
+
params.set("type", "screenshot");
|
|
661
|
+
if (request.isActive !== void 0) params.set("isActive", request.isActive.toString());
|
|
662
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
663
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
664
|
+
const query = params.toString();
|
|
665
|
+
const response = await this.http.get(
|
|
666
|
+
`/webdata/schedules${query ? `?${query}` : ""}`
|
|
667
|
+
);
|
|
668
|
+
return {
|
|
669
|
+
...response,
|
|
670
|
+
items: response.items.map((item) => this.convertScheduleDates(item))
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
/**
|
|
674
|
+
* Delete a schedule
|
|
675
|
+
*/
|
|
676
|
+
async deleteSchedule(request) {
|
|
677
|
+
const params = new URLSearchParams();
|
|
678
|
+
if (request.environment) params.set("environment", request.environment);
|
|
679
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
680
|
+
const query = params.toString();
|
|
681
|
+
return this.http.delete(
|
|
682
|
+
`/webdata/schedules/${request.id}${query ? `?${query}` : ""}`
|
|
683
|
+
);
|
|
684
|
+
}
|
|
685
|
+
/**
|
|
686
|
+
* Toggle a schedule on or off
|
|
687
|
+
*/
|
|
688
|
+
async toggleSchedule(request) {
|
|
689
|
+
const params = new URLSearchParams();
|
|
690
|
+
if (request.environment) params.set("environment", request.environment);
|
|
691
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
692
|
+
const query = params.toString();
|
|
693
|
+
return this.http.post(
|
|
694
|
+
`/webdata/schedules/${request.id}/toggle${query ? `?${query}` : ""}`,
|
|
695
|
+
{}
|
|
696
|
+
);
|
|
697
|
+
}
|
|
698
|
+
// ==========================================================================
|
|
699
|
+
// HELPERS
|
|
700
|
+
// ==========================================================================
|
|
701
|
+
convertDates(screenshot) {
|
|
702
|
+
if (typeof screenshot.createdAt === "string") {
|
|
703
|
+
screenshot.createdAt = new Date(screenshot.createdAt);
|
|
704
|
+
}
|
|
705
|
+
if (screenshot.completedAt && typeof screenshot.completedAt === "string") {
|
|
706
|
+
screenshot.completedAt = new Date(screenshot.completedAt);
|
|
707
|
+
}
|
|
708
|
+
return screenshot;
|
|
709
|
+
}
|
|
710
|
+
convertBatchJobDates(job) {
|
|
711
|
+
if (typeof job.createdAt === "string") {
|
|
712
|
+
job.createdAt = new Date(job.createdAt);
|
|
713
|
+
}
|
|
714
|
+
if (job.startedAt && typeof job.startedAt === "string") {
|
|
715
|
+
job.startedAt = new Date(job.startedAt);
|
|
716
|
+
}
|
|
717
|
+
if (job.completedAt && typeof job.completedAt === "string") {
|
|
718
|
+
job.completedAt = new Date(job.completedAt);
|
|
719
|
+
}
|
|
720
|
+
return job;
|
|
721
|
+
}
|
|
722
|
+
convertScheduleDates(schedule) {
|
|
723
|
+
if (typeof schedule.createdAt === "string") {
|
|
724
|
+
schedule.createdAt = new Date(schedule.createdAt);
|
|
725
|
+
}
|
|
726
|
+
if (typeof schedule.updatedAt === "string") {
|
|
727
|
+
schedule.updatedAt = new Date(schedule.updatedAt);
|
|
728
|
+
}
|
|
729
|
+
if (schedule.lastRunAt && typeof schedule.lastRunAt === "string") {
|
|
730
|
+
schedule.lastRunAt = new Date(schedule.lastRunAt);
|
|
731
|
+
}
|
|
732
|
+
if (schedule.nextRunAt && typeof schedule.nextRunAt === "string") {
|
|
733
|
+
schedule.nextRunAt = new Date(schedule.nextRunAt);
|
|
734
|
+
}
|
|
735
|
+
return schedule;
|
|
736
|
+
}
|
|
737
|
+
};
|
|
738
|
+
|
|
739
|
+
// src/extraction/client.ts
|
|
740
|
+
var Extraction = class {
|
|
741
|
+
http;
|
|
742
|
+
constructor(config) {
|
|
743
|
+
this.http = new HttpClient(config);
|
|
744
|
+
}
|
|
745
|
+
// ==========================================================================
|
|
746
|
+
// EXTRACTIONS
|
|
747
|
+
// ==========================================================================
|
|
748
|
+
/**
|
|
749
|
+
* Extract content from a URL
|
|
750
|
+
*
|
|
751
|
+
* @example
|
|
752
|
+
* ```typescript
|
|
753
|
+
* const { id, status } = await extraction.extract({
|
|
754
|
+
* url: 'https://example.com/article',
|
|
755
|
+
* mode: 'markdown',
|
|
756
|
+
* includeMetadata: true,
|
|
757
|
+
* });
|
|
758
|
+
*
|
|
759
|
+
* // Poll for completion
|
|
760
|
+
* const result = await extraction.get({ id });
|
|
761
|
+
* console.log(result.markdown);
|
|
762
|
+
* ```
|
|
763
|
+
*/
|
|
764
|
+
async extract(request) {
|
|
765
|
+
return this.http.post("/webdata/extractions", request);
|
|
766
|
+
}
|
|
767
|
+
/**
|
|
768
|
+
* Get an extraction by ID
|
|
769
|
+
*
|
|
770
|
+
* @example
|
|
771
|
+
* ```typescript
|
|
772
|
+
* const extraction = await extraction.get({ id: 'extraction-id' });
|
|
773
|
+
* if (extraction.status === 'completed') {
|
|
774
|
+
* console.log(extraction.markdown);
|
|
775
|
+
* console.log(extraction.pageMetadata);
|
|
776
|
+
* }
|
|
777
|
+
* ```
|
|
778
|
+
*/
|
|
779
|
+
async get(request) {
|
|
780
|
+
const params = new URLSearchParams();
|
|
781
|
+
if (request.environment) params.set("environment", request.environment);
|
|
782
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
783
|
+
const query = params.toString();
|
|
784
|
+
const path = `/webdata/extractions/${request.id}${query ? `?${query}` : ""}`;
|
|
785
|
+
const response = await this.http.get(path);
|
|
786
|
+
return this.convertDates(response);
|
|
787
|
+
}
|
|
788
|
+
/**
|
|
789
|
+
* List extractions with pagination and filters
|
|
790
|
+
*
|
|
791
|
+
* @example
|
|
792
|
+
* ```typescript
|
|
793
|
+
* const { items, nextCursor } = await extraction.list({
|
|
794
|
+
* status: 'completed',
|
|
795
|
+
* limit: 20,
|
|
796
|
+
* });
|
|
797
|
+
* ```
|
|
798
|
+
*/
|
|
799
|
+
async list(request = {}) {
|
|
800
|
+
const params = new URLSearchParams();
|
|
801
|
+
if (request.environment) params.set("environment", request.environment);
|
|
802
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
803
|
+
if (request.status) params.set("status", request.status);
|
|
804
|
+
if (request.url) params.set("url", request.url);
|
|
805
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
806
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
807
|
+
const query = params.toString();
|
|
808
|
+
const response = await this.http.get(
|
|
809
|
+
`/webdata/extractions${query ? `?${query}` : ""}`
|
|
810
|
+
);
|
|
811
|
+
return {
|
|
812
|
+
...response,
|
|
813
|
+
items: response.items.map((item) => this.convertDates(item))
|
|
814
|
+
};
|
|
815
|
+
}
|
|
816
|
+
/**
|
|
817
|
+
* Delete an extraction
|
|
818
|
+
*
|
|
819
|
+
* @example
|
|
820
|
+
* ```typescript
|
|
821
|
+
* await extraction.delete({ id: 'extraction-id' });
|
|
822
|
+
* ```
|
|
823
|
+
*/
|
|
824
|
+
async delete(request) {
|
|
825
|
+
const params = new URLSearchParams();
|
|
826
|
+
if (request.environment) params.set("environment", request.environment);
|
|
827
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
828
|
+
const query = params.toString();
|
|
829
|
+
return this.http.delete(
|
|
830
|
+
`/webdata/extractions/${request.id}${query ? `?${query}` : ""}`
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
/**
|
|
834
|
+
* Extract content and wait for completion
|
|
835
|
+
*
|
|
836
|
+
* @example
|
|
837
|
+
* ```typescript
|
|
838
|
+
* const extraction = await extraction.extractAndWait({
|
|
839
|
+
* url: 'https://example.com/article',
|
|
840
|
+
* mode: 'markdown',
|
|
841
|
+
* });
|
|
842
|
+
* console.log(extraction.markdown);
|
|
843
|
+
* ```
|
|
844
|
+
*/
|
|
845
|
+
async extractAndWait(request, options = {}) {
|
|
846
|
+
const { pollInterval = 1e3, timeout = 6e4 } = options;
|
|
847
|
+
const startTime = Date.now();
|
|
848
|
+
const { id } = await this.extract(request);
|
|
849
|
+
while (Date.now() - startTime < timeout) {
|
|
850
|
+
const extraction = await this.get({
|
|
851
|
+
id,
|
|
852
|
+
environment: request.environment,
|
|
853
|
+
projectId: request.projectId
|
|
854
|
+
});
|
|
855
|
+
if (extraction.status === "completed" || extraction.status === "failed") {
|
|
856
|
+
if (extraction.status === "failed") {
|
|
857
|
+
throw new Error(extraction.error || "Extraction failed");
|
|
858
|
+
}
|
|
859
|
+
return extraction;
|
|
860
|
+
}
|
|
861
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
862
|
+
}
|
|
863
|
+
throw new Error("Extraction timed out");
|
|
864
|
+
}
|
|
865
|
+
// ==========================================================================
|
|
866
|
+
// BATCH JOBS
|
|
867
|
+
// ==========================================================================
|
|
868
|
+
/**
|
|
869
|
+
* Create a batch extraction job for multiple URLs
|
|
870
|
+
*
|
|
871
|
+
* @example
|
|
872
|
+
* ```typescript
|
|
873
|
+
* const { id, totalUrls } = await extraction.batch({
|
|
874
|
+
* urls: [
|
|
875
|
+
* 'https://example.com/article1',
|
|
876
|
+
* 'https://example.com/article2',
|
|
877
|
+
* ],
|
|
878
|
+
* config: { mode: 'markdown' },
|
|
879
|
+
* });
|
|
880
|
+
* ```
|
|
881
|
+
*/
|
|
882
|
+
async batch(request) {
|
|
883
|
+
return this.http.post("/webdata/batch/extractions", request);
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* Get a batch job by ID
|
|
887
|
+
*/
|
|
888
|
+
async getBatchJob(request) {
|
|
889
|
+
const params = new URLSearchParams();
|
|
890
|
+
if (request.environment) params.set("environment", request.environment);
|
|
891
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
892
|
+
const query = params.toString();
|
|
893
|
+
const path = `/webdata/batch/${request.id}${query ? `?${query}` : ""}`;
|
|
894
|
+
const response = await this.http.get(path);
|
|
895
|
+
return this.convertBatchJobDates(response);
|
|
896
|
+
}
|
|
897
|
+
/**
|
|
898
|
+
* List batch jobs with pagination and filters
|
|
899
|
+
*/
|
|
900
|
+
async listBatchJobs(request = {}) {
|
|
901
|
+
const params = new URLSearchParams();
|
|
902
|
+
if (request.environment) params.set("environment", request.environment);
|
|
903
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
904
|
+
if (request.status) params.set("status", request.status);
|
|
905
|
+
params.set("type", "extraction");
|
|
906
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
907
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
908
|
+
const query = params.toString();
|
|
909
|
+
const response = await this.http.get(
|
|
910
|
+
`/webdata/batch${query ? `?${query}` : ""}`
|
|
911
|
+
);
|
|
912
|
+
return {
|
|
913
|
+
...response,
|
|
914
|
+
items: response.items.map((item) => this.convertBatchJobDates(item))
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
/**
|
|
918
|
+
* Cancel a batch job
|
|
919
|
+
*/
|
|
920
|
+
async cancelBatchJob(request) {
|
|
921
|
+
const params = new URLSearchParams();
|
|
922
|
+
if (request.environment) params.set("environment", request.environment);
|
|
923
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
924
|
+
const query = params.toString();
|
|
925
|
+
return this.http.post(
|
|
926
|
+
`/webdata/batch/${request.id}/cancel${query ? `?${query}` : ""}`,
|
|
927
|
+
{}
|
|
928
|
+
);
|
|
929
|
+
}
|
|
930
|
+
/**
|
|
931
|
+
* Create a batch extraction job and wait for completion
|
|
932
|
+
*/
|
|
933
|
+
async batchAndWait(request, options = {}) {
|
|
934
|
+
const { pollInterval = 2e3, timeout = 3e5 } = options;
|
|
935
|
+
const startTime = Date.now();
|
|
936
|
+
const { id } = await this.batch(request);
|
|
937
|
+
while (Date.now() - startTime < timeout) {
|
|
938
|
+
const job = await this.getBatchJob({
|
|
939
|
+
id,
|
|
940
|
+
environment: request.environment,
|
|
941
|
+
projectId: request.projectId
|
|
942
|
+
});
|
|
943
|
+
if (job.status === "completed" || job.status === "failed" || job.status === "cancelled") {
|
|
944
|
+
return job;
|
|
945
|
+
}
|
|
946
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
947
|
+
}
|
|
948
|
+
throw new Error("Batch job timed out");
|
|
949
|
+
}
|
|
950
|
+
// ==========================================================================
|
|
951
|
+
// SCHEDULES
|
|
952
|
+
// ==========================================================================
|
|
953
|
+
/**
|
|
954
|
+
* Create a scheduled extraction job
|
|
955
|
+
*
|
|
956
|
+
* @example
|
|
957
|
+
* ```typescript
|
|
958
|
+
* const { id } = await extraction.createSchedule({
|
|
959
|
+
* name: 'Daily price monitoring',
|
|
960
|
+
* url: 'https://competitor.com/pricing',
|
|
961
|
+
* frequency: 'daily',
|
|
962
|
+
* config: { mode: 'schema', schema: { ... } },
|
|
963
|
+
* });
|
|
964
|
+
* ```
|
|
965
|
+
*/
|
|
966
|
+
async createSchedule(request) {
|
|
967
|
+
return this.http.post("/webdata/schedules", {
|
|
968
|
+
...request,
|
|
969
|
+
type: "extraction"
|
|
970
|
+
});
|
|
971
|
+
}
|
|
972
|
+
/**
|
|
973
|
+
* Update a schedule
|
|
974
|
+
*/
|
|
975
|
+
async updateSchedule(request) {
|
|
976
|
+
const { id, environment, projectId, ...data } = request;
|
|
977
|
+
const params = new URLSearchParams();
|
|
978
|
+
if (environment) params.set("environment", environment);
|
|
979
|
+
if (projectId) params.set("projectId", projectId);
|
|
980
|
+
const query = params.toString();
|
|
981
|
+
return this.http.post(
|
|
982
|
+
`/webdata/schedules/${id}${query ? `?${query}` : ""}`,
|
|
983
|
+
data
|
|
984
|
+
);
|
|
985
|
+
}
|
|
986
|
+
/**
|
|
987
|
+
* Get a schedule by ID
|
|
988
|
+
*/
|
|
989
|
+
async getSchedule(request) {
|
|
990
|
+
const params = new URLSearchParams();
|
|
991
|
+
if (request.environment) params.set("environment", request.environment);
|
|
992
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
993
|
+
const query = params.toString();
|
|
994
|
+
const path = `/webdata/schedules/${request.id}${query ? `?${query}` : ""}`;
|
|
995
|
+
const response = await this.http.get(path);
|
|
996
|
+
return this.convertScheduleDates(response);
|
|
997
|
+
}
|
|
998
|
+
/**
|
|
999
|
+
* List schedules with pagination and filters
|
|
1000
|
+
*/
|
|
1001
|
+
async listSchedules(request = {}) {
|
|
1002
|
+
const params = new URLSearchParams();
|
|
1003
|
+
if (request.environment) params.set("environment", request.environment);
|
|
1004
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
1005
|
+
params.set("type", "extraction");
|
|
1006
|
+
if (request.isActive !== void 0) params.set("isActive", request.isActive.toString());
|
|
1007
|
+
if (request.limit) params.set("limit", request.limit.toString());
|
|
1008
|
+
if (request.cursor) params.set("cursor", request.cursor);
|
|
1009
|
+
const query = params.toString();
|
|
1010
|
+
const response = await this.http.get(
|
|
1011
|
+
`/webdata/schedules${query ? `?${query}` : ""}`
|
|
1012
|
+
);
|
|
1013
|
+
return {
|
|
1014
|
+
...response,
|
|
1015
|
+
items: response.items.map((item) => this.convertScheduleDates(item))
|
|
1016
|
+
};
|
|
1017
|
+
}
|
|
1018
|
+
/**
|
|
1019
|
+
* Delete a schedule
|
|
1020
|
+
*/
|
|
1021
|
+
async deleteSchedule(request) {
|
|
1022
|
+
const params = new URLSearchParams();
|
|
1023
|
+
if (request.environment) params.set("environment", request.environment);
|
|
1024
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
1025
|
+
const query = params.toString();
|
|
1026
|
+
return this.http.delete(
|
|
1027
|
+
`/webdata/schedules/${request.id}${query ? `?${query}` : ""}`
|
|
1028
|
+
);
|
|
1029
|
+
}
|
|
1030
|
+
/**
|
|
1031
|
+
* Toggle a schedule on or off
|
|
1032
|
+
*/
|
|
1033
|
+
async toggleSchedule(request) {
|
|
1034
|
+
const params = new URLSearchParams();
|
|
1035
|
+
if (request.environment) params.set("environment", request.environment);
|
|
1036
|
+
if (request.projectId) params.set("projectId", request.projectId);
|
|
1037
|
+
const query = params.toString();
|
|
1038
|
+
return this.http.post(
|
|
1039
|
+
`/webdata/schedules/${request.id}/toggle${query ? `?${query}` : ""}`,
|
|
1040
|
+
{}
|
|
1041
|
+
);
|
|
1042
|
+
}
|
|
1043
|
+
// ==========================================================================
|
|
1044
|
+
// USAGE
|
|
1045
|
+
// ==========================================================================
|
|
1046
|
+
/**
|
|
1047
|
+
* Get usage statistics
|
|
1048
|
+
*
|
|
1049
|
+
* @example
|
|
1050
|
+
* ```typescript
|
|
1051
|
+
* const usage = await extraction.getUsage({
|
|
1052
|
+
* periodStart: '2024-01-01T00:00:00Z',
|
|
1053
|
+
* periodEnd: '2024-01-31T23:59:59Z',
|
|
1054
|
+
* });
|
|
1055
|
+
* console.log(`Extractions: ${usage.extractionsTotal}`);
|
|
1056
|
+
* console.log(`Tokens used: ${usage.extractionTokensUsed}`);
|
|
1057
|
+
* ```
|
|
1058
|
+
*/
|
|
1059
|
+
async getUsage(request = {}) {
|
|
1060
|
+
const params = new URLSearchParams();
|
|
1061
|
+
if (request.environment) params.set("environment", request.environment);
|
|
1062
|
+
if (request.periodStart) params.set("periodStart", request.periodStart);
|
|
1063
|
+
if (request.periodEnd) params.set("periodEnd", request.periodEnd);
|
|
1064
|
+
const query = params.toString();
|
|
1065
|
+
const response = await this.http.get(
|
|
1066
|
+
`/webdata/usage${query ? `?${query}` : ""}`
|
|
1067
|
+
);
|
|
1068
|
+
return this.convertUsageDates(response);
|
|
1069
|
+
}
|
|
1070
|
+
// ==========================================================================
|
|
1071
|
+
// HELPERS
|
|
1072
|
+
// ==========================================================================
|
|
1073
|
+
convertDates(extraction) {
|
|
1074
|
+
if (typeof extraction.createdAt === "string") {
|
|
1075
|
+
extraction.createdAt = new Date(extraction.createdAt);
|
|
1076
|
+
}
|
|
1077
|
+
if (extraction.completedAt && typeof extraction.completedAt === "string") {
|
|
1078
|
+
extraction.completedAt = new Date(extraction.completedAt);
|
|
1079
|
+
}
|
|
1080
|
+
return extraction;
|
|
1081
|
+
}
|
|
1082
|
+
convertBatchJobDates(job) {
|
|
1083
|
+
if (typeof job.createdAt === "string") {
|
|
1084
|
+
job.createdAt = new Date(job.createdAt);
|
|
1085
|
+
}
|
|
1086
|
+
if (job.startedAt && typeof job.startedAt === "string") {
|
|
1087
|
+
job.startedAt = new Date(job.startedAt);
|
|
1088
|
+
}
|
|
1089
|
+
if (job.completedAt && typeof job.completedAt === "string") {
|
|
1090
|
+
job.completedAt = new Date(job.completedAt);
|
|
1091
|
+
}
|
|
1092
|
+
return job;
|
|
1093
|
+
}
|
|
1094
|
+
convertScheduleDates(schedule) {
|
|
1095
|
+
if (typeof schedule.createdAt === "string") {
|
|
1096
|
+
schedule.createdAt = new Date(schedule.createdAt);
|
|
1097
|
+
}
|
|
1098
|
+
if (typeof schedule.updatedAt === "string") {
|
|
1099
|
+
schedule.updatedAt = new Date(schedule.updatedAt);
|
|
1100
|
+
}
|
|
1101
|
+
if (schedule.lastRunAt && typeof schedule.lastRunAt === "string") {
|
|
1102
|
+
schedule.lastRunAt = new Date(schedule.lastRunAt);
|
|
1103
|
+
}
|
|
1104
|
+
if (schedule.nextRunAt && typeof schedule.nextRunAt === "string") {
|
|
1105
|
+
schedule.nextRunAt = new Date(schedule.nextRunAt);
|
|
1106
|
+
}
|
|
1107
|
+
return schedule;
|
|
1108
|
+
}
|
|
1109
|
+
convertUsageDates(usage) {
|
|
1110
|
+
if (typeof usage.periodStart === "string") {
|
|
1111
|
+
usage.periodStart = new Date(usage.periodStart);
|
|
1112
|
+
}
|
|
1113
|
+
if (typeof usage.periodEnd === "string") {
|
|
1114
|
+
usage.periodEnd = new Date(usage.periodEnd);
|
|
1115
|
+
}
|
|
1116
|
+
return usage;
|
|
1117
|
+
}
|
|
1118
|
+
};
|
|
1119
|
+
|
|
390
1120
|
// src/webdata/client.ts
|
|
391
1121
|
var Webdata = class {
|
|
392
1122
|
http;
|
|
@@ -991,6 +1721,11 @@ var Webdata = class {
|
|
|
991
1721
|
var Stack0 = class {
|
|
992
1722
|
mail;
|
|
993
1723
|
cdn;
|
|
1724
|
+
screenshots;
|
|
1725
|
+
extraction;
|
|
1726
|
+
/**
|
|
1727
|
+
* @deprecated Use `screenshots` and `extraction` instead. Will be removed in a future version.
|
|
1728
|
+
*/
|
|
994
1729
|
webdata;
|
|
995
1730
|
constructor(config) {
|
|
996
1731
|
const clientConfig = {
|
|
@@ -999,13 +1734,17 @@ var Stack0 = class {
|
|
|
999
1734
|
};
|
|
1000
1735
|
this.mail = new Mail(clientConfig);
|
|
1001
1736
|
this.cdn = new CDN(clientConfig);
|
|
1737
|
+
this.screenshots = new Screenshots(clientConfig);
|
|
1738
|
+
this.extraction = new Extraction(clientConfig);
|
|
1002
1739
|
this.webdata = new Webdata(clientConfig);
|
|
1003
1740
|
}
|
|
1004
1741
|
};
|
|
1005
1742
|
var src_default = Stack0;
|
|
1006
1743
|
|
|
1007
1744
|
exports.CDN = CDN;
|
|
1745
|
+
exports.Extraction = Extraction;
|
|
1008
1746
|
exports.Mail = Mail;
|
|
1747
|
+
exports.Screenshots = Screenshots;
|
|
1009
1748
|
exports.Stack0 = Stack0;
|
|
1010
1749
|
exports.Webdata = Webdata;
|
|
1011
1750
|
exports.default = src_default;
|