@airtop/sdk 0.1.18 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/resources/profiles/client/Client.js +2 -2
- package/api/resources/sessions/client/Client.js +14 -14
- package/api/resources/sessions/types/SessionsEventsResponse.d.ts +4 -4
- package/api/resources/sessions/types/index.d.ts +1 -1
- package/api/resources/sessions/types/index.js +1 -1
- package/api/resources/windows/client/Client.js +26 -26
- package/api/types/AirtopProxyConfiguration.d.ts +9 -0
- package/api/types/ExternalSessionWithConnectionInfo.d.ts +1 -1
- package/api/types/PageQueryConfig.d.ts +2 -0
- package/api/types/PaginatedExtractionConfig.d.ts +3 -0
- package/api/types/Proxy.d.ts +9 -6
- package/api/types/ProxyConfigurationKind.d.ts +9 -0
- package/api/types/ProxyCredentials.d.ts +11 -0
- package/api/types/ScrapeConfig.d.ts +7 -0
- package/api/types/SessionConfigV1.d.ts +1 -1
- package/api/types/SessionConfigV1Proxy.d.ts +4 -4
- package/api/types/VisualAnalysisConfig.d.ts +1 -1
- package/api/types/VisualAnalysisConfigResultSelectionStrategy.d.ts +3 -2
- package/api/types/VisualAnalysisConfigResultSelectionStrategy.js +1 -0
- package/api/types/Window.d.ts +2 -0
- package/api/types/index.d.ts +4 -2
- package/api/types/index.js +4 -2
- package/biome.json +66 -0
- package/config/tsconfig-nobase.json +16 -0
- package/dist/api/resources/profiles/client/Client.js +2 -2
- package/dist/api/resources/sessions/client/Client.js +14 -14
- package/dist/api/resources/sessions/types/SessionsEventsResponse.d.ts +4 -4
- package/dist/api/resources/sessions/types/index.d.ts +1 -1
- package/dist/api/resources/sessions/types/index.js +1 -1
- package/dist/api/resources/windows/client/Client.js +26 -26
- package/dist/api/types/AirtopProxyConfiguration.d.ts +9 -0
- package/dist/api/types/AirtopProxyConfiguration.js +5 -0
- package/dist/api/types/ExternalSessionWithConnectionInfo.d.ts +1 -1
- package/dist/api/types/PageQueryConfig.d.ts +2 -0
- package/dist/api/types/PaginatedExtractionConfig.d.ts +3 -0
- package/dist/api/types/Proxy.d.ts +9 -6
- package/dist/api/types/ProxyConfigurationKind.d.ts +9 -0
- package/dist/api/types/ProxyConfigurationKind.js +5 -0
- package/dist/api/types/ProxyCredentials.d.ts +11 -0
- package/dist/api/types/ProxyCredentials.js +5 -0
- package/dist/api/types/ScrapeConfig.d.ts +7 -0
- package/dist/api/types/ScrapeConfig.js +5 -0
- package/dist/api/types/SessionConfigV1.d.ts +1 -1
- package/dist/api/types/SessionConfigV1Proxy.d.ts +4 -4
- package/dist/api/types/VisualAnalysisConfig.d.ts +1 -1
- package/dist/api/types/VisualAnalysisConfigResultSelectionStrategy.d.ts +3 -2
- package/dist/api/types/VisualAnalysisConfigResultSelectionStrategy.js +1 -0
- package/dist/api/types/Window.d.ts +2 -0
- package/dist/api/types/index.d.ts +4 -2
- package/dist/api/types/index.js +4 -2
- package/dist/serialization/resources/sessions/types/SessionsEventsResponse.d.ts +5 -5
- package/dist/serialization/resources/sessions/types/SessionsEventsResponse.js +2 -2
- package/dist/serialization/resources/sessions/types/index.d.ts +1 -1
- package/dist/serialization/resources/sessions/types/index.js +1 -1
- package/dist/serialization/types/AirtopProxyConfiguration.d.ts +13 -0
- package/dist/serialization/types/{SessionConfigV1ProxyItem.js → AirtopProxyConfiguration.js} +4 -5
- package/dist/serialization/types/PageQueryConfig.d.ts +2 -0
- package/dist/serialization/types/PageQueryConfig.js +2 -0
- package/dist/serialization/types/PaginatedExtractionConfig.d.ts +2 -0
- package/dist/serialization/types/PaginatedExtractionConfig.js +2 -0
- package/dist/serialization/types/Proxy.d.ts +4 -6
- package/dist/serialization/types/Proxy.js +3 -5
- package/dist/serialization/types/ProxyConfigurationKind.d.ts +14 -0
- package/dist/serialization/types/{CustomProxy.js → ProxyConfigurationKind.js} +5 -2
- package/dist/serialization/types/ProxyCredentials.d.ts +14 -0
- package/{serialization/types/SessionConfigV1ProxyItem.js → dist/serialization/types/ProxyCredentials.js} +5 -5
- package/dist/serialization/types/ScrapeConfig.d.ts +12 -0
- package/dist/serialization/types/ScrapeConfig.js +33 -0
- package/dist/serialization/types/SessionConfigV1Proxy.d.ts +3 -3
- package/dist/serialization/types/SessionConfigV1Proxy.js +4 -4
- package/dist/serialization/types/VisualAnalysisConfigResultSelectionStrategy.d.ts +1 -1
- package/dist/serialization/types/VisualAnalysisConfigResultSelectionStrategy.js +1 -1
- package/dist/serialization/types/Window.d.ts +1 -0
- package/dist/serialization/types/Window.js +1 -0
- package/dist/serialization/types/index.d.ts +4 -2
- package/dist/serialization/types/index.js +4 -2
- package/dist/utils/batch-operate/SessionQueue.d.ts +4 -4
- package/dist/utils/batch-operate/SessionQueue.js +11 -12
- package/dist/utils/batch-operate/WindowQueue.d.ts +3 -3
- package/dist/utils/batch-operate/WindowQueue.js +23 -11
- package/dist/utils/batch-operate/batch-util.d.ts +2 -2
- package/dist/utils/batch-operate/batch-util.js +5 -5
- package/dist/utils/batch-operate/helpers.d.ts +1 -1
- package/dist/utils/batch-operate/types.d.ts +1 -1
- package/dist/utils/index.d.ts +3 -3
- package/dist/utils/process-screenshots.d.ts +1 -1
- package/dist/utils/process-screenshots.js +4 -4
- package/dist/wrapper/AirtopClient.d.ts +1 -1
- package/dist/wrapper/AirtopClient.js +3 -3
- package/dist/wrapper/AirtopSessions.d.ts +2 -2
- package/dist/wrapper/AirtopSessions.js +4 -4
- package/dist/wrapper/AirtopWindows.d.ts +9 -9
- package/dist/wrapper/AirtopWindows.js +11 -11
- package/lefthook.yml +23 -0
- package/package.json +9 -3
- package/serialization/resources/sessions/types/SessionsEventsResponse.d.ts +5 -5
- package/serialization/resources/sessions/types/SessionsEventsResponse.js +2 -2
- package/serialization/resources/sessions/types/index.d.ts +1 -1
- package/serialization/resources/sessions/types/index.js +1 -1
- package/serialization/types/AirtopProxyConfiguration.d.ts +13 -0
- package/serialization/types/AirtopProxyConfiguration.js +34 -0
- package/serialization/types/PageQueryConfig.d.ts +2 -0
- package/serialization/types/PageQueryConfig.js +2 -0
- package/serialization/types/PaginatedExtractionConfig.d.ts +2 -0
- package/serialization/types/PaginatedExtractionConfig.js +2 -0
- package/serialization/types/Proxy.d.ts +4 -6
- package/serialization/types/Proxy.js +3 -5
- package/serialization/types/ProxyConfigurationKind.d.ts +14 -0
- package/serialization/types/{CustomProxy.js → ProxyConfigurationKind.js} +5 -2
- package/serialization/types/ProxyCredentials.d.ts +14 -0
- package/serialization/types/ProxyCredentials.js +35 -0
- package/serialization/types/ScrapeConfig.d.ts +12 -0
- package/serialization/types/ScrapeConfig.js +33 -0
- package/serialization/types/SessionConfigV1Proxy.d.ts +3 -3
- package/serialization/types/SessionConfigV1Proxy.js +4 -4
- package/serialization/types/VisualAnalysisConfigResultSelectionStrategy.d.ts +1 -1
- package/serialization/types/VisualAnalysisConfigResultSelectionStrategy.js +1 -1
- package/serialization/types/Window.d.ts +1 -0
- package/serialization/types/Window.js +1 -0
- package/serialization/types/index.d.ts +4 -2
- package/serialization/types/index.js +4 -2
- package/utils/batch-operate/SessionQueue.d.ts +4 -4
- package/utils/batch-operate/SessionQueue.js +11 -12
- package/utils/batch-operate/WindowQueue.d.ts +3 -3
- package/utils/batch-operate/WindowQueue.js +23 -11
- package/utils/batch-operate/batch-util.d.ts +2 -2
- package/utils/batch-operate/batch-util.js +5 -5
- package/utils/batch-operate/helpers.d.ts +1 -1
- package/utils/batch-operate/types.d.ts +1 -1
- package/utils/index.d.ts +3 -3
- package/utils/process-screenshots.d.ts +1 -1
- package/utils/process-screenshots.js +4 -4
- package/wrapper/AirtopClient.d.ts +1 -1
- package/wrapper/AirtopClient.js +3 -3
- package/wrapper/AirtopSessions.d.ts +2 -2
- package/wrapper/AirtopSessions.js +4 -4
- package/wrapper/AirtopWindows.d.ts +9 -9
- package/wrapper/AirtopWindows.js +11 -11
- package/api/types/CustomProxy.d.ts +0 -14
- package/api/types/SessionConfigV1ProxyItem.d.ts +0 -12
- package/dist/api/types/CustomProxy.d.ts +0 -14
- package/dist/api/types/SessionConfigV1ProxyItem.d.ts +0 -12
- package/dist/serialization/types/CustomProxy.d.ts +0 -11
- package/dist/serialization/types/SessionConfigV1ProxyItem.d.ts +0 -14
- package/serialization/types/CustomProxy.d.ts +0 -11
- package/serialization/types/SessionConfigV1ProxyItem.d.ts +0 -14
- /package/api/types/{CustomProxy.js → AirtopProxyConfiguration.js} +0 -0
- /package/api/types/{SessionConfigV1ProxyItem.js → ProxyConfigurationKind.js} +0 -0
- /package/{dist/api/types/CustomProxy.js → api/types/ProxyCredentials.js} +0 -0
- /package/{dist/api/types/SessionConfigV1ProxyItem.js → api/types/ScrapeConfig.js} +0 -0
@@ -29,7 +29,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
29
|
exports.PageQueryConfig = void 0;
|
30
30
|
const core = __importStar(require("../../core"));
|
31
31
|
const PageQueryExperimentalConfig_1 = require("./PageQueryExperimentalConfig");
|
32
|
+
const ScrapeConfig_1 = require("./ScrapeConfig");
|
32
33
|
exports.PageQueryConfig = core.serialization.object({
|
33
34
|
experimental: PageQueryExperimentalConfig_1.PageQueryExperimentalConfig.optional(),
|
34
35
|
outputSchema: core.serialization.string().optional(),
|
36
|
+
scrape: ScrapeConfig_1.ScrapeConfig.optional(),
|
35
37
|
});
|
@@ -4,11 +4,13 @@
|
|
4
4
|
import * as serializers from "../index";
|
5
5
|
import * as Airtop from "../../api/index";
|
6
6
|
import * as core from "../../core";
|
7
|
+
import { ScrapeConfig } from "./ScrapeConfig";
|
7
8
|
export declare const PaginatedExtractionConfig: core.serialization.ObjectSchema<serializers.PaginatedExtractionConfig.Raw, Airtop.PaginatedExtractionConfig>;
|
8
9
|
export declare namespace PaginatedExtractionConfig {
|
9
10
|
interface Raw {
|
10
11
|
interactionMode?: string | null;
|
11
12
|
outputSchema?: string | null;
|
12
13
|
paginationMode?: string | null;
|
14
|
+
scrape?: ScrapeConfig.Raw | null;
|
13
15
|
}
|
14
16
|
}
|
@@ -28,8 +28,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
29
29
|
exports.PaginatedExtractionConfig = void 0;
|
30
30
|
const core = __importStar(require("../../core"));
|
31
|
+
const ScrapeConfig_1 = require("./ScrapeConfig");
|
31
32
|
exports.PaginatedExtractionConfig = core.serialization.object({
|
32
33
|
interactionMode: core.serialization.string().optional(),
|
33
34
|
outputSchema: core.serialization.string().optional(),
|
34
35
|
paginationMode: core.serialization.string().optional(),
|
36
|
+
scrape: ScrapeConfig_1.ScrapeConfig.optional(),
|
35
37
|
});
|
@@ -4,11 +4,9 @@
|
|
4
4
|
import * as serializers from "../index";
|
5
5
|
import * as Airtop from "../../api/index";
|
6
6
|
import * as core from "../../core";
|
7
|
-
|
7
|
+
import { AirtopProxyConfiguration } from "./AirtopProxyConfiguration";
|
8
|
+
import { ProxyCredentials } from "./ProxyCredentials";
|
9
|
+
export declare const Proxy: core.serialization.Schema<serializers.Proxy.Raw, Airtop.Proxy>;
|
8
10
|
export declare namespace Proxy {
|
9
|
-
|
10
|
-
password?: string | null;
|
11
|
-
url: string;
|
12
|
-
username?: string | null;
|
13
|
-
}
|
11
|
+
type Raw = string | AirtopProxyConfiguration.Raw | ProxyCredentials.Raw;
|
14
12
|
}
|
@@ -28,8 +28,6 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
29
29
|
exports.Proxy = void 0;
|
30
30
|
const core = __importStar(require("../../core"));
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
username: core.serialization.string().optional(),
|
35
|
-
});
|
31
|
+
const AirtopProxyConfiguration_1 = require("./AirtopProxyConfiguration");
|
32
|
+
const ProxyCredentials_1 = require("./ProxyCredentials");
|
33
|
+
exports.Proxy = core.serialization.undiscriminatedUnion([core.serialization.string(), AirtopProxyConfiguration_1.AirtopProxyConfiguration, ProxyCredentials_1.ProxyCredentials]);
|
@@ -0,0 +1,14 @@
|
|
1
|
+
/**
|
2
|
+
* This file was auto-generated by Fern from our API Definition.
|
3
|
+
*/
|
4
|
+
import * as serializers from "../index";
|
5
|
+
import * as Airtop from "../../api/index";
|
6
|
+
import * as core from "../../core";
|
7
|
+
import { Proxy } from "./Proxy";
|
8
|
+
export declare const ProxyConfigurationKind: core.serialization.ObjectSchema<serializers.ProxyConfigurationKind.Raw, Airtop.ProxyConfigurationKind>;
|
9
|
+
export declare namespace ProxyConfigurationKind {
|
10
|
+
interface Raw {
|
11
|
+
domainPattern: string;
|
12
|
+
relay: Proxy.Raw;
|
13
|
+
}
|
14
|
+
}
|
@@ -26,7 +26,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
26
26
|
return result;
|
27
27
|
};
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
29
|
-
exports.
|
29
|
+
exports.ProxyConfigurationKind = void 0;
|
30
30
|
const core = __importStar(require("../../core"));
|
31
31
|
const Proxy_1 = require("./Proxy");
|
32
|
-
exports.
|
32
|
+
exports.ProxyConfigurationKind = core.serialization.object({
|
33
|
+
domainPattern: core.serialization.string(),
|
34
|
+
relay: Proxy_1.Proxy,
|
35
|
+
});
|
@@ -0,0 +1,14 @@
|
|
1
|
+
/**
|
2
|
+
* This file was auto-generated by Fern from our API Definition.
|
3
|
+
*/
|
4
|
+
import * as serializers from "../index";
|
5
|
+
import * as Airtop from "../../api/index";
|
6
|
+
import * as core from "../../core";
|
7
|
+
export declare const ProxyCredentials: core.serialization.ObjectSchema<serializers.ProxyCredentials.Raw, Airtop.ProxyCredentials>;
|
8
|
+
export declare namespace ProxyCredentials {
|
9
|
+
interface Raw {
|
10
|
+
password?: string | null;
|
11
|
+
url: string;
|
12
|
+
username?: string | null;
|
13
|
+
}
|
14
|
+
}
|
@@ -26,10 +26,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
26
26
|
return result;
|
27
27
|
};
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
29
|
-
exports.
|
29
|
+
exports.ProxyCredentials = void 0;
|
30
30
|
const core = __importStar(require("../../core"));
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
31
|
+
exports.ProxyCredentials = core.serialization.object({
|
32
|
+
password: core.serialization.string().optional(),
|
33
|
+
url: core.serialization.string(),
|
34
|
+
username: core.serialization.string().optional(),
|
35
35
|
});
|
@@ -0,0 +1,12 @@
|
|
1
|
+
/**
|
2
|
+
* This file was auto-generated by Fern from our API Definition.
|
3
|
+
*/
|
4
|
+
import * as serializers from "../index";
|
5
|
+
import * as Airtop from "../../api/index";
|
6
|
+
import * as core from "../../core";
|
7
|
+
export declare const ScrapeConfig: core.serialization.ObjectSchema<serializers.ScrapeConfig.Raw, Airtop.ScrapeConfig>;
|
8
|
+
export declare namespace ScrapeConfig {
|
9
|
+
interface Raw {
|
10
|
+
optimizeUrls?: string | null;
|
11
|
+
}
|
12
|
+
}
|
@@ -0,0 +1,33 @@
|
|
1
|
+
"use strict";
|
2
|
+
/**
|
3
|
+
* This file was auto-generated by Fern from our API Definition.
|
4
|
+
*/
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
6
|
+
if (k2 === undefined) k2 = k;
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
10
|
+
}
|
11
|
+
Object.defineProperty(o, k2, desc);
|
12
|
+
}) : (function(o, m, k, k2) {
|
13
|
+
if (k2 === undefined) k2 = k;
|
14
|
+
o[k2] = m[k];
|
15
|
+
}));
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
18
|
+
}) : function(o, v) {
|
19
|
+
o["default"] = v;
|
20
|
+
});
|
21
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
22
|
+
if (mod && mod.__esModule) return mod;
|
23
|
+
var result = {};
|
24
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
25
|
+
__setModuleDefault(result, mod);
|
26
|
+
return result;
|
27
|
+
};
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
29
|
+
exports.ScrapeConfig = void 0;
|
30
|
+
const core = __importStar(require("../../core"));
|
31
|
+
exports.ScrapeConfig = core.serialization.object({
|
32
|
+
optimizeUrls: core.serialization.string().optional(),
|
33
|
+
});
|
@@ -4,9 +4,9 @@
|
|
4
4
|
import * as serializers from "../index";
|
5
5
|
import * as Airtop from "../../api/index";
|
6
6
|
import * as core from "../../core";
|
7
|
-
import {
|
8
|
-
import {
|
7
|
+
import { Proxy } from "./Proxy";
|
8
|
+
import { ProxyConfigurationKind } from "./ProxyConfigurationKind";
|
9
9
|
export declare const SessionConfigV1Proxy: core.serialization.Schema<serializers.SessionConfigV1Proxy.Raw, Airtop.SessionConfigV1Proxy>;
|
10
10
|
export declare namespace SessionConfigV1Proxy {
|
11
|
-
type Raw = boolean |
|
11
|
+
type Raw = boolean | Proxy.Raw | ProxyConfigurationKind.Raw[];
|
12
12
|
}
|
@@ -28,10 +28,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
29
29
|
exports.SessionConfigV1Proxy = void 0;
|
30
30
|
const core = __importStar(require("../../core"));
|
31
|
-
const
|
32
|
-
const
|
31
|
+
const Proxy_1 = require("./Proxy");
|
32
|
+
const ProxyConfigurationKind_1 = require("./ProxyConfigurationKind");
|
33
33
|
exports.SessionConfigV1Proxy = core.serialization.undiscriminatedUnion([
|
34
34
|
core.serialization.boolean(),
|
35
|
-
|
36
|
-
core.serialization.list(
|
35
|
+
Proxy_1.Proxy,
|
36
|
+
core.serialization.list(ProxyConfigurationKind_1.ProxyConfigurationKind),
|
37
37
|
]);
|
@@ -6,5 +6,5 @@ import * as Airtop from "../../api/index";
|
|
6
6
|
import * as core from "../../core";
|
7
7
|
export declare const VisualAnalysisConfigResultSelectionStrategy: core.serialization.Schema<serializers.VisualAnalysisConfigResultSelectionStrategy.Raw, Airtop.VisualAnalysisConfigResultSelectionStrategy>;
|
8
8
|
export declare namespace VisualAnalysisConfigResultSelectionStrategy {
|
9
|
-
type Raw = "first" | "bestMatch";
|
9
|
+
type Raw = "first" | "bestMatch" | "auto";
|
10
10
|
}
|
@@ -28,4 +28,4 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
29
29
|
exports.VisualAnalysisConfigResultSelectionStrategy = void 0;
|
30
30
|
const core = __importStar(require("../../core"));
|
31
|
-
exports.VisualAnalysisConfigResultSelectionStrategy = core.serialization.enum_(["first", "bestMatch"]);
|
31
|
+
exports.VisualAnalysisConfigResultSelectionStrategy = core.serialization.enum_(["first", "bestMatch", "auto"]);
|
@@ -23,6 +23,7 @@ export * from "./PageQueryConfig";
|
|
23
23
|
export * from "./PageQueryExperimentalConfig";
|
24
24
|
export * from "./PaginatedExtractionConfig";
|
25
25
|
export * from "./Pagination";
|
26
|
+
export * from "./ScrapeConfig";
|
26
27
|
export * from "./ScrapeResponse";
|
27
28
|
export * from "./ScrapeResponseContent";
|
28
29
|
export * from "./ScrapeResponseEnvelope";
|
@@ -30,9 +31,10 @@ export * from "./ScrapeResponseOutput";
|
|
30
31
|
export * from "./ScreenshotConfig";
|
31
32
|
export * from "./ScreenshotMetadata";
|
32
33
|
export * from "./ScreenshotRequestConfig";
|
34
|
+
export * from "./AirtopProxyConfiguration";
|
35
|
+
export * from "./ProxyCredentials";
|
33
36
|
export * from "./Proxy";
|
34
|
-
export * from "./
|
35
|
-
export * from "./SessionConfigV1ProxyItem";
|
37
|
+
export * from "./ProxyConfigurationKind";
|
36
38
|
export * from "./SessionConfigV1Proxy";
|
37
39
|
export * from "./SessionConfigV1";
|
38
40
|
export * from "./SessionEventMessage";
|
@@ -39,6 +39,7 @@ __exportStar(require("./PageQueryConfig"), exports);
|
|
39
39
|
__exportStar(require("./PageQueryExperimentalConfig"), exports);
|
40
40
|
__exportStar(require("./PaginatedExtractionConfig"), exports);
|
41
41
|
__exportStar(require("./Pagination"), exports);
|
42
|
+
__exportStar(require("./ScrapeConfig"), exports);
|
42
43
|
__exportStar(require("./ScrapeResponse"), exports);
|
43
44
|
__exportStar(require("./ScrapeResponseContent"), exports);
|
44
45
|
__exportStar(require("./ScrapeResponseEnvelope"), exports);
|
@@ -46,9 +47,10 @@ __exportStar(require("./ScrapeResponseOutput"), exports);
|
|
46
47
|
__exportStar(require("./ScreenshotConfig"), exports);
|
47
48
|
__exportStar(require("./ScreenshotMetadata"), exports);
|
48
49
|
__exportStar(require("./ScreenshotRequestConfig"), exports);
|
50
|
+
__exportStar(require("./AirtopProxyConfiguration"), exports);
|
51
|
+
__exportStar(require("./ProxyCredentials"), exports);
|
49
52
|
__exportStar(require("./Proxy"), exports);
|
50
|
-
__exportStar(require("./
|
51
|
-
__exportStar(require("./SessionConfigV1ProxyItem"), exports);
|
53
|
+
__exportStar(require("./ProxyConfigurationKind"), exports);
|
52
54
|
__exportStar(require("./SessionConfigV1Proxy"), exports);
|
53
55
|
__exportStar(require("./SessionConfigV1"), exports);
|
54
56
|
__exportStar(require("./SessionEventMessage"), exports);
|
@@ -1,7 +1,7 @@
|
|
1
|
-
import type {
|
2
|
-
import type {
|
3
|
-
import type {
|
4
|
-
import type {
|
1
|
+
import type { EventEmitter } from 'eventemitter3';
|
2
|
+
import type { AirtopClient } from '../../wrapper/AirtopClient';
|
3
|
+
import type { AirtopSessionConfigV1 } from '../../wrapper/AirtopSessions';
|
4
|
+
import type { BatchOperationError, BatchOperationInput, BatchOperationResponse, BatchOperationUrl } from './types';
|
5
5
|
export declare class SessionQueue<T> {
|
6
6
|
private activePromises;
|
7
7
|
private activePromisesMutex;
|
@@ -10,9 +10,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
10
10
|
};
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
12
12
|
exports.SessionQueue = void 0;
|
13
|
-
const helpers_1 = require("./helpers");
|
14
|
-
const WindowQueue_1 = require("./WindowQueue");
|
15
13
|
const async_mutex_1 = require("async-mutex");
|
14
|
+
const WindowQueue_1 = require("./WindowQueue");
|
15
|
+
const helpers_1 = require("./helpers");
|
16
16
|
class SessionQueue {
|
17
17
|
constructor({ maxConcurrentSessions, runEmitter, maxWindowsPerSession, initialBatches, operation, client, sessionConfig, onError, }) {
|
18
18
|
this.activePromises = [];
|
@@ -24,9 +24,8 @@ class SessionQueue {
|
|
24
24
|
this.processingPromisesCount = 0;
|
25
25
|
this.sessionPool = [];
|
26
26
|
this.sessionPoolMutex = new async_mutex_1.Mutex();
|
27
|
-
if (!Number.isInteger(maxConcurrentSessions) ||
|
28
|
-
maxConcurrentSessions
|
29
|
-
throw new Error("maxConcurrentSessions must be a positive integer");
|
27
|
+
if (!Number.isInteger(maxConcurrentSessions) || maxConcurrentSessions <= 0) {
|
28
|
+
throw new Error('maxConcurrentSessions must be a positive integer');
|
30
29
|
}
|
31
30
|
this.maxConcurrentSessions = maxConcurrentSessions;
|
32
31
|
this.runEmitter = runEmitter;
|
@@ -41,7 +40,7 @@ class SessionQueue {
|
|
41
40
|
this.isHalted = false;
|
42
41
|
}
|
43
42
|
handleHaltEvent() {
|
44
|
-
this.client.log(
|
43
|
+
this.client.log('Halt event received');
|
45
44
|
this.isHalted = true;
|
46
45
|
}
|
47
46
|
addUrlsToBatchQueue(newBatch) {
|
@@ -64,7 +63,7 @@ class SessionQueue {
|
|
64
63
|
this.batchQueue = [...this.initialBatches];
|
65
64
|
});
|
66
65
|
this.processingPromisesCount++;
|
67
|
-
this.runEmitter.on(
|
66
|
+
this.runEmitter.on('halt', this.handleHaltEvent.bind(this));
|
68
67
|
this.latestProcessingPromise = this.processPendingBatches();
|
69
68
|
yield this.latestProcessingPromise;
|
70
69
|
});
|
@@ -75,7 +74,7 @@ class SessionQueue {
|
|
75
74
|
yield this.latestProcessingPromise;
|
76
75
|
}
|
77
76
|
yield this.terminateAllSessions();
|
78
|
-
this.runEmitter.removeListener(
|
77
|
+
this.runEmitter.removeListener('halt', this.handleHaltEvent);
|
79
78
|
return this.results;
|
80
79
|
});
|
81
80
|
}
|
@@ -108,7 +107,7 @@ class SessionQueue {
|
|
108
107
|
break;
|
109
108
|
const promise = (() => __awaiter(this, void 0, void 0, function* () {
|
110
109
|
if (this.isHalted) {
|
111
|
-
this.client.log(
|
110
|
+
this.client.log('Halt event received, skipping batch');
|
112
111
|
return;
|
113
112
|
}
|
114
113
|
let sessionId;
|
@@ -121,7 +120,7 @@ class SessionQueue {
|
|
121
120
|
});
|
122
121
|
// Otherwise, create a new session
|
123
122
|
if (!sessionId) {
|
124
|
-
const { data: session, warnings, errors } = yield this.client.sessions.create({
|
123
|
+
const { data: session, warnings, errors, } = yield this.client.sessions.create({
|
125
124
|
configuration: this.sessionConfig,
|
126
125
|
});
|
127
126
|
sessionId = session.id;
|
@@ -133,7 +132,7 @@ class SessionQueue {
|
|
133
132
|
// Return the session to the pool
|
134
133
|
yield this.sessionPoolMutex.runExclusive(() => {
|
135
134
|
if (!sessionId) {
|
136
|
-
throw new Error(
|
135
|
+
throw new Error('Missing sessionId, cannot return to pool');
|
137
136
|
}
|
138
137
|
this.sessionPool.push(sessionId);
|
139
138
|
});
|
@@ -202,7 +201,7 @@ class SessionQueue {
|
|
202
201
|
formatError(error) {
|
203
202
|
return error instanceof Error ? error.message : String(error);
|
204
203
|
}
|
205
|
-
handleErrorAndWarningResponses({ warnings, errors, sessionId, batch }) {
|
204
|
+
handleErrorAndWarningResponses({ warnings, errors, sessionId, batch, }) {
|
206
205
|
if (!warnings && !errors)
|
207
206
|
return;
|
208
207
|
const details = {
|
@@ -1,6 +1,6 @@
|
|
1
|
-
import type {
|
2
|
-
import type {
|
3
|
-
import type {
|
1
|
+
import type { EventEmitter } from 'eventemitter3';
|
2
|
+
import type { AirtopClient } from '../../wrapper/AirtopClient';
|
3
|
+
import type { BatchOperationError, BatchOperationInput, BatchOperationResponse, BatchOperationUrl } from './types';
|
4
4
|
export declare class WindowQueue<T> {
|
5
5
|
private activePromises;
|
6
6
|
private urlQueue;
|
@@ -18,7 +18,7 @@ class WindowQueue {
|
|
18
18
|
this.activePromisesMutex = new async_mutex_1.Mutex();
|
19
19
|
this.urlQueueMutex = new async_mutex_1.Mutex();
|
20
20
|
if (!Number.isInteger(maxWindowsPerSession) || maxWindowsPerSession <= 0) {
|
21
|
-
throw new Error(
|
21
|
+
throw new Error('maxWindowsPerSession must be a positive integer');
|
22
22
|
}
|
23
23
|
this.maxWindowsPerSession = maxWindowsPerSession;
|
24
24
|
this.runEmitter = runEmitter;
|
@@ -36,13 +36,13 @@ class WindowQueue {
|
|
36
36
|
});
|
37
37
|
}
|
38
38
|
handleHaltEvent() {
|
39
|
-
this.client.log(
|
39
|
+
this.client.log('Halt event received');
|
40
40
|
this.isHalted = true;
|
41
41
|
}
|
42
42
|
processInBatches(urls) {
|
43
43
|
return __awaiter(this, void 0, void 0, function* () {
|
44
44
|
const results = [];
|
45
|
-
this.runEmitter.on(
|
45
|
+
this.runEmitter.on('halt', this.handleHaltEvent.bind(this));
|
46
46
|
yield this.urlQueueMutex.runExclusive(() => {
|
47
47
|
this.urlQueue = [...urls];
|
48
48
|
});
|
@@ -80,13 +80,25 @@ class WindowQueue {
|
|
80
80
|
url: urlData.url,
|
81
81
|
});
|
82
82
|
windowId = data.windowId;
|
83
|
-
this.handleErrorAndWarningResponses({
|
83
|
+
this.handleErrorAndWarningResponses({
|
84
|
+
warnings,
|
85
|
+
errors,
|
86
|
+
sessionId: this.sessionId,
|
87
|
+
url: urlData,
|
88
|
+
operation: 'window creation',
|
89
|
+
});
|
84
90
|
if (!windowId) {
|
85
91
|
throw new Error(`WindowId not found, errors: ${JSON.stringify(errors)}`);
|
86
92
|
}
|
87
|
-
const { data: windowInfo, warnings: windowWarnings, errors: windowErrors } = yield this.client.windows.getWindowInfo(this.sessionId, windowId);
|
93
|
+
const { data: windowInfo, warnings: windowWarnings, errors: windowErrors, } = yield this.client.windows.getWindowInfo(this.sessionId, windowId);
|
88
94
|
liveViewUrl = windowInfo.liveViewUrl;
|
89
|
-
this.handleErrorAndWarningResponses({
|
95
|
+
this.handleErrorAndWarningResponses({
|
96
|
+
warnings: windowWarnings,
|
97
|
+
errors: windowErrors,
|
98
|
+
sessionId: this.sessionId,
|
99
|
+
url: urlData,
|
100
|
+
operation: 'window info retrieval',
|
101
|
+
});
|
90
102
|
// Run the operation on the window
|
91
103
|
const result = yield this.operation({
|
92
104
|
windowId,
|
@@ -100,12 +112,12 @@ class WindowQueue {
|
|
100
112
|
results.push(data);
|
101
113
|
}
|
102
114
|
if (shouldHaltBatch) {
|
103
|
-
this.client.log(
|
104
|
-
this.runEmitter.emit(
|
115
|
+
this.client.log('Emitting halt event');
|
116
|
+
this.runEmitter.emit('halt');
|
105
117
|
}
|
106
118
|
if (additionalUrls && additionalUrls.length > 0) {
|
107
119
|
this.client.log(`Emitting addUrls event with urls: ${JSON.stringify(additionalUrls)}`);
|
108
|
-
this.runEmitter.emit(
|
120
|
+
this.runEmitter.emit('addUrls', additionalUrls);
|
109
121
|
}
|
110
122
|
}
|
111
123
|
}
|
@@ -147,7 +159,7 @@ class WindowQueue {
|
|
147
159
|
// Wait for all processes to complete
|
148
160
|
yield Promise.allSettled(this.activePromises);
|
149
161
|
// Remove the halt listener
|
150
|
-
this.runEmitter.removeListener(
|
162
|
+
this.runEmitter.removeListener('halt', this.handleHaltEvent);
|
151
163
|
return results;
|
152
164
|
});
|
153
165
|
}
|
@@ -181,7 +193,7 @@ class WindowQueue {
|
|
181
193
|
formatError(error) {
|
182
194
|
return error instanceof Error ? error.message : String(error);
|
183
195
|
}
|
184
|
-
handleErrorAndWarningResponses({ warnings, errors, sessionId, url, operation }) {
|
196
|
+
handleErrorAndWarningResponses({ warnings, errors, sessionId, url, operation, }) {
|
185
197
|
if (!warnings && !errors)
|
186
198
|
return;
|
187
199
|
const details = {
|
@@ -1,3 +1,3 @@
|
|
1
|
-
import type { AirtopClient } from
|
2
|
-
import type { BatchOperateConfig, BatchOperationInput, BatchOperationResponse, BatchOperationUrl } from
|
1
|
+
import type { AirtopClient } from '../../wrapper/AirtopClient';
|
2
|
+
import type { BatchOperateConfig, BatchOperationInput, BatchOperationResponse, BatchOperationUrl } from './types';
|
3
3
|
export declare const batchOperate: <T>(urls: BatchOperationUrl[], operation: (input: BatchOperationInput) => Promise<BatchOperationResponse<T>>, client: AirtopClient, config?: BatchOperateConfig | undefined) => Promise<T[]>;
|
@@ -11,19 +11,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
12
12
|
exports.batchOperate = void 0;
|
13
13
|
const eventemitter3_1 = require("eventemitter3");
|
14
|
-
const helpers_1 = require("./helpers");
|
15
14
|
const SessionQueue_1 = require("./SessionQueue");
|
15
|
+
const helpers_1 = require("./helpers");
|
16
16
|
const DEFAULT_MAX_WINDOWS_PER_SESSION = 1;
|
17
17
|
const DEFAULT_MAX_CONCURRENT_SESSIONS = 30;
|
18
18
|
const batchOperate = (urls, operation, // operation to invoke on each url
|
19
19
|
client, config) => __awaiter(void 0, void 0, void 0, function* () {
|
20
20
|
// Validate the urls before proceeding
|
21
21
|
if (!Array.isArray(urls)) {
|
22
|
-
throw new Error(
|
22
|
+
throw new Error('Please provide a valid list of urls');
|
23
23
|
}
|
24
24
|
for (const url of urls) {
|
25
|
-
if (!url || typeof url !==
|
26
|
-
throw new Error(
|
25
|
+
if (!url || typeof url !== 'object' || !('url' in url)) {
|
26
|
+
throw new Error('Please provide a valid list of urls');
|
27
27
|
}
|
28
28
|
}
|
29
29
|
const runEmitter = new eventemitter3_1.EventEmitter();
|
@@ -40,7 +40,7 @@ client, config) => __awaiter(void 0, void 0, void 0, function* () {
|
|
40
40
|
sessionConfig,
|
41
41
|
onError,
|
42
42
|
});
|
43
|
-
runEmitter.on(
|
43
|
+
runEmitter.on('addUrls', (additionalUrls) => {
|
44
44
|
sessionQueue.addUrlsToBatchQueue(additionalUrls);
|
45
45
|
});
|
46
46
|
yield sessionQueue.processInitialBatches();
|
@@ -1,2 +1,2 @@
|
|
1
|
-
import type { BatchOperationUrl } from
|
1
|
+
import type { BatchOperationUrl } from './types';
|
2
2
|
export declare const distributeUrlsToBatches: (urls: BatchOperationUrl[], maxConcurrentSessions: number) => BatchOperationUrl[][];
|
package/dist/utils/index.d.ts
CHANGED
@@ -1,3 +1,3 @@
|
|
1
|
-
export * from
|
2
|
-
export * from
|
3
|
-
export * from
|
1
|
+
export * from './batch-operate/batch-util';
|
2
|
+
export * from './batch-operate/types';
|
3
|
+
export * from './process-screenshots';
|
@@ -16,8 +16,8 @@ function extractMimeAndBase64(dataUrl) {
|
|
16
16
|
return { mimeType: match[1], base64Data: match[2] };
|
17
17
|
}
|
18
18
|
return {
|
19
|
-
mimeType:
|
20
|
-
base64Data: dataUrl.replace(/^data:image\/jpeg;base64,/,
|
19
|
+
mimeType: 'image/jpeg',
|
20
|
+
base64Data: dataUrl.replace(/^data:image\/jpeg;base64,/, ''),
|
21
21
|
};
|
22
22
|
}
|
23
23
|
function processScreenshots(response) {
|
@@ -30,13 +30,13 @@ function processScreenshots(response) {
|
|
30
30
|
if (!screenshot.dataUrl) {
|
31
31
|
return {
|
32
32
|
index,
|
33
|
-
error: new Error(
|
33
|
+
error: new Error('Screenshot data URL not found'),
|
34
34
|
success: false,
|
35
35
|
};
|
36
36
|
}
|
37
37
|
try {
|
38
38
|
const { mimeType, base64Data } = extractMimeAndBase64(screenshot.dataUrl);
|
39
|
-
const binaryData = Buffer.from(base64Data,
|
39
|
+
const binaryData = Buffer.from(base64Data, 'base64');
|
40
40
|
return {
|
41
41
|
index,
|
42
42
|
binaryData,
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import { AirtopClient as FernClient } from '../Client';
|
2
|
+
import { type BatchOperateConfig, type BatchOperationInput, type BatchOperationResponse, type BatchOperationUrl } from '../utils';
|
2
3
|
import { AirtopSessions } from './AirtopSessions';
|
3
4
|
import { AirtopWindows } from './AirtopWindows';
|
4
|
-
import { type BatchOperateConfig, type BatchOperationInput, type BatchOperationResponse, type BatchOperationUrl } from "../utils";
|
5
5
|
declare type AugmentedOptions = FernClient.Options & {
|
6
6
|
debug?: boolean;
|
7
7
|
};
|
@@ -34,10 +34,10 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
34
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
35
35
|
exports.AirtopClient = void 0;
|
36
36
|
const Client_1 = require("../Client"); // alias the Fern generated client
|
37
|
+
const core = __importStar(require("../core"));
|
38
|
+
const utils_1 = require("../utils");
|
37
39
|
const AirtopSessions_1 = require("./AirtopSessions");
|
38
40
|
const AirtopWindows_1 = require("./AirtopWindows");
|
39
|
-
const utils_1 = require("../utils");
|
40
|
-
const core = __importStar(require("../core"));
|
41
41
|
class AirtopClient {
|
42
42
|
constructor(_options) {
|
43
43
|
this._options = _options;
|
@@ -47,7 +47,7 @@ class AirtopClient {
|
|
47
47
|
const version = require('../package.json').version; //we don't control tsconfig.json so we can't use resolveJsonModule
|
48
48
|
if (!_options.fetcher) {
|
49
49
|
_options.fetcher = (req) => {
|
50
|
-
return core.fetcher(Object.assign(Object.assign({}, req), { headers: Object.assign(Object.assign({}, req.headers), {
|
50
|
+
return core.fetcher(Object.assign(Object.assign({}, req), { headers: Object.assign(Object.assign({}, req.headers), { 'x-airtop-sdk-source': 'javascript', 'x-airtop-sdk-version': version }) }));
|
51
51
|
};
|
52
52
|
}
|
53
53
|
this._client = new Client_1.AirtopClient(_options);
|
@@ -1,5 +1,5 @@
|
|
1
|
-
import * as Airtop from '../api';
|
2
|
-
import { Sessions as SessionsClass, Sessions as SessionsNamespace } from '../api/resources/sessions/client/Client';
|
1
|
+
import type * as Airtop from '../api';
|
2
|
+
import { Sessions as SessionsClass, type Sessions as SessionsNamespace } from '../api/resources/sessions/client/Client';
|
3
3
|
export interface AirtopSessionConfigV1 extends Airtop.SessionConfigV1 {
|
4
4
|
skipWaitSessionReady?: boolean;
|
5
5
|
}
|