@airtop/sdk 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Client.d.ts +3 -3
- package/Client.js +9 -9
- package/README.md +6 -6
- package/api/resources/index.d.ts +3 -3
- package/api/resources/index.js +4 -4
- package/api/resources/profiles/client/Client.js +4 -4
- package/api/resources/sessions/client/Client.js +10 -10
- package/api/resources/windows/client/Client.d.ts +3 -3
- package/api/resources/windows/client/Client.js +25 -25
- package/api/resources/windows/client/requests/CreateWindowInputV1Body.d.ts +1 -1
- package/api/resources/windows/client/requests/WindowLoadUrlV1Body.d.ts +1 -1
- package/api/resources/windows/types/CreateWindowInputV1BodyWaitUntil.d.ts +3 -2
- package/api/resources/windows/types/CreateWindowInputV1BodyWaitUntil.js +1 -0
- package/api/resources/windows/types/WindowLoadUrlV1BodyWaitUntil.d.ts +3 -2
- package/api/resources/windows/types/WindowLoadUrlV1BodyWaitUntil.js +1 -0
- package/api/types/ClickConfig.d.ts +5 -1
- package/api/types/Issue.d.ts +2 -0
- package/{dist/api/types/EmptyResponseJson.d.ts → api/types/OperationOutcome.d.ts} +1 -1
- package/api/types/{EmptyResponse.d.ts → OperationOutcomeResponse.d.ts} +3 -3
- package/api/types/PageQueryConfig.d.ts +3 -0
- package/api/types/SummaryConfig.d.ts +2 -0
- package/api/types/VisualAnalysisConfig.d.ts +12 -0
- package/api/types/VisualAnalysisConfigPartitionDirection.d.ts +12 -0
- package/api/types/VisualAnalysisConfigPartitionDirection.js +11 -0
- package/api/types/VisualAnalysisConfigScope.d.ts +11 -0
- package/api/types/VisualAnalysisConfigScope.js +10 -0
- package/api/types/index.d.ts +5 -4
- package/api/types/index.js +5 -4
- package/dist/Client.d.ts +3 -3
- package/dist/Client.js +9 -9
- package/dist/api/resources/index.d.ts +3 -3
- package/dist/api/resources/index.js +4 -4
- package/dist/api/resources/profiles/client/Client.js +4 -4
- package/dist/api/resources/sessions/client/Client.js +10 -10
- package/dist/api/resources/windows/client/Client.d.ts +3 -3
- package/dist/api/resources/windows/client/Client.js +25 -25
- package/dist/api/resources/windows/client/requests/CreateWindowInputV1Body.d.ts +1 -1
- package/dist/api/resources/windows/client/requests/WindowLoadUrlV1Body.d.ts +1 -1
- package/dist/api/resources/windows/types/CreateWindowInputV1BodyWaitUntil.d.ts +3 -2
- package/dist/api/resources/windows/types/CreateWindowInputV1BodyWaitUntil.js +1 -0
- package/dist/api/resources/windows/types/WindowLoadUrlV1BodyWaitUntil.d.ts +3 -2
- package/dist/api/resources/windows/types/WindowLoadUrlV1BodyWaitUntil.js +1 -0
- package/dist/api/types/ClickConfig.d.ts +5 -1
- package/dist/api/types/Issue.d.ts +2 -0
- package/{api/types/EmptyResponseJson.d.ts → dist/api/types/OperationOutcome.d.ts} +1 -1
- package/dist/api/types/{EmptyResponse.d.ts → OperationOutcomeResponse.d.ts} +3 -3
- package/dist/api/types/PageQueryConfig.d.ts +3 -0
- package/dist/api/types/SummaryConfig.d.ts +2 -0
- package/dist/api/types/VisualAnalysisConfig.d.ts +12 -0
- package/dist/api/types/VisualAnalysisConfigPartitionDirection.d.ts +12 -0
- package/dist/api/types/VisualAnalysisConfigPartitionDirection.js +11 -0
- package/dist/api/types/VisualAnalysisConfigScope.d.ts +11 -0
- package/dist/api/types/VisualAnalysisConfigScope.js +10 -0
- package/dist/api/types/index.d.ts +5 -4
- package/dist/api/types/index.js +5 -4
- package/dist/index.d.ts +1 -0
- package/dist/index.js +4 -0
- package/dist/serialization/resources/index.d.ts +3 -3
- package/dist/serialization/resources/index.js +4 -4
- package/dist/serialization/resources/windows/types/CreateWindowInputV1BodyWaitUntil.d.ts +1 -1
- package/dist/serialization/resources/windows/types/CreateWindowInputV1BodyWaitUntil.js +1 -1
- package/dist/serialization/resources/windows/types/WindowLoadUrlV1BodyWaitUntil.d.ts +1 -1
- package/dist/serialization/resources/windows/types/WindowLoadUrlV1BodyWaitUntil.js +1 -1
- package/dist/serialization/types/ClickConfig.d.ts +5 -2
- package/dist/serialization/types/ClickConfig.js +4 -1
- package/dist/serialization/types/Issue.d.ts +1 -0
- package/dist/serialization/types/Issue.js +1 -0
- package/{serialization/types/EmptyResponseJson.d.ts → dist/serialization/types/OperationOutcome.d.ts} +2 -2
- package/{serialization/types/EmptyResponseJson.js → dist/serialization/types/OperationOutcome.js} +2 -2
- package/dist/serialization/types/OperationOutcomeResponse.d.ts +18 -0
- package/dist/serialization/types/{EmptyResponse.js → OperationOutcomeResponse.js} +6 -6
- package/dist/serialization/types/PageQueryConfig.d.ts +2 -0
- package/dist/serialization/types/PageQueryConfig.js +2 -0
- package/dist/serialization/types/SummaryConfig.d.ts +2 -0
- package/dist/serialization/types/SummaryConfig.js +2 -0
- package/dist/serialization/types/VisualAnalysisConfig.d.ts +16 -0
- package/dist/serialization/types/{EnvelopeStatusDefaultMeta.js → VisualAnalysisConfig.js} +7 -5
- package/dist/serialization/types/VisualAnalysisConfigPartitionDirection.d.ts +10 -0
- package/dist/serialization/types/VisualAnalysisConfigPartitionDirection.js +31 -0
- package/dist/serialization/types/VisualAnalysisConfigScope.d.ts +10 -0
- package/dist/serialization/types/{EnvelopeStatusDefaultMetaStatus.js → VisualAnalysisConfigScope.js} +2 -2
- package/dist/serialization/types/index.d.ts +5 -4
- package/dist/serialization/types/index.js +5 -4
- package/dist/utils/batch-operate/SessionQueue.d.ts +46 -0
- package/dist/utils/batch-operate/SessionQueue.js +223 -0
- package/dist/utils/batch-operate/WindowQueue.d.ts +25 -0
- package/dist/utils/batch-operate/WindowQueue.js +201 -0
- package/dist/utils/batch-operate/batch-util.d.ts +3 -0
- package/dist/utils/batch-operate/batch-util.js +51 -0
- package/dist/utils/batch-operate/helpers.d.ts +2 -0
- package/dist/utils/batch-operate/helpers.js +19 -0
- package/dist/utils/batch-operate/types.d.ts +29 -0
- package/dist/utils/batch-operate/types.js +2 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +18 -0
- package/dist/wrapper/AirtopClient.d.ts +3 -0
- package/dist/wrapper/AirtopClient.js +16 -0
- package/index.d.ts +1 -0
- package/index.js +4 -0
- package/package.json +3 -2
- package/reference.md +243 -243
- package/serialization/resources/index.d.ts +3 -3
- package/serialization/resources/index.js +4 -4
- package/serialization/resources/windows/types/CreateWindowInputV1BodyWaitUntil.d.ts +1 -1
- package/serialization/resources/windows/types/CreateWindowInputV1BodyWaitUntil.js +1 -1
- package/serialization/resources/windows/types/WindowLoadUrlV1BodyWaitUntil.d.ts +1 -1
- package/serialization/resources/windows/types/WindowLoadUrlV1BodyWaitUntil.js +1 -1
- package/serialization/types/ClickConfig.d.ts +5 -2
- package/serialization/types/ClickConfig.js +4 -1
- package/serialization/types/Issue.d.ts +1 -0
- package/serialization/types/Issue.js +1 -0
- package/{dist/serialization/types/EmptyResponseJson.d.ts → serialization/types/OperationOutcome.d.ts} +2 -2
- package/{dist/serialization/types/EmptyResponseJson.js → serialization/types/OperationOutcome.js} +2 -2
- package/serialization/types/OperationOutcomeResponse.d.ts +18 -0
- package/serialization/types/{EmptyResponse.js → OperationOutcomeResponse.js} +6 -6
- package/serialization/types/PageQueryConfig.d.ts +2 -0
- package/serialization/types/PageQueryConfig.js +2 -0
- package/serialization/types/SummaryConfig.d.ts +2 -0
- package/serialization/types/SummaryConfig.js +2 -0
- package/serialization/types/VisualAnalysisConfig.d.ts +16 -0
- package/serialization/types/VisualAnalysisConfig.js +37 -0
- package/serialization/types/VisualAnalysisConfigPartitionDirection.d.ts +10 -0
- package/serialization/types/VisualAnalysisConfigPartitionDirection.js +31 -0
- package/serialization/types/VisualAnalysisConfigScope.d.ts +10 -0
- package/serialization/types/{EnvelopeStatusDefaultMetaStatus.js → VisualAnalysisConfigScope.js} +2 -2
- package/serialization/types/index.d.ts +5 -4
- package/serialization/types/index.js +5 -4
- package/utils/batch-operate/SessionQueue.d.ts +46 -0
- package/utils/batch-operate/SessionQueue.js +223 -0
- package/utils/batch-operate/WindowQueue.d.ts +25 -0
- package/utils/batch-operate/WindowQueue.js +201 -0
- package/utils/batch-operate/batch-util.d.ts +3 -0
- package/utils/batch-operate/batch-util.js +51 -0
- package/utils/batch-operate/helpers.d.ts +2 -0
- package/utils/batch-operate/helpers.js +19 -0
- package/utils/batch-operate/types.d.ts +29 -0
- package/utils/batch-operate/types.js +2 -0
- package/utils/index.d.ts +2 -0
- package/utils/index.js +18 -0
- package/wrapper/AirtopClient.d.ts +3 -0
- package/wrapper/AirtopClient.js +16 -0
- package/api/types/EnvelopeStatusDefaultMeta.d.ts +0 -8
- package/api/types/EnvelopeStatusDefaultMetaStatus.d.ts +0 -9
- package/api/types/EnvelopeStatusDefaultMetaStatus.js +0 -11
- package/dist/api/types/EnvelopeStatusDefaultMeta.d.ts +0 -8
- package/dist/api/types/EnvelopeStatusDefaultMetaStatus.d.ts +0 -9
- package/dist/api/types/EnvelopeStatusDefaultMetaStatus.js +0 -11
- package/dist/serialization/types/EmptyResponse.d.ts +0 -18
- package/dist/serialization/types/EnvelopeStatusDefaultMeta.d.ts +0 -14
- package/dist/serialization/types/EnvelopeStatusDefaultMetaStatus.d.ts +0 -10
- package/serialization/types/EmptyResponse.d.ts +0 -18
- package/serialization/types/EnvelopeStatusDefaultMeta.d.ts +0 -14
- package/serialization/types/EnvelopeStatusDefaultMeta.js +0 -35
- package/serialization/types/EnvelopeStatusDefaultMetaStatus.d.ts +0 -10
- /package/api/types/{EmptyResponse.js → OperationOutcome.js} +0 -0
- /package/api/types/{EmptyResponseJson.js → OperationOutcomeResponse.js} +0 -0
- /package/api/types/{EnvelopeStatusDefaultMeta.js → VisualAnalysisConfig.js} +0 -0
- /package/dist/api/types/{EmptyResponse.js → OperationOutcome.js} +0 -0
- /package/dist/api/types/{EmptyResponseJson.js → OperationOutcomeResponse.js} +0 -0
- /package/dist/api/types/{EnvelopeStatusDefaultMeta.js → VisualAnalysisConfig.js} +0 -0
@@ -0,0 +1,223 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
12
|
+
exports.SessionQueue = void 0;
|
13
|
+
const helpers_1 = require("./helpers");
|
14
|
+
const WindowQueue_1 = require("./WindowQueue");
|
15
|
+
const async_mutex_1 = require("async-mutex");
|
16
|
+
class SessionQueue {
|
17
|
+
constructor({ maxConcurrentSessions, runEmitter, maxWindowsPerSession, initialBatches, operation, client, sessionConfig, onError, }) {
|
18
|
+
this.activePromises = [];
|
19
|
+
this.activePromisesMutex = new async_mutex_1.Mutex();
|
20
|
+
this.initialBatches = [];
|
21
|
+
this.batchQueue = [];
|
22
|
+
this.batchQueueMutex = new async_mutex_1.Mutex();
|
23
|
+
this.latestProcessingPromise = null;
|
24
|
+
this.processingPromisesCount = 0;
|
25
|
+
this.sessionPool = [];
|
26
|
+
this.sessionPoolMutex = new async_mutex_1.Mutex();
|
27
|
+
if (!Number.isInteger(maxConcurrentSessions) ||
|
28
|
+
maxConcurrentSessions <= 0) {
|
29
|
+
throw new Error("maxConcurrentSessions must be a positive integer");
|
30
|
+
}
|
31
|
+
this.maxConcurrentSessions = maxConcurrentSessions;
|
32
|
+
this.runEmitter = runEmitter;
|
33
|
+
this.maxWindowsPerSession = maxWindowsPerSession;
|
34
|
+
this.sessionConfig = sessionConfig;
|
35
|
+
this.initialBatches = initialBatches;
|
36
|
+
this.operation = operation;
|
37
|
+
this.onError = onError;
|
38
|
+
this.latestProcessingPromise = null;
|
39
|
+
this.results = [];
|
40
|
+
this.client = client;
|
41
|
+
this.isHalted = false;
|
42
|
+
}
|
43
|
+
handleHaltEvent() {
|
44
|
+
this.client.log("Halt event received");
|
45
|
+
this.isHalted = true;
|
46
|
+
}
|
47
|
+
addUrlsToBatchQueue(newBatch) {
|
48
|
+
return __awaiter(this, void 0, void 0, function* () {
|
49
|
+
// Distribute new URLs across batches
|
50
|
+
const newBatches = (0, helpers_1.distributeUrlsToBatches)(newBatch, this.maxConcurrentSessions);
|
51
|
+
this.client.log(`Adding new batches to queue: ${JSON.stringify(newBatches)}`);
|
52
|
+
// Add new batches to the queue
|
53
|
+
yield this.batchQueueMutex.runExclusive(() => {
|
54
|
+
this.batchQueue.push(...newBatches);
|
55
|
+
});
|
56
|
+
// Update existing processing promise
|
57
|
+
this.processingPromisesCount++;
|
58
|
+
this.latestProcessingPromise = this.processPendingBatches();
|
59
|
+
});
|
60
|
+
}
|
61
|
+
processInitialBatches() {
|
62
|
+
return __awaiter(this, void 0, void 0, function* () {
|
63
|
+
yield this.batchQueueMutex.runExclusive(() => {
|
64
|
+
this.batchQueue = [...this.initialBatches];
|
65
|
+
});
|
66
|
+
this.processingPromisesCount++;
|
67
|
+
this.runEmitter.on("halt", this.handleHaltEvent.bind(this));
|
68
|
+
this.latestProcessingPromise = this.processPendingBatches();
|
69
|
+
yield this.latestProcessingPromise;
|
70
|
+
});
|
71
|
+
}
|
72
|
+
waitForProcessingToComplete() {
|
73
|
+
return __awaiter(this, void 0, void 0, function* () {
|
74
|
+
while (this.processingPromisesCount > 0) {
|
75
|
+
yield this.latestProcessingPromise;
|
76
|
+
}
|
77
|
+
yield this.terminateAllSessions();
|
78
|
+
this.runEmitter.removeListener("halt", this.handleHaltEvent);
|
79
|
+
return this.results;
|
80
|
+
});
|
81
|
+
}
|
82
|
+
terminateAllSessions() {
|
83
|
+
return __awaiter(this, void 0, void 0, function* () {
|
84
|
+
for (const sessionId of this.sessionPool) {
|
85
|
+
this.safelyTerminateSession(sessionId);
|
86
|
+
}
|
87
|
+
});
|
88
|
+
}
|
89
|
+
processPendingBatches() {
|
90
|
+
return __awaiter(this, void 0, void 0, function* () {
|
91
|
+
try {
|
92
|
+
while (this.batchQueue.length > 0) {
|
93
|
+
// Wait for any session to complete before starting a new one
|
94
|
+
let shouldContinue = false;
|
95
|
+
yield this.activePromisesMutex.runExclusive(() => __awaiter(this, void 0, void 0, function* () {
|
96
|
+
if (this.activePromises.length >= this.maxConcurrentSessions) {
|
97
|
+
yield Promise.race(this.activePromises);
|
98
|
+
shouldContinue = true;
|
99
|
+
}
|
100
|
+
}));
|
101
|
+
if (shouldContinue)
|
102
|
+
continue;
|
103
|
+
let batch;
|
104
|
+
yield this.batchQueueMutex.runExclusive(() => {
|
105
|
+
batch = this.batchQueue.shift();
|
106
|
+
});
|
107
|
+
if (!batch || batch.length === 0)
|
108
|
+
break;
|
109
|
+
const promise = (() => __awaiter(this, void 0, void 0, function* () {
|
110
|
+
if (this.isHalted) {
|
111
|
+
this.client.log("Halt event received, skipping batch");
|
112
|
+
return;
|
113
|
+
}
|
114
|
+
let sessionId;
|
115
|
+
try {
|
116
|
+
// Check if there's an available session in the pool
|
117
|
+
yield this.sessionPoolMutex.runExclusive(() => {
|
118
|
+
if (this.sessionPool.length > 0) {
|
119
|
+
sessionId = this.sessionPool.pop();
|
120
|
+
}
|
121
|
+
});
|
122
|
+
// Otherwise, create a new session
|
123
|
+
if (!sessionId) {
|
124
|
+
const { data: session, warnings, errors } = yield this.client.sessions.create({
|
125
|
+
configuration: this.sessionConfig,
|
126
|
+
});
|
127
|
+
sessionId = session.id;
|
128
|
+
this.handleErrorAndWarningResponses({ warnings, errors, sessionId, batch });
|
129
|
+
}
|
130
|
+
const queue = new WindowQueue_1.WindowQueue(this.maxWindowsPerSession, this.runEmitter, sessionId, this.client, this.operation, this.onError, this.isHalted);
|
131
|
+
const windowResults = yield queue.processInBatches(batch);
|
132
|
+
this.results.push(...windowResults);
|
133
|
+
// Return the session to the pool
|
134
|
+
yield this.sessionPoolMutex.runExclusive(() => {
|
135
|
+
if (!sessionId) {
|
136
|
+
throw new Error("Missing sessionId, cannot return to pool");
|
137
|
+
}
|
138
|
+
this.sessionPool.push(sessionId);
|
139
|
+
});
|
140
|
+
}
|
141
|
+
catch (error) {
|
142
|
+
if (this.onError) {
|
143
|
+
yield this.handleErrorWithCallback({ originalError: error, batch, sessionId, callback: this.onError });
|
144
|
+
}
|
145
|
+
else {
|
146
|
+
// By default, log the error and continue
|
147
|
+
const urls = batch.map((url) => url.url);
|
148
|
+
this.logErrorForUrls(urls, error);
|
149
|
+
}
|
150
|
+
// Clean up the session in case of error
|
151
|
+
if (sessionId) {
|
152
|
+
this.safelyTerminateSession(sessionId);
|
153
|
+
}
|
154
|
+
}
|
155
|
+
}))();
|
156
|
+
yield this.activePromisesMutex.runExclusive(() => {
|
157
|
+
this.activePromises.push(promise);
|
158
|
+
});
|
159
|
+
// Remove the promise when it completes
|
160
|
+
promise.finally(() => __awaiter(this, void 0, void 0, function* () {
|
161
|
+
yield this.activePromisesMutex.runExclusive(() => {
|
162
|
+
const index = this.activePromises.indexOf(promise);
|
163
|
+
if (index > -1) {
|
164
|
+
this.activePromises.splice(index, 1);
|
165
|
+
}
|
166
|
+
});
|
167
|
+
}));
|
168
|
+
}
|
169
|
+
// Wait for all remaining sessions to complete
|
170
|
+
yield Promise.allSettled(this.activePromises);
|
171
|
+
}
|
172
|
+
finally {
|
173
|
+
this.processingPromisesCount--;
|
174
|
+
}
|
175
|
+
});
|
176
|
+
}
|
177
|
+
handleErrorWithCallback({ originalError, batch, sessionId, callback, }) {
|
178
|
+
return __awaiter(this, void 0, void 0, function* () {
|
179
|
+
// Catch any errors in the onError callback to avoid halting the entire process
|
180
|
+
try {
|
181
|
+
yield callback({
|
182
|
+
error: this.formatError(originalError),
|
183
|
+
operationUrls: batch,
|
184
|
+
sessionId,
|
185
|
+
});
|
186
|
+
}
|
187
|
+
catch (newError) {
|
188
|
+
this.client.error(`Error in onError callback: ${this.formatError(newError)}. Original error: ${this.formatError(originalError)}`);
|
189
|
+
}
|
190
|
+
});
|
191
|
+
}
|
192
|
+
logErrorForUrls(urls, error) {
|
193
|
+
const message = `Error for URLs ${JSON.stringify(urls)}: ${this.formatError(error)}`;
|
194
|
+
this.client.error(message);
|
195
|
+
}
|
196
|
+
safelyTerminateSession(sessionId) {
|
197
|
+
// Do not await since we don't want to block the main thread
|
198
|
+
this.client.sessions.terminate(sessionId).catch((error) => {
|
199
|
+
this.client.error(`Error terminating session ${sessionId}: ${this.formatError(error)}`);
|
200
|
+
});
|
201
|
+
}
|
202
|
+
formatError(error) {
|
203
|
+
return error instanceof Error ? error.message : String(error);
|
204
|
+
}
|
205
|
+
handleErrorAndWarningResponses({ warnings, errors, sessionId, batch }) {
|
206
|
+
if (!warnings && !errors)
|
207
|
+
return;
|
208
|
+
const details = {
|
209
|
+
sessionId,
|
210
|
+
urls: batch,
|
211
|
+
};
|
212
|
+
if (warnings) {
|
213
|
+
details.warnings = warnings;
|
214
|
+
this.client.warn(`Received warnings creating session: ${JSON.stringify(details)}`);
|
215
|
+
}
|
216
|
+
// Log an object with the errors and the URL
|
217
|
+
if (errors) {
|
218
|
+
details.errors = errors;
|
219
|
+
this.client.error(`Received errors creating session: ${JSON.stringify(details)}`);
|
220
|
+
}
|
221
|
+
}
|
222
|
+
}
|
223
|
+
exports.SessionQueue = SessionQueue;
|
@@ -0,0 +1,25 @@
|
|
1
|
+
/// <reference types="node" />
|
2
|
+
import type { AirtopClient } from "wrapper/AirtopClient";
|
3
|
+
import type { BatchOperationError, BatchOperationInput, BatchOperationResponse, BatchOperationUrl } from "./types";
|
4
|
+
import type { EventEmitter } from "node:events";
|
5
|
+
export declare class WindowQueue<T> {
|
6
|
+
private activePromises;
|
7
|
+
private urlQueue;
|
8
|
+
private activePromisesMutex;
|
9
|
+
private urlQueueMutex;
|
10
|
+
private maxWindowsPerSession;
|
11
|
+
private runEmitter;
|
12
|
+
private sessionId;
|
13
|
+
private client;
|
14
|
+
private operation;
|
15
|
+
private onError?;
|
16
|
+
private isHalted;
|
17
|
+
constructor(maxWindowsPerSession: number, runEmitter: EventEmitter, sessionId: string, client: AirtopClient, operation: (input: BatchOperationInput) => Promise<BatchOperationResponse<T>>, onError?: (error: BatchOperationError) => Promise<void>, isHalted?: boolean);
|
18
|
+
addUrlToQueue(url: BatchOperationUrl): Promise<void>;
|
19
|
+
private handleHaltEvent;
|
20
|
+
processInBatches(urls: BatchOperationUrl[]): Promise<T[]>;
|
21
|
+
private handleErrorWithCallback;
|
22
|
+
private safelyTerminateWindow;
|
23
|
+
private formatError;
|
24
|
+
private handleErrorAndWarningResponses;
|
25
|
+
}
|
@@ -0,0 +1,201 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
12
|
+
exports.WindowQueue = void 0;
|
13
|
+
const async_mutex_1 = require("async-mutex");
|
14
|
+
class WindowQueue {
|
15
|
+
constructor(maxWindowsPerSession, runEmitter, sessionId, client, operation, onError, isHalted = false) {
|
16
|
+
this.activePromises = [];
|
17
|
+
this.urlQueue = [];
|
18
|
+
this.activePromisesMutex = new async_mutex_1.Mutex();
|
19
|
+
this.urlQueueMutex = new async_mutex_1.Mutex();
|
20
|
+
if (!Number.isInteger(maxWindowsPerSession) || maxWindowsPerSession <= 0) {
|
21
|
+
throw new Error("maxWindowsPerSession must be a positive integer");
|
22
|
+
}
|
23
|
+
this.maxWindowsPerSession = maxWindowsPerSession;
|
24
|
+
this.runEmitter = runEmitter;
|
25
|
+
this.sessionId = sessionId;
|
26
|
+
this.client = client;
|
27
|
+
this.operation = operation;
|
28
|
+
this.onError = onError;
|
29
|
+
this.isHalted = isHalted;
|
30
|
+
}
|
31
|
+
addUrlToQueue(url) {
|
32
|
+
return __awaiter(this, void 0, void 0, function* () {
|
33
|
+
yield this.urlQueueMutex.runExclusive(() => {
|
34
|
+
this.urlQueue.push(url);
|
35
|
+
});
|
36
|
+
});
|
37
|
+
}
|
38
|
+
handleHaltEvent() {
|
39
|
+
this.client.log("Halt event received");
|
40
|
+
this.isHalted = true;
|
41
|
+
}
|
42
|
+
processInBatches(urls) {
|
43
|
+
return __awaiter(this, void 0, void 0, function* () {
|
44
|
+
const results = [];
|
45
|
+
this.runEmitter.on("halt", this.handleHaltEvent.bind(this));
|
46
|
+
yield this.urlQueueMutex.runExclusive(() => {
|
47
|
+
this.urlQueue = [...urls];
|
48
|
+
});
|
49
|
+
this.client.log(`Processing batch: ${JSON.stringify(urls)} for session ${this.sessionId}`);
|
50
|
+
while (this.urlQueue.length > 0) {
|
51
|
+
// Wait for any window to complete before starting a new one
|
52
|
+
let shouldContinue = false;
|
53
|
+
yield this.activePromisesMutex.runExclusive(() => __awaiter(this, void 0, void 0, function* () {
|
54
|
+
if (this.activePromises.length >= this.maxWindowsPerSession) {
|
55
|
+
yield Promise.race(this.activePromises);
|
56
|
+
shouldContinue = true;
|
57
|
+
}
|
58
|
+
}));
|
59
|
+
if (shouldContinue)
|
60
|
+
continue;
|
61
|
+
let urlData;
|
62
|
+
yield this.urlQueueMutex.runExclusive(() => {
|
63
|
+
urlData = this.urlQueue.shift(); // Take the next url from the queue
|
64
|
+
});
|
65
|
+
if (!urlData)
|
66
|
+
break; // No more urls to process
|
67
|
+
// If we have less than the max concurrent operations, start a new one
|
68
|
+
const promise = (() => __awaiter(this, void 0, void 0, function* () {
|
69
|
+
// Do not process any more urls if the processing has been halted
|
70
|
+
if (this.isHalted) {
|
71
|
+
this.client.log(`Processing halted, skipping window creation for ${urlData.url}`);
|
72
|
+
return;
|
73
|
+
}
|
74
|
+
let windowId;
|
75
|
+
let liveViewUrl;
|
76
|
+
try {
|
77
|
+
// Create a new window pointed to the url
|
78
|
+
this.client.log(`Creating window for ${urlData.url} in session ${this.sessionId}`);
|
79
|
+
const { data, errors, warnings } = yield this.client.windows.create(this.sessionId, {
|
80
|
+
url: urlData.url,
|
81
|
+
});
|
82
|
+
windowId = data.windowId;
|
83
|
+
this.handleErrorAndWarningResponses({ warnings, errors, sessionId: this.sessionId, url: urlData, operation: "window creation" });
|
84
|
+
if (!windowId) {
|
85
|
+
throw new Error(`WindowId not found, errors: ${JSON.stringify(errors)}`);
|
86
|
+
}
|
87
|
+
const { data: windowInfo, warnings: windowWarnings, errors: windowErrors } = yield this.client.windows.getWindowInfo(this.sessionId, windowId);
|
88
|
+
liveViewUrl = windowInfo.liveViewUrl;
|
89
|
+
this.handleErrorAndWarningResponses({ warnings: windowWarnings, errors: windowErrors, sessionId: this.sessionId, url: urlData, operation: "window info retrieval" });
|
90
|
+
// Run the operation on the window
|
91
|
+
const result = yield this.operation({
|
92
|
+
windowId,
|
93
|
+
sessionId: this.sessionId,
|
94
|
+
liveViewUrl,
|
95
|
+
operationUrl: urlData,
|
96
|
+
});
|
97
|
+
if (result) {
|
98
|
+
const { shouldHaltBatch, additionalUrls, data } = result;
|
99
|
+
if (data) {
|
100
|
+
results.push(data);
|
101
|
+
}
|
102
|
+
if (shouldHaltBatch) {
|
103
|
+
this.client.log("Emitting halt event");
|
104
|
+
this.runEmitter.emit("halt");
|
105
|
+
}
|
106
|
+
if (additionalUrls && additionalUrls.length > 0) {
|
107
|
+
this.client.log(`Emitting addUrls event with urls: ${JSON.stringify(additionalUrls)}`);
|
108
|
+
this.runEmitter.emit("addUrls", additionalUrls);
|
109
|
+
}
|
110
|
+
}
|
111
|
+
}
|
112
|
+
catch (error) {
|
113
|
+
if (this.onError) {
|
114
|
+
yield this.handleErrorWithCallback({
|
115
|
+
originalError: error,
|
116
|
+
url: urlData,
|
117
|
+
callback: this.onError,
|
118
|
+
windowId,
|
119
|
+
liveViewUrl,
|
120
|
+
});
|
121
|
+
}
|
122
|
+
else {
|
123
|
+
// By default, log the error and continue
|
124
|
+
const message = `Error for URL ${urlData.url}: ${this.formatError(error)}`;
|
125
|
+
this.client.error(message);
|
126
|
+
}
|
127
|
+
}
|
128
|
+
finally {
|
129
|
+
if (windowId) {
|
130
|
+
yield this.safelyTerminateWindow(windowId);
|
131
|
+
}
|
132
|
+
}
|
133
|
+
}))();
|
134
|
+
yield this.activePromisesMutex.runExclusive(() => {
|
135
|
+
this.activePromises.push(promise);
|
136
|
+
});
|
137
|
+
// Remove the promise from the active list when it resolves
|
138
|
+
promise.finally(() => __awaiter(this, void 0, void 0, function* () {
|
139
|
+
yield this.activePromisesMutex.runExclusive(() => {
|
140
|
+
const index = this.activePromises.indexOf(promise);
|
141
|
+
if (index > -1) {
|
142
|
+
this.activePromises.splice(index, 1);
|
143
|
+
}
|
144
|
+
});
|
145
|
+
}));
|
146
|
+
}
|
147
|
+
// Wait for all processes to complete
|
148
|
+
yield Promise.allSettled(this.activePromises);
|
149
|
+
// Remove the halt listener
|
150
|
+
this.runEmitter.removeListener("halt", this.handleHaltEvent);
|
151
|
+
return results;
|
152
|
+
});
|
153
|
+
}
|
154
|
+
handleErrorWithCallback({ originalError, url, windowId, liveViewUrl, callback, }) {
|
155
|
+
return __awaiter(this, void 0, void 0, function* () {
|
156
|
+
// Catch any errors in the onError callback to avoid halting the entire process
|
157
|
+
try {
|
158
|
+
yield callback({
|
159
|
+
error: this.formatError(originalError),
|
160
|
+
operationUrls: [url],
|
161
|
+
sessionId: this.sessionId,
|
162
|
+
windowId,
|
163
|
+
liveViewUrl,
|
164
|
+
});
|
165
|
+
}
|
166
|
+
catch (newError) {
|
167
|
+
this.client.error(`Error in onError callback: ${this.formatError(newError)}. Original error: ${this.formatError(originalError)}`);
|
168
|
+
}
|
169
|
+
});
|
170
|
+
}
|
171
|
+
safelyTerminateWindow(windowId) {
|
172
|
+
return __awaiter(this, void 0, void 0, function* () {
|
173
|
+
try {
|
174
|
+
yield this.client.windows.close(this.sessionId, windowId);
|
175
|
+
}
|
176
|
+
catch (error) {
|
177
|
+
this.client.error(`Error closing window ${windowId}: ${this.formatError(error)}`);
|
178
|
+
}
|
179
|
+
});
|
180
|
+
}
|
181
|
+
formatError(error) {
|
182
|
+
return error instanceof Error ? error.message : String(error);
|
183
|
+
}
|
184
|
+
handleErrorAndWarningResponses({ warnings, errors, sessionId, url, operation }) {
|
185
|
+
if (!warnings && !errors)
|
186
|
+
return;
|
187
|
+
const details = {
|
188
|
+
sessionId,
|
189
|
+
url,
|
190
|
+
};
|
191
|
+
if (warnings) {
|
192
|
+
details.warnings = warnings;
|
193
|
+
this.client.warn(`Received warnings for ${operation}: ${JSON.stringify(details)}`);
|
194
|
+
}
|
195
|
+
if (errors) {
|
196
|
+
details.errors = errors;
|
197
|
+
this.client.error(`Received errors for ${operation}: ${JSON.stringify(details)}`);
|
198
|
+
}
|
199
|
+
}
|
200
|
+
}
|
201
|
+
exports.WindowQueue = WindowQueue;
|
@@ -0,0 +1,3 @@
|
|
1
|
+
import type { AirtopClient } from "wrapper/AirtopClient";
|
2
|
+
import type { BatchOperateConfig, BatchOperationInput, BatchOperationResponse, BatchOperationUrl } from "./types";
|
3
|
+
export declare const batchOperate: <T>(urls: BatchOperationUrl[], operation: (input: BatchOperationInput) => Promise<BatchOperationResponse<T>>, client: AirtopClient, config?: BatchOperateConfig | undefined) => Promise<T[]>;
|
@@ -0,0 +1,51 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
12
|
+
exports.batchOperate = void 0;
|
13
|
+
const node_events_1 = require("node:events");
|
14
|
+
const helpers_1 = require("./helpers");
|
15
|
+
const SessionQueue_1 = require("./SessionQueue");
|
16
|
+
const DEFAULT_MAX_WINDOWS_PER_SESSION = 1;
|
17
|
+
const DEFAULT_MAX_CONCURRENT_SESSIONS = 30;
|
18
|
+
const batchOperate = (urls, operation, // operation to invoke on each url
|
19
|
+
client, config) => __awaiter(void 0, void 0, void 0, function* () {
|
20
|
+
// Validate the urls before proceeding
|
21
|
+
if (!Array.isArray(urls)) {
|
22
|
+
throw new Error("Please provide a valid list of urls");
|
23
|
+
}
|
24
|
+
for (const url of urls) {
|
25
|
+
if (!url || typeof url !== "object" || !("url" in url)) {
|
26
|
+
throw new Error("Please provide a valid list of urls");
|
27
|
+
}
|
28
|
+
}
|
29
|
+
const runEmitter = new node_events_1.EventEmitter();
|
30
|
+
const { maxConcurrentSessions = DEFAULT_MAX_CONCURRENT_SESSIONS, maxWindowsPerSession = DEFAULT_MAX_WINDOWS_PER_SESSION, sessionConfig, onError, } = config !== null && config !== void 0 ? config : {};
|
31
|
+
// Set the maximum number of listeners to accommodate all concurrent sessions and windows
|
32
|
+
runEmitter.setMaxListeners(maxConcurrentSessions + (maxConcurrentSessions * maxWindowsPerSession) + 1);
|
33
|
+
// Split the urls into batches
|
34
|
+
const initialBatches = (0, helpers_1.distributeUrlsToBatches)(urls, maxConcurrentSessions);
|
35
|
+
const sessionQueue = new SessionQueue_1.SessionQueue({
|
36
|
+
maxConcurrentSessions,
|
37
|
+
runEmitter,
|
38
|
+
maxWindowsPerSession,
|
39
|
+
initialBatches,
|
40
|
+
operation,
|
41
|
+
client,
|
42
|
+
sessionConfig,
|
43
|
+
onError,
|
44
|
+
});
|
45
|
+
runEmitter.on("addUrls", (additionalUrls) => {
|
46
|
+
sessionQueue.addUrlsToBatchQueue(additionalUrls);
|
47
|
+
});
|
48
|
+
yield sessionQueue.processInitialBatches();
|
49
|
+
return yield sessionQueue.waitForProcessingToComplete();
|
50
|
+
});
|
51
|
+
exports.batchOperate = batchOperate;
|
@@ -0,0 +1,19 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.distributeUrlsToBatches = void 0;
|
4
|
+
const distributeUrlsToBatches = (urls, maxConcurrentSessions) => {
|
5
|
+
if (urls.length === 0)
|
6
|
+
return [];
|
7
|
+
// Calculate optimal number of batches
|
8
|
+
const batchCount = Math.min(maxConcurrentSessions, urls.length);
|
9
|
+
const batches = Array.from({ length: batchCount }, () => []);
|
10
|
+
urls.forEach((url, index) => {
|
11
|
+
const batchIndex = index % batchCount;
|
12
|
+
if (!batches[batchIndex]) {
|
13
|
+
batches[batchIndex] = [];
|
14
|
+
}
|
15
|
+
batches[batchIndex].push(url);
|
16
|
+
});
|
17
|
+
return batches;
|
18
|
+
};
|
19
|
+
exports.distributeUrlsToBatches = distributeUrlsToBatches;
|
@@ -0,0 +1,29 @@
|
|
1
|
+
import type { AirtopSessionConfigV1 } from "wrapper/AirtopSessions";
|
2
|
+
export declare type BatchOperateConfig = {
|
3
|
+
maxConcurrentSessions?: number;
|
4
|
+
maxWindowsPerSession?: number;
|
5
|
+
sessionConfig?: AirtopSessionConfigV1;
|
6
|
+
onError?: (error: BatchOperationError) => Promise<void>;
|
7
|
+
};
|
8
|
+
export declare type BatchOperationUrl = {
|
9
|
+
url: string;
|
10
|
+
context?: Record<string, unknown>;
|
11
|
+
};
|
12
|
+
export declare type BatchOperationInput = {
|
13
|
+
windowId: string;
|
14
|
+
sessionId: string;
|
15
|
+
liveViewUrl: string;
|
16
|
+
operationUrl: BatchOperationUrl;
|
17
|
+
};
|
18
|
+
export declare type BatchOperationResponse<T> = {
|
19
|
+
shouldHaltBatch?: boolean;
|
20
|
+
additionalUrls?: BatchOperationUrl[];
|
21
|
+
data?: T;
|
22
|
+
};
|
23
|
+
export declare type BatchOperationError = {
|
24
|
+
error: Error | string;
|
25
|
+
operationUrls: BatchOperationUrl[];
|
26
|
+
sessionId?: string;
|
27
|
+
windowId?: string;
|
28
|
+
liveViewUrl?: string;
|
29
|
+
};
|
@@ -0,0 +1,18 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
3
|
+
if (k2 === undefined) k2 = k;
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
7
|
+
}
|
8
|
+
Object.defineProperty(o, k2, desc);
|
9
|
+
}) : (function(o, m, k, k2) {
|
10
|
+
if (k2 === undefined) k2 = k;
|
11
|
+
o[k2] = m[k];
|
12
|
+
}));
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
15
|
+
};
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
17
|
+
__exportStar(require("./batch-operate/batch-util"), exports);
|
18
|
+
__exportStar(require("./batch-operate/types"), exports);
|
@@ -1,6 +1,7 @@
|
|
1
1
|
import { AirtopClient as FernClient } from '../Client';
|
2
2
|
import { AirtopSessions } from './AirtopSessions';
|
3
3
|
import { AirtopWindows } from './AirtopWindows';
|
4
|
+
import { type BatchOperateConfig, type BatchOperationInput, type BatchOperationResponse, type BatchOperationUrl } from "../utils";
|
4
5
|
declare type AugmentedOptions = FernClient.Options & {
|
5
6
|
debug?: boolean;
|
6
7
|
};
|
@@ -15,6 +16,8 @@ export declare class AirtopClient {
|
|
15
16
|
get windows(): AirtopWindows;
|
16
17
|
get profiles(): import("../api/resources/profiles/client/Client").Profiles;
|
17
18
|
log(message: string): void;
|
19
|
+
warn(message: string): void;
|
18
20
|
error(err: any): void;
|
21
|
+
batchOperate: <T>(urls: BatchOperationUrl[], operation: (input: BatchOperationInput) => Promise<BatchOperationResponse<T>>, config?: BatchOperateConfig | undefined) => Promise<T[]>;
|
19
22
|
}
|
20
23
|
export {};
|
@@ -1,12 +1,25 @@
|
|
1
1
|
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
12
|
exports.AirtopClient = void 0;
|
4
13
|
const Client_1 = require("../Client"); // alias the Fern generated client
|
5
14
|
const AirtopSessions_1 = require("./AirtopSessions");
|
6
15
|
const AirtopWindows_1 = require("./AirtopWindows");
|
16
|
+
const utils_1 = require("../utils");
|
7
17
|
class AirtopClient {
|
8
18
|
constructor(_options) {
|
9
19
|
this._options = _options;
|
20
|
+
this.batchOperate = (urls, operation, config) => __awaiter(this, void 0, void 0, function* () {
|
21
|
+
return yield (0, utils_1.batchOperate)(urls, operation, this, config);
|
22
|
+
});
|
10
23
|
this._client = new Client_1.AirtopClient(_options);
|
11
24
|
this.debug = (_options === null || _options === void 0 ? void 0 : _options.debug) || false;
|
12
25
|
}
|
@@ -26,6 +39,9 @@ class AirtopClient {
|
|
26
39
|
console.log(message);
|
27
40
|
}
|
28
41
|
}
|
42
|
+
warn(message) {
|
43
|
+
console.warn(message);
|
44
|
+
}
|
29
45
|
error(err) {
|
30
46
|
console.error(err);
|
31
47
|
}
|
package/index.d.ts
CHANGED
package/index.js
CHANGED
@@ -22,6 +22,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
22
22
|
__setModuleDefault(result, mod);
|
23
23
|
return result;
|
24
24
|
};
|
25
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
26
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
27
|
+
};
|
25
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
26
29
|
exports.LiveViewEventName = exports.AirtopClient = exports.AirtopTimeoutError = exports.AirtopError = exports.AirtopEnvironment = exports.Resources = exports.Airtop = void 0;
|
27
30
|
exports.Airtop = __importStar(require("./api"));
|
@@ -35,3 +38,4 @@ var AirtopClient_1 = require("./wrapper/AirtopClient");
|
|
35
38
|
Object.defineProperty(exports, "AirtopClient", { enumerable: true, get: function () { return AirtopClient_1.AirtopClient; } });
|
36
39
|
var types_1 = require("./live-view/types");
|
37
40
|
Object.defineProperty(exports, "LiveViewEventName", { enumerable: true, get: function () { return types_1.LiveViewEventName; } });
|
41
|
+
__exportStar(require("./utils"), exports);
|