@crawlee/core 3.8.2-beta.13 → 3.8.2-beta.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.mjs +2 -0
- package/package.json +5 -5
- package/storages/access_checking.d.ts +13 -0
- package/storages/access_checking.d.ts.map +1 -0
- package/storages/access_checking.js +19 -0
- package/storages/access_checking.js.map +1 -0
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +12 -0
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +1 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +1 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +9 -0
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +10 -0
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +8 -0
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +10 -0
- package/storages/request_queue_v2.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.RequestQueue = void 0;
|
|
4
4
|
const promises_1 = require("node:timers/promises");
|
|
5
5
|
const consts_1 = require("@apify/consts");
|
|
6
|
+
const access_checking_1 = require("./access_checking");
|
|
6
7
|
const request_provider_1 = require("./request_provider");
|
|
7
8
|
const utils_1 = require("./utils");
|
|
8
9
|
const configuration_1 = require("../configuration");
|
|
@@ -101,6 +102,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
101
102
|
* Returns the request object or `null` if there are no more pending requests.
|
|
102
103
|
*/
|
|
103
104
|
async fetchNextRequest() {
|
|
105
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
104
106
|
await this.ensureHeadIsNonEmpty();
|
|
105
107
|
const nextRequestId = this.queueHeadIds.removeFirst();
|
|
106
108
|
// We are likely done at this point.
|
|
@@ -243,6 +245,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
243
245
|
}
|
|
244
246
|
// RequestQueue v1 behavior overrides below
|
|
245
247
|
async isFinished() {
|
|
248
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
246
249
|
if ((Date.now() - +this.lastActivity) > this.internalTimeoutMillis) {
|
|
247
250
|
const message = `The request queue seems to be stuck for ${this.internalTimeoutMillis / 1e3}s, resetting internal state.`;
|
|
248
251
|
this.log.warning(message, { inProgress: [...this.inProgress] });
|
|
@@ -254,22 +257,27 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
254
257
|
return isHeadConsistent && this.queueHeadIds.length() === 0 && this.inProgressCount() === 0;
|
|
255
258
|
}
|
|
256
259
|
async addRequest(...args) {
|
|
260
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
257
261
|
this.lastActivity = new Date();
|
|
258
262
|
return super.addRequest(...args);
|
|
259
263
|
}
|
|
260
264
|
async addRequests(...args) {
|
|
265
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
261
266
|
this.lastActivity = new Date();
|
|
262
267
|
return super.addRequests(...args);
|
|
263
268
|
}
|
|
264
269
|
async addRequestsBatched(...args) {
|
|
270
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
265
271
|
this.lastActivity = new Date();
|
|
266
272
|
return super.addRequestsBatched(...args);
|
|
267
273
|
}
|
|
268
274
|
async markRequestHandled(...args) {
|
|
275
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
269
276
|
this.lastActivity = new Date();
|
|
270
277
|
return super.markRequestHandled(...args);
|
|
271
278
|
}
|
|
272
279
|
async reclaimRequest(...args) {
|
|
280
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
273
281
|
this.lastActivity = new Date();
|
|
274
282
|
return super.reclaimRequest(...args);
|
|
275
283
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":";;;AAAA,mDAA2D;AAE3D,0CAA6D;
|
|
1
|
+
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":";;;AAAA,mDAA2D;AAE3D,0CAA6D;AAI7D,uDAAuD;AAEvD,yDAAqD;AACrD,mCAOiB;AACjB,oDAAiD;AAGjD,MAAM,mBAAmB,GAAG,OAAS,CAAC;AAEtC;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,MAAa,YAAa,SAAQ,kCAAe;IAW7C;;OAEG;IACH,YAAY,OAA+B,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE;QACjF,KAAK,CAAC;YACF,GAAG,OAAO;YACV,SAAS,EAAE,cAAc;YACzB,8BAA8B,EAAE,2BAA2B;YAC3D,mBAAmB,EAAE,mBAAmB;SAC3C,EAAE,MAAM,CAAC,CAAC;QAnBP;;;;mBAMI,IAAI;WAAC;QAET;;;;mBAAe,IAAI,IAAI,EAAE;WAAC;IAYlC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACM,KAAK,CAAC,gBAAgB;QAC3B,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAEtD,oCAAoC;QACpC,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAEhC,mCAAmC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;YAC7F,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6DAA6D,EAAE;gBAC5E,aAAa;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC9C,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC;aAC1E,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,IAAI,OAAuB,CAAC;QAC5B,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,qFAAqF;YACrF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtC,MAAM,CAAC,CAAC;QACZ,CAAC;QAED,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0EAA0E,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YAC9G,UAAU,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAC1C,CAAC,EAAE,wCAAgC,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC3D,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEkB,KAAK,CAAC,oBAAoB;QACzC,oCAAoC;QACpC,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACvC,CAAC;IAED;;;;;;;;;;OAUG;IACO,KAAK,CAAC,qBAAqB,CACjC,iBAAiB,GAAG,KAAK,EACzB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,yBAAiB,EAAE,6BAAqB,CAAC,EACnF,SAAS,GAAG,CAAC;QAEb,uDAAuD;QACvD,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,sCAAsC;QACtC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC9B,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC;YAElC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,MAAM;iBACnC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC;iBACnB,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,EAAE,EAAE;gBACrD,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE;oBAC3C,uFAAuF;oBACvF,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAU,CAAC;wBAAE,OAAO;oBAE5H,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;oBACnD,IAAI,CAAC,aAAa,CAAC,IAAA,oBAAY,EAAC,SAAS,CAAC,EAAE;wBACxC,SAAS;wBACT,iBAAiB,EAAE,KAAK;wBACxB,iBAAiB,EAAE,IAAI;wBACvB,SAAS;qBACZ,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;gBAEH,mGAAmG;gBACnG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;gBAElC,OAAO;oBACH,eAAe,EAAE,KAAK,CAAC,MAAM,IAAI,KAAK;oBACtC,SAAS,EAAE,KAAK;oBAChB,eAAe,EAAE,IAAI,IAAI,CAAC,eAAe,CAAC;oBAC1C,cAAc;oBACd,kBAAkB;iBACrB,CAAC;YACN,CAAC,CAAC,CAAC;QACX,CAAC;QAED,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;QAE7H,4DAA4D;QAE5D,2DAA2D;QAC3D,wFAAwF;QACxF,kEAAkE;QAClE,0DAA0D;QAC1D,kDAAkD;QAElD,uFAAuF;QACvF,IAAI,SAAS,IAAI,qCAA4B,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uDAAuD,qCAA4B,GAAG,CAAC,CAAC;QAC7G,CAAC;QACD,MAAM,2BAA2B,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC;eAC7D,eAAe;eACf,SAAS,GAAG,qCAA4B,CAAC;QAEhD,6DAA6D;QAC7D,iGAAiG;QACjG,kFAAkF;QAClF,MAAM,oBAAoB,GAAG,CAAC,cAAc,GAAG,CAAC,eAAe,IAAI,2CAAmC,CAAC;QACvG,MAAM,mBAAmB,GAAG,CAAC,kBAAkB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,mBAAmB,CAAC;QACtG,uFAAuF;QACvF,MAAM,0BAA0B,GAAG,iBAAiB,IAAI,CAAC,oBAAoB,IAAI,CAAC,mBAAmB,CAAC;QAEtG,6DAA6D;QAC7D,IAAI,CAAC,2BAA2B,IAAI,CAAC,0BAA0B;YAAE,OAAO,IAAI,CAAC;QAE7E,yGAAyG;QACzG,0GAA0G;QAC1G,IAAI,CAAC,2BAA2B,IAAI,SAAS,GAAG,mCAA2B;YAAE,OAAO,KAAK,CAAC;QAE1F,MAAM,SAAS,GAAG,2BAA2B;YACzC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC;QAEhB,+DAA+D;QAC/D,IAAI,0BAA0B,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,2CAAmC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC1F,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,WAAW,oFAAoF,CAAC,CAAC;YAC9H,MAAM,IAAA,qBAAK,EAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,SAAS,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,2CAA2C;IAClC,KAAK,CAAC,UAAU;QACrB,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;YACjE,MAAM,OAAO,GAAG,2CAA2C,IAAI,CAAC,qBAAqB,GAAG,GAAG,8BAA8B,CAAC;YAC1H,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/E,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAChG,CAAC;IAEQ,KAAK,CAAC,UAAU,CAAC,GAAG,IAA+C;QACxE,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAEQ,KAAK,CAAC,WAAW,CAAC,GAAG,IAAgD;QAC1E,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,CAAC;IACtC,CAAC;IAEQ,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAuD;QACxF,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC,CAAC;IAC7C,CAAC;IAEQ,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAuD;QACxF,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC,CAAC;IAC7C,CAAC;IAEQ,KAAK,CAAC,cAAc,CAAC,GAAG,IAAmD;QAChF,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC;IACzC,CAAC;IAEkB,MAAM;QACrB,KAAK,CAAC,MAAM,EAAE,CAAC;QACf,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,IAAI,CAAC,GAAG,IAA6C;QACvE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAA0B,CAAC;IACxD,CAAC;CACJ;AA7QD,oCA6QC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue_v2.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"request_queue_v2.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,OAAO,KAAK,EAAE,yBAAyB,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAKrD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAY1C,cAAM,YAAa,SAAQ,eAAe;IACtC,OAAO,CAAC,uBAAuB,CAA8B;gBAEjD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAkC;IAmBrF;;OAEG;cACgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAcvG;;;;;;;;;;;;;;;;OAgBG;IACY,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAgEjF,cAAc,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC,GAAG,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;cAmBpH,oBAAoB;YAoBtB,gBAAgB;YAqBhB,mBAAmB;YAoFnB,mBAAmB;cAgBd,MAAM;cAON,2BAA2B;cAI9B,mBAAmB;WAgBb,IAAI,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;CAGtG;AAED,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,CAAC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.RequestQueueV2 = void 0;
|
|
4
|
+
const access_checking_1 = require("./access_checking");
|
|
4
5
|
const request_provider_1 = require("./request_provider");
|
|
5
6
|
const utils_1 = require("./utils");
|
|
6
7
|
const configuration_1 = require("../configuration");
|
|
@@ -38,6 +39,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
38
39
|
* Caches information about request to beware of unneeded addRequest() calls.
|
|
39
40
|
*/
|
|
40
41
|
_cacheRequest(cacheKey, queueOperationInfo) {
|
|
42
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
41
43
|
super._cacheRequest(cacheKey, queueOperationInfo);
|
|
42
44
|
this.requestCache.add(queueOperationInfo.requestId, {
|
|
43
45
|
id: queueOperationInfo.requestId,
|
|
@@ -65,6 +67,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
65
67
|
* Returns the request object or `null` if there are no more pending requests.
|
|
66
68
|
*/
|
|
67
69
|
async fetchNextRequest() {
|
|
70
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
68
71
|
await this.ensureHeadIsNonEmpty();
|
|
69
72
|
const nextRequestId = this.queueHeadIds.removeFirst();
|
|
70
73
|
// We are likely done at this point.
|
|
@@ -115,6 +118,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
115
118
|
return request;
|
|
116
119
|
}
|
|
117
120
|
async reclaimRequest(...args) {
|
|
121
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
118
122
|
const res = await super.reclaimRequest(...args);
|
|
119
123
|
if (res) {
|
|
120
124
|
const [request, options] = args;
|
|
@@ -129,6 +133,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
129
133
|
return res;
|
|
130
134
|
}
|
|
131
135
|
async ensureHeadIsNonEmpty() {
|
|
136
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
132
137
|
// Stop fetching if we are paused for migration
|
|
133
138
|
if (this.queuePausedForMigration) {
|
|
134
139
|
return;
|
|
@@ -143,6 +148,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
143
148
|
await this._listHeadAndLockPromise;
|
|
144
149
|
}
|
|
145
150
|
async _listHeadAndLock() {
|
|
151
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
146
152
|
const headData = await this.client.listAndLockHead({ limit: 25, lockSecs: this.requestLockSecs });
|
|
147
153
|
for (const { id, uniqueKey } of headData.items) {
|
|
148
154
|
// Queue head index might be behind the main table, so ensure we don't recycle requests
|
|
@@ -159,6 +165,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
159
165
|
}
|
|
160
166
|
}
|
|
161
167
|
async getOrHydrateRequest(requestId) {
|
|
168
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
162
169
|
const cachedEntry = this.requestCache.get(requestId);
|
|
163
170
|
if (!cachedEntry) {
|
|
164
171
|
// 2.1. Attempt to prolong the request lock to see if we still own the request
|
|
@@ -224,6 +231,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
224
231
|
return hydratedRequest;
|
|
225
232
|
}
|
|
226
233
|
async _prolongRequestLock(requestId) {
|
|
234
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
227
235
|
try {
|
|
228
236
|
const res = await this.client.prolongRequestLock(requestId, { lockSecs: this.requestLockSecs });
|
|
229
237
|
return res.lockExpiresAt;
|
|
@@ -237,6 +245,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
237
245
|
}
|
|
238
246
|
}
|
|
239
247
|
_reset() {
|
|
248
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
240
249
|
super._reset();
|
|
241
250
|
this._listHeadAndLockPromise = null;
|
|
242
251
|
}
|
|
@@ -244,6 +253,7 @@ class RequestQueue extends request_provider_1.RequestProvider {
|
|
|
244
253
|
// Do nothing for request queue v2, as we are only able to lock requests when listing the head
|
|
245
254
|
}
|
|
246
255
|
async _clearPossibleLocks() {
|
|
256
|
+
(0, access_checking_1.checkStorageAccess)();
|
|
247
257
|
this.queuePausedForMigration = true;
|
|
248
258
|
let requestId;
|
|
249
259
|
// eslint-disable-next-line no-cond-assign
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue_v2.js","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"request_queue_v2.js","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":";;;AAEA,uDAAuD;AAEvD,yDAAqD;AACrD,mCAGiB;AACjB,oDAAiD;AAIjD,gHAAgH;AAChH,MAAM,mBAAmB,GAAG,OAAS,CAAC;AAEtC;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC,MAAM,YAAa,SAAQ,kCAAe;IAGtC,YAAY,OAA+B,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE;QACjF,KAAK,CAAC;YACF,GAAG,OAAO;YACV,SAAS,EAAE,eAAe;YAC1B,8BAA8B,EAAE,2BAA2B;YAC3D,mBAAmB,EAAE,mBAAmB;SAC3C,EAAE,MAAM,CAAC,CAAC;QARP;;;;mBAAgD,IAAI;WAAC;QAUzD,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAE9C,YAAY,CAAC,EAAE,wCAAsB,KAAK,IAAI,EAAE;YAC5C,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,YAAY,CAAC,EAAE,sCAAqB,KAAK,IAAI,EAAE;YAC3C,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACgB,aAAa,CAAC,QAAgB,EAAE,kBAA6C;QAC5F,IAAA,oCAAkB,GAAE,CAAC;QAErB,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAElD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,kBAAkB,CAAC,SAAS,EAAE;YAChD,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,QAAQ,EAAE,IAAI;YACd,aAAa,EAAE,IAAI;SACtB,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACM,KAAK,CAAC,gBAAgB;QAC3B,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAEtD,oCAAoC;QACpC,IAAI,CAAC,aAAa,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,mCAAmC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;YAC7F,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6DAA6D,EAAE;gBAC5E,aAAa;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC9C,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC;aAC1E,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAEnC,IAAI,OAAuB,CAAC;QAE5B,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,qFAAqF;YACrF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtC,MAAM,CAAC,CAAC;QACZ,CAAC;QAED,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uFAAuF,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YAE3H,UAAU,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAC1C,CAAC,EAAE,wCAAgC,CAAC,CAAC;YAErC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC3D,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEQ,KAAK,CAAC,cAAc,CAAC,GAAG,IAAmD;QAChF,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC;QAEhD,IAAI,GAAG,EAAE,CAAC;YACN,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;YAEhC,6CAA6C;YAC7C,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,KAAK,EAAE,CAAC,CAAC;YACjG,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,6CAA6C,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACvF,CAAC;QACL,CAAC;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAES,KAAK,CAAC,oBAAoB;QAChC,IAAA,oCAAkB,GAAE,CAAC;QAErB,+CAA+C;QAC/C,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO;QACX,CAAC;QAED,uDAAuD;QACvD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,uBAAuB,KAA5B,IAAI,CAAC,uBAAuB,GAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAClE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,EAAC;QAEH,MAAM,IAAI,CAAC,uBAAuB,CAAC;IACvC,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAElG,KAAK,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC7C,uFAAuF;YACvF,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC5F,SAAS;YACb,CAAC;YAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,aAAa,CAAC,IAAA,oBAAY,EAAC,SAAS,CAAC,EAAE;gBACxC,SAAS,EAAE,EAAE;gBACb,SAAS;gBACT,iBAAiB,EAAE,IAAI;gBACvB,iBAAiB,EAAE,KAAK;aAC3B,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAoC,SAAiB;QAClF,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAErD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,8EAA8E;YAC9E,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;YAEhE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,0DAA0D;YAC1D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAEzD,sIAAsI;YACtI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnB,8EAA8E;gBAC9E,4CAA4C;gBAC5C,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;gBACnD,CAAC;gBAAC,MAAM,CAAC;oBACL,SAAS;gBACb,CAAC;gBAED,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC7B,EAAE,EAAE,SAAS;gBACb,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,QAAQ,EAAE,eAAe;gBACzB,SAAS,EAAE,eAAe,CAAC,SAAS,KAAK,IAAI;gBAC7C,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE;aACzC,CAAC,CAAC;YAEH,OAAO,eAAe,CAAC;QAC3B,CAAC;QAED,wDAAwD;QACxD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;YACvB,mIAAmI;YACnI,IAAI,WAAW,CAAC,aAAa,IAAI,WAAW,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;gBACtE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gBAEjE,IAAI,CAAC,SAAS,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBAChB,CAAC;gBAED,WAAW,CAAC,aAAa,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;YACpD,CAAC;YAED,OAAO,WAAW,CAAC,QAAQ,CAAC;QAChC,CAAC;QAED,iHAAiH;QACjH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAEjE,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,4FAA4F;QAC5F,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAE9D,WAAW,CAAC,QAAQ,GAAG,eAAe,CAAC;QAEvC,sIAAsI;QACtI,IAAI,CAAC,eAAe,EAAE,CAAC;YACnB,8EAA8E;YAC9E,4CAA4C;YAC5C,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YACxD,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;YAED,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,eAAe,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,SAAiB;QAC/C,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;YAChG,OAAO,GAAG,CAAC,aAAa,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,6CAA6C;YAC7C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6CAA6C,SAAS,6DAA6D,EAAE;gBAClI,GAAG;aACN,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAEkB,MAAM;QACrB,IAAA,oCAAkB,GAAE,CAAC;QAErB,KAAK,CAAC,MAAM,EAAE,CAAC;QACf,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;IACxC,CAAC;IAEkB,2BAA2B;QAC1C,8FAA8F;IAClG,CAAC;IAES,KAAK,CAAC,mBAAmB;QAC/B,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACpC,IAAI,SAAwB,CAAC;QAE7B,0CAA0C;QAC1C,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5D,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACnD,CAAC;YAAC,MAAM,CAAC;gBACL,gFAAgF;YACpF,CAAC;QACL,CAAC;IACL,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,IAAI,CAAC,GAAG,IAA6C;QACvE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAA0B,CAAC;IACxD,CAAC;CACJ;AAEwB,sCAAc"}
|