@webda/gcp 2.0.0-beta.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.d.ts +4 -1
- package/lib/index.js +4 -17
- package/lib/index.js.map +1 -1
- package/lib/index.spec.d.ts +1 -1
- package/lib/index.spec.js +4 -22
- package/lib/index.spec.js.map +1 -1
- package/lib/services/firestore.d.ts +1 -1
- package/lib/services/firestore.js +207 -255
- package/lib/services/firestore.js.map +1 -1
- package/lib/services/firestore.spec.js +132 -169
- package/lib/services/firestore.spec.js.map +1 -1
- package/lib/services/index.d.ts +1 -1
- package/lib/services/index.js +1 -17
- package/lib/services/index.js.map +1 -1
- package/lib/services/pubsub.d.ts +1 -1
- package/lib/services/pubsub.js +19 -39
- package/lib/services/pubsub.js.map +1 -1
- package/lib/services/pubsub.spec.js +45 -58
- package/lib/services/pubsub.spec.js.map +1 -1
- package/lib/services/queue.d.ts +1 -1
- package/lib/services/queue.js +63 -89
- package/lib/services/queue.js.map +1 -1
- package/lib/services/queue.spec.js +75 -92
- package/lib/services/queue.spec.js.map +1 -1
- package/lib/services/storage.d.ts +2 -0
- package/lib/services/storage.js +93 -126
- package/lib/services/storage.js.map +1 -1
- package/lib/services/storage.spec.js +113 -169
- package/lib/services/storage.spec.js.map +1 -1
- package/package.json +14 -11
- package/webda.module.json +216 -74
|
@@ -1,73 +1,60 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
2
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
3
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
4
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
5
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
6
|
};
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
basic() {
|
|
25
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
26
|
-
let pubsub = this.webda.getService("pubsub");
|
|
27
|
-
let counter = 0;
|
|
28
|
-
let consumers = [];
|
|
29
|
-
let subscription;
|
|
30
|
-
yield new Promise((resolve, reject) => {
|
|
31
|
-
consumers.push(pubsub.consume((evt) => __awaiter(this, void 0, void 0, function* () {
|
|
32
|
-
counter++;
|
|
33
|
-
})));
|
|
34
|
-
consumers.push(pubsub.consume((evt) => __awaiter(this, void 0, void 0, function* () {
|
|
35
|
-
counter++;
|
|
36
|
-
throw new Error("Should not fail");
|
|
37
|
-
}), undefined, sub => {
|
|
38
|
-
subscription = sub;
|
|
39
|
-
resolve();
|
|
40
|
-
}));
|
|
41
|
-
});
|
|
42
|
-
yield pubsub.sendMessage("plop");
|
|
43
|
-
yield (0, core_1.WaitFor)((resolve) => __awaiter(this, void 0, void 0, function* () {
|
|
44
|
-
if (counter === 2) {
|
|
45
|
-
resolve();
|
|
46
|
-
return true;
|
|
47
|
-
}
|
|
48
|
-
return false;
|
|
49
|
-
}), 10, "Events", undefined, (0, core_1.WaitLinearDelay)(1000));
|
|
50
|
-
// We might have concurrence in unit test
|
|
51
|
-
assert.ok(counter >= 2);
|
|
52
|
-
subscription.emit("error", "Fake server error");
|
|
53
|
-
yield assert.rejects(() => consumers[1], /Fake server error/);
|
|
54
|
-
yield consumers[0].cancel();
|
|
55
|
-
// Hack our way to test exception within the main loop
|
|
56
|
-
let stub = sinon.stub(pubsub.pubsub, "subscription").callsFake(() => {
|
|
57
|
-
throw new Error("Bad code?");
|
|
58
|
-
});
|
|
59
|
-
// Should reject
|
|
60
|
-
yield assert.rejects(() => pubsub.consume((evt) => __awaiter(this, void 0, void 0, function* () {
|
|
7
|
+
import { suite, test } from "@testdeck/mocha";
|
|
8
|
+
import { WaitFor, WaitLinearDelay } from "@webda/core";
|
|
9
|
+
import { WebdaTest } from "@webda/core/lib/test.js";
|
|
10
|
+
import * as assert from "assert";
|
|
11
|
+
import * as sinon from "sinon";
|
|
12
|
+
let GCPPubSubTest = class GCPPubSubTest extends WebdaTest {
|
|
13
|
+
async basic() {
|
|
14
|
+
let pubsub = this.webda.getService("pubsub");
|
|
15
|
+
let counter = 0;
|
|
16
|
+
let consumers = [];
|
|
17
|
+
let subscription;
|
|
18
|
+
await new Promise((resolve, reject) => {
|
|
19
|
+
consumers.push(pubsub.consume(async (evt) => {
|
|
20
|
+
counter++;
|
|
21
|
+
}));
|
|
22
|
+
consumers.push(pubsub.consume(async (evt) => {
|
|
61
23
|
counter++;
|
|
62
|
-
|
|
63
|
-
|
|
24
|
+
throw new Error("Should not fail");
|
|
25
|
+
}, undefined, sub => {
|
|
26
|
+
subscription = sub;
|
|
27
|
+
resolve();
|
|
28
|
+
}));
|
|
29
|
+
});
|
|
30
|
+
await pubsub.sendMessage("plop");
|
|
31
|
+
await WaitFor(async (resolve) => {
|
|
32
|
+
if (counter === 2) {
|
|
33
|
+
resolve();
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
return false;
|
|
37
|
+
}, 10, "Events", undefined, WaitLinearDelay(1000));
|
|
38
|
+
// We might have concurrence in unit test
|
|
39
|
+
assert.ok(counter >= 2);
|
|
40
|
+
subscription.emit("error", "Fake server error");
|
|
41
|
+
await assert.rejects(() => consumers[1], /Fake server error/);
|
|
42
|
+
await consumers[0].cancel();
|
|
43
|
+
// Hack our way to test exception within the main loop
|
|
44
|
+
let stub = sinon.stub(pubsub.pubsub, "subscription").callsFake(() => {
|
|
45
|
+
throw new Error("Bad code?");
|
|
64
46
|
});
|
|
47
|
+
// Should reject
|
|
48
|
+
await assert.rejects(() => pubsub.consume(async (evt) => {
|
|
49
|
+
counter++;
|
|
50
|
+
}), /Bad code\?/);
|
|
51
|
+
stub.restore();
|
|
65
52
|
}
|
|
66
53
|
};
|
|
67
54
|
__decorate([
|
|
68
|
-
|
|
55
|
+
test
|
|
69
56
|
], GCPPubSubTest.prototype, "basic", null);
|
|
70
57
|
GCPPubSubTest = __decorate([
|
|
71
|
-
|
|
58
|
+
suite
|
|
72
59
|
], GCPPubSubTest);
|
|
73
60
|
//# sourceMappingURL=pubsub.spec.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pubsub.spec.js","sourceRoot":"","sources":["../../src/services/pubsub.spec.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"pubsub.spec.js","sourceRoot":"","sources":["../../src/services/pubsub.spec.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAqB,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAI/B,IAAM,aAAa,GAAnB,MAAM,aAAc,SAAQ,SAAS;IAEnC,KAAK,CAAC,KAAK;QACT,IAAI,MAAM,GAAqB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAmB,QAAQ,CAAC,CAAC;QACjF,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAwB,EAAE,CAAC;QACxC,IAAI,YAAY,CAAC;QACjB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,SAAS,CAAC,IAAI,CACZ,MAAM,CAAC,OAAO,CAAC,KAAK,EAAC,GAAG,EAAC,EAAE;gBACzB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CACH,CAAC;YACF,SAAS,CAAC,IAAI,CACZ,MAAM,CAAC,OAAO,CACZ,KAAK,EAAC,GAAG,EAAC,EAAE;gBACV,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACrC,CAAC,EACD,SAAS,EACT,GAAG,CAAC,EAAE;gBACJ,YAAY,GAAG,GAAG,CAAC;gBACnB,OAAO,EAAE,CAAC;YACZ,CAAC,CACF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,OAAO,CACX,KAAK,EAAC,OAAO,EAAC,EAAE;YACd,IAAI,OAAO,KAAK,CAAC,EAAE;gBACjB,OAAO,EAAE,CAAC;gBACV,OAAO,IAAI,CAAC;aACb;YACD,OAAO,KAAK,CAAC;QACf,CAAC,EACD,EAAE,EACF,QAAQ,EACR,SAAS,EACT,eAAe,CAAC,IAAI,CAAC,CACtB,CAAC;QACF,yCAAyC;QACzC,MAAM,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;QACxB,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;QAChD,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAC9D,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC5B,sDAAsD;QACtD,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE;YAClE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QACH,gBAAgB;QAChB,MAAM,MAAM,CAAC,OAAO,CAClB,GAAG,EAAE,CACH,MAAM,CAAC,OAAO,CAAC,KAAK,EAAC,GAAG,EAAC,EAAE;YACzB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,EACJ,YAAY,CACb,CAAC;QACF,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;CACF,CAAA;AA1DC;IADC,IAAI;0CA0DJ;AA3DG,aAAa;IADlB,KAAK;GACA,aAAa,CA4DlB"}
|
package/lib/services/queue.d.ts
CHANGED
package/lib/services/queue.js
CHANGED
|
@@ -1,29 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.GCPQueue = exports.GCPQueueParameters = void 0;
|
|
13
|
-
const pubsub_1 = require("@google-cloud/pubsub");
|
|
14
|
-
const core_1 = require("@webda/core");
|
|
1
|
+
import { PubSub } from "@google-cloud/pubsub";
|
|
2
|
+
import { CancelablePromise, Queue, QueueParameters } from "@webda/core";
|
|
15
3
|
/**
|
|
16
4
|
* GCPQueue Parameters
|
|
17
5
|
*/
|
|
18
|
-
class GCPQueueParameters extends
|
|
6
|
+
export class GCPQueueParameters extends QueueParameters {
|
|
19
7
|
}
|
|
20
|
-
exports.GCPQueueParameters = GCPQueueParameters;
|
|
21
8
|
/**
|
|
22
9
|
* GCP Queue implementation on top of Pub/Sub
|
|
23
10
|
*
|
|
24
11
|
* @WebdaModda GoogleCloudQueue
|
|
25
12
|
*/
|
|
26
|
-
class GCPQueue extends
|
|
13
|
+
export default class GCPQueue extends Queue {
|
|
27
14
|
constructor() {
|
|
28
15
|
super(...arguments);
|
|
29
16
|
this.messages = {};
|
|
@@ -37,95 +24,83 @@ class GCPQueue extends core_1.Queue {
|
|
|
37
24
|
/**
|
|
38
25
|
* @override
|
|
39
26
|
*/
|
|
40
|
-
init() {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
return
|
|
45
|
-
yield _super.init.call(this);
|
|
46
|
-
this.pubsub = new pubsub_1.PubSub();
|
|
47
|
-
this.projectId = this.pubsub.auth.getProjectId();
|
|
48
|
-
});
|
|
27
|
+
async init() {
|
|
28
|
+
await super.init();
|
|
29
|
+
this.pubsub = new PubSub();
|
|
30
|
+
this.projectId = this.pubsub.auth.getProjectId();
|
|
31
|
+
return this;
|
|
49
32
|
}
|
|
50
33
|
/**
|
|
51
34
|
* The queue size is not available within this service
|
|
52
35
|
*
|
|
53
36
|
* @returns
|
|
54
37
|
*/
|
|
55
|
-
size() {
|
|
56
|
-
return
|
|
57
|
-
return 0;
|
|
58
|
-
});
|
|
38
|
+
async size() {
|
|
39
|
+
return 0;
|
|
59
40
|
}
|
|
60
41
|
/**
|
|
61
42
|
* Acknowledge a message
|
|
62
43
|
* @param id
|
|
63
44
|
*/
|
|
64
|
-
deleteMessage(id) {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
}),
|
|
72
|
-
});
|
|
73
|
-
this.messages[id].ack();
|
|
74
|
-
delete this.messages[id];
|
|
45
|
+
async deleteMessage(id) {
|
|
46
|
+
await this.pubsub.auth.request({
|
|
47
|
+
method: "POST",
|
|
48
|
+
url: `https://pubsub.googleapis.com/v1/projects/${await this.projectId}/subscriptions/${this.parameters.subscription}:acknowledge`,
|
|
49
|
+
body: JSON.stringify({
|
|
50
|
+
ackIds: [id],
|
|
51
|
+
}),
|
|
75
52
|
});
|
|
53
|
+
this.messages[id].ack();
|
|
54
|
+
delete this.messages[id];
|
|
76
55
|
}
|
|
77
56
|
/**
|
|
78
57
|
* Send a message to the queue
|
|
79
58
|
* @param msg
|
|
80
59
|
*/
|
|
81
|
-
sendMessage(msg) {
|
|
82
|
-
|
|
83
|
-
yield this.pubsub.topic(this.parameters.topic).publishMessage({ data: Buffer.from(JSON.stringify(msg)) });
|
|
84
|
-
});
|
|
60
|
+
async sendMessage(msg) {
|
|
61
|
+
await this.pubsub.topic(this.parameters.topic).publishMessage({ data: Buffer.from(JSON.stringify(msg)) });
|
|
85
62
|
}
|
|
86
63
|
/**
|
|
87
64
|
* Retrieve just one message from the queue
|
|
88
65
|
* @param proto
|
|
89
66
|
* @returns
|
|
90
67
|
*/
|
|
91
|
-
receiveMessage(proto) {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
timeoutId = setTimeout(() => resolve([]), this.parameters.timeout);
|
|
105
|
-
}
|
|
106
|
-
errorHandler = err => {
|
|
107
|
-
reject(err);
|
|
108
|
-
};
|
|
109
|
-
msgHandler = (message) => {
|
|
110
|
-
this.messages[message.ackId] = message;
|
|
111
|
-
resolve([
|
|
112
|
-
{
|
|
113
|
-
Message: this.unserialize(message.data.toString(), proto),
|
|
114
|
-
ReceiptHandle: message.ackId,
|
|
115
|
-
},
|
|
116
|
-
]);
|
|
117
|
-
};
|
|
118
|
-
subscription.on("error", errorHandler);
|
|
119
|
-
subscription.on("message", msgHandler);
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
|
-
finally {
|
|
123
|
-
subscription.close();
|
|
124
|
-
if (timeoutId) {
|
|
125
|
-
clearTimeout(timeoutId);
|
|
68
|
+
async receiveMessage(proto) {
|
|
69
|
+
let timeoutId;
|
|
70
|
+
let errorHandler;
|
|
71
|
+
let msgHandler;
|
|
72
|
+
const subscription = this.pubsub.subscription(this.parameters.subscription, {
|
|
73
|
+
flowControl: {
|
|
74
|
+
maxMessages: 1,
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
try {
|
|
78
|
+
return await new Promise((resolve, reject) => {
|
|
79
|
+
if (this.parameters.timeout) {
|
|
80
|
+
timeoutId = setTimeout(() => resolve([]), this.parameters.timeout);
|
|
126
81
|
}
|
|
82
|
+
errorHandler = err => {
|
|
83
|
+
reject(err);
|
|
84
|
+
};
|
|
85
|
+
msgHandler = (message) => {
|
|
86
|
+
this.messages[message.ackId] = message;
|
|
87
|
+
resolve([
|
|
88
|
+
{
|
|
89
|
+
Message: this.unserialize(message.data.toString(), proto),
|
|
90
|
+
ReceiptHandle: message.ackId,
|
|
91
|
+
},
|
|
92
|
+
]);
|
|
93
|
+
};
|
|
94
|
+
subscription.on("error", errorHandler);
|
|
95
|
+
subscription.on("message", msgHandler);
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
finally {
|
|
99
|
+
subscription.close();
|
|
100
|
+
if (timeoutId) {
|
|
101
|
+
clearTimeout(timeoutId);
|
|
127
102
|
}
|
|
128
|
-
}
|
|
103
|
+
}
|
|
129
104
|
}
|
|
130
105
|
/**
|
|
131
106
|
* Work a queue calling the callback with every Event received
|
|
@@ -139,21 +114,20 @@ class GCPQueue extends core_1.Queue {
|
|
|
139
114
|
maxMessages: this.getMaxConsumers(),
|
|
140
115
|
},
|
|
141
116
|
});
|
|
142
|
-
return new
|
|
143
|
-
subscription.on("message", (message) =>
|
|
117
|
+
return new CancelablePromise(async () => {
|
|
118
|
+
subscription.on("message", async (message) => {
|
|
144
119
|
try {
|
|
145
|
-
|
|
120
|
+
await callback(this.unserialize(message.data.toString(), eventPrototype));
|
|
146
121
|
message.ack();
|
|
147
122
|
}
|
|
148
123
|
catch (err) {
|
|
149
124
|
this.log("ERROR", `Message ${message.ackId}`, err);
|
|
150
125
|
}
|
|
151
|
-
})
|
|
152
|
-
}
|
|
126
|
+
});
|
|
127
|
+
}, async () => {
|
|
153
128
|
subscription.close();
|
|
154
|
-
})
|
|
129
|
+
});
|
|
155
130
|
}
|
|
156
131
|
}
|
|
157
|
-
|
|
158
|
-
exports.GCPQueue = GCPQueue;
|
|
132
|
+
export { GCPQueue };
|
|
159
133
|
//# sourceMappingURL=queue.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../src/services/queue.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../src/services/queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAA+B,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAErG;;GAEG;AACH,MAAM,OAAO,kBAAmB,SAAQ,eAAe;CAiBtD;AAED;;;;GAIG;AACH,MAAM,CAAC,OAAO,OAAO,QAAqE,SAAQ,KAAW;IAA7G;;QASE,aAAQ,GAA+B,EAAE,CAAC;IA2H5C,CAAC;IA1HC;;OAEG;IACH,cAAc,CAAC,MAAsB;QACnC,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,OAAO,CAAC,CAAC;IACX,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa,CAAC,EAAU;QAC5B,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,MAAM;YACd,GAAG,EAAE,6CAA6C,MAAM,IAAI,CAAC,SAAS,kBACpE,IAAI,CAAC,UAAU,CAAC,YAClB,cAAc;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,MAAM,EAAE,CAAC,EAAE,CAAC;aACb,CAAC;SACH,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,GAAM;QACtB,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAI,KAAmB;QACzC,IAAI,SAAS,CAAC;QACd,IAAI,YAAY,CAAC;QACjB,IAAI,UAAU,CAAC;QACf,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;YAC1E,WAAW,EAAE;gBACX,WAAW,EAAE,CAAC;aACf;SACF,CAAC,CAAC;QACH,IAAI;YACF,OAAO,MAAM,IAAI,OAAO,CAAsB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAChE,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE;oBAC3B,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;iBACpE;gBACD,YAAY,GAAG,GAAG,CAAC,EAAE;oBACnB,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC,CAAC;gBACF,UAAU,GAAG,CAAC,OAAgB,EAAE,EAAE;oBAChC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;oBACvC,OAAO,CAAC;wBACN;4BACE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC;4BACzD,aAAa,EAAE,OAAO,CAAC,KAAK;yBAC7B;qBACF,CAAC,CAAC;gBACL,CAAC,CAAC;gBACF,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;gBACvC,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;SACJ;gBAAS;YACR,YAAY,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,SAAS,EAAE;gBACb,YAAY,CAAC,SAAS,CAAC,CAAC;aACzB;SACF;IACH,CAAC;IAED;;;;;OAKG;IACH,OAAO,CAAC,QAAqC,EAAE,cAA8B;QAC3E,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;YAC1E,WAAW,EAAE;gBACX,WAAW,EAAE,IAAI,CAAC,eAAe,EAAE;aACpC;SACF,CAAC,CAAC;QACH,OAAO,IAAI,iBAAiB,CAC1B,KAAK,IAAI,EAAE;YACT,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAAgB,EAAE,EAAE;gBACpD,IAAI;oBACF,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;oBAC1E,OAAO,CAAC,GAAG,EAAE,CAAC;iBACf;gBAAC,OAAO,GAAG,EAAE;oBACZ,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;iBACpD;YACH,CAAC,CAAC,CAAC;QACL,CAAC,EACD,KAAK,IAAI,EAAE;YACT,YAAY,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AAED,OAAO,EAAE,QAAQ,EAAE,CAAC"}
|
|
@@ -1,120 +1,103 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
2
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
3
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
4
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
5
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
6
|
};
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
15
|
-
});
|
|
16
|
-
};
|
|
17
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
-
const mocha_1 = require("@testdeck/mocha");
|
|
19
|
-
const queue_spec_1 = require("@webda/core/lib/queues/queue.spec");
|
|
20
|
-
const assert = require("assert");
|
|
21
|
-
const events_1 = require("events");
|
|
22
|
-
const sinon = require("sinon");
|
|
23
|
-
class FakeSubscription extends events_1.EventEmitter {
|
|
7
|
+
import { suite, test } from "@testdeck/mocha";
|
|
8
|
+
import { QueueTest } from "@webda/core/lib/queues/queue.spec.js";
|
|
9
|
+
import * as assert from "assert";
|
|
10
|
+
import { EventEmitter } from "events";
|
|
11
|
+
import * as sinon from "sinon";
|
|
12
|
+
class FakeSubscription extends EventEmitter {
|
|
24
13
|
emitError() {
|
|
25
14
|
this.emit("error", "Fake error");
|
|
26
15
|
}
|
|
27
16
|
close() { }
|
|
28
17
|
}
|
|
29
|
-
let GCPQueueTest = class GCPQueueTest extends
|
|
30
|
-
basic() {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
enableExactlyOnceDelivery: true,
|
|
37
|
-
});
|
|
38
|
-
try {
|
|
39
|
-
yield this.simple(queue, true, 12000);
|
|
40
|
-
// Seems to be penalized by no promise on ACK
|
|
41
|
-
yield this.sleep(5000);
|
|
42
|
-
this.log("DEBUG", "Verify receiveMessage is now empty");
|
|
43
|
-
queue.getParameters().timeout = 3000;
|
|
44
|
-
// TODO Investigate this one
|
|
45
|
-
if ((yield queue.receiveMessage()).length > 0) {
|
|
46
|
-
this.log("ERROR", "Queue should be empty - not an assert to avoid random");
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
finally {
|
|
50
|
-
yield queue.pubsub.subscription(queue.getParameters().subscription).delete();
|
|
51
|
-
}
|
|
18
|
+
let GCPQueueTest = class GCPQueueTest extends QueueTest {
|
|
19
|
+
async basic() {
|
|
20
|
+
let queue = this.webda.getService("queue");
|
|
21
|
+
queue.getParameters().subscription = `${queue.getParameters().subscription}_${this.webda.getUuid()}`;
|
|
22
|
+
await queue.pubsub.createSubscription(queue.getParameters().topic, queue.getParameters().subscription, {
|
|
23
|
+
ackDeadlineSeconds: 10,
|
|
24
|
+
enableExactlyOnceDelivery: true,
|
|
52
25
|
});
|
|
26
|
+
try {
|
|
27
|
+
await this.simple(queue, true, 12000);
|
|
28
|
+
// Seems to be penalized by no promise on ACK
|
|
29
|
+
await this.sleep(5000);
|
|
30
|
+
this.log("DEBUG", "Verify receiveMessage is now empty");
|
|
31
|
+
queue.getParameters().timeout = 3000;
|
|
32
|
+
// TODO Investigate this one
|
|
33
|
+
if ((await queue.receiveMessage()).length > 0) {
|
|
34
|
+
this.log("ERROR", "Queue should be empty - not an assert to avoid random");
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
finally {
|
|
38
|
+
await queue.pubsub.subscription(queue.getParameters().subscription).delete();
|
|
39
|
+
}
|
|
53
40
|
}
|
|
54
|
-
consumers() {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
throw new Error();
|
|
70
|
-
}
|
|
71
|
-
msg = dt;
|
|
72
|
-
}));
|
|
73
|
-
yield this.sleep(1000);
|
|
74
|
-
this.log("DEBUG", "Consume sendMessage");
|
|
75
|
-
yield queue.sendMessage({ plop: 1 });
|
|
76
|
-
this.log("DEBUG", "Consume cancel");
|
|
77
|
-
yield queue.sendMessage({ plop: 2 });
|
|
78
|
-
// Need to wait before cancel to ensure message are received
|
|
79
|
-
while (consumed < 2) {
|
|
80
|
-
yield this.sleep(500);
|
|
41
|
+
async consumers() {
|
|
42
|
+
let queue = this.webda.getService("queue");
|
|
43
|
+
queue.getParameters().subscription = `${queue.getParameters().subscription}_${this.webda.getUuid()}`;
|
|
44
|
+
await queue.pubsub.createSubscription(queue.getParameters().topic, queue.getParameters().subscription, {
|
|
45
|
+
ackDeadlineSeconds: 10,
|
|
46
|
+
enableMessageOrdering: true,
|
|
47
|
+
});
|
|
48
|
+
try {
|
|
49
|
+
// Test consumer
|
|
50
|
+
let msg;
|
|
51
|
+
let consumed = 0;
|
|
52
|
+
let consumer = queue.consume(async (dt) => {
|
|
53
|
+
consumed++;
|
|
54
|
+
if (msg) {
|
|
55
|
+
throw new Error();
|
|
81
56
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
|
|
57
|
+
msg = dt;
|
|
58
|
+
});
|
|
59
|
+
await this.sleep(1000);
|
|
60
|
+
this.log("DEBUG", "Consume sendMessage");
|
|
61
|
+
await queue.sendMessage({ plop: 1 });
|
|
62
|
+
this.log("DEBUG", "Consume cancel");
|
|
63
|
+
await queue.sendMessage({ plop: 2 });
|
|
64
|
+
// Need to wait before cancel to ensure message are received
|
|
65
|
+
while (consumed < 2) {
|
|
66
|
+
await this.sleep(500);
|
|
91
67
|
}
|
|
92
|
-
|
|
68
|
+
await consumer.cancel();
|
|
69
|
+
this.log("DEBUG", "Consume assert");
|
|
70
|
+
assert.notStrictEqual(msg, undefined);
|
|
71
|
+
// The second message throw an exception so it can come back
|
|
72
|
+
await queue.receiveMessage();
|
|
73
|
+
await queue.receiveMessage();
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
await queue.pubsub.subscription(queue.getParameters().subscription).delete();
|
|
77
|
+
}
|
|
93
78
|
}
|
|
94
|
-
receiveError() {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
return service;
|
|
101
|
-
});
|
|
102
|
-
let consumer = queue.receiveMessage();
|
|
103
|
-
service.emitError();
|
|
104
|
-
yield assert.rejects(() => consumer, /Fake error/);
|
|
79
|
+
async receiveError() {
|
|
80
|
+
this.log("DEBUG", "Receive Error test");
|
|
81
|
+
let queue = this.webda.getService("queue");
|
|
82
|
+
let service = new FakeSubscription();
|
|
83
|
+
sinon.stub(queue.pubsub, "subscription").callsFake(() => {
|
|
84
|
+
return service;
|
|
105
85
|
});
|
|
86
|
+
let consumer = queue.receiveMessage();
|
|
87
|
+
service.emitError();
|
|
88
|
+
await assert.rejects(() => consumer, /Fake error/);
|
|
106
89
|
}
|
|
107
90
|
};
|
|
108
91
|
__decorate([
|
|
109
|
-
|
|
92
|
+
test
|
|
110
93
|
], GCPQueueTest.prototype, "basic", null);
|
|
111
94
|
__decorate([
|
|
112
|
-
|
|
95
|
+
test
|
|
113
96
|
], GCPQueueTest.prototype, "consumers", null);
|
|
114
97
|
__decorate([
|
|
115
|
-
|
|
98
|
+
test
|
|
116
99
|
], GCPQueueTest.prototype, "receiveError", null);
|
|
117
100
|
GCPQueueTest = __decorate([
|
|
118
|
-
|
|
101
|
+
suite
|
|
119
102
|
], GCPQueueTest);
|
|
120
103
|
//# sourceMappingURL=queue.spec.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.spec.js","sourceRoot":"","sources":["../../src/services/queue.spec.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"queue.spec.js","sourceRoot":"","sources":["../../src/services/queue.spec.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,mCAAmC,CAAC;AAC9D,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAG/B,MAAM,gBAAiB,SAAQ,YAAY;IACzC,SAAS;QACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACnC,CAAC;IACD,KAAK,KAAI,CAAC;CACX;AAGD,IAAM,YAAY,GAAlB,MAAM,YAAa,SAAQ,SAAS;IAElC,KAAK,CAAC,KAAK;QACT,IAAI,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,UAAU,CAAW,OAAO,CAAC,CAAC;QAC/D,KAAK,CAAC,aAAa,EAAE,CAAC,YAAY,GAAG,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACrG,MAAM,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC,YAAY,EAAE;YACrG,kBAAkB,EAAE,EAAE;YACtB,yBAAyB,EAAE,IAAI;SAChC,CAAC,CAAC;QACH,IAAI;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YACtC,6CAA6C;YAC7C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,oCAAoC,CAAC,CAAC;YACxD,KAAK,CAAC,aAAa,EAAE,CAAC,OAAO,GAAG,IAAI,CAAC;YACrC,4BAA4B;YAC5B,IAAI,CAAC,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC7C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,uDAAuD,CAAC,CAAC;aAC5E;SACF;gBAAS;YACR,MAAM,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE,CAAC;SAC9E;IACH,CAAC;IAGD,KAAK,CAAC,SAAS;QACb,IAAI,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,UAAU,CAAW,OAAO,CAAC,CAAC;QAC/D,KAAK,CAAC,aAAa,EAAE,CAAC,YAAY,GAAG,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACrG,MAAM,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC,YAAY,EAAE;YACrG,kBAAkB,EAAE,EAAE;YACtB,qBAAqB,EAAE,IAAI;SAC5B,CAAC,CAAC;QACH,IAAI;YACF,gBAAgB;YAChB,IAAI,GAAG,CAAC;YACR,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAC,EAAE,EAAC,EAAE;gBACtC,QAAQ,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE;oBACP,MAAM,IAAI,KAAK,EAAE,CAAC;iBACnB;gBACD,GAAG,GAAG,EAAE,CAAC;YACX,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YACrC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;YACpC,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YACrC,4DAA4D;YAC5D,OAAO,QAAQ,GAAG,CAAC,EAAE;gBACnB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;aACvB;YACD,MAAM,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;YACpC,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YACtC,4DAA4D;YAC5D,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;SAC9B;gBAAS;YACR,MAAM,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE,CAAC;SAC9E;IACH,CAAC;IAGD,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QACxC,IAAI,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,UAAU,CAAW,OAAO,CAAC,CAAC;QAC/D,IAAI,OAAO,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE;YACtD,OAAY,OAAO,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,IAAI,QAAQ,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC;QACtC,OAAO,CAAC,SAAS,EAAE,CAAC;QACpB,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACrD,CAAC;CACF,CAAA;AAzEC;IADC,IAAI;yCAqBJ;AAGD;IADC,IAAI;6CAqCJ;AAGD;IADC,IAAI;gDAWJ;AA1EG,YAAY;IADjB,KAAK;GACA,YAAY,CA2EjB"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
/// <reference types="node" />
|
|
2
4
|
import { Bucket } from "@google-cloud/storage";
|
|
3
5
|
import { BinaryFile, BinaryMap, BinaryParameters, CloudBinary, Context, CoreModel, DeepPartial } from "@webda/core";
|
|
4
6
|
import { Readable } from "stream";
|