@abyss-project/monitor 2.2.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/dist/__tests__/alert-log-url.test.d.ts +1 -0
- package/dist/__tests__/alert-log-url.test.js +21 -0
- package/dist/__tests__/alert-log-url.test.js.map +1 -0
- package/dist/__tests__/application-log-bulk-api.test.d.ts +1 -0
- package/dist/__tests__/application-log-bulk-api.test.js +125 -0
- package/dist/__tests__/application-log-bulk-api.test.js.map +1 -0
- package/dist/__tests__/body-redaction.test.d.ts +1 -0
- package/dist/__tests__/body-redaction.test.js +104 -0
- package/dist/__tests__/body-redaction.test.js.map +1 -0
- package/dist/__tests__/graceful-shutdown-flush.test.d.ts +1 -0
- package/dist/__tests__/graceful-shutdown-flush.test.js +253 -0
- package/dist/__tests__/graceful-shutdown-flush.test.js.map +1 -0
- package/dist/__tests__/header-pattern.test.js +3 -0
- package/dist/__tests__/header-pattern.test.js.map +1 -1
- package/dist/__tests__/log-queue.test.d.ts +1 -0
- package/dist/__tests__/log-queue.test.js +97 -0
- package/dist/__tests__/log-queue.test.js.map +1 -0
- package/dist/__tests__/log-shipper-preconfig.test.d.ts +1 -0
- package/dist/__tests__/log-shipper-preconfig.test.js +54 -0
- package/dist/__tests__/log-shipper-preconfig.test.js.map +1 -0
- package/dist/__tests__/log-shipper.test.d.ts +1 -0
- package/dist/__tests__/log-shipper.test.js +1040 -0
- package/dist/__tests__/log-shipper.test.js.map +1 -0
- package/dist/__tests__/log-spool.test.d.ts +1 -0
- package/dist/__tests__/log-spool.test.js +297 -0
- package/dist/__tests__/log-spool.test.js.map +1 -0
- package/dist/__tests__/logger-body-size.test.d.ts +1 -0
- package/dist/__tests__/logger-body-size.test.js +12 -0
- package/dist/__tests__/logger-body-size.test.js.map +1 -0
- package/dist/__tests__/logger-enqueues.test.d.ts +1 -0
- package/dist/__tests__/logger-enqueues.test.js +276 -0
- package/dist/__tests__/logger-enqueues.test.js.map +1 -0
- package/dist/__tests__/logger-returns-promise.test.d.ts +1 -0
- package/dist/__tests__/logger-returns-promise.test.js +27 -0
- package/dist/__tests__/logger-returns-promise.test.js.map +1 -0
- package/dist/__tests__/route-decorator-body-flags.test.d.ts +1 -0
- package/dist/__tests__/route-decorator-body-flags.test.js +117 -0
- package/dist/__tests__/route-decorator-body-flags.test.js.map +1 -0
- package/dist/__tests__/route-flags-loader.test.d.ts +1 -0
- package/dist/__tests__/route-flags-loader.test.js +153 -0
- package/dist/__tests__/route-flags-loader.test.js.map +1 -0
- package/dist/__tests__/setconfig-keeps-queued-logs.test.d.ts +1 -0
- package/dist/__tests__/setconfig-keeps-queued-logs.test.js +59 -0
- package/dist/__tests__/setconfig-keeps-queued-logs.test.js.map +1 -0
- package/dist/__tests__/shipper-replay-on-config.test.d.ts +1 -0
- package/dist/__tests__/shipper-replay-on-config.test.js +59 -0
- package/dist/__tests__/shipper-replay-on-config.test.js.map +1 -0
- package/dist/__tests__/utils/setup-global.d.ts +1 -0
- package/dist/__tests__/utils/setup-global.js +35 -0
- package/dist/__tests__/utils/setup-global.js.map +1 -0
- package/dist/__tests__/utils/teardown-global.d.ts +1 -0
- package/dist/__tests__/utils/teardown-global.js +36 -0
- package/dist/__tests__/utils/teardown-global.js.map +1 -0
- package/dist/api/application-log.api.d.ts +5 -1
- package/dist/api/application-log.api.js +10 -1
- package/dist/api/application-log.api.js.map +1 -1
- package/dist/frameworks/express/decorators.d.ts +2 -0
- package/dist/frameworks/express/decorators.js +6 -0
- package/dist/frameworks/express/decorators.js.map +1 -1
- package/dist/frameworks/express/index.js +15 -5
- package/dist/frameworks/express/index.js.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.js +22 -0
- package/dist/index.js.map +1 -1
- package/dist/middlewares/logger-endpoint.middleware.js +5 -4
- package/dist/middlewares/logger-endpoint.middleware.js.map +1 -1
- package/dist/models/index.d.ts +3 -0
- package/dist/models/index.js +3 -0
- package/dist/models/index.js.map +1 -1
- package/dist/models/log-queue.model.d.ts +25 -0
- package/dist/models/log-queue.model.js +68 -0
- package/dist/models/log-queue.model.js.map +1 -0
- package/dist/models/log-shipper.model.d.ts +75 -0
- package/dist/models/log-shipper.model.js +452 -0
- package/dist/models/log-shipper.model.js.map +1 -0
- package/dist/models/log-spool.model.d.ts +32 -0
- package/dist/models/log-spool.model.js +202 -0
- package/dist/models/log-spool.model.js.map +1 -0
- package/dist/models/logger.model.d.ts +2 -0
- package/dist/models/logger.model.js +49 -70
- package/dist/models/logger.model.js.map +1 -1
- package/dist/types/interface/api/requests/application-log.request.d.ts +10 -0
- package/dist/types/interface/api/responses/application-log.response.d.ts +5 -0
- package/dist/utils/alert.utils.d.ts +1 -0
- package/dist/utils/alert.utils.js +8 -5
- package/dist/utils/alert.utils.js.map +1 -1
- package/dist/utils/body-redaction.d.ts +9 -0
- package/dist/utils/body-redaction.js +57 -0
- package/dist/utils/body-redaction.js.map +1 -0
- package/dist/utils/graceful-shutdown.utils.d.ts +1 -0
- package/dist/utils/graceful-shutdown.utils.js +53 -8
- package/dist/utils/graceful-shutdown.utils.js.map +1 -1
- package/dist/utils/header-pattern.js +1 -0
- package/dist/utils/header-pattern.js.map +1 -1
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.js +1 -0
- package/dist/utils/index.js.map +1 -1
- package/jest.config.ts +5 -0
- package/package.json +1 -1
|
@@ -0,0 +1,1040 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
const fs = __importStar(require("fs"));
|
|
27
|
+
const os = __importStar(require("os"));
|
|
28
|
+
const path = __importStar(require("path"));
|
|
29
|
+
const uuid_1 = require("uuid");
|
|
30
|
+
const log_shipper_model_1 = require("../models/log-shipper.model");
|
|
31
|
+
const log_spool_model_1 = require("../models/log-spool.model");
|
|
32
|
+
const __1 = require("..");
|
|
33
|
+
const applicationLogApi = __importStar(require("../api/application-log.api"));
|
|
34
|
+
const types_1 = require("../types");
|
|
35
|
+
const APPLICATION_ID = 'app-1';
|
|
36
|
+
const ACK = { data: { created: 1, deduped: 0 } };
|
|
37
|
+
const REFUSED = '[refusé]';
|
|
38
|
+
const makeLog = (message, dateValue = new Date()) => ({ message, level: types_1.LogLevel.LOG, processId: '1', dateValue, clientLogId: (0, uuid_1.v4)() });
|
|
39
|
+
const httpError = (status) => Object.assign(new Error(`HTTP ${status}`), { response: { status } });
|
|
40
|
+
const spooledFiles = (base) => {
|
|
41
|
+
const directory = path.join(base, APPLICATION_ID);
|
|
42
|
+
return fs.existsSync(directory)
|
|
43
|
+
? fs.readdirSync(directory).filter((f) => !f.startsWith('.'))
|
|
44
|
+
: [];
|
|
45
|
+
};
|
|
46
|
+
const settle = () => new Promise((resolve) => setImmediate(resolve));
|
|
47
|
+
const useFakeClock = () => {
|
|
48
|
+
jest.useFakeTimers({
|
|
49
|
+
now: Date.now(),
|
|
50
|
+
doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'],
|
|
51
|
+
});
|
|
52
|
+
};
|
|
53
|
+
const deferred = () => {
|
|
54
|
+
let resolve;
|
|
55
|
+
const promise = new Promise((res) => {
|
|
56
|
+
resolve = res;
|
|
57
|
+
});
|
|
58
|
+
return { promise, resolve };
|
|
59
|
+
};
|
|
60
|
+
const ORIGINAL_SPOOL_DIRECTORY_ENV = process.env[log_shipper_model_1.SPOOL_DIRECTORY_ENV];
|
|
61
|
+
describe('LogShipper', () => {
|
|
62
|
+
let base;
|
|
63
|
+
let sendBulk;
|
|
64
|
+
const shippers = [];
|
|
65
|
+
const messagesOf = (call) => sendBulk.mock.calls[call][2].logs.map((l) => l.message);
|
|
66
|
+
const allShipped = () => sendBulk.mock.calls.flatMap((call) => call[2].logs.map((l) => l.message));
|
|
67
|
+
let accepted = [];
|
|
68
|
+
const refusingMarked = (status) => async (_params, _query, body) => {
|
|
69
|
+
if (body.logs.some((log) => log.message.includes(REFUSED))) {
|
|
70
|
+
throw httpError(status);
|
|
71
|
+
}
|
|
72
|
+
accepted.push(body);
|
|
73
|
+
return ACK;
|
|
74
|
+
};
|
|
75
|
+
const deliveredMessages = () => accepted.flatMap((body) => body.logs.map((log) => log.message)).sort();
|
|
76
|
+
beforeEach(() => {
|
|
77
|
+
accepted = [];
|
|
78
|
+
base = fs.mkdtempSync(path.join(os.tmpdir(), 'abyss-shipper-'));
|
|
79
|
+
__1.AbyssMonitorCore.setConfig({ applicationId: APPLICATION_ID, secretPublishToken: 'secret' });
|
|
80
|
+
sendBulk = jest
|
|
81
|
+
.spyOn(applicationLogApi, 'createBulkApplicationLog')
|
|
82
|
+
.mockResolvedValue({ data: { created: 0, deduped: 0 } });
|
|
83
|
+
jest.spyOn(console, 'error').mockImplementation(() => undefined);
|
|
84
|
+
});
|
|
85
|
+
afterEach(async () => {
|
|
86
|
+
shippers.splice(0).forEach((shipper) => shipper.stop());
|
|
87
|
+
await settle();
|
|
88
|
+
jest.useRealTimers();
|
|
89
|
+
jest.restoreAllMocks();
|
|
90
|
+
fs.rmSync(base, { recursive: true, force: true });
|
|
91
|
+
if (ORIGINAL_SPOOL_DIRECTORY_ENV === undefined) {
|
|
92
|
+
delete process.env[log_shipper_model_1.SPOOL_DIRECTORY_ENV];
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
process.env[log_shipper_model_1.SPOOL_DIRECTORY_ENV] = ORIGINAL_SPOOL_DIRECTORY_ENV;
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
const shipperOn = (options = {}) => {
|
|
99
|
+
const shipper = new log_shipper_model_1.LogShipper({ spoolDirectory: base, flushIntervalMs: 60_000, ...options });
|
|
100
|
+
shippers.push(shipper);
|
|
101
|
+
return shipper;
|
|
102
|
+
};
|
|
103
|
+
it('regroupe plusieurs logs dans un seul POST', async () => {
|
|
104
|
+
const shipper = shipperOn();
|
|
105
|
+
shipper.enqueue(makeLog('a'));
|
|
106
|
+
shipper.enqueue(makeLog('b'));
|
|
107
|
+
await shipper.flush();
|
|
108
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
109
|
+
expect(sendBulk.mock.calls[0][0]).toEqual({ applicationId: APPLICATION_ID });
|
|
110
|
+
expect(sendBulk.mock.calls[0][1]).toEqual({ token: 'secret' });
|
|
111
|
+
expect(sendBulk.mock.calls[0][2].logs).toHaveLength(2);
|
|
112
|
+
expect(sendBulk.mock.calls[0][2].droppedCount).toBeNull();
|
|
113
|
+
});
|
|
114
|
+
it('un lot porte au moins une ligne, quel que soit maxBatchEntries', async () => {
|
|
115
|
+
useFakeClock();
|
|
116
|
+
const shipper = shipperOn({ maxBatchEntries: 0 });
|
|
117
|
+
shipper.enqueue(makeLog('a'));
|
|
118
|
+
shipper.enqueue(makeLog('b'));
|
|
119
|
+
await shipper.flush();
|
|
120
|
+
expect(messagesOf(0)).toEqual(['a']);
|
|
121
|
+
expect(messagesOf(1)).toEqual(['b']);
|
|
122
|
+
});
|
|
123
|
+
it('une file garde au moins une ligne, quel que soit maxQueueEntries', async () => {
|
|
124
|
+
const shipper = shipperOn({ maxQueueEntries: 0 });
|
|
125
|
+
shipper.enqueue(makeLog('a'));
|
|
126
|
+
shipper.enqueue(makeLog('b'));
|
|
127
|
+
await shipper.flush();
|
|
128
|
+
expect(allShipped()).toEqual(['b']);
|
|
129
|
+
expect(sendBulk.mock.calls[0][2].droppedCount).toBe(1);
|
|
130
|
+
});
|
|
131
|
+
it('remonte le nombre de logs jetés par la file dans le lot suivant', async () => {
|
|
132
|
+
const shipper = shipperOn({ maxQueueEntries: 1 });
|
|
133
|
+
shipper.enqueue(makeLog('a'));
|
|
134
|
+
shipper.enqueue(makeLog('b'));
|
|
135
|
+
await shipper.flush();
|
|
136
|
+
expect(sendBulk.mock.calls[0][2].droppedCount).toBe(1);
|
|
137
|
+
});
|
|
138
|
+
it('supprime le fichier de spool après un ACK', async () => {
|
|
139
|
+
const shipper = shipperOn();
|
|
140
|
+
shipper.enqueue(makeLog('a'));
|
|
141
|
+
await shipper.flush();
|
|
142
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
143
|
+
});
|
|
144
|
+
it('garde le lot sur disque sur un échec transitoire', async () => {
|
|
145
|
+
sendBulk.mockRejectedValue(httpError(503));
|
|
146
|
+
const shipper = shipperOn();
|
|
147
|
+
shipper.enqueue(makeLog('a'));
|
|
148
|
+
await shipper.flush();
|
|
149
|
+
expect(spooledFiles(base)).toHaveLength(1);
|
|
150
|
+
});
|
|
151
|
+
it('jette un lot refusé définitivement, et le dit dans le lot suivant', async () => {
|
|
152
|
+
sendBulk.mockImplementation(refusingMarked(400));
|
|
153
|
+
const shipper = shipperOn();
|
|
154
|
+
shipper.enqueue(makeLog(`a ${REFUSED}`));
|
|
155
|
+
shipper.enqueue(makeLog(`b ${REFUSED}`));
|
|
156
|
+
await shipper.flush();
|
|
157
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
158
|
+
shipper.enqueue(makeLog('c'));
|
|
159
|
+
await shipper.flush();
|
|
160
|
+
expect(sendBulk).toHaveBeenCalledTimes(4);
|
|
161
|
+
expect(messagesOf(3)).toEqual(['c']);
|
|
162
|
+
expect(sendBulk.mock.calls[3][2].droppedCount).toBe(2);
|
|
163
|
+
});
|
|
164
|
+
it("un lot refusé pour une seule ligne livre les autres et ne jette qu'elle", async () => {
|
|
165
|
+
sendBulk.mockImplementation(refusingMarked(400));
|
|
166
|
+
const shipper = shipperOn();
|
|
167
|
+
['a', 'b', `c ${REFUSED}`, 'd'].forEach((message) => shipper.enqueue(makeLog(message)));
|
|
168
|
+
await shipper.flush();
|
|
169
|
+
expect(deliveredMessages()).toEqual(['a', 'b', 'd']);
|
|
170
|
+
expect(sendBulk.mock.calls.map((call) => call[2].logs.length)).toEqual([4, 2, 2, 1, 1]);
|
|
171
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
172
|
+
shipper.enqueue(makeLog('e'));
|
|
173
|
+
await shipper.flush();
|
|
174
|
+
expect(messagesOf(5)).toEqual(['e']);
|
|
175
|
+
expect(sendBulk.mock.calls[5][2].droppedCount).toBe(1);
|
|
176
|
+
});
|
|
177
|
+
it('une ligne sans message (logger.error(undefined)) refusée en 422 est isolée, jetée et comptée', async () => {
|
|
178
|
+
sendBulk.mockImplementation(async (_params, _query, body) => {
|
|
179
|
+
if (body.logs.some((log) => log.message === undefined)) {
|
|
180
|
+
throw httpError(422);
|
|
181
|
+
}
|
|
182
|
+
accepted.push(body);
|
|
183
|
+
return ACK;
|
|
184
|
+
});
|
|
185
|
+
const shipper = shipperOn();
|
|
186
|
+
const withoutMessage = { ...makeLog('x'), message: undefined };
|
|
187
|
+
shipper.enqueue(makeLog('a'));
|
|
188
|
+
shipper.enqueue(withoutMessage);
|
|
189
|
+
shipper.enqueue(makeLog('b'));
|
|
190
|
+
await shipper.flush();
|
|
191
|
+
shipper.enqueue(makeLog('c'));
|
|
192
|
+
await shipper.flush();
|
|
193
|
+
expect(deliveredMessages()).toEqual(['a', 'b', 'c']);
|
|
194
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
195
|
+
expect(accepted[accepted.length - 1].droppedCount).toBe(1);
|
|
196
|
+
});
|
|
197
|
+
it('une ligne seule refusée est jetée sans découpe', async () => {
|
|
198
|
+
sendBulk.mockImplementation(refusingMarked(422));
|
|
199
|
+
const shipper = shipperOn();
|
|
200
|
+
shipper.enqueue(makeLog(`seule ${REFUSED}`));
|
|
201
|
+
await shipper.flush();
|
|
202
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
203
|
+
shipper.enqueue(makeLog('suivante'));
|
|
204
|
+
await shipper.flush();
|
|
205
|
+
expect(sendBulk).toHaveBeenCalledTimes(2);
|
|
206
|
+
expect(sendBulk.mock.calls[1][2].droppedCount).toBe(1);
|
|
207
|
+
});
|
|
208
|
+
it('un 413 sur un lot de deux le coupe en deux envois d une ligne', async () => {
|
|
209
|
+
sendBulk.mockImplementation(async (_params, _query, body) => {
|
|
210
|
+
if (body.logs.length > 1) {
|
|
211
|
+
throw httpError(413);
|
|
212
|
+
}
|
|
213
|
+
return ACK;
|
|
214
|
+
});
|
|
215
|
+
const shipper = shipperOn();
|
|
216
|
+
shipper.enqueue(makeLog('a'));
|
|
217
|
+
shipper.enqueue(makeLog('b'));
|
|
218
|
+
await shipper.flush();
|
|
219
|
+
expect(sendBulk.mock.calls.map((call) => call[2].logs.length)).toEqual([2, 1, 1]);
|
|
220
|
+
expect(messagesOf(1)).toEqual(['a']);
|
|
221
|
+
expect(messagesOf(2)).toEqual(['b']);
|
|
222
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
223
|
+
shipper.enqueue(makeLog('c'));
|
|
224
|
+
await shipper.flush();
|
|
225
|
+
expect(sendBulk.mock.calls[3][2].droppedCount).toBeNull();
|
|
226
|
+
});
|
|
227
|
+
it('le rejeu d un fichier refusé pour une ligne livre les autres, puis supprime le fichier', async () => {
|
|
228
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
229
|
+
const first = shipperOn();
|
|
230
|
+
['a', `b ${REFUSED}`, 'c', 'd'].forEach((message) => first.enqueue(makeLog(message)));
|
|
231
|
+
await first.flush();
|
|
232
|
+
first.stop();
|
|
233
|
+
expect(spooledFiles(base)).toHaveLength(1);
|
|
234
|
+
sendBulk.mockImplementation(refusingMarked(400));
|
|
235
|
+
const second = shipperOn();
|
|
236
|
+
expect(await second.replaySpool()).toBe(1);
|
|
237
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
238
|
+
expect(deliveredMessages()).toEqual(['a', 'c', 'd']);
|
|
239
|
+
second.enqueue(makeLog('e'));
|
|
240
|
+
await second.flush();
|
|
241
|
+
expect(accepted[accepted.length - 1].droppedCount).toBe(1);
|
|
242
|
+
});
|
|
243
|
+
it.each([
|
|
244
|
+
{
|
|
245
|
+
half: 'la seconde moitié',
|
|
246
|
+
failing: 'c,d',
|
|
247
|
+
deliveredFirst: ['a'],
|
|
248
|
+
deliveredInTheEnd: ['a', 'a', 'c', 'd', 'e', 'f'],
|
|
249
|
+
},
|
|
250
|
+
{
|
|
251
|
+
half: 'la première moitié',
|
|
252
|
+
failing: 'a',
|
|
253
|
+
deliveredFirst: [],
|
|
254
|
+
deliveredInTheEnd: ['a', 'c', 'd', 'e', 'f'],
|
|
255
|
+
},
|
|
256
|
+
])('un échec transitoire dans $half de la découpe garde tout le lot, sans compter deux fois', async ({ failing, deliveredFirst, deliveredInTheEnd }) => {
|
|
257
|
+
let serverDown = true;
|
|
258
|
+
const server = refusingMarked(400);
|
|
259
|
+
sendBulk.mockImplementation(async (params, query, body) => {
|
|
260
|
+
if (serverDown && body.logs.map((log) => log.message).join() === failing) {
|
|
261
|
+
throw httpError(503);
|
|
262
|
+
}
|
|
263
|
+
return server(params, query, body);
|
|
264
|
+
});
|
|
265
|
+
const shipper = shipperOn();
|
|
266
|
+
['a', `X ${REFUSED}`, 'c', 'd'].forEach((message) => shipper.enqueue(makeLog(message)));
|
|
267
|
+
await shipper.flush();
|
|
268
|
+
expect(spooledFiles(base)).toHaveLength(1);
|
|
269
|
+
expect(deliveredMessages()).toEqual(deliveredFirst);
|
|
270
|
+
serverDown = false;
|
|
271
|
+
shipper.enqueue(makeLog('e'));
|
|
272
|
+
await shipper.flush(3_000, { ignoreBackoff: true });
|
|
273
|
+
await settle();
|
|
274
|
+
shipper.enqueue(makeLog('f'));
|
|
275
|
+
await shipper.flush();
|
|
276
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
277
|
+
expect(deliveredMessages()).toEqual(deliveredInTheEnd);
|
|
278
|
+
expect(accepted.reduce((sum, body) => sum + (body.droppedCount ?? 0), 0)).toBe(1);
|
|
279
|
+
});
|
|
280
|
+
it("un fichier dont toutes les lignes sont refusées est jeté, compté, et n'est pas « rejoué »", async () => {
|
|
281
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
282
|
+
const first = shipperOn();
|
|
283
|
+
first.enqueue(makeLog(`x ${REFUSED}`));
|
|
284
|
+
first.enqueue(makeLog(`y ${REFUSED}`));
|
|
285
|
+
await first.flush();
|
|
286
|
+
first.stop();
|
|
287
|
+
sendBulk.mockImplementation(refusingMarked(400));
|
|
288
|
+
const second = shipperOn();
|
|
289
|
+
expect(await second.replaySpool()).toBe(0);
|
|
290
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
291
|
+
second.enqueue(makeLog('z'));
|
|
292
|
+
await second.flush();
|
|
293
|
+
expect(accepted.map((body) => body.droppedCount)).toEqual([2]);
|
|
294
|
+
});
|
|
295
|
+
it(`un refus qui ne vise pas une ligne s'arrête à ${log_shipper_model_1.MAX_REFUSED_LINES_PER_BATCH} lignes refusées`, async () => {
|
|
296
|
+
useFakeClock();
|
|
297
|
+
sendBulk.mockRejectedValue(httpError(400));
|
|
298
|
+
const shipper = shipperOn();
|
|
299
|
+
for (let line = 0; line < 100; line += 1) {
|
|
300
|
+
shipper.enqueue(makeLog(`ligne ${line}`));
|
|
301
|
+
}
|
|
302
|
+
await shipper.flush();
|
|
303
|
+
expect(sendBulk.mock.calls.map((call) => call[2].logs.length)).toEqual([
|
|
304
|
+
100, 50, 25, 13, 7, 4, 2, 1, 1, 2, 1, 1, 3, 2, 1,
|
|
305
|
+
]);
|
|
306
|
+
expect(sendBulk.mock.calls.length).toBeLessThanOrEqual(7 + 2 * log_shipper_model_1.MAX_REFUSED_LINES_PER_BATCH);
|
|
307
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
308
|
+
sendBulk.mockImplementation(refusingMarked(400));
|
|
309
|
+
shipper.enqueue(makeLog('suivante'));
|
|
310
|
+
await shipper.flush();
|
|
311
|
+
expect(accepted.map((body) => body.droppedCount)).toEqual([100]);
|
|
312
|
+
});
|
|
313
|
+
it("à la sortie, le délai qui tombe au milieu d'une découpe garde le lot sur disque", async () => {
|
|
314
|
+
useFakeClock();
|
|
315
|
+
const server = refusingMarked(400);
|
|
316
|
+
sendBulk.mockImplementation(async (params, query, body) => {
|
|
317
|
+
jest.setSystemTime(Date.now() + 1_000);
|
|
318
|
+
return server(params, query, body);
|
|
319
|
+
});
|
|
320
|
+
const shipper = shipperOn();
|
|
321
|
+
['a', 'b', `c ${REFUSED}`, 'd'].forEach((message) => shipper.enqueue(makeLog(message)));
|
|
322
|
+
await shipper.flush(1_500, { ignoreBackoff: true });
|
|
323
|
+
expect(sendBulk.mock.calls.map((call) => call[2].logs.length)).toEqual([4, 2]);
|
|
324
|
+
const [file] = spooledFiles(base);
|
|
325
|
+
const onDisk = fs
|
|
326
|
+
.readFileSync(path.join(base, APPLICATION_ID, file), 'utf8')
|
|
327
|
+
.split('\n')
|
|
328
|
+
.map((line) => JSON.parse(line).message);
|
|
329
|
+
expect(onDisk).toEqual(['a', 'b', `c ${REFUSED}`, 'd']);
|
|
330
|
+
sendBulk.mockImplementation(server);
|
|
331
|
+
const next = shipperOn();
|
|
332
|
+
expect(await next.replaySpool()).toBe(1);
|
|
333
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
334
|
+
expect(deliveredMessages()).toEqual(['a', 'a', 'b', 'b', 'd']);
|
|
335
|
+
next.enqueue(makeLog('e'));
|
|
336
|
+
await next.flush();
|
|
337
|
+
expect(accepted[accepted.length - 1].droppedCount).toBe(1);
|
|
338
|
+
});
|
|
339
|
+
it('un flush ordinaire finit sa découpe même si son délai tombe au milieu', async () => {
|
|
340
|
+
useFakeClock();
|
|
341
|
+
const server = refusingMarked(400);
|
|
342
|
+
sendBulk.mockImplementation(async (params, query, body) => {
|
|
343
|
+
jest.setSystemTime(Date.now() + 1_000);
|
|
344
|
+
return server(params, query, body);
|
|
345
|
+
});
|
|
346
|
+
const shipper = shipperOn();
|
|
347
|
+
['a', 'b', `c ${REFUSED}`, 'd'].forEach((message) => shipper.enqueue(makeLog(message)));
|
|
348
|
+
await shipper.flush(1_500);
|
|
349
|
+
expect(deliveredMessages()).toEqual(['a', 'b', 'd']);
|
|
350
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
351
|
+
});
|
|
352
|
+
it("n'envoie rien pendant le recul, puis relit le spool sans attendre de redémarrage", async () => {
|
|
353
|
+
useFakeClock();
|
|
354
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
355
|
+
const shipper = shipperOn();
|
|
356
|
+
shipper.enqueue(makeLog('pendant la panne'));
|
|
357
|
+
await shipper.flush();
|
|
358
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
359
|
+
shipper.enqueue(makeLog('toujours pendant la panne'));
|
|
360
|
+
await shipper.flush();
|
|
361
|
+
expect(await shipper.replaySpool()).toBe(0);
|
|
362
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
363
|
+
expect(spooledFiles(base)).toHaveLength(2);
|
|
364
|
+
jest.setSystemTime(Date.now() + log_shipper_model_1.BACKOFF_BASE_MS + 1);
|
|
365
|
+
shipper.enqueue(makeLog('après'));
|
|
366
|
+
await shipper.flush();
|
|
367
|
+
await settle();
|
|
368
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
369
|
+
expect(allShipped()).toEqual(expect.arrayContaining(['pendant la panne', 'toujours pendant la panne', 'après']));
|
|
370
|
+
});
|
|
371
|
+
it('relit le spool au tick qui suit la fin du recul, sans nouveau log', async () => {
|
|
372
|
+
useFakeClock();
|
|
373
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
374
|
+
const shipper = shipperOn({ flushIntervalMs: 500 });
|
|
375
|
+
shipper.enqueue(makeLog('pendant la panne'));
|
|
376
|
+
await shipper.flush();
|
|
377
|
+
expect(spooledFiles(base)).toHaveLength(1);
|
|
378
|
+
await jest.advanceTimersByTimeAsync(log_shipper_model_1.BACKOFF_BASE_MS + 500);
|
|
379
|
+
await settle();
|
|
380
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
381
|
+
expect(sendBulk).toHaveBeenCalledTimes(2);
|
|
382
|
+
expect(messagesOf(1)).toEqual(['pendant la panne']);
|
|
383
|
+
});
|
|
384
|
+
it('rejoue un lot avec sa date d émission et son clientLogId d origine', async () => {
|
|
385
|
+
const emittedAt = new Date('2026-09-10T08:00:00.000Z');
|
|
386
|
+
const log = makeLog('a', emittedAt);
|
|
387
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
388
|
+
const first = shipperOn();
|
|
389
|
+
first.enqueue(log);
|
|
390
|
+
await first.flush();
|
|
391
|
+
first.stop();
|
|
392
|
+
const second = shipperOn();
|
|
393
|
+
const replayed = await second.replaySpool();
|
|
394
|
+
expect(replayed).toBe(1);
|
|
395
|
+
const resent = sendBulk.mock.calls[1][2].logs[0];
|
|
396
|
+
expect(new Date(resent.dateValue).toISOString()).toBe(emittedAt.toISOString());
|
|
397
|
+
expect(resent.clientLogId).toBe(log.clientLogId);
|
|
398
|
+
});
|
|
399
|
+
it("ne rejoue jamais le spool d'une autre application", async () => {
|
|
400
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
401
|
+
const first = shipperOn();
|
|
402
|
+
first.enqueue(makeLog('à app-1'));
|
|
403
|
+
await first.flush();
|
|
404
|
+
first.stop();
|
|
405
|
+
__1.AbyssMonitorCore.setConfig({ applicationId: 'app-2', secretPublishToken: 'other' });
|
|
406
|
+
const second = shipperOn();
|
|
407
|
+
const replayed = await second.replaySpool();
|
|
408
|
+
expect(replayed).toBe(0);
|
|
409
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
410
|
+
});
|
|
411
|
+
it('sans spool, remet le lot en tête de file sur un échec transitoire', async () => {
|
|
412
|
+
useFakeClock();
|
|
413
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
414
|
+
const shipper = shipperOn({ spoolDirectory: null });
|
|
415
|
+
shipper.enqueue(makeLog('a'));
|
|
416
|
+
await shipper.flush();
|
|
417
|
+
shipper.enqueue(makeLog('b'));
|
|
418
|
+
jest.setSystemTime(Date.now() + log_shipper_model_1.BACKOFF_BASE_MS + 1);
|
|
419
|
+
await shipper.flush();
|
|
420
|
+
expect(sendBulk).toHaveBeenCalledTimes(2);
|
|
421
|
+
expect(messagesOf(1)).toEqual(['a', 'b']);
|
|
422
|
+
expect(fs.existsSync(path.join(base, APPLICATION_ID))).toBe(false);
|
|
423
|
+
});
|
|
424
|
+
it('un flush de sortie envoie malgré un recul en cours', async () => {
|
|
425
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
426
|
+
const shipper = shipperOn();
|
|
427
|
+
shipper.enqueue(makeLog('avant'));
|
|
428
|
+
await shipper.flush();
|
|
429
|
+
shipper.enqueue(makeLog('à la sortie'));
|
|
430
|
+
await shipper.flush(3_000, { ignoreBackoff: true });
|
|
431
|
+
await settle();
|
|
432
|
+
expect(messagesOf(1)).toEqual(['à la sortie']);
|
|
433
|
+
expect(allShipped()).toEqual(['avant', 'à la sortie', 'avant']);
|
|
434
|
+
});
|
|
435
|
+
it('à la sortie, après un échec, le reste descend sur disque sans nouvel envoi', async () => {
|
|
436
|
+
useFakeClock();
|
|
437
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
438
|
+
const shipper = shipperOn({ maxBatchEntries: 1 });
|
|
439
|
+
shipper.enqueue(makeLog('a'));
|
|
440
|
+
shipper.enqueue(makeLog('b'));
|
|
441
|
+
shipper.enqueue(makeLog('c'));
|
|
442
|
+
await shipper.flush(3_000, { ignoreBackoff: true });
|
|
443
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
444
|
+
expect(spooledFiles(base)).toHaveLength(3);
|
|
445
|
+
});
|
|
446
|
+
it('flush ne lève jamais, même si le transport échoue', async () => {
|
|
447
|
+
sendBulk.mockRejectedValue(new Error('monitor down'));
|
|
448
|
+
const shipper = shipperOn();
|
|
449
|
+
shipper.enqueue(makeLog('a'));
|
|
450
|
+
await expect(shipper.flush()).resolves.toBeUndefined();
|
|
451
|
+
});
|
|
452
|
+
it('replaySpool ne rejette jamais', async () => {
|
|
453
|
+
jest.spyOn(log_spool_model_1.LogSpool.prototype, 'listPendingBatches').mockRejectedValueOnce(new Error('disk'));
|
|
454
|
+
const shipper = shipperOn();
|
|
455
|
+
await expect(shipper.replaySpool()).resolves.toBe(0);
|
|
456
|
+
});
|
|
457
|
+
it('classe les refus', () => {
|
|
458
|
+
expect((0, log_shipper_model_1.classifySendError)(httpError(400))).toBe(log_shipper_model_1.SendOutcome.DROP);
|
|
459
|
+
expect((0, log_shipper_model_1.classifySendError)(httpError(413))).toBe(log_shipper_model_1.SendOutcome.DROP);
|
|
460
|
+
expect((0, log_shipper_model_1.classifySendError)(httpError(422))).toBe(log_shipper_model_1.SendOutcome.DROP);
|
|
461
|
+
expect((0, log_shipper_model_1.classifySendError)(httpError(404))).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
462
|
+
expect((0, log_shipper_model_1.classifySendError)(Object.assign(new Error('timeout of 10000ms exceeded'), { code: 'ECONNABORTED' }))).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
463
|
+
expect((0, log_shipper_model_1.classifySendError)(httpError(500))).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
464
|
+
expect((0, log_shipper_model_1.classifySendError)(httpError(503))).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
465
|
+
expect((0, log_shipper_model_1.classifySendError)(httpError(429))).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
466
|
+
expect((0, log_shipper_model_1.classifySendError)(httpError(401))).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
467
|
+
expect((0, log_shipper_model_1.classifySendError)(httpError(403))).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
468
|
+
expect((0, log_shipper_model_1.classifySendError)(new Error('ECONNREFUSED'))).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
469
|
+
expect((0, log_shipper_model_1.classifySendError)(null)).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
470
|
+
expect((0, log_shipper_model_1.classifySendError)('down')).toBe(log_shipper_model_1.SendOutcome.RETRY);
|
|
471
|
+
});
|
|
472
|
+
it('prend le répertoire du spool dans ABYSS_MONITOR_SPOOL_DIR, sinon dans le répertoire temporaire', () => {
|
|
473
|
+
try {
|
|
474
|
+
process.env[log_shipper_model_1.SPOOL_DIRECTORY_ENV] = '/var/spool/abyss';
|
|
475
|
+
expect((0, log_shipper_model_1.resolveSpoolBase)()).toBe('/var/spool/abyss');
|
|
476
|
+
expect((0, log_shipper_model_1.resolveSpoolBase)('/explicit')).toBe('/explicit');
|
|
477
|
+
delete process.env[log_shipper_model_1.SPOOL_DIRECTORY_ENV];
|
|
478
|
+
expect((0, log_shipper_model_1.resolveSpoolBase)()).toBe(path.join(os.tmpdir(), 'abyss-monitor-spool'));
|
|
479
|
+
}
|
|
480
|
+
finally {
|
|
481
|
+
if (ORIGINAL_SPOOL_DIRECTORY_ENV === undefined) {
|
|
482
|
+
delete process.env[log_shipper_model_1.SPOOL_DIRECTORY_ENV];
|
|
483
|
+
}
|
|
484
|
+
else {
|
|
485
|
+
process.env[log_shipper_model_1.SPOOL_DIRECTORY_ENV] = ORIGINAL_SPOOL_DIRECTORY_ENV;
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
});
|
|
489
|
+
it('restaure le sandbox de spool après le test précédent, au lieu de le laisser supprimé', () => {
|
|
490
|
+
expect(ORIGINAL_SPOOL_DIRECTORY_ENV).toEqual(expect.any(String));
|
|
491
|
+
expect(process.env[log_shipper_model_1.SPOOL_DIRECTORY_ENV]).toBe(ORIGINAL_SPOOL_DIRECTORY_ENV);
|
|
492
|
+
});
|
|
493
|
+
it('hors Node : ni spool ni exception', async () => {
|
|
494
|
+
const globals = globalThis;
|
|
495
|
+
const originalBuffer = globals.Buffer;
|
|
496
|
+
globals.window = {};
|
|
497
|
+
try {
|
|
498
|
+
expect((0, log_shipper_model_1.resolveSpoolBase)('/explicit')).toBeNull();
|
|
499
|
+
const shipper = shipperOn();
|
|
500
|
+
globals.Buffer = undefined;
|
|
501
|
+
try {
|
|
502
|
+
shipper.enqueue(makeLog('navigateur'));
|
|
503
|
+
}
|
|
504
|
+
finally {
|
|
505
|
+
globals.Buffer = originalBuffer;
|
|
506
|
+
}
|
|
507
|
+
await shipper.flush();
|
|
508
|
+
expect(messagesOf(0)).toEqual(['navigateur']);
|
|
509
|
+
expect(fs.existsSync(path.join(base, APPLICATION_ID))).toBe(false);
|
|
510
|
+
}
|
|
511
|
+
finally {
|
|
512
|
+
delete globals.window;
|
|
513
|
+
}
|
|
514
|
+
});
|
|
515
|
+
it('prévient une seule fois quand le spool est inutilisable', async () => {
|
|
516
|
+
const blocker = path.join(base, 'not-a-directory');
|
|
517
|
+
fs.writeFileSync(blocker, '');
|
|
518
|
+
const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined);
|
|
519
|
+
const shipper = shipperOn({ spoolDirectory: blocker });
|
|
520
|
+
shipper.enqueue(makeLog('a'));
|
|
521
|
+
await shipper.flush();
|
|
522
|
+
shipper.enqueue(makeLog('b'));
|
|
523
|
+
await shipper.flush();
|
|
524
|
+
expect(warn).toHaveBeenCalledTimes(1);
|
|
525
|
+
expect(warn.mock.calls[0][0]).toContain(log_shipper_model_1.SPOOL_DIRECTORY_ENV);
|
|
526
|
+
expect(sendBulk).toHaveBeenCalledTimes(2);
|
|
527
|
+
});
|
|
528
|
+
it('enqueue ne fait aucune E/S : un lot plein part au tour suivant de la boucle', async () => {
|
|
529
|
+
useFakeClock();
|
|
530
|
+
const shipper = shipperOn({ maxBatchEntries: 2 });
|
|
531
|
+
shipper.enqueue(makeLog('a'));
|
|
532
|
+
shipper.enqueue(makeLog('b'));
|
|
533
|
+
shipper.enqueue(makeLog('c'));
|
|
534
|
+
expect(fs.existsSync(path.join(base, APPLICATION_ID))).toBe(false);
|
|
535
|
+
expect(sendBulk).not.toHaveBeenCalled();
|
|
536
|
+
expect(jest.getTimerCount()).toBe(2);
|
|
537
|
+
await jest.advanceTimersByTimeAsync(0);
|
|
538
|
+
await settle();
|
|
539
|
+
expect(allShipped()).toEqual(['a', 'b', 'c']);
|
|
540
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
541
|
+
});
|
|
542
|
+
it("prend la taille déjà mesurée par l'appelant, sans resérialiser la ligne", async () => {
|
|
543
|
+
const shipper = shipperOn({ maxQueueBytes: 1_000 });
|
|
544
|
+
const stringify = jest.spyOn(JSON, 'stringify');
|
|
545
|
+
shipper.enqueue(makeLog('a'), 600);
|
|
546
|
+
shipper.enqueue(makeLog('b'), 600);
|
|
547
|
+
expect(stringify).not.toHaveBeenCalled();
|
|
548
|
+
stringify.mockRestore();
|
|
549
|
+
await shipper.flush();
|
|
550
|
+
expect(allShipped()).toEqual(['b']);
|
|
551
|
+
expect(sendBulk.mock.calls[0][2].droppedCount).toBe(1);
|
|
552
|
+
});
|
|
553
|
+
it('un log impossible à sérialiser ne fait pas lever enqueue, et il est compté', async () => {
|
|
554
|
+
const shipper = shipperOn();
|
|
555
|
+
const data = {};
|
|
556
|
+
data.self = data;
|
|
557
|
+
const circular = { ...makeLog('circulaire'), data };
|
|
558
|
+
expect(() => shipper.enqueue(circular)).not.toThrow();
|
|
559
|
+
shipper.enqueue(makeLog('ok'));
|
|
560
|
+
await shipper.flush();
|
|
561
|
+
expect(messagesOf(0)).toEqual(['ok']);
|
|
562
|
+
expect(sendBulk.mock.calls[0][2].droppedCount).toBe(1);
|
|
563
|
+
});
|
|
564
|
+
it("un envoi qui échoue transitoirement ne perd pas le décompte qu'il portait", async () => {
|
|
565
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
566
|
+
const shipper = shipperOn({ maxQueueEntries: 1 });
|
|
567
|
+
shipper.enqueue(makeLog('a'));
|
|
568
|
+
shipper.enqueue(makeLog('b'));
|
|
569
|
+
await shipper.flush();
|
|
570
|
+
expect(sendBulk.mock.calls[0][2].droppedCount).toBe(1);
|
|
571
|
+
shipper.enqueue(makeLog('c'));
|
|
572
|
+
await shipper.flush(3_000, { ignoreBackoff: true });
|
|
573
|
+
expect(messagesOf(1)).toEqual(['c']);
|
|
574
|
+
expect(sendBulk.mock.calls[1][2].droppedCount).toBe(1);
|
|
575
|
+
});
|
|
576
|
+
it("un lot refusé définitivement rend le décompte qu'il portait, en plus de ses lignes", async () => {
|
|
577
|
+
sendBulk.mockRejectedValueOnce(httpError(422));
|
|
578
|
+
const shipper = shipperOn({ maxQueueEntries: 1 });
|
|
579
|
+
shipper.enqueue(makeLog('a'));
|
|
580
|
+
shipper.enqueue(makeLog('b'));
|
|
581
|
+
await shipper.flush();
|
|
582
|
+
shipper.enqueue(makeLog('c'));
|
|
583
|
+
await shipper.flush();
|
|
584
|
+
expect(sendBulk.mock.calls[1][2].droppedCount).toBe(2);
|
|
585
|
+
});
|
|
586
|
+
it('remonte les lignes évincées du spool plein', async () => {
|
|
587
|
+
sendBulk.mockRejectedValueOnce(httpError(503)).mockRejectedValueOnce(httpError(503));
|
|
588
|
+
const shipper = shipperOn({ spoolMaxBytes: 500 });
|
|
589
|
+
shipper.enqueue(makeLog('a'.repeat(200)));
|
|
590
|
+
await shipper.flush();
|
|
591
|
+
shipper.enqueue(makeLog('b'.repeat(200)));
|
|
592
|
+
await shipper.flush(3_000, { ignoreBackoff: true });
|
|
593
|
+
expect(spooledFiles(base)).toHaveLength(1);
|
|
594
|
+
shipper.enqueue(makeLog('c'));
|
|
595
|
+
await shipper.flush(3_000, { ignoreBackoff: true });
|
|
596
|
+
expect(sendBulk.mock.calls[1][2].droppedCount).toBe(1);
|
|
597
|
+
expect(messagesOf(2)).toEqual(['c']);
|
|
598
|
+
expect(sendBulk.mock.calls[2][2].droppedCount).toBe(1);
|
|
599
|
+
});
|
|
600
|
+
it('un lot illisible sur disque est écarté et compté', async () => {
|
|
601
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
602
|
+
const first = shipperOn();
|
|
603
|
+
first.enqueue(makeLog('a'));
|
|
604
|
+
first.enqueue(makeLog('b'));
|
|
605
|
+
await first.flush();
|
|
606
|
+
first.stop();
|
|
607
|
+
const [file] = spooledFiles(base);
|
|
608
|
+
fs.writeFileSync(path.join(base, APPLICATION_ID, file), '{"message":"tronq');
|
|
609
|
+
const second = shipperOn();
|
|
610
|
+
expect(await second.replaySpool()).toBe(0);
|
|
611
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
612
|
+
second.enqueue(makeLog('c'));
|
|
613
|
+
await second.flush();
|
|
614
|
+
expect(sendBulk.mock.calls[1][2].droppedCount).toBe(2);
|
|
615
|
+
});
|
|
616
|
+
it("un lot livré entre le listage et la lecture du rejeu n'est pas compté comme perdu", async () => {
|
|
617
|
+
useFakeClock();
|
|
618
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
619
|
+
const shipper = shipperOn();
|
|
620
|
+
shipper.enqueue(makeLog('g'));
|
|
621
|
+
await shipper.flush();
|
|
622
|
+
jest.setSystemTime(Date.now() + log_shipper_model_1.BACKOFF_BASE_MS + 1);
|
|
623
|
+
const fSend = deferred();
|
|
624
|
+
sendBulk.mockReturnValueOnce(fSend.promise);
|
|
625
|
+
shipper.enqueue(makeLog('f'));
|
|
626
|
+
const flushing = shipper.flush();
|
|
627
|
+
await settle();
|
|
628
|
+
const gSend = deferred();
|
|
629
|
+
sendBulk.mockReturnValueOnce(gSend.promise);
|
|
630
|
+
const replay = shipper.replaySpool();
|
|
631
|
+
await settle();
|
|
632
|
+
expect(allShipped()).toEqual(['g', 'f', 'g']);
|
|
633
|
+
fSend.resolve(ACK);
|
|
634
|
+
await flushing;
|
|
635
|
+
gSend.resolve(ACK);
|
|
636
|
+
expect(await replay).toBe(1);
|
|
637
|
+
shipper.enqueue(makeLog('h'));
|
|
638
|
+
await shipper.flush();
|
|
639
|
+
expect(messagesOf(3)).toEqual(['h']);
|
|
640
|
+
expect(sendBulk.mock.calls[3][2].droppedCount).toBeNull();
|
|
641
|
+
});
|
|
642
|
+
it("pendant le recul, un lot que le spool refuse d'écrire reste en mémoire", async () => {
|
|
643
|
+
useFakeClock();
|
|
644
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
645
|
+
const shipper = shipperOn();
|
|
646
|
+
shipper.enqueue(makeLog('a'));
|
|
647
|
+
await shipper.flush();
|
|
648
|
+
const writeBatch = jest.spyOn(log_spool_model_1.LogSpool.prototype, 'writeBatch').mockResolvedValueOnce(null);
|
|
649
|
+
shipper.enqueue(makeLog('b'));
|
|
650
|
+
await shipper.flush();
|
|
651
|
+
expect(writeBatch).toHaveBeenCalledTimes(1);
|
|
652
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
653
|
+
jest.setSystemTime(Date.now() + log_shipper_model_1.BACKOFF_BASE_MS + 1);
|
|
654
|
+
await shipper.flush();
|
|
655
|
+
await settle();
|
|
656
|
+
expect(messagesOf(1)).toEqual(['b']);
|
|
657
|
+
expect(sendBulk.mock.calls[1][2].droppedCount).toBeNull();
|
|
658
|
+
expect(allShipped()).toEqual(['a', 'b', 'a']);
|
|
659
|
+
});
|
|
660
|
+
it("le premier tick d'un shipper neuf ne POSTe son lot qu'une fois", async () => {
|
|
661
|
+
useFakeClock();
|
|
662
|
+
const shipper = shipperOn({ flushIntervalMs: 500 });
|
|
663
|
+
shipper.enqueue(makeLog('first'));
|
|
664
|
+
await jest.advanceTimersByTimeAsync(500);
|
|
665
|
+
await settle();
|
|
666
|
+
expect(allShipped()).toEqual(['first']);
|
|
667
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
668
|
+
});
|
|
669
|
+
it("au premier tick, une ligne refusée n'est découpée et comptée qu'une fois", async () => {
|
|
670
|
+
useFakeClock();
|
|
671
|
+
sendBulk.mockImplementation(refusingMarked(400));
|
|
672
|
+
const shipper = shipperOn({ flushIntervalMs: 500 });
|
|
673
|
+
shipper.enqueue(makeLog('a'));
|
|
674
|
+
shipper.enqueue(makeLog(`bad ${REFUSED}`));
|
|
675
|
+
await jest.advanceTimersByTimeAsync(500);
|
|
676
|
+
await settle();
|
|
677
|
+
shipper.enqueue(makeLog('next'));
|
|
678
|
+
await shipper.flush();
|
|
679
|
+
expect(sendBulk.mock.calls.map((call) => call[2].logs.length)).toEqual([2, 1, 1, 1]);
|
|
680
|
+
expect(accepted.map((body) => body.droppedCount)).toEqual([null, 1]);
|
|
681
|
+
});
|
|
682
|
+
it('un flush suivi d un rejeu dans le même tour ne POSTe pas le lot deux fois', async () => {
|
|
683
|
+
const shipper = shipperOn();
|
|
684
|
+
shipper.enqueue(makeLog('first'));
|
|
685
|
+
const flushing = shipper.flush();
|
|
686
|
+
const replaying = shipper.replaySpool();
|
|
687
|
+
await Promise.all([flushing, replaying]);
|
|
688
|
+
await settle();
|
|
689
|
+
expect(allShipped()).toEqual(['first']);
|
|
690
|
+
});
|
|
691
|
+
it('à la sortie, un flush en vol bloqué ne retarde pas le déversement au-delà du délai', async () => {
|
|
692
|
+
useFakeClock();
|
|
693
|
+
const hung = deferred();
|
|
694
|
+
sendBulk.mockReturnValueOnce(hung.promise);
|
|
695
|
+
const shipper = shipperOn();
|
|
696
|
+
shipper.enqueue(makeLog('a'));
|
|
697
|
+
const regular = shipper.flush();
|
|
698
|
+
await settle();
|
|
699
|
+
shipper.enqueue(makeLog('b'));
|
|
700
|
+
let exited = false;
|
|
701
|
+
const exit = shipper.flush(100, { ignoreBackoff: true }).then(() => {
|
|
702
|
+
exited = true;
|
|
703
|
+
});
|
|
704
|
+
await jest.advanceTimersByTimeAsync(100);
|
|
705
|
+
await settle();
|
|
706
|
+
expect(exited).toBe(true);
|
|
707
|
+
expect(spooledFiles(base)).toHaveLength(2);
|
|
708
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
709
|
+
hung.resolve(ACK);
|
|
710
|
+
await Promise.all([regular, exit]);
|
|
711
|
+
});
|
|
712
|
+
it('à la sortie, un flush en vol qui finit à temps laisse le reste partir par le réseau', async () => {
|
|
713
|
+
useFakeClock();
|
|
714
|
+
const slow = deferred();
|
|
715
|
+
sendBulk.mockReturnValueOnce(slow.promise);
|
|
716
|
+
const shipper = shipperOn();
|
|
717
|
+
shipper.enqueue(makeLog('a'));
|
|
718
|
+
const regular = shipper.flush();
|
|
719
|
+
await settle();
|
|
720
|
+
shipper.enqueue(makeLog('b'));
|
|
721
|
+
const exit = shipper.flush(1_000, { ignoreBackoff: true });
|
|
722
|
+
await jest.advanceTimersByTimeAsync(50);
|
|
723
|
+
slow.resolve(ACK);
|
|
724
|
+
await Promise.all([regular, exit]);
|
|
725
|
+
expect(allShipped()).toEqual(['a', 'b']);
|
|
726
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
727
|
+
expect(jest.getTimerCount()).toBe(1);
|
|
728
|
+
});
|
|
729
|
+
it("à la sortie sans identité, le déversement n'a nulle part où écrire et ne lève rien", async () => {
|
|
730
|
+
useFakeClock();
|
|
731
|
+
const hung = deferred();
|
|
732
|
+
sendBulk.mockReturnValueOnce(hung.promise);
|
|
733
|
+
const shipper = shipperOn();
|
|
734
|
+
shipper.enqueue(makeLog('a'));
|
|
735
|
+
const regular = shipper.flush();
|
|
736
|
+
await settle();
|
|
737
|
+
shipper.enqueue(makeLog('b'));
|
|
738
|
+
__1.AbyssMonitorCore.setConfig({ applicationId: '' });
|
|
739
|
+
const exit = shipper.flush(100, { ignoreBackoff: true });
|
|
740
|
+
await jest.advanceTimersByTimeAsync(100);
|
|
741
|
+
await exit;
|
|
742
|
+
expect(console.error).not.toHaveBeenCalled();
|
|
743
|
+
hung.resolve(ACK);
|
|
744
|
+
await regular;
|
|
745
|
+
__1.AbyssMonitorCore.setConfig({ applicationId: APPLICATION_ID });
|
|
746
|
+
await shipper.flush();
|
|
747
|
+
expect(allShipped()).toEqual(['a', 'b']);
|
|
748
|
+
});
|
|
749
|
+
it('un fichier momentanément illisible est gardé, puis livré au rejeu suivant', async () => {
|
|
750
|
+
useFakeClock();
|
|
751
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
752
|
+
const first = shipperOn();
|
|
753
|
+
first.enqueue(makeLog('a'));
|
|
754
|
+
await first.flush();
|
|
755
|
+
first.stop();
|
|
756
|
+
const realFs = jest.requireActual('fs');
|
|
757
|
+
jest.spyOn(realFs, 'readFileSync').mockImplementationOnce(() => {
|
|
758
|
+
throw Object.assign(new Error('EBUSY: resource busy or locked'), { code: 'EBUSY' });
|
|
759
|
+
});
|
|
760
|
+
const second = shipperOn({ flushIntervalMs: 500 });
|
|
761
|
+
expect(await second.replaySpool()).toBe(0);
|
|
762
|
+
expect(spooledFiles(base)).toHaveLength(1);
|
|
763
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
764
|
+
await jest.advanceTimersByTimeAsync(500);
|
|
765
|
+
await settle();
|
|
766
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
767
|
+
expect(messagesOf(1)).toEqual(['a']);
|
|
768
|
+
expect(sendBulk.mock.calls[1][2].droppedCount).toBeNull();
|
|
769
|
+
});
|
|
770
|
+
it('un fichier illisible en tête ne retient pas les suivants dans le même rejeu', async () => {
|
|
771
|
+
const previous = new log_spool_model_1.LogSpool({
|
|
772
|
+
directory: path.join(base, APPLICATION_ID),
|
|
773
|
+
maxBytes: 1_000_000,
|
|
774
|
+
});
|
|
775
|
+
const head = await previous.writeBatch([makeLog('tête')]);
|
|
776
|
+
await previous.writeBatch([makeLog('suivant')]);
|
|
777
|
+
const realFs = jest.requireActual('fs');
|
|
778
|
+
jest.spyOn(realFs, 'readFileSync').mockImplementationOnce(() => {
|
|
779
|
+
throw Object.assign(new Error('EBUSY: resource busy or locked'), { code: 'EBUSY' });
|
|
780
|
+
});
|
|
781
|
+
const shipper = shipperOn();
|
|
782
|
+
expect(await shipper.replaySpool()).toBe(1);
|
|
783
|
+
expect(allShipped()).toEqual(['suivant']);
|
|
784
|
+
expect(spooledFiles(base)).toEqual([head.fileName]);
|
|
785
|
+
expect(shipper.mayHavePendingSpool()).toBe(true);
|
|
786
|
+
});
|
|
787
|
+
it("ne rejoue pas le fichier temporaire d'un lot encore en cours d'écriture", async () => {
|
|
788
|
+
const directory = path.join(base, APPLICATION_ID);
|
|
789
|
+
fs.mkdirSync(directory, { recursive: true });
|
|
790
|
+
const temporary = path.join(directory, '.tmp-batch-1-99999-000001-n1.jsonl');
|
|
791
|
+
fs.writeFileSync(temporary, JSON.stringify(makeLog('à moitié écrit')));
|
|
792
|
+
const shipper = shipperOn();
|
|
793
|
+
expect(await shipper.replaySpool()).toBe(0);
|
|
794
|
+
expect(sendBulk).not.toHaveBeenCalled();
|
|
795
|
+
expect(fs.existsSync(temporary)).toBe(true);
|
|
796
|
+
});
|
|
797
|
+
it("ne rejoue pas un lot en cours d'envoi", async () => {
|
|
798
|
+
const pending = deferred();
|
|
799
|
+
sendBulk.mockReturnValueOnce(pending.promise);
|
|
800
|
+
const shipper = shipperOn();
|
|
801
|
+
shipper.enqueue(makeLog('en vol'));
|
|
802
|
+
const flushing = shipper.flush();
|
|
803
|
+
await settle();
|
|
804
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
805
|
+
expect(spooledFiles(base)).toHaveLength(1);
|
|
806
|
+
expect(await shipper.replaySpool()).toBe(0);
|
|
807
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
808
|
+
pending.resolve(ACK);
|
|
809
|
+
await flushing;
|
|
810
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
811
|
+
});
|
|
812
|
+
it('un lot laissé sur disque pendant un rejeu est relu ensuite', async () => {
|
|
813
|
+
useFakeClock();
|
|
814
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
815
|
+
const shipper = shipperOn({ flushIntervalMs: 500 });
|
|
816
|
+
shipper.enqueue(makeLog('ancien'));
|
|
817
|
+
await shipper.flush();
|
|
818
|
+
jest.setSystemTime(Date.now() + log_shipper_model_1.BACKOFF_BASE_MS + 1);
|
|
819
|
+
const replayedSend = deferred();
|
|
820
|
+
sendBulk.mockReturnValueOnce(replayedSend.promise);
|
|
821
|
+
const replay = shipper.replaySpool();
|
|
822
|
+
await settle();
|
|
823
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
824
|
+
shipper.enqueue(makeLog('nouveau'));
|
|
825
|
+
await shipper.flush();
|
|
826
|
+
replayedSend.resolve(ACK);
|
|
827
|
+
expect(await replay).toBe(1);
|
|
828
|
+
expect(spooledFiles(base)).toHaveLength(1);
|
|
829
|
+
await jest.advanceTimersByTimeAsync(500);
|
|
830
|
+
await settle();
|
|
831
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
832
|
+
expect(allShipped()).toEqual(['ancien', 'ancien', 'nouveau', 'nouveau']);
|
|
833
|
+
});
|
|
834
|
+
it('des flush concurrents ne font jamais deux envois à la fois', async () => {
|
|
835
|
+
useFakeClock();
|
|
836
|
+
let outstanding = 0;
|
|
837
|
+
let maxOutstanding = 0;
|
|
838
|
+
const releases = [];
|
|
839
|
+
sendBulk.mockImplementation(() => {
|
|
840
|
+
outstanding += 1;
|
|
841
|
+
maxOutstanding = Math.max(maxOutstanding, outstanding);
|
|
842
|
+
return new Promise((resolve) => releases.push(() => {
|
|
843
|
+
outstanding -= 1;
|
|
844
|
+
jest.setSystemTime(Date.now() + 10);
|
|
845
|
+
resolve(ACK);
|
|
846
|
+
}));
|
|
847
|
+
});
|
|
848
|
+
const shipper = shipperOn({ spoolDirectory: null, maxBatchEntries: 1 });
|
|
849
|
+
shipper.enqueue(makeLog('a'));
|
|
850
|
+
const first = shipper.flush(5);
|
|
851
|
+
await settle();
|
|
852
|
+
shipper.enqueue(makeLog('b'));
|
|
853
|
+
shipper.enqueue(makeLog('c'));
|
|
854
|
+
const second = shipper.flush();
|
|
855
|
+
const third = shipper.flush();
|
|
856
|
+
for (let turn = 0; turn < 10 && releases.length > 0; turn += 1) {
|
|
857
|
+
releases.shift()?.();
|
|
858
|
+
await settle();
|
|
859
|
+
}
|
|
860
|
+
await Promise.all([first, second, third]);
|
|
861
|
+
expect(allShipped()).toEqual(['a', 'b', 'c']);
|
|
862
|
+
expect(maxOutstanding).toBe(1);
|
|
863
|
+
});
|
|
864
|
+
it("le délai d'un flush court depuis son appel, attente comprise", async () => {
|
|
865
|
+
useFakeClock();
|
|
866
|
+
const hung = deferred();
|
|
867
|
+
sendBulk.mockReturnValueOnce(hung.promise);
|
|
868
|
+
const shipper = shipperOn({ spoolDirectory: null });
|
|
869
|
+
shipper.enqueue(makeLog('a'));
|
|
870
|
+
const first = shipper.flush(500);
|
|
871
|
+
await settle();
|
|
872
|
+
shipper.enqueue(makeLog('b'));
|
|
873
|
+
const second = shipper.flush(1_000);
|
|
874
|
+
jest.setSystemTime(Date.now() + 2_000);
|
|
875
|
+
hung.resolve(ACK);
|
|
876
|
+
await Promise.all([first, second]);
|
|
877
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
878
|
+
});
|
|
879
|
+
it('reconfigure garde la file et applique la nouvelle taille de lot', async () => {
|
|
880
|
+
const shipper = shipperOn();
|
|
881
|
+
['a', 'b', 'c'].forEach((message) => shipper.enqueue(makeLog(message)));
|
|
882
|
+
shipper.reconfigure({ spoolDirectory: base, flushIntervalMs: 60_000, maxBatchEntries: 1 });
|
|
883
|
+
await shipper.flush();
|
|
884
|
+
expect(sendBulk.mock.calls.map((call) => call[2].logs.length)).toEqual([1, 1, 1]);
|
|
885
|
+
expect(allShipped()).toEqual(['a', 'b', 'c']);
|
|
886
|
+
});
|
|
887
|
+
it("reconfigure vers un autre répertoire de spool y écrit, et rejoue ce qu'il y trouve", async () => {
|
|
888
|
+
const other = fs.mkdtempSync(path.join(os.tmpdir(), 'abyss-shipper-other-'));
|
|
889
|
+
try {
|
|
890
|
+
await new log_spool_model_1.LogSpool({
|
|
891
|
+
directory: path.join(other, APPLICATION_ID),
|
|
892
|
+
maxBytes: 1_000_000,
|
|
893
|
+
}).writeBatch([makeLog('laissé là')]);
|
|
894
|
+
const shipper = shipperOn();
|
|
895
|
+
expect(await shipper.replaySpool()).toBe(0);
|
|
896
|
+
expect(shipper.mayHavePendingSpool()).toBe(false);
|
|
897
|
+
shipper.reconfigure({ spoolDirectory: other, flushIntervalMs: 60_000 });
|
|
898
|
+
expect(shipper.mayHavePendingSpool()).toBe(true);
|
|
899
|
+
expect(await shipper.replaySpool()).toBe(1);
|
|
900
|
+
expect(allShipped()).toEqual(['laissé là']);
|
|
901
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
902
|
+
shipper.enqueue(makeLog('après'));
|
|
903
|
+
await shipper.flush();
|
|
904
|
+
expect(spooledFiles(other)).toHaveLength(1);
|
|
905
|
+
expect(spooledFiles(base)).toHaveLength(0);
|
|
906
|
+
}
|
|
907
|
+
finally {
|
|
908
|
+
fs.rmSync(other, { recursive: true, force: true });
|
|
909
|
+
}
|
|
910
|
+
});
|
|
911
|
+
it('reconfigure réarme le flush périodique sur le nouvel intervalle', async () => {
|
|
912
|
+
useFakeClock();
|
|
913
|
+
const shipper = shipperOn();
|
|
914
|
+
shipper.enqueue(makeLog('a'));
|
|
915
|
+
shipper.reconfigure({ spoolDirectory: base, flushIntervalMs: 100 });
|
|
916
|
+
await jest.advanceTimersByTimeAsync(100);
|
|
917
|
+
await settle();
|
|
918
|
+
expect(allShipped()).toEqual(['a']);
|
|
919
|
+
expect(jest.getTimerCount()).toBe(1);
|
|
920
|
+
});
|
|
921
|
+
it('un shipper arrêté reste arrêté quand on le reconfigure', async () => {
|
|
922
|
+
useFakeClock();
|
|
923
|
+
const shipper = shipperOn();
|
|
924
|
+
shipper.enqueue(makeLog('a'));
|
|
925
|
+
shipper.stop();
|
|
926
|
+
shipper.reconfigure({ spoolDirectory: base, flushIntervalMs: 100 });
|
|
927
|
+
await jest.advanceTimersByTimeAsync(1_000);
|
|
928
|
+
await settle();
|
|
929
|
+
expect(sendBulk).not.toHaveBeenCalled();
|
|
930
|
+
expect(jest.getTimerCount()).toBe(0);
|
|
931
|
+
});
|
|
932
|
+
it('reconfigure applique tout de suite des plafonds de file plus serrés, en comptant', async () => {
|
|
933
|
+
const shipper = shipperOn();
|
|
934
|
+
['a', 'b', 'c'].forEach((message) => shipper.enqueue(makeLog(message)));
|
|
935
|
+
shipper.reconfigure({ spoolDirectory: base, flushIntervalMs: 60_000, maxQueueEntries: 1 });
|
|
936
|
+
await shipper.flush();
|
|
937
|
+
expect(allShipped()).toEqual(['c']);
|
|
938
|
+
expect(sendBulk.mock.calls[0][2].droppedCount).toBe(2);
|
|
939
|
+
});
|
|
940
|
+
it("les évictions d'un spool abandonné par reconfigure sont quand même remontées", async () => {
|
|
941
|
+
useFakeClock();
|
|
942
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
943
|
+
const shipper = shipperOn({ spoolMaxBytes: 500 });
|
|
944
|
+
shipper.enqueue(makeLog('a'.repeat(200)));
|
|
945
|
+
await shipper.flush();
|
|
946
|
+
shipper.enqueue(makeLog('b'.repeat(200)));
|
|
947
|
+
await shipper.flush();
|
|
948
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
949
|
+
expect(spooledFiles(base)).toHaveLength(1);
|
|
950
|
+
shipper.reconfigure({ spoolDirectory: null, flushIntervalMs: 60_000 });
|
|
951
|
+
jest.setSystemTime(Date.now() + log_shipper_model_1.BACKOFF_BASE_MS + 1);
|
|
952
|
+
shipper.enqueue(makeLog('c'));
|
|
953
|
+
await shipper.flush();
|
|
954
|
+
expect(messagesOf(1)).toEqual(['c']);
|
|
955
|
+
expect(sendBulk.mock.calls[1][2].droppedCount).toBe(1);
|
|
956
|
+
});
|
|
957
|
+
it('à la sortie, le délai écoulé, ce qui reste en file descend sur disque sans POST', async () => {
|
|
958
|
+
useFakeClock();
|
|
959
|
+
const shipper = shipperOn({ maxBatchEntries: 2 });
|
|
960
|
+
['a', 'b', 'c', 'd', 'e'].forEach((message) => shipper.enqueue(makeLog(message)));
|
|
961
|
+
await shipper.flush(0, { ignoreBackoff: true });
|
|
962
|
+
expect(sendBulk).not.toHaveBeenCalled();
|
|
963
|
+
const spooled = spooledFiles(base)
|
|
964
|
+
.sort()
|
|
965
|
+
.flatMap((file) => fs
|
|
966
|
+
.readFileSync(path.join(base, APPLICATION_ID, file), 'utf8')
|
|
967
|
+
.split('\n')
|
|
968
|
+
.map((line) => JSON.parse(line).message));
|
|
969
|
+
expect(spooled).toEqual(['a', 'b', 'c', 'd', 'e']);
|
|
970
|
+
});
|
|
971
|
+
it('à la sortie, un POST qui ne répond jamais finit dans le délai, et toute la file est sur disque', async () => {
|
|
972
|
+
useFakeClock();
|
|
973
|
+
sendBulk.mockImplementation((_params, _query, _body, options) => new Promise((_resolve, reject) => {
|
|
974
|
+
setTimeout(() => reject(Object.assign(new Error('timeout exceeded'), { code: 'ECONNABORTED' })), options?.timeoutMs ?? applicationLogApi.BULK_REQUEST_TIMEOUT_MS);
|
|
975
|
+
}));
|
|
976
|
+
const shipper = shipperOn();
|
|
977
|
+
const emitted = Array.from({ length: 250 }, (_, index) => `line-${index}`);
|
|
978
|
+
emitted.forEach((message) => shipper.enqueue(makeLog(message)));
|
|
979
|
+
const deadlineMs = 300;
|
|
980
|
+
const exit = Promise.race([
|
|
981
|
+
shipper.flush(deadlineMs, { ignoreBackoff: true }),
|
|
982
|
+
new Promise((resolve) => setTimeout(resolve, deadlineMs + 250)),
|
|
983
|
+
]);
|
|
984
|
+
await jest.advanceTimersByTimeAsync(deadlineMs + 250);
|
|
985
|
+
await exit;
|
|
986
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
987
|
+
expect(sendBulk.mock.calls[0][3]).toEqual({ timeoutMs: deadlineMs });
|
|
988
|
+
const onDisk = spooledFiles(base).flatMap((file) => fs
|
|
989
|
+
.readFileSync(path.join(base, APPLICATION_ID, file), 'utf8')
|
|
990
|
+
.split('\n')
|
|
991
|
+
.map((line) => JSON.parse(line).message));
|
|
992
|
+
expect(onDisk.sort()).toEqual([...emitted].sort());
|
|
993
|
+
});
|
|
994
|
+
it('un flush ordinaire garde le délai de requête par défaut', async () => {
|
|
995
|
+
const shipper = shipperOn();
|
|
996
|
+
shipper.enqueue(makeLog('a'));
|
|
997
|
+
await shipper.flush();
|
|
998
|
+
expect(sendBulk.mock.calls[0][3]).toEqual({ timeoutMs: undefined });
|
|
999
|
+
});
|
|
1000
|
+
it('à la sortie sans spool, ce que le délai laisse en file est perdu et compté', async () => {
|
|
1001
|
+
useFakeClock();
|
|
1002
|
+
const shipper = shipperOn({ spoolDirectory: null });
|
|
1003
|
+
['a', 'b', 'c'].forEach((message) => shipper.enqueue(makeLog(message)));
|
|
1004
|
+
await shipper.flush(0, { ignoreBackoff: true });
|
|
1005
|
+
expect(sendBulk).not.toHaveBeenCalled();
|
|
1006
|
+
shipper.enqueue(makeLog('x'));
|
|
1007
|
+
await shipper.flush();
|
|
1008
|
+
expect(messagesOf(0)).toEqual(['x']);
|
|
1009
|
+
expect(sendBulk.mock.calls[0][2].droppedCount).toBe(3);
|
|
1010
|
+
});
|
|
1011
|
+
it("à la sortie sans spool, un échec fait compter ce qui reste au lieu de l'abandonner en file", async () => {
|
|
1012
|
+
useFakeClock();
|
|
1013
|
+
sendBulk.mockRejectedValueOnce(httpError(503));
|
|
1014
|
+
const shipper = shipperOn({ spoolDirectory: null, maxBatchEntries: 1 });
|
|
1015
|
+
shipper.enqueue(makeLog('a'));
|
|
1016
|
+
shipper.enqueue(makeLog('b'));
|
|
1017
|
+
await shipper.flush(3_000, { ignoreBackoff: true });
|
|
1018
|
+
expect(sendBulk).toHaveBeenCalledTimes(1);
|
|
1019
|
+
shipper.enqueue(makeLog('x'));
|
|
1020
|
+
await shipper.flush(3_000, { ignoreBackoff: true });
|
|
1021
|
+
expect(messagesOf(1)).toEqual(['x']);
|
|
1022
|
+
expect(sendBulk.mock.calls[1][2].droppedCount).toBe(2);
|
|
1023
|
+
});
|
|
1024
|
+
it("un envoi bloqué n'empile pas un flush par tick", async () => {
|
|
1025
|
+
useFakeClock();
|
|
1026
|
+
const hung = deferred();
|
|
1027
|
+
sendBulk.mockReturnValueOnce(hung.promise);
|
|
1028
|
+
const drainAll = jest.spyOn(log_shipper_model_1.LogShipper.prototype, 'drainAll');
|
|
1029
|
+
const shipper = shipperOn({ flushIntervalMs: 500 });
|
|
1030
|
+
shipper.enqueue(makeLog('a'));
|
|
1031
|
+
const flushing = shipper.flush();
|
|
1032
|
+
await settle();
|
|
1033
|
+
await jest.advanceTimersByTimeAsync(5_000);
|
|
1034
|
+
hung.resolve(ACK);
|
|
1035
|
+
await flushing;
|
|
1036
|
+
await settle();
|
|
1037
|
+
expect(drainAll).toHaveBeenCalledTimes(1);
|
|
1038
|
+
});
|
|
1039
|
+
});
|
|
1040
|
+
//# sourceMappingURL=log-shipper.test.js.map
|