@unito/integration-sdk 1.0.24 → 1.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.cjs +62 -9
- package/dist/src/resources/logger.d.ts +2 -1
- package/dist/src/resources/logger.js +59 -7
- package/dist/src/resources/provider.js +3 -2
- package/dist/test/resources/logger.test.js +71 -45
- package/dist/test/resources/provider.test.js +4 -1
- package/package.json +1 -1
- package/src/resources/logger.ts +66 -8
- package/src/resources/provider.ts +6 -2
- package/test/resources/logger.test.ts +82 -47
- package/test/resources/provider.test.ts +4 -1
package/dist/src/index.cjs
CHANGED
|
@@ -34,6 +34,35 @@ var LogLevel;
|
|
|
34
34
|
LogLevel["LOG"] = "log";
|
|
35
35
|
LogLevel["DEBUG"] = "debug";
|
|
36
36
|
})(LogLevel || (LogLevel = {}));
|
|
37
|
+
/**
|
|
38
|
+
* See https://docs.datadoghq.com/logs/log_collection/?tab=host#custom-log-forwarding
|
|
39
|
+
* - Datadog Agent splits at 256kB (256000 bytes)...
|
|
40
|
+
* - ... but the same docs say that "for optimal performance, it is
|
|
41
|
+
* recommended that an individual log be no greater than 25kB"
|
|
42
|
+
* -> Truncating at 25kB - a bit of wiggle room for metadata = 20kB.
|
|
43
|
+
*/
|
|
44
|
+
const MAX_LOG_MESSAGE_SIZE = parseInt(process.env.MAX_LOG_MESSAGE_SIZE ?? '20000', 10);
|
|
45
|
+
const LOG_LINE_TRUNCATED_SUFFIX = ' - LOG LINE TRUNCATED';
|
|
46
|
+
/**
|
|
47
|
+
* For *LogMeta* sanitization, we let in anything that was passed, except for clearly-problematic keys
|
|
48
|
+
*/
|
|
49
|
+
const LOGMETA_BLACKLIST = [
|
|
50
|
+
// Security
|
|
51
|
+
'access_token',
|
|
52
|
+
'bot_auth_code',
|
|
53
|
+
'client_secret',
|
|
54
|
+
'jwt',
|
|
55
|
+
'oauth_token',
|
|
56
|
+
'password',
|
|
57
|
+
'refresh_token',
|
|
58
|
+
'shared_secret',
|
|
59
|
+
'token',
|
|
60
|
+
// Privacy
|
|
61
|
+
'billing_email',
|
|
62
|
+
'email',
|
|
63
|
+
'first_name',
|
|
64
|
+
'last_name',
|
|
65
|
+
];
|
|
37
66
|
/**
|
|
38
67
|
* Logger class that can be configured with metadata add creation and when logging to add additional context to your logs.
|
|
39
68
|
*/
|
|
@@ -113,20 +142,27 @@ class Logger {
|
|
|
113
142
|
this.metadata = {};
|
|
114
143
|
}
|
|
115
144
|
send(logLevel, message, metadata) {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
145
|
+
// We need to provide the date to Datadog. Otherwise, the date is set to when they receive the log.
|
|
146
|
+
const date = Date.now();
|
|
147
|
+
if (message.length > MAX_LOG_MESSAGE_SIZE) {
|
|
148
|
+
message = `${message.substring(0, MAX_LOG_MESSAGE_SIZE)}${LOG_LINE_TRUNCATED_SUFFIX}`;
|
|
149
|
+
}
|
|
150
|
+
let processedMetadata = Logger.snakifyKeys({ ...this.metadata, ...metadata, logMessageSize: message.length });
|
|
151
|
+
processedMetadata = Logger.pruneSensitiveMetadata(processedMetadata);
|
|
152
|
+
const processedLogs = {
|
|
153
|
+
...processedMetadata,
|
|
119
154
|
message,
|
|
155
|
+
date,
|
|
120
156
|
status: logLevel,
|
|
121
|
-
}
|
|
157
|
+
};
|
|
122
158
|
if (process.env.NODE_ENV === 'development') {
|
|
123
|
-
console[logLevel](JSON.stringify(
|
|
159
|
+
console[logLevel](JSON.stringify(processedLogs, null, 2));
|
|
124
160
|
}
|
|
125
161
|
else {
|
|
126
|
-
console[logLevel](JSON.stringify(
|
|
162
|
+
console[logLevel](JSON.stringify(processedLogs));
|
|
127
163
|
}
|
|
128
164
|
}
|
|
129
|
-
snakifyKeys(value) {
|
|
165
|
+
static snakifyKeys(value) {
|
|
130
166
|
const result = {};
|
|
131
167
|
for (const key in value) {
|
|
132
168
|
const deepValue = typeof value[key] === 'object' ? this.snakifyKeys(value[key]) : value[key];
|
|
@@ -135,6 +171,22 @@ class Logger {
|
|
|
135
171
|
}
|
|
136
172
|
return result;
|
|
137
173
|
}
|
|
174
|
+
static pruneSensitiveMetadata(metadata, topLevelMeta) {
|
|
175
|
+
const prunedMetadata = {};
|
|
176
|
+
for (const key in metadata) {
|
|
177
|
+
if (LOGMETA_BLACKLIST.includes(key)) {
|
|
178
|
+
prunedMetadata[key] = '[REDACTED]';
|
|
179
|
+
(topLevelMeta ?? prunedMetadata).has_sensitive_attribute = true;
|
|
180
|
+
}
|
|
181
|
+
else if (typeof metadata[key] === 'object') {
|
|
182
|
+
prunedMetadata[key] = Logger.pruneSensitiveMetadata(metadata[key], topLevelMeta ?? prunedMetadata);
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
prunedMetadata[key] = metadata[key];
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return prunedMetadata;
|
|
189
|
+
}
|
|
138
190
|
}
|
|
139
191
|
|
|
140
192
|
/**
|
|
@@ -1305,8 +1357,9 @@ class Provider {
|
|
|
1305
1357
|
case 'TimeoutError':
|
|
1306
1358
|
throw this.handleError(408, 'Request timeout');
|
|
1307
1359
|
}
|
|
1360
|
+
throw this.handleError(500, `Unexpected error while calling the provider: name: "${error.name}" \n message: "${error.message}" \n stack: ${error.stack}`);
|
|
1308
1361
|
}
|
|
1309
|
-
throw this.handleError(500,
|
|
1362
|
+
throw this.handleError(500, 'Unexpected error while calling the provider - this is not normal, investigate');
|
|
1310
1363
|
}
|
|
1311
1364
|
if (response.status >= 400) {
|
|
1312
1365
|
const textResult = await response.text();
|
|
@@ -1335,7 +1388,7 @@ class Provider {
|
|
|
1335
1388
|
body = response.body;
|
|
1336
1389
|
}
|
|
1337
1390
|
else {
|
|
1338
|
-
throw this.handleError(500, 'Unsupported
|
|
1391
|
+
throw this.handleError(500, 'Unsupported Content-Type');
|
|
1339
1392
|
}
|
|
1340
1393
|
return { status: response.status, headers: response.headers, body };
|
|
1341
1394
|
};
|
|
@@ -6,6 +6,35 @@ var LogLevel;
|
|
|
6
6
|
LogLevel["LOG"] = "log";
|
|
7
7
|
LogLevel["DEBUG"] = "debug";
|
|
8
8
|
})(LogLevel || (LogLevel = {}));
|
|
9
|
+
/**
|
|
10
|
+
* See https://docs.datadoghq.com/logs/log_collection/?tab=host#custom-log-forwarding
|
|
11
|
+
* - Datadog Agent splits at 256kB (256000 bytes)...
|
|
12
|
+
* - ... but the same docs say that "for optimal performance, it is
|
|
13
|
+
* recommended that an individual log be no greater than 25kB"
|
|
14
|
+
* -> Truncating at 25kB - a bit of wiggle room for metadata = 20kB.
|
|
15
|
+
*/
|
|
16
|
+
const MAX_LOG_MESSAGE_SIZE = parseInt(process.env.MAX_LOG_MESSAGE_SIZE ?? '20000', 10);
|
|
17
|
+
const LOG_LINE_TRUNCATED_SUFFIX = ' - LOG LINE TRUNCATED';
|
|
18
|
+
/**
|
|
19
|
+
* For *LogMeta* sanitization, we let in anything that was passed, except for clearly-problematic keys
|
|
20
|
+
*/
|
|
21
|
+
const LOGMETA_BLACKLIST = [
|
|
22
|
+
// Security
|
|
23
|
+
'access_token',
|
|
24
|
+
'bot_auth_code',
|
|
25
|
+
'client_secret',
|
|
26
|
+
'jwt',
|
|
27
|
+
'oauth_token',
|
|
28
|
+
'password',
|
|
29
|
+
'refresh_token',
|
|
30
|
+
'shared_secret',
|
|
31
|
+
'token',
|
|
32
|
+
// Privacy
|
|
33
|
+
'billing_email',
|
|
34
|
+
'email',
|
|
35
|
+
'first_name',
|
|
36
|
+
'last_name',
|
|
37
|
+
];
|
|
9
38
|
/**
|
|
10
39
|
* Logger class that can be configured with metadata add creation and when logging to add additional context to your logs.
|
|
11
40
|
*/
|
|
@@ -85,20 +114,27 @@ export default class Logger {
|
|
|
85
114
|
this.metadata = {};
|
|
86
115
|
}
|
|
87
116
|
send(logLevel, message, metadata) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
117
|
+
// We need to provide the date to Datadog. Otherwise, the date is set to when they receive the log.
|
|
118
|
+
const date = Date.now();
|
|
119
|
+
if (message.length > MAX_LOG_MESSAGE_SIZE) {
|
|
120
|
+
message = `${message.substring(0, MAX_LOG_MESSAGE_SIZE)}${LOG_LINE_TRUNCATED_SUFFIX}`;
|
|
121
|
+
}
|
|
122
|
+
let processedMetadata = Logger.snakifyKeys({ ...this.metadata, ...metadata, logMessageSize: message.length });
|
|
123
|
+
processedMetadata = Logger.pruneSensitiveMetadata(processedMetadata);
|
|
124
|
+
const processedLogs = {
|
|
125
|
+
...processedMetadata,
|
|
91
126
|
message,
|
|
127
|
+
date,
|
|
92
128
|
status: logLevel,
|
|
93
|
-
}
|
|
129
|
+
};
|
|
94
130
|
if (process.env.NODE_ENV === 'development') {
|
|
95
|
-
console[logLevel](JSON.stringify(
|
|
131
|
+
console[logLevel](JSON.stringify(processedLogs, null, 2));
|
|
96
132
|
}
|
|
97
133
|
else {
|
|
98
|
-
console[logLevel](JSON.stringify(
|
|
134
|
+
console[logLevel](JSON.stringify(processedLogs));
|
|
99
135
|
}
|
|
100
136
|
}
|
|
101
|
-
snakifyKeys(value) {
|
|
137
|
+
static snakifyKeys(value) {
|
|
102
138
|
const result = {};
|
|
103
139
|
for (const key in value) {
|
|
104
140
|
const deepValue = typeof value[key] === 'object' ? this.snakifyKeys(value[key]) : value[key];
|
|
@@ -107,4 +143,20 @@ export default class Logger {
|
|
|
107
143
|
}
|
|
108
144
|
return result;
|
|
109
145
|
}
|
|
146
|
+
static pruneSensitiveMetadata(metadata, topLevelMeta) {
|
|
147
|
+
const prunedMetadata = {};
|
|
148
|
+
for (const key in metadata) {
|
|
149
|
+
if (LOGMETA_BLACKLIST.includes(key)) {
|
|
150
|
+
prunedMetadata[key] = '[REDACTED]';
|
|
151
|
+
(topLevelMeta ?? prunedMetadata).has_sensitive_attribute = true;
|
|
152
|
+
}
|
|
153
|
+
else if (typeof metadata[key] === 'object') {
|
|
154
|
+
prunedMetadata[key] = Logger.pruneSensitiveMetadata(metadata[key], topLevelMeta ?? prunedMetadata);
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
prunedMetadata[key] = metadata[key];
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return prunedMetadata;
|
|
161
|
+
}
|
|
110
162
|
}
|
|
@@ -266,8 +266,9 @@ export class Provider {
|
|
|
266
266
|
case 'TimeoutError':
|
|
267
267
|
throw this.handleError(408, 'Request timeout');
|
|
268
268
|
}
|
|
269
|
+
throw this.handleError(500, `Unexpected error while calling the provider: name: "${error.name}" \n message: "${error.message}" \n stack: ${error.stack}`);
|
|
269
270
|
}
|
|
270
|
-
throw this.handleError(500,
|
|
271
|
+
throw this.handleError(500, 'Unexpected error while calling the provider - this is not normal, investigate');
|
|
271
272
|
}
|
|
272
273
|
if (response.status >= 400) {
|
|
273
274
|
const textResult = await response.text();
|
|
@@ -296,7 +297,7 @@ export class Provider {
|
|
|
296
297
|
body = response.body;
|
|
297
298
|
}
|
|
298
299
|
else {
|
|
299
|
-
throw this.handleError(500, 'Unsupported
|
|
300
|
+
throw this.handleError(500, 'Unsupported Content-Type');
|
|
300
301
|
}
|
|
301
302
|
return { status: response.status, headers: response.headers, body };
|
|
302
303
|
};
|
|
@@ -32,86 +32,112 @@ describe('Logger', () => {
|
|
|
32
32
|
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
33
33
|
logger.log('test');
|
|
34
34
|
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
]);
|
|
35
|
+
let actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
36
|
+
assert.ok(actual['date']);
|
|
37
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
38
|
+
assert.equal(actual['message'], 'test');
|
|
39
|
+
assert.equal(actual['status'], 'log');
|
|
38
40
|
const errorSpy = testContext.mock.method(global.console, 'error', () => { });
|
|
39
41
|
assert.strictEqual(errorSpy.mock.calls.length, 0);
|
|
40
42
|
logger.error('test');
|
|
41
43
|
assert.strictEqual(errorSpy.mock.calls.length, 1);
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
]);
|
|
44
|
+
actual = JSON.parse(errorSpy.mock.calls[0]?.arguments[0]);
|
|
45
|
+
assert.ok(actual['date']);
|
|
46
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
47
|
+
assert.equal(actual['message'], 'test');
|
|
48
|
+
assert.equal(actual['status'], 'error');
|
|
45
49
|
const warnSpy = testContext.mock.method(global.console, 'warn', () => { });
|
|
46
50
|
assert.strictEqual(warnSpy.mock.calls.length, 0);
|
|
47
51
|
logger.warn('test');
|
|
48
52
|
assert.strictEqual(warnSpy.mock.calls.length, 1);
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
]);
|
|
53
|
+
actual = JSON.parse(warnSpy.mock.calls[0]?.arguments[0]);
|
|
54
|
+
assert.ok(actual['date']);
|
|
55
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
56
|
+
assert.equal(actual['message'], 'test');
|
|
57
|
+
assert.equal(actual['status'], 'warn');
|
|
52
58
|
const infoSpy = testContext.mock.method(global.console, 'info', () => { });
|
|
53
59
|
assert.strictEqual(infoSpy.mock.calls.length, 0);
|
|
54
60
|
logger.info('test');
|
|
55
61
|
assert.strictEqual(infoSpy.mock.calls.length, 1);
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
]);
|
|
62
|
+
actual = JSON.parse(infoSpy.mock.calls[0]?.arguments[0]);
|
|
63
|
+
assert.ok(actual['date']);
|
|
64
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
65
|
+
assert.equal(actual['message'], 'test');
|
|
66
|
+
assert.equal(actual['status'], 'info');
|
|
59
67
|
const debugSpy = testContext.mock.method(global.console, 'debug', () => { });
|
|
60
68
|
assert.strictEqual(debugSpy.mock.calls.length, 0);
|
|
61
69
|
logger.debug('test');
|
|
62
70
|
assert.strictEqual(debugSpy.mock.calls.length, 1);
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
]);
|
|
71
|
+
actual = JSON.parse(debugSpy.mock.calls[0]?.arguments[0]);
|
|
72
|
+
assert.ok(actual['date']);
|
|
73
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
74
|
+
assert.equal(actual['message'], 'test');
|
|
75
|
+
assert.equal(actual['status'], 'debug');
|
|
66
76
|
});
|
|
67
77
|
it('merges message payload with metadata', testContext => {
|
|
68
|
-
const metadata = { correlation_id: '123456789', http: { method: 'GET' } };
|
|
69
|
-
const logger = new Logger(metadata);
|
|
70
78
|
const logSpy = testContext.mock.method(global.console, 'log', () => { });
|
|
71
79
|
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
80
|
+
const metadata = { correlation_id: '123456789', http: { method: 'GET' } };
|
|
81
|
+
const logger = new Logger(metadata);
|
|
72
82
|
logger.log('test', { error: { code: '200', message: 'Page Not Found' } });
|
|
73
83
|
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}),
|
|
82
|
-
]);
|
|
84
|
+
const actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
85
|
+
assert.ok(actual['date']);
|
|
86
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
87
|
+
assert.equal(actual['message'], 'test');
|
|
88
|
+
assert.equal(actual['status'], 'log');
|
|
89
|
+
assert.deepEqual(actual['http'], { method: 'GET' });
|
|
90
|
+
assert.deepEqual(actual['error'], { code: '200', message: 'Page Not Found' });
|
|
83
91
|
});
|
|
84
92
|
it('overwrites conflicting metadata keys (1st level) with message payload', testContext => {
|
|
85
|
-
const metadata = { correlation_id: '123456789', http: { method: 'GET' } };
|
|
86
|
-
const logger = new Logger(metadata);
|
|
87
93
|
const logSpy = testContext.mock.method(global.console, 'log', () => { });
|
|
88
94
|
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
95
|
+
const metadata = { correlation_id: '123456789', http: { method: 'GET' } };
|
|
96
|
+
const logger = new Logger(metadata);
|
|
89
97
|
logger.log('test', { http: { status_code: 200 } });
|
|
90
98
|
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
}),
|
|
98
|
-
]);
|
|
99
|
+
const actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
100
|
+
assert.ok(actual['date']);
|
|
101
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
102
|
+
assert.equal(actual['message'], 'test');
|
|
103
|
+
assert.equal(actual['status'], 'log');
|
|
104
|
+
assert.deepEqual(actual['http'], { status_code: 200 });
|
|
99
105
|
});
|
|
100
106
|
it('snakify keys of Message and Metadata', testContext => {
|
|
107
|
+
const logSpy = testContext.mock.method(global.console, 'log', () => { });
|
|
108
|
+
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
101
109
|
const metadata = { correlationId: '123456789', http: { method: 'GET', statusCode: 200 } };
|
|
102
110
|
const logger = new Logger(metadata);
|
|
111
|
+
logger.log('test', { errorContext: { errorCode: 200, errorMessage: 'Page Not Found' } });
|
|
112
|
+
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
113
|
+
const actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
114
|
+
assert.ok(actual['date']);
|
|
115
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
116
|
+
assert.equal(actual['message'], 'test');
|
|
117
|
+
assert.equal(actual['status'], 'log');
|
|
118
|
+
assert.deepEqual(actual['http'], { method: 'GET', status_code: 200 });
|
|
119
|
+
assert.deepEqual(actual['error_context'], { error_code: 200, error_message: 'Page Not Found' });
|
|
120
|
+
});
|
|
121
|
+
it('prunes sensitive Metadata', testContext => {
|
|
103
122
|
const logSpy = testContext.mock.method(global.console, 'log', () => { });
|
|
104
123
|
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
124
|
+
const metadata = {
|
|
125
|
+
correlationId: '123456789',
|
|
126
|
+
http: { method: 'GET', statusCode: 200, jwt: 'deepSecret' },
|
|
127
|
+
user: { contact: { email: 'deep_deep_deep@email.address', firstName: 'should be snakify then hidden' } },
|
|
128
|
+
access_token: 'secret',
|
|
129
|
+
};
|
|
130
|
+
const logger = new Logger(metadata);
|
|
105
131
|
logger.log('test', { errorContext: { errorCode: 200, errorMessage: 'Page Not Found' } });
|
|
106
132
|
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
]);
|
|
133
|
+
const actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
134
|
+
assert.ok(actual['date']);
|
|
135
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
136
|
+
assert.equal(actual['message'], 'test');
|
|
137
|
+
assert.equal(actual['status'], 'log');
|
|
138
|
+
assert.equal(actual['has_sensitive_attribute'], true);
|
|
139
|
+
assert.equal(actual['access_token'], '[REDACTED]');
|
|
140
|
+
assert.deepEqual(actual['http'], { method: 'GET', status_code: 200, jwt: '[REDACTED]' });
|
|
141
|
+
assert.deepEqual(actual['user']['contact'], { email: '[REDACTED]', first_name: '[REDACTED]' });
|
|
116
142
|
});
|
|
117
143
|
});
|
|
@@ -478,7 +478,10 @@ describe('Provider', () => {
|
|
|
478
478
|
error = e;
|
|
479
479
|
}
|
|
480
480
|
assert.ok(error instanceof HttpErrors.HttpError);
|
|
481
|
-
assert.
|
|
481
|
+
assert.ok(error.message.startsWith('Unexpected error while calling the provider:'));
|
|
482
|
+
assert.ok(error.message.includes('name: "Error"'));
|
|
483
|
+
assert.ok(error.message.includes('message: "foo"'));
|
|
484
|
+
assert.ok(error.message.includes('stack:'));
|
|
482
485
|
});
|
|
483
486
|
it('throws on status 429', async (context) => {
|
|
484
487
|
const response = new Response('response body', {
|
package/package.json
CHANGED
package/src/resources/logger.ts
CHANGED
|
@@ -14,6 +14,37 @@ type Value = {
|
|
|
14
14
|
export type Metadata = Value & { message?: never };
|
|
15
15
|
type ForbidenMetadataKey = 'message';
|
|
16
16
|
|
|
17
|
+
/**
|
|
18
|
+
* See https://docs.datadoghq.com/logs/log_collection/?tab=host#custom-log-forwarding
|
|
19
|
+
* - Datadog Agent splits at 256kB (256000 bytes)...
|
|
20
|
+
* - ... but the same docs say that "for optimal performance, it is
|
|
21
|
+
* recommended that an individual log be no greater than 25kB"
|
|
22
|
+
* -> Truncating at 25kB - a bit of wiggle room for metadata = 20kB.
|
|
23
|
+
*/
|
|
24
|
+
const MAX_LOG_MESSAGE_SIZE = parseInt(process.env.MAX_LOG_MESSAGE_SIZE ?? '20000', 10);
|
|
25
|
+
const LOG_LINE_TRUNCATED_SUFFIX = ' - LOG LINE TRUNCATED';
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* For *LogMeta* sanitization, we let in anything that was passed, except for clearly-problematic keys
|
|
29
|
+
*/
|
|
30
|
+
const LOGMETA_BLACKLIST = [
|
|
31
|
+
// Security
|
|
32
|
+
'access_token',
|
|
33
|
+
'bot_auth_code',
|
|
34
|
+
'client_secret',
|
|
35
|
+
'jwt',
|
|
36
|
+
'oauth_token',
|
|
37
|
+
'password',
|
|
38
|
+
'refresh_token',
|
|
39
|
+
'shared_secret',
|
|
40
|
+
'token',
|
|
41
|
+
// Privacy
|
|
42
|
+
'billing_email',
|
|
43
|
+
'email',
|
|
44
|
+
'first_name',
|
|
45
|
+
'last_name',
|
|
46
|
+
];
|
|
47
|
+
|
|
17
48
|
/**
|
|
18
49
|
* Logger class that can be configured with metadata add creation and when logging to add additional context to your logs.
|
|
19
50
|
*/
|
|
@@ -107,21 +138,31 @@ export default class Logger {
|
|
|
107
138
|
}
|
|
108
139
|
|
|
109
140
|
private send(logLevel: LogLevel, message: string, metadata?: Metadata): void {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
141
|
+
// We need to provide the date to Datadog. Otherwise, the date is set to when they receive the log.
|
|
142
|
+
const date = Date.now();
|
|
143
|
+
|
|
144
|
+
if (message.length > MAX_LOG_MESSAGE_SIZE) {
|
|
145
|
+
message = `${message.substring(0, MAX_LOG_MESSAGE_SIZE)}${LOG_LINE_TRUNCATED_SUFFIX}`;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
let processedMetadata = Logger.snakifyKeys({ ...this.metadata, ...metadata, logMessageSize: message.length });
|
|
149
|
+
processedMetadata = Logger.pruneSensitiveMetadata(processedMetadata);
|
|
150
|
+
|
|
151
|
+
const processedLogs = {
|
|
152
|
+
...processedMetadata,
|
|
113
153
|
message,
|
|
154
|
+
date,
|
|
114
155
|
status: logLevel,
|
|
115
|
-
}
|
|
156
|
+
};
|
|
116
157
|
|
|
117
158
|
if (process.env.NODE_ENV === 'development') {
|
|
118
|
-
console[logLevel](JSON.stringify(
|
|
159
|
+
console[logLevel](JSON.stringify(processedLogs, null, 2));
|
|
119
160
|
} else {
|
|
120
|
-
console[logLevel](JSON.stringify(
|
|
161
|
+
console[logLevel](JSON.stringify(processedLogs));
|
|
121
162
|
}
|
|
122
163
|
}
|
|
123
164
|
|
|
124
|
-
private snakifyKeys
|
|
165
|
+
private static snakifyKeys(value: Value): Value {
|
|
125
166
|
const result: Value = {};
|
|
126
167
|
|
|
127
168
|
for (const key in value) {
|
|
@@ -130,6 +171,23 @@ export default class Logger {
|
|
|
130
171
|
result[snakifiedKey] = deepValue;
|
|
131
172
|
}
|
|
132
173
|
|
|
133
|
-
return result
|
|
174
|
+
return result;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
private static pruneSensitiveMetadata(metadata: Value, topLevelMeta?: Value): Value {
|
|
178
|
+
const prunedMetadata: Value = {};
|
|
179
|
+
|
|
180
|
+
for (const key in metadata) {
|
|
181
|
+
if (LOGMETA_BLACKLIST.includes(key)) {
|
|
182
|
+
prunedMetadata[key] = '[REDACTED]';
|
|
183
|
+
(topLevelMeta ?? prunedMetadata).has_sensitive_attribute = true;
|
|
184
|
+
} else if (typeof metadata[key] === 'object') {
|
|
185
|
+
prunedMetadata[key] = Logger.pruneSensitiveMetadata(metadata[key] as Value, topLevelMeta ?? prunedMetadata);
|
|
186
|
+
} else {
|
|
187
|
+
prunedMetadata[key] = metadata[key];
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
return prunedMetadata;
|
|
134
192
|
}
|
|
135
193
|
}
|
|
@@ -357,9 +357,13 @@ export class Provider {
|
|
|
357
357
|
case 'TimeoutError':
|
|
358
358
|
throw this.handleError(408, 'Request timeout');
|
|
359
359
|
}
|
|
360
|
+
throw this.handleError(
|
|
361
|
+
500,
|
|
362
|
+
`Unexpected error while calling the provider: name: "${error.name}" \n message: "${error.message}" \n stack: ${error.stack}`,
|
|
363
|
+
);
|
|
360
364
|
}
|
|
361
365
|
|
|
362
|
-
throw this.handleError(500,
|
|
366
|
+
throw this.handleError(500, 'Unexpected error while calling the provider - this is not normal, investigate');
|
|
363
367
|
}
|
|
364
368
|
|
|
365
369
|
if (response.status >= 400) {
|
|
@@ -392,7 +396,7 @@ export class Provider {
|
|
|
392
396
|
// When we expect octet-stream, we accept any Content-Type the provider sends us, we just want to stream it.
|
|
393
397
|
body = response.body as T;
|
|
394
398
|
} else {
|
|
395
|
-
throw this.handleError(500, 'Unsupported
|
|
399
|
+
throw this.handleError(500, 'Unsupported Content-Type');
|
|
396
400
|
}
|
|
397
401
|
|
|
398
402
|
return { status: response.status, headers: response.headers, body };
|
|
@@ -41,96 +41,131 @@ describe('Logger', () => {
|
|
|
41
41
|
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
42
42
|
logger.log('test');
|
|
43
43
|
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
]);
|
|
44
|
+
let actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
45
|
+
assert.ok(actual['date']);
|
|
46
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
47
|
+
assert.equal(actual['message'], 'test');
|
|
48
|
+
assert.equal(actual['status'], 'log');
|
|
47
49
|
|
|
48
50
|
const errorSpy = testContext.mock.method(global.console, 'error', () => {});
|
|
49
51
|
assert.strictEqual(errorSpy.mock.calls.length, 0);
|
|
50
52
|
logger.error('test');
|
|
51
53
|
assert.strictEqual(errorSpy.mock.calls.length, 1);
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
]);
|
|
54
|
+
actual = JSON.parse(errorSpy.mock.calls[0]?.arguments[0]);
|
|
55
|
+
assert.ok(actual['date']);
|
|
56
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
57
|
+
assert.equal(actual['message'], 'test');
|
|
58
|
+
assert.equal(actual['status'], 'error');
|
|
55
59
|
|
|
56
60
|
const warnSpy = testContext.mock.method(global.console, 'warn', () => {});
|
|
57
61
|
assert.strictEqual(warnSpy.mock.calls.length, 0);
|
|
58
62
|
logger.warn('test');
|
|
59
63
|
assert.strictEqual(warnSpy.mock.calls.length, 1);
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
]);
|
|
64
|
+
actual = JSON.parse(warnSpy.mock.calls[0]?.arguments[0]);
|
|
65
|
+
assert.ok(actual['date']);
|
|
66
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
67
|
+
assert.equal(actual['message'], 'test');
|
|
68
|
+
assert.equal(actual['status'], 'warn');
|
|
63
69
|
|
|
64
70
|
const infoSpy = testContext.mock.method(global.console, 'info', () => {});
|
|
65
71
|
assert.strictEqual(infoSpy.mock.calls.length, 0);
|
|
66
72
|
logger.info('test');
|
|
67
73
|
assert.strictEqual(infoSpy.mock.calls.length, 1);
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
]);
|
|
74
|
+
actual = JSON.parse(infoSpy.mock.calls[0]?.arguments[0]);
|
|
75
|
+
assert.ok(actual['date']);
|
|
76
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
77
|
+
assert.equal(actual['message'], 'test');
|
|
78
|
+
assert.equal(actual['status'], 'info');
|
|
71
79
|
|
|
72
80
|
const debugSpy = testContext.mock.method(global.console, 'debug', () => {});
|
|
73
81
|
assert.strictEqual(debugSpy.mock.calls.length, 0);
|
|
74
82
|
logger.debug('test');
|
|
75
83
|
assert.strictEqual(debugSpy.mock.calls.length, 1);
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
]);
|
|
84
|
+
actual = JSON.parse(debugSpy.mock.calls[0]?.arguments[0]);
|
|
85
|
+
assert.ok(actual['date']);
|
|
86
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
87
|
+
assert.equal(actual['message'], 'test');
|
|
88
|
+
assert.equal(actual['status'], 'debug');
|
|
79
89
|
});
|
|
80
90
|
|
|
81
91
|
it('merges message payload with metadata', testContext => {
|
|
82
|
-
const metadata = { correlation_id: '123456789', http: { method: 'GET' } };
|
|
83
|
-
const logger = new Logger(metadata);
|
|
84
|
-
|
|
85
92
|
const logSpy = testContext.mock.method(global.console, 'log', () => {});
|
|
86
93
|
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
94
|
+
|
|
95
|
+
const metadata = { correlation_id: '123456789', http: { method: 'GET' } };
|
|
96
|
+
const logger = new Logger(metadata);
|
|
87
97
|
logger.log('test', { error: { code: '200', message: 'Page Not Found' } });
|
|
88
98
|
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
]);
|
|
99
|
+
|
|
100
|
+
const actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
101
|
+
|
|
102
|
+
assert.ok(actual['date']);
|
|
103
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
104
|
+
assert.equal(actual['message'], 'test');
|
|
105
|
+
assert.equal(actual['status'], 'log');
|
|
106
|
+
assert.deepEqual(actual['http'], { method: 'GET' });
|
|
107
|
+
assert.deepEqual(actual['error'], { code: '200', message: 'Page Not Found' });
|
|
98
108
|
});
|
|
99
109
|
|
|
100
110
|
it('overwrites conflicting metadata keys (1st level) with message payload', testContext => {
|
|
101
|
-
const metadata = { correlation_id: '123456789', http: { method: 'GET' } };
|
|
102
|
-
const logger = new Logger(metadata);
|
|
103
|
-
|
|
104
111
|
const logSpy = testContext.mock.method(global.console, 'log', () => {});
|
|
105
112
|
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
113
|
+
|
|
114
|
+
const metadata = { correlation_id: '123456789', http: { method: 'GET' } };
|
|
115
|
+
const logger = new Logger(metadata);
|
|
106
116
|
logger.log('test', { http: { status_code: 200 } });
|
|
107
117
|
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
]);
|
|
118
|
+
|
|
119
|
+
const actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
120
|
+
|
|
121
|
+
assert.ok(actual['date']);
|
|
122
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
123
|
+
assert.equal(actual['message'], 'test');
|
|
124
|
+
assert.equal(actual['status'], 'log');
|
|
125
|
+
assert.deepEqual(actual['http'], { status_code: 200 });
|
|
116
126
|
});
|
|
117
127
|
|
|
118
128
|
it('snakify keys of Message and Metadata', testContext => {
|
|
129
|
+
const logSpy = testContext.mock.method(global.console, 'log', () => {});
|
|
130
|
+
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
131
|
+
|
|
119
132
|
const metadata = { correlationId: '123456789', http: { method: 'GET', statusCode: 200 } };
|
|
120
133
|
const logger = new Logger(metadata);
|
|
134
|
+
logger.log('test', { errorContext: { errorCode: 200, errorMessage: 'Page Not Found' } });
|
|
135
|
+
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
121
136
|
|
|
137
|
+
const actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
138
|
+
|
|
139
|
+
assert.ok(actual['date']);
|
|
140
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
141
|
+
assert.equal(actual['message'], 'test');
|
|
142
|
+
assert.equal(actual['status'], 'log');
|
|
143
|
+
assert.deepEqual(actual['http'], { method: 'GET', status_code: 200 });
|
|
144
|
+
assert.deepEqual(actual['error_context'], { error_code: 200, error_message: 'Page Not Found' });
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
it('prunes sensitive Metadata', testContext => {
|
|
122
148
|
const logSpy = testContext.mock.method(global.console, 'log', () => {});
|
|
123
149
|
assert.strictEqual(logSpy.mock.calls.length, 0);
|
|
150
|
+
|
|
151
|
+
const metadata = {
|
|
152
|
+
correlationId: '123456789',
|
|
153
|
+
http: { method: 'GET', statusCode: 200, jwt: 'deepSecret' },
|
|
154
|
+
user: { contact: { email: 'deep_deep_deep@email.address', firstName: 'should be snakify then hidden' } },
|
|
155
|
+
access_token: 'secret',
|
|
156
|
+
};
|
|
157
|
+
const logger = new Logger(metadata);
|
|
124
158
|
logger.log('test', { errorContext: { errorCode: 200, errorMessage: 'Page Not Found' } });
|
|
125
159
|
assert.strictEqual(logSpy.mock.calls.length, 1);
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
]);
|
|
160
|
+
|
|
161
|
+
const actual = JSON.parse(logSpy.mock.calls[0]?.arguments[0]);
|
|
162
|
+
assert.ok(actual['date']);
|
|
163
|
+
assert.equal(actual['correlation_id'], '123456789');
|
|
164
|
+
assert.equal(actual['message'], 'test');
|
|
165
|
+
assert.equal(actual['status'], 'log');
|
|
166
|
+
assert.equal(actual['has_sensitive_attribute'], true);
|
|
167
|
+
assert.equal(actual['access_token'], '[REDACTED]');
|
|
168
|
+
assert.deepEqual(actual['http'], { method: 'GET', status_code: 200, jwt: '[REDACTED]' });
|
|
169
|
+
assert.deepEqual(actual['user']['contact'], { email: '[REDACTED]', first_name: '[REDACTED]' });
|
|
135
170
|
});
|
|
136
171
|
});
|
|
@@ -569,7 +569,10 @@ describe('Provider', () => {
|
|
|
569
569
|
}
|
|
570
570
|
|
|
571
571
|
assert.ok(error instanceof HttpErrors.HttpError);
|
|
572
|
-
assert.
|
|
572
|
+
assert.ok(error.message.startsWith('Unexpected error while calling the provider:'));
|
|
573
|
+
assert.ok(error.message.includes('name: "Error"'));
|
|
574
|
+
assert.ok(error.message.includes('message: "foo"'));
|
|
575
|
+
assert.ok(error.message.includes('stack:'));
|
|
573
576
|
});
|
|
574
577
|
|
|
575
578
|
it('throws on status 429', async context => {
|