@contrast/agent 4.12.2 → 4.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/bootstrap.js +2 -3
  2. package/esm.mjs +9 -35
  3. package/lib/assess/membrane/debraner.js +0 -2
  4. package/lib/assess/membrane/index.js +1 -3
  5. package/lib/assess/models/tag-range/util.js +1 -2
  6. package/lib/assess/policy/propagators.json +13 -4
  7. package/lib/assess/policy/rules.json +42 -0
  8. package/lib/assess/policy/signatures.json +18 -0
  9. package/lib/assess/policy/util.js +3 -2
  10. package/lib/assess/propagators/JSON/stringify.js +6 -11
  11. package/lib/assess/propagators/ajv/conditionals.js +0 -3
  12. package/lib/assess/propagators/ajv/json-schema-type-evaluators.js +5 -4
  13. package/lib/assess/propagators/ajv/refs.js +1 -2
  14. package/lib/assess/propagators/ajv/schema-context.js +2 -3
  15. package/lib/assess/propagators/joi/any.js +1 -1
  16. package/lib/assess/propagators/joi/object.js +1 -1
  17. package/lib/assess/propagators/joi/string-base.js +16 -3
  18. package/lib/assess/propagators/mongoose/map.js +1 -1
  19. package/lib/assess/propagators/mongoose/mixed.js +1 -1
  20. package/lib/assess/propagators/mongoose/string.js +1 -1
  21. package/lib/assess/propagators/path/common.js +38 -29
  22. package/lib/assess/propagators/path/resolve.js +1 -0
  23. package/lib/assess/propagators/sequelize/utils.js +1 -2
  24. package/lib/assess/propagators/v8/init-hooks.js +0 -1
  25. package/lib/assess/sinks/dynamo.js +65 -30
  26. package/lib/assess/static/hardcoded.js +3 -3
  27. package/lib/assess/static/read-findings-from-cache.js +40 -0
  28. package/lib/assess/technologies/index.js +12 -13
  29. package/lib/cli-rewriter/index.js +65 -6
  30. package/lib/core/async-storage/hooks/mysql.js +57 -6
  31. package/lib/core/config/options.js +12 -6
  32. package/lib/core/config/util.js +15 -33
  33. package/lib/core/exclusions/input.js +6 -1
  34. package/lib/core/express/index.js +2 -4
  35. package/lib/core/logger/debug-logger.js +2 -2
  36. package/lib/core/stacktrace.js +2 -1
  37. package/lib/hooks/http.js +81 -81
  38. package/lib/hooks/require.js +1 -0
  39. package/lib/instrumentation.js +17 -0
  40. package/lib/protect/analysis/aho-corasick.js +1 -1
  41. package/lib/protect/errors/handler-async-errors.js +66 -0
  42. package/lib/protect/input-analysis.js +7 -13
  43. package/lib/protect/listeners.js +27 -23
  44. package/lib/protect/rules/base-scanner/index.js +2 -2
  45. package/lib/protect/rules/bot-blocker/bot-blocker-rule.js +4 -2
  46. package/lib/protect/rules/cmd-injection/cmdinjection-rule.js +57 -2
  47. package/lib/protect/rules/cmd-injection-semantic-chained-commands/cmd-injection-semantic-chained-commands-rule.js +31 -2
  48. package/lib/protect/rules/cmd-injection-semantic-dangerous-paths/cmd-injection-semantic-dangerous-paths-rule.js +32 -2
  49. package/lib/protect/rules/index.js +42 -21
  50. package/lib/protect/rules/ip-denylist/ip-denylist-rule.js +2 -2
  51. package/lib/protect/rules/nosqli/nosql-injection-rule.js +104 -39
  52. package/lib/protect/rules/path-traversal/path-traversal-rule.js +3 -0
  53. package/lib/protect/rules/rule-factory.js +6 -7
  54. package/lib/protect/rules/signatures/signature.js +3 -0
  55. package/lib/protect/rules/sqli/sql-injection-rule.js +98 -5
  56. package/lib/protect/rules/sqli/sql-scanner/labels.json +0 -3
  57. package/lib/protect/rules/xss/reflected-xss-rule.js +3 -3
  58. package/lib/protect/sample-aggregator.js +65 -57
  59. package/lib/protect/service.js +709 -104
  60. package/lib/reporter/models/app-activity/sample.js +6 -0
  61. package/lib/reporter/speedracer/unknown-connection-state.js +20 -32
  62. package/lib/reporter/translations/to-protobuf/settings/assess-features.js +4 -6
  63. package/lib/reporter/ts-reporter.js +1 -1
  64. package/lib/util/get-file-type.js +43 -0
  65. package/package.json +11 -11
  66. package/perf-logs.js +2 -5
@@ -36,7 +36,6 @@ const headerValidators = require('./validators');
36
36
  const UserInputKit = require('../reporter/models/utils/user-input-kit');
37
37
  const UserInputFactory = require('../reporter/models/utils/user-input-factory');
38
38
  const blockRequest = require('../util/block-request');
39
- const Analyzer = require('./analysis/dfsa-analyzer');
40
39
 
41
40
  class ProtectService {
42
41
  /**
@@ -48,7 +47,21 @@ class ProtectService {
48
47
  this.config = agent.config;
49
48
  this.enabled = agent.isInDefendMode();
50
49
  this.assessEnabled = agent.isInAssessMode();
51
- this.nativeAnalysis = this.config.agent.node.native_input_analysis;
50
+
51
+ this.agentLibAnalysis =
52
+ this.config.agent.node.native_input_analysis &&
53
+ this.config.agent.node.speedracer_input_analysis;
54
+
55
+ // if agentLib is present it will be used (for the "speedracer" variant of
56
+ // protect).
57
+ this.agentLib = agent.agentLib;
58
+ // map the rule-id in this.rules to the constant name for agentLib.RuleType values.
59
+ // are these mappings needed elsewhere? if so, yet another module...
60
+ if (this.agentLib && reporter.speedracer) {
61
+ this.agentLibRuleTypeToName = {
62
+ 'nosql-injection-mongo': 'nosql-injection',
63
+ };
64
+ }
52
65
 
53
66
  this._exclusionFactory = new ExclusionFactory({
54
67
  featureSet: agent.tsFeatureSet,
@@ -57,7 +70,8 @@ class ProtectService {
57
70
  });
58
71
  this._ruleFactory = new RuleFactory({
59
72
  featureSet: agent.tsFeatureSet,
60
- enabled: this.enabled
73
+ enabled: this.enabled,
74
+ agent
61
75
  });
62
76
  this.rules = this._ruleFactory.getRules();
63
77
  this.updateIpAllowlist(agent.tsFeatureSet.serverFeatures);
@@ -65,6 +79,9 @@ class ProtectService {
65
79
  this.urlExclusions = this._exclusionFactory.getUrlExclusions();
66
80
  this.inputExclusions = this._exclusionFactory.getInputExclusions();
67
81
  this.rules = this._ruleFactory.getRules();
82
+ if (this.agentLibAnalysis) {
83
+ this.addAgentLibBitToRules();
84
+ }
68
85
 
69
86
  agentEmitter.on('server-features', (serverFeatures) => {
70
87
  this.enabled = agent.isInDefendMode();
@@ -81,16 +98,32 @@ class ProtectService {
81
98
  }
82
99
 
83
100
  /**
84
- * Sends Connection/Header/URI data to SR to perform input analysis.
101
+ * Sends Connection/Header/URI data to SR or agent-lib to perform input analysis.
102
+ * @param {} meta
85
103
  * @param {IncomingMessage} req The current request
86
104
  * @param {ServerResponse} res The current response
87
105
  * @returns {Boolean} Returning `true` allows instrumentation to resume app code
88
106
  */
89
107
  analyzeRequest({ meta, req, res, appContext }) {
108
+ if (this.agentLibAnalysis) {
109
+ const agentLibResults = this.analyzeWithAgentLib(meta, req);
110
+
111
+ const analysis = this.handleAgentLibAnalysis({
112
+ asyncStorageContext: meta.asyncStorageContext,
113
+ appContext,
114
+ agentSettings: agentLibResults,
115
+ req,
116
+ res
117
+ });
118
+ return Promise.resolve(analysis);
119
+ }
120
+
121
+ // if not doing native analysis (i.e., agent-lib) then send a message to
122
+ // SR and wait for the reply.
90
123
  return this.reporter
91
124
  .sendMessage('request', { incomingMessage: req })
92
125
  .then((agentSettings) => {
93
- meta.requestId = _.get(agentSettings, 'protectState.uuid');
126
+ meta.requestId = agentSettings.protectState.uuid;
94
127
 
95
128
  return this.handleAnalysisResponse({
96
129
  asyncStorageContext: meta.asyncStorageContext,
@@ -113,17 +146,32 @@ class ProtectService {
113
146
  analyzeRequestStream({ meta, req, res, appContext }) {
114
147
  const { requestId, chunks } = meta;
115
148
 
116
- if (this.nativeAnalysis) {
117
- const analyzer = new Analyzer();
118
- // if nothing is suspicious then don't send the body for analysis
119
- if (!chunks.some((b) => analyzer.suspicious(b))) {
120
- return Promise.resolve(true);
149
+ // use agentLib?
150
+ if (this.agentLibAnalysis) {
151
+ // don't try to analyze multipart bodies; agent-lib does not parse because the
152
+ // interpretation is framework dependent, like query params.
153
+ let multipart = false;
154
+ if (req.headers['content-type']) {
155
+ multipart = req.headers['content-type'].toLowerCase().includes('multipart');
156
+ }
157
+ let agentLibResults;
158
+ if (multipart) {
159
+ agentLibResults = {};
121
160
  } else {
122
- // do some processing on the data. tbd in wasm/rust/napi. potentially
123
- // replace entire "send to service".
161
+ agentLibResults = this.analyzeBodyWithAgentLib(meta, chunks);
124
162
  }
163
+
164
+ const analysis = this.handleAgentLibAnalysis({
165
+ asyncStorageContext: meta.asyncStorageContext,
166
+ appContext,
167
+ agentSettings: agentLibResults,
168
+ req,
169
+ res
170
+ });
171
+ return Promise.resolve(analysis);
125
172
  }
126
173
 
174
+ // use SR, not agentLib.
127
175
  return this.reporter
128
176
  .sendMessage('request', { requestId, chunks })
129
177
  .then((agentSettings) =>
@@ -137,6 +185,64 @@ class ProtectService {
137
185
  );
138
186
  }
139
187
 
188
+ //
189
+ // note that agent-lib returns "trackRequest" which is the logical-not
190
+ // of SR's "permit" return.
191
+ //
192
+ analyzeWithAgentLib(meta, req) {
193
+ const rules = this.getRulesMask(meta.asyncStorageContext.defend.rules);
194
+ if (!rules) {
195
+ return {};
196
+ }
197
+
198
+ const arg = {
199
+ rules,
200
+ preferWorthWatching: true,
201
+ // header names must be lowercase. should this be done in agent-lib?
202
+ headers: req.rawHeaders.map((h, ix) => (ix & 1 ? h : h.toLowerCase()))
203
+ };
204
+
205
+ const questionMark = req.url.indexOf('?');
206
+ if (questionMark >= 0) {
207
+ arg.queries = req.url.slice(questionMark + 1);
208
+ }
209
+
210
+ const findings = this.agentLib.scoreRequestConnect(arg);
211
+
212
+ return findings;
213
+ }
214
+
215
+ analyzeBodyWithAgentLib(meta, chunks) {
216
+ const rules = this.getRulesMask(meta.asyncStorageContext.defend.rules);
217
+ if (!rules) {
218
+ return {};
219
+ }
220
+ // also, if content-type has multipart...
221
+ const options = { preferWorthWatching: true };
222
+
223
+ const bodyBuffer = Buffer.concat(chunks);
224
+
225
+ const findings = this.agentLib.scoreRequestUnknownBody(
226
+ rules,
227
+ bodyBuffer,
228
+ options
229
+ );
230
+
231
+ // store body buffer on findings for nosqli sink.
232
+ findings.bodyBuffer = bodyBuffer;
233
+ return findings;
234
+ }
235
+
236
+ getRulesMask(rules) {
237
+ return rules.reduce((mask, rule) => {
238
+ if (!rule.agentLibBit) {
239
+ logger.trace(`rule ${rule.id} missing agentLibBit`);
240
+ return mask;
241
+ }
242
+ return mask | rule.agentLibBit;
243
+ }, 0);
244
+ }
245
+
140
246
  /**
141
247
  * Independent of the part(s) of the HTTP message being analyzed, there is a
142
248
  * common process for handling the analysis response from S-R.
@@ -167,9 +273,219 @@ class ProtectService {
167
273
  }
168
274
 
169
275
  /**
170
- * Block at perimeter when instructed to do so by S-R.
276
+ * Handle the analysis response from agent-lib
277
+ *
278
+ * @param {AgentSettings} agentSettings agentLib findings
171
279
  * @param {IncomingMessage} req The current request
172
280
  * @param {ServerResponse} res The current response
281
+ * @returns {Boolean}
282
+ *
283
+ * agentLib findings are an object:
284
+ * {trackRequest: true|false, resultsList: [result]}
285
+ *
286
+ * a result is an object:
287
+ * {
288
+ * ruleId: string,
289
+ * inputType: string,
290
+ * path: [string],
291
+ * key: string,
292
+ * value: string,
293
+ * score: number
294
+ * }
295
+ */
296
+ // eslint-disable-next-line complexity
297
+ handleAgentLibAnalysis({
298
+ asyncStorageContext,
299
+ appContext,
300
+ agentSettings: agentLibResults,
301
+ res
302
+ }) {
303
+ if (!agentLibResults.resultsList) {
304
+ return true;
305
+ }
306
+
307
+ // at this point rules that are excluded by URL have been removed but
308
+ // none of the user-input exclusions have been applied; those exclusions
309
+ // are only applied for the protect.source event and this is (indirectly)
310
+ // invoked by the request.start and request.end events.
311
+
312
+ // determine if user input is excluded now that we have the results.
313
+ const { defend: { exclusions } } = asyncStorageContext;
314
+
315
+ let securityException = false;
316
+ // map the resultsList to the srResultsList (SR legacy format)
317
+ const srResultsList = [];
318
+
319
+ for (const r of agentLibResults.resultsList) {
320
+ // it's a little ugly but not all names returned correspond. this duplicates work
321
+ // in resultItemToSrResultItem() but allows us to avoid the conversion if the
322
+ // rule was excluded. i'm not sure it is a good trade because i'm presuming most
323
+ // items are not excluded, so it's a little bit of extra work to do this before
324
+ // the conversion.
325
+ const ruleId = this.agentLibRuleTypeToName[r.ruleId] || r.ruleId;
326
+ if (exclusions.length) {
327
+ const exclusionId = this.shouldExclude(exclusions, ruleId, r.inputType, r.key);
328
+ // don't add this to srResultsList if it is excluded.
329
+ // check null - can an exclusion name be an empty string?
330
+ if (exclusionId !== null) {
331
+ logger.debug(`EXCLUSION: ${exclusionId} - ${r.inputType} '${r.key}'`);
332
+ continue;
333
+ }
334
+ }
335
+
336
+ const mapped = this.resultItemToSrResultItem(r);
337
+ // nosqli requires the object at the key returned by the object to be stored
338
+ // on the sample so that it can be accessed at sink time.
339
+ if (mapped.ruleId === 'nosql-injection' && agentLibResults.bodyBuffer) {
340
+ this.captureMongoObject(mapped, agentLibResults.bodyBuffer);
341
+ }
342
+ // is the rule BAP?
343
+ if (mapped.scoreLevel === 'DEFINITE' && this.getRuleMode(mapped.ruleId) === 'BLOCK_AT_PERIMETER') {
344
+ securityException = true;
345
+ }
346
+
347
+ srResultsList.push(mapped);
348
+ }
349
+
350
+ /*
351
+ // this is the message SR returns but there is no need to create that format.
352
+ // only the resultsList is used by collectSamples(). securityException has
353
+ // already been synthesized.
354
+ const srAnalysisFmt = {
355
+ sentMs: Date.now(),
356
+ serverFeatures: undefined,
357
+ applicationSettings: undefined,
358
+ accumulatorSettings: undefined,
359
+ protectState: {
360
+ uuid: 'dead-beef-feed-a-fad-b4-a-fade',
361
+ trackRequest: agentLibResults.trackRequest,
362
+ securityException,
363
+ securityMessage: ''
364
+ },
365
+ inputAnalysis: { resultsList: srResultsList },
366
+ // tack on raw agent lib results so they can be used at sinks. this will
367
+ // facilitate removing the SR format when SR is removed.
368
+ agentLibResults
369
+ };
370
+ // */
371
+
372
+ // hack this into agentLibResults for now. it's needed to set sample.blocked
373
+ // when one had a DEFINITE score. previously, this was not needed because SR
374
+ // did reporting and only returned a securityException flag, indicating that
375
+ // the agent needed to block the request.
376
+ agentLibResults.securityException = securityException;
377
+
378
+ // save results for input tracing. collectSamples() is called with the
379
+ // additional parameter, agentLibResults, in this case. (see the implementation
380
+ // of collectSamples()).
381
+ this.collectSamples(
382
+ asyncStorageContext,
383
+ srResultsList,
384
+ appContext,
385
+ agentLibResults
386
+ );
387
+
388
+ // if there is a security exception there used to be no need to do anything more
389
+ // because SR would report it; when SR sent us a "securityException" it had already
390
+ // been reported, so the agent needed only to block the request. but with agentLib
391
+ // the sample must always be collected so it will be reported.
392
+ if (securityException) {
393
+ return this.handleBlockAtPerimeter(res);
394
+ }
395
+
396
+ return true;
397
+ }
398
+
399
+ /**
400
+ * map an agent-lib result to an SR format result.
401
+ *
402
+ * @param {result} r see handleAgentLibAnalysis above; an item returned by scoreRequestConnect
403
+ * in the resultsList array.
404
+ *
405
+ * @returns an SR-formatted result.
406
+ */
407
+ resultItemToSrResultItem(r) {
408
+ const copy = Object.assign({}, r, { attackCount: 1 });
409
+ // the ruleIds are not the same. kind of ugly.
410
+ if (copy.ruleId in this.agentLibRuleTypeToName) {
411
+ copy.ruleId = this.agentLibRuleTypeToName[copy.ruleId];
412
+ }
413
+
414
+ // user-input serialization wants a string. it replaces
415
+ // '.' with '>'; it really shouldn't do that - a '.' could
416
+ // be in a key, but that's how it works.
417
+ copy.path = copy.path.join('>');
418
+ // agent-lib doesn't return the pattern IDs that matched. they're not used, but the
419
+ // array cannot be empty for TS (rumor has it).
420
+ copy.idsList = ['agent-lib'];
421
+ if (copy.score >= 90) {
422
+ copy.scoreLevel = 'DEFINITE';
423
+ } else if (copy.score >= 10) {
424
+ copy.scoreLevel = 'WATCH';
425
+ } else {
426
+ // it really shouldn't be in this list...
427
+ copy.scoreLevel = 'NONE';
428
+ }
429
+ // get rid of the score property because it's not part of the SR
430
+ // resultsList items.
431
+ delete copy.score;
432
+
433
+ return copy;
434
+ }
435
+
436
+ /**
437
+ * Capture document object sample for Mongo. Right now applies blanketly
438
+ * to all nosqli rules because there is not a translation layer.
439
+ * For more information on how this applies to mongo injection & expansion,
440
+ * See `mongo.md' in the agent-lib-core repo.
441
+ *
442
+ * @param {Object} libResult result object from library representing mongo injection/expansion
443
+ * @param {Buffer} bodyBuffer buffer form of the request body (concat'd from chunks)
444
+ */
445
+ captureMongoObject(libResult, bodyBuffer) {
446
+ try {
447
+ // matches Sample's _inputInfoForSink
448
+ if (!libResult.inputInfo) {
449
+ libResult.inputInfo = {};
450
+ }
451
+
452
+ // parse the body as json.
453
+ const { path } = libResult;
454
+ const obj = JSON.parse(bodyBuffer.toString());
455
+ let doc = obj;
456
+ // returned path from lib is array of keys to traverse.
457
+ for (const entry of path) {
458
+ doc = doc[entry];
459
+ }
460
+
461
+ libResult.inputInfo.docObject = doc;
462
+ // the query clause (eg: $ne) is always the last entry in the path.
463
+ libResult.inputInfo.queryClause = path[path.length - 1];
464
+ } catch (e) {
465
+ logger.debug(`Failed to parse body buffer on nosqli libResult ${e}`);
466
+ }
467
+ }
468
+
469
+ /**
470
+ * Get the mode for a given rule
471
+ * @param {string} ruleId rule to get mode of
472
+ * @returns {string} the mode of the given rule
473
+ */
474
+ getRuleMode(ruleId) {
475
+ // must filter every time because teamserver can update these
476
+ // at any time.
477
+ for (const rule of this.rules) {
478
+ if (rule.id === ruleId) {
479
+ return rule.mode;
480
+ }
481
+ }
482
+
483
+ return null;
484
+ }
485
+
486
+ /**
487
+ * Block at perimeter when instructed to do so by S-R.
488
+ * @param {ServerResponse} res The current response
173
489
  * @returns {Boolean} false which halts executing of original method
174
490
  */
175
491
  handleBlockAtPerimeter(res) {
@@ -179,41 +495,55 @@ class ProtectService {
179
495
  }
180
496
 
181
497
  /**
182
- * When results are returned from S-R, save them to to input tracing.
183
- * @param {Object[]} resultsList
184
- * @param {ApplicationContext} appContext
498
+ * When results are returned from S-R, save them for input tracing.
499
+ *
500
+ * @param {AsyncContext} asyncContext
501
+ * @param {[Object]} resultsList SR-format results list (see handleAgentLibAnalysis)
502
+ * @param {ApplicationContext} appContext request is added to this if not present
503
+ * @param {Object} agentLibResults used only for securityException
185
504
  */
186
- collectSamples(asyncStorageContext, resultsList = [], appContext = {}) {
187
- if (!resultsList.length) {
505
+ collectSamples(asyncContext, resultsList, appContext, agentLibResults) {
506
+ if (!resultsList || !resultsList.length) {
188
507
  return;
189
508
  }
190
509
 
191
- /*
192
- * this shouldn't happen as this is retrieved when the request event
193
- * is processed, and will be available.
194
- */
195
- if (!asyncStorageContext) {
196
- logger.error(
197
- 'StorageContext not found - Unable to create samples from results list'
198
- );
510
+ // this shouldn't happen as this is retrieved when the request event
511
+ // is processed, and will be available.
512
+ if (!asyncContext) {
513
+ logger.error('StorageContext not found - Unable to create samples from results list');
199
514
  return;
200
515
  }
201
516
 
202
- const {
203
- request,
204
- defend: { samples }
205
- } = asyncStorageContext;
517
+ const { request, defend } = asyncContext;
206
518
 
207
519
  if (!appContext.request) {
208
520
  appContext.request = request;
209
521
  }
210
522
 
523
+ this._collectSamples(defend.samples, resultsList, appContext, agentLibResults);
524
+ }
525
+
526
+ /**
527
+ * Collect samples from already checked and present arguments
528
+ */
529
+ _collectSamples(samples, resultsList, appContext, agentLibResults) {
530
+ let blocked = false;
531
+
532
+ if (agentLibResults) {
533
+ blocked = !!agentLibResults.securityException;
534
+ }
535
+
211
536
  for (const result of resultsList) {
212
537
  // Coerce custom rule id
213
538
  if (result.ruleId === RULES.NOSQL_EXPANSION) {
214
539
  result.ruleId = RULES.NOSQL_INJECTION;
215
540
  }
216
541
 
542
+ // don't bind all the following vars unless we need to
543
+ if (result.scoreLevel === IMPORTANCE.NONE) {
544
+ continue;
545
+ }
546
+
217
547
  const {
218
548
  scoreLevel,
219
549
  ruleId,
@@ -224,16 +554,20 @@ class ProtectService {
224
554
  idsList
225
555
  } = result;
226
556
 
227
- if (scoreLevel === IMPORTANCE.NONE) {
228
- return;
229
- }
230
-
231
557
  const sample = samples.addRuleSample({
232
558
  id: ruleId,
233
559
  input: UserInputFactory.makeOne({ name, path, type, value }),
234
560
  evaluation: { results: { importance: scoreLevel } },
235
561
  appContext
236
562
  });
563
+
564
+ sample.blocked = blocked;
565
+
566
+ // copy over custom info for sink.
567
+ if (result.inputInfo) {
568
+ Object.assign(sample._inputInfoForSink, result.inputInfo);
569
+ }
570
+
237
571
  sample.filters.push(...idsList);
238
572
  }
239
573
  }
@@ -242,6 +576,15 @@ class ProtectService {
242
576
  if (settings) {
243
577
  this._ruleFactory.updateSettings(settings, this.enabled);
244
578
  this.rules = this._ruleFactory.getRules();
579
+ if (this.agentLibAnalysis) {
580
+ this.addAgentLibBitToRules();
581
+ }
582
+ }
583
+ }
584
+
585
+ addAgentLibBitToRules() {
586
+ for (const rule of this.rules) {
587
+ rule.agentLibBit = this.agentLib.RuleType[rule.id];
245
588
  }
246
589
  }
247
590
 
@@ -273,7 +616,7 @@ class ProtectService {
273
616
  // hack; we don't have a proper rule to create the inputs from
274
617
  const inputs = inputKit.create({}, data, ipEvent.type);
275
618
  // length should always just be 1
276
- const input = inputs[0];
619
+ const [input] = inputs;
277
620
  return this.ipAllowlist.evaluate(input);
278
621
  }
279
622
 
@@ -295,10 +638,31 @@ class ProtectService {
295
638
  }
296
639
 
297
640
  /**
298
- * Loads the rules for context storage based on current url exclusions
641
+ * Loads the rules for context storage based on current url exclusions.
642
+ * This is only called by protect/listeners.js and probably belongs there
643
+ * rather than here, but it's here. In any case, listeners sets async
644
+ * context rules based on the return value of this function.
299
645
  *
300
646
  * @param {string} path
301
- * @param {UserInput} ipInput
647
+ * @param {SourceEvent} ipEvent created when an http 'request' event occurs
648
+ * @returns {[Rule]} the array of rules that applies to this URL
649
+ *
650
+ * exclusions are an array of exclusion objects.
651
+ * [{
652
+ * assess: boolean,
653
+ * assessmentRulesList: [],
654
+ * defend: boolean,
655
+ * inputName: string,
656
+ * inputType: string enum 'PARAMETER', ? (<= querystring & parameter)
657
+ * isNamed: boolean,
658
+ * matchStrategy: string enum 'ALL', ?,
659
+ * name: 'parameter-input', // name of exclusion
660
+ * urls: [],
661
+ * }]
662
+ *
663
+ * exclusion inputTypes: BODY, COOKIE, HEADER, PARAMETER - all input types
664
+ * are mapped to one of these four.
665
+ *
302
666
  */
303
667
  getEnabledRules(path, ipEvent) {
304
668
  if (!this.enabled) {
@@ -323,7 +687,7 @@ class ProtectService {
323
687
  }
324
688
 
325
689
  /**
326
- * Loads the input exclusions for context storage based on current request url
690
+ * returns an array of the input exclusions applicable to the current url
327
691
  *
328
692
  * @param {string} path
329
693
  */
@@ -340,51 +704,264 @@ class ProtectService {
340
704
  }
341
705
 
342
706
  /**
343
- * Dispatches to the appropriate <code>preFilter</code> handler
344
- * based on the <code>SourceEvent</code>'s input type.
707
+ * Dispatches to the appropriate preFilter handler based on the SourceEvent
708
+ * input type. If the event type is an URL_PARAMETER and agent-lib analysis
709
+ * is being used, dispatches to a different handler because agent-lib needs
710
+ * to check url params after the framework has parsed them.
345
711
  *
346
- * @param {SourceEvent} event Source event providing data and context
712
+ * @param {SourceEvent} event Source event providing data and context (from lib/protect/listeners).
713
+ * @param {[Rule]} rules enabled rules
714
+ * @param {[InputExclusions]} inputExclusions input exclusions
715
+ * @param {Samples} samples Samples object for this request
347
716
  */
717
+ // eslint-disable-next-line complexity
348
718
  handleSourceEvent(event, rules, inputExclusions, samples) {
349
- if (this.skipEventHandling(rules)) {
719
+ // reduce the number of rules and exclusions that need to be checked because
720
+ // the event.type does not change.
721
+ rules = rules.filter((rule) => rule.appliesToInputType(event.type));
722
+ if (rules.length === 0) {
723
+ return;
724
+ }
725
+ inputExclusions = inputExclusions.filter((iex) => iex.appliesToInputType(event.type));
726
+
727
+ // agent-lib handles raw URLs, bodies, querystrings, headers, etc. but cannot
728
+ // handle URL parameter (e.g., /path/:param/action) because only the framework
729
+ // is aware of them. this function is invoked after the framework has parsed
730
+ // the URL and created the params object. this is important because the params,
731
+ // as represented in the URL, is URI encoded so the normal regexes will not
732
+ // match until the framework has decoded the param.
733
+ if (this.agentLibAnalysis) {
734
+ switch (event.type) {
735
+ case 'URL_PARAMETER': {
736
+ this.handleUrlParametersWithAgentLib(event, rules, inputExclusions, samples);
737
+ break;
738
+ }
739
+ case 'MULTIPART_NAME': {
740
+ this.handleMultipartFilenameWithAgentLib(event, rules, inputExclusions, samples);
741
+ break;
742
+ }
743
+ case 'MULTIPART_VALUE':
744
+ case 'BODY': {
745
+ this.handleMultipartBodyWithAgentLib(event, rules, inputExclusions, samples);
746
+ break;
747
+ }
748
+ case 'COOKIE_VALUE': {
749
+ this.handleCookiesWithAgentLib(event, rules, inputExclusions, samples);
750
+ break;
751
+ }
752
+ }
753
+ }
754
+
755
+ // remove agent-lib rules from the list to be handled by node. node handles rules
756
+ // that are not implemented by agent-lib. remove the agent-lib rules so those rules
757
+ // are not executed by both agent-lib and node.
758
+ rules = rules.filter((r) => !r.agentLibBit);
759
+ if (rules.length === 0) {
350
760
  return;
351
761
  }
352
762
 
353
763
  const data = this.filterSafeData(event);
354
-
355
- if (_.isEmpty(data)) {
764
+ if (data.length === 0) {
356
765
  return;
357
766
  }
358
767
 
359
768
  const inputKit = new UserInputKit();
360
- rules = rules.filter((rule) => rule.appliesToInputType(event.type));
361
769
 
362
770
  for (const rule of rules) {
363
771
  const inputs = inputKit.create(rule, data, event.type);
364
-
365
772
  for (const input of inputs) {
366
773
  if (this.isUserInputExcluded({ inputExclusions, rule, event, input })) {
367
774
  continue;
368
775
  }
776
+ // for all rules that do not use library input analysis.
777
+ if (!(rule.usesLibInputAnalysis && this.agentLibAnalysis)) {
778
+ logger.debug(`Starting rule analysis: ${input.type} ${input.name}`);
779
+ rule.preFilterUserInput(input, event, samples);
780
+ }
781
+ }
782
+ }
783
+ }
784
+
785
+ /**
786
+ * handle protect.source events for URL parameters when agent lib is enabled.
787
+ *
788
+ * @param {SourceEvent} event Source event providing data and context (from lib/protect/listeners).
789
+ * @param {[Rule]} rules enabled rules
790
+ * @param {[InputExclusions]} inputExclusions input exclusions
791
+ * @param {Samples} samples Samples object for this request
792
+ */
793
+ // eslint-disable-next-line complexity
794
+ handleUrlParametersWithAgentLib(event, rules, inputExclusions, samples) {
795
+ const res = event._serverResponse;
796
+ const params = event.data;
797
+ // if it's URL_PARAMETER and there are not params, then why are
798
+ // we here?
799
+ if (!params) {
800
+ logger.debug('handleUrlParametersWithAgentLib - no params found');
801
+ return;
802
+ }
803
+
804
+ const srResultsList = [];
805
+ let securityException = false;
806
+ const type = this.agentLib.InputType.UrlParameter;
807
+ const libRules = this.getRulesMask(rules);
808
+
809
+ if (!libRules) {
810
+ logger.debug('handleUrlParametersWithAgentLib - no rules');
811
+ return;
812
+ }
813
+
814
+ // for each key, check out the value. the key is set in the code so
815
+ // is not vulnerable.
816
+ for (const key in params) {
817
+ // items from scoreAtom() are only [{ruleId, score}, ...] because the key
818
+ // and inputType are already known and there is no path.
819
+ const items = this.agentLib.scoreAtom(params[key], type, libRules);
820
+ if (!items) {
821
+ continue;
822
+ }
823
+ for (const item of items) {
824
+ item.inputType = type;
825
+ const resultItem = Object.assign({ path: [key], value: params[key] }, item);
826
+ const mapped = this.resultItemToSrResultItem(resultItem);
827
+ const input = { type, name: key };
828
+ if (this.isUserInputExcluded({ inputExclusions, rule: { id: mapped.ruleId }, event, input })) {
829
+ continue;
830
+ }
831
+ if (mapped.scoreLevel === 'DEFINITE' && this.getRuleMode(mapped.ruleId) === 'BLOCK_AT_PERIMETER') {
832
+ securityException = true;
833
+ }
834
+ srResultsList.push(mapped);
835
+ }
836
+ }
837
+
838
+ this._collectSamples(samples, srResultsList, {}, { securityException });
839
+
840
+
841
+ if (securityException) {
842
+ this.handleBlockAtPerimeter(res);
843
+ }
844
+ }
845
+
846
+ // event.type === MULTIPART_NAME, data: {newrelic.js: 'newrelic.js'}
847
+ handleMultipartFilenameWithAgentLib(event, rules, inputExclusions, samples) {
848
+ const res = event._serverResponse;
849
+ const srResultsList = [];
850
+ let securityException = false;
851
+ // 'MULTIPART_NAME' is apparently used only for filenames; 'MULTIPART_VALUE'
852
+ // is used for multipart KV pairs (and we can just use PARAMETER_KEY/PARAMETER_VALUE).
853
+ const type = this.agentLib.InputType.MultipartName;
854
+ const libRules = this.getRulesMask(rules);
855
+
856
+ if (!libRules) {
857
+ logger.debug('handleUrlParametersWithAgentLib - no rules');
858
+ return;
859
+ }
860
+
861
+ // why these aren't {filename: 'newrelic.js'} instead of {newrelic.js: 'newrelic.js'}
862
+ // escapes me.
863
+ if (typeof event.data !== 'object') {
864
+ return;
865
+ }
866
+ const filenames = Object.keys(event.data);
867
+
868
+ for (const filename of filenames) {
869
+ const items = this.agentLib.scoreAtom(filename, type, libRules);
870
+ if (!items) {
871
+ continue;
872
+ }
873
+ for (const item of items) {
874
+ item.inputType = type;
875
+ const resultItem = Object.assign({ path: [filename], value: filename }, item);
876
+ const mapped = this.resultItemToSrResultItem(resultItem);
877
+ if (mapped.scoreLevel === 'DEFINITE' && this.getRuleMode(mapped.ruleId) === 'BLOCK_AT_PERIMETER') {
878
+ securityException = true;
879
+ }
880
+ srResultsList.push(mapped);
881
+ }
882
+ }
883
+
884
+ this._collectSamples(samples, srResultsList, {}, { securityException });
885
+
886
+ if (securityException) {
887
+ this.handleBlockAtPerimeter(res);
888
+ }
889
+ }
890
+
891
+ handleMultipartBodyWithAgentLib(event, rules, inputExclusions, samples) {
892
+ const rulesMask = this.getRulesMask(rules);
893
+ if (!rulesMask || typeof event.data !== 'object' || !event._ctxt) {
894
+ return;
895
+ }
896
+ // just treat these as an array of query params.
897
+ const queries = Object.entries(event.data)
898
+ .filter(i => typeof i[1] === 'string')
899
+ .reduce((queries, q) => {
900
+ queries.unshift(...q); return queries;
901
+ }, []);
902
+
903
+ const arg = {
904
+ rules: rulesMask,
905
+ preferWorthWatching: true,
906
+ queries,
907
+ };
908
+
909
+ const findings = this.agentLib.scoreRequestConnect(arg);
910
+
911
+ this.handleAgentLibAnalysis({
912
+ asyncStorageContext: event._ctxt,
913
+ appContext: {},
914
+ agentSettings: findings,
915
+ req: event._incomingMessage,
916
+ res: event._serverResponse,
917
+ });
918
+ }
919
+
920
+ handleCookiesWithAgentLib(event, rules, inputExclusions, samples) {
921
+ const cookies = Object.entries(event.data).reduce((acc, [key, value]) => {
922
+ acc.unshift(key, value);
923
+ return acc;
924
+ }, []);
925
+ const findings = this.agentLib.scoreRequestConnect({
926
+ preferWorthWatching: true,
927
+ rules: this.getRulesMask(rules),
928
+ cookies
929
+ });
930
+ this.handleAgentLibAnalysis({
931
+ asyncStorageContext: event._ctxt,
932
+ appContext: {},
933
+ agentSettings: findings,
934
+ req: event._incomingMessage,
935
+ res: event._serverResponse,
936
+ });
937
+
938
+ }
369
939
 
370
- logger.debug(`Starting rule analysis: ${input.type} ${input.name}`);
371
- rule.preFilterUserInput(input, event, samples);
940
+ /**
941
+ * check a rule/input combination against the specified exclusions.
942
+ *
943
+ * @param {[Exclusion]} exclusions array of exclusions to check against
944
+ * @param {String} ruleId the rule ID
945
+ * @param {String} inputType the type of the input
946
+ * @param {String} inputName the key for JSON objects and KV pairs
947
+ *
948
+ * @returns {String|null} the name of the exclusion that applied, or null.
949
+ */
950
+ shouldExclude(exclusions, ruleId, inputType, inputName) {
951
+ for (const exclusion of exclusions) {
952
+ if (exclusion.shouldExclude(ruleId, inputType, inputName)) {
953
+ return exclusion.name;
372
954
  }
373
955
  }
956
+ return null;
374
957
  }
375
958
 
376
959
  isUserInputExcluded({ inputExclusions, rule, event, input }) {
377
960
  let excluded;
378
961
  for (const exclusion of inputExclusions) {
379
- excluded =
380
- exclusion.appliesToInputType(event.type) &&
381
- exclusion.appliesToProtectRule(rule.id) &&
382
- exclusion.matches(input.name);
383
-
962
+ excluded = exclusion.shouldExclude(rule.id, input.type, input.name);
384
963
  if (excluded) {
385
- logger.debug(
386
- `EXCLUSION: ${exclusion.name} - ${input.type} '${input.name}'`
387
- );
964
+ logger.debug(`EXCLUSION: ${exclusion.name} - ${input.type} '${input.name}'`);
388
965
  break;
389
966
  }
390
967
  }
@@ -392,7 +969,7 @@ class ProtectService {
392
969
  }
393
970
 
394
971
  skipEventHandling(rules) {
395
- return _.isEmpty(rules) || !this._ruleFactory.enabled;
972
+ return _.isEmpty(rules);
396
973
  }
397
974
 
398
975
  /**
@@ -403,7 +980,7 @@ class ProtectService {
403
980
  * @param {[Samples]} params.samples worthWatching/definite
404
981
  */
405
982
  handleSinkEvent({ event, rules, samples }) {
406
- if (this.skipEventHandling(rules)) {
983
+ if (_.isEmpty(rules)) {
407
984
  return;
408
985
  }
409
986
 
@@ -414,7 +991,23 @@ class ProtectService {
414
991
  }
415
992
 
416
993
  const applicableSamples = samples.getAll(rule.id);
417
- rule.evaluateAtSink({ event, samples, applicableSamples, request });
994
+ // this should be tested here as opposed to constructing an object
995
+ // and passing it to evaluateAtSink*(). but tests expect that
996
+ // evaluateAtSink*() gets called and they don't bother to set up
997
+ // appopriate samples and event data. so, comment it out for now.
998
+ //if (applicableSamples.size === 0 || !event.data) {
999
+ // continue;
1000
+ //}
1001
+
1002
+ // Do we want to use the standard node evaluator or the library sink
1003
+ // evaluation (which requires data from the library's input analysis stage)?
1004
+ const args = { event, samples, applicableSamples, request };
1005
+
1006
+ if (!this.agentLibAnalysis || !rule.evaluateAtSinkForLib) {
1007
+ rule.evaluateAtSink(args);
1008
+ } else {
1009
+ rule.evaluateAtSinkForLib(args);
1010
+ }
418
1011
  }
419
1012
  }
420
1013
 
@@ -477,25 +1070,12 @@ class ProtectService {
477
1070
  */
478
1071
  submitFindings(rules, samples) {
479
1072
  const findings = this.createFindings(rules, samples);
480
- const aggregated = SampleAggregator.aggregate(findings);
1073
+ const aggregated = SampleAggregator.aggregate(findings, (finding) => this.wwFilter(finding));
481
1074
  for (const finding of aggregated) {
482
1075
  agentEmitter.emit('attack', finding);
483
1076
  }
484
1077
  }
485
1078
 
486
- /**
487
- * checks if samples already contain input types of PARAMETER_VALUE or PARAMETER_NAME
488
- *
489
- * @param {boolean} isEffective
490
- * @param {string} inputType
491
- * @return {boolean}
492
- */
493
- static paramInputs(inputType) {
494
- return [INPUT_TYPES.PARAMETER_VALUE, INPUT_TYPES.PARAMETER_NAME].includes(
495
- inputType
496
- );
497
- }
498
-
499
1079
  /**
500
1080
  * See: https://contrast.atlassian.net/browse/NODE-670
501
1081
  * The way we collect findings in SR vs node
@@ -506,22 +1086,30 @@ class ProtectService {
506
1086
  *
507
1087
  * @param {Object} params
508
1088
  * @param {Array} findings to report
509
- * @param {Set} set of all samples for a given rule
1089
+ * @param {Set} ruleSamples of all samples for a given rule
510
1090
  * @param {Rule} protect rule object
1091
+ * @param {Boolean} speedracer speedracer analysis is being used. this
1092
+ * includes agent-lib, which uses the speedracer logic.
511
1093
  */
512
- addSRFindings({ findings, ruleSamples, rule }) {
513
- let qsSample = null;
1094
+ addFindings({ findings, ruleSamples, rule, speedracer }) {
1095
+ const qsSamples = [];
514
1096
  let hasEffectiveParamInputs = false;
515
1097
 
516
1098
  for (const sample of ruleSamples) {
517
- hasEffectiveParamInputs =
518
- hasEffectiveParamInputs ||
519
- ProtectService.paramInputs(sample.input.type);
1099
+ if (sample.input.type === INPUT_TYPES.URI) {
1100
+ // forget about URL things
1101
+ continue;
1102
+ }
1103
+
1104
+ // is the sample a parameter name or value?
1105
+ hasEffectiveParamInputs = hasEffectiveParamInputs ||
1106
+ INPUT_TYPES.PARAMETER_VALUE === sample.input.type ||
1107
+ INPUT_TYPES.PARAMETER_NAME === sample.input.type;
520
1108
 
521
1109
  // saving reference to QUERYSTRING sample in case
522
1110
  // there are no Parameter type samples for rule
523
1111
  if (sample.input.type === INPUT_TYPES.QUERYSTRING) {
524
- qsSample = sample;
1112
+ qsSamples.push(sample);
525
1113
  } else {
526
1114
  findings.push({
527
1115
  rule,
@@ -533,14 +1121,16 @@ class ProtectService {
533
1121
  }
534
1122
 
535
1123
  // https://contrast.atlassian.net/browse/NODE-660 - only report one attack
536
- // when there are both QUERYSTRING and PARAMETER_VALUE types for a given rule
537
- if (qsSample && !hasEffectiveParamInputs) {
538
- findings.push({
539
- rule,
540
- ruleId: rule.id,
541
- sample: qsSample,
542
- status: qsSample.getStatus()
543
- });
1124
+ // when there are both QUERYSTRING and PARAMETER_VALUE types for a given rule.
1125
+ if (qsSamples.length > 0 && !hasEffectiveParamInputs) {
1126
+ for (const qsSample of qsSamples) {
1127
+ findings.push({
1128
+ rule,
1129
+ ruleId: rule.id,
1130
+ sample: qsSample,
1131
+ status: qsSample.getStatus()
1132
+ });
1133
+ }
544
1134
  }
545
1135
  }
546
1136
 
@@ -551,31 +1141,46 @@ class ProtectService {
551
1141
  * @param {Rule[]} rules Rules from which to build findings
552
1142
  * @returns {Object[]} The findings from the rules
553
1143
  */
554
- createFindings(rules = [], samples) {
1144
+ createFindings(rules, samples) {
555
1145
  const findings = [];
1146
+ const speedracer = this.reporter.speedracer &&
1147
+ this.config.agent.node.speedracer_input_analysis;
1148
+
556
1149
  for (const rule of rules) {
557
1150
  const { id } = rule;
558
1151
  const ruleSamples = samples.getAll(id);
559
-
560
- if (
561
- this.reporter.speedracer &&
562
- this.config.agent.node.speedracer_input_analysis
563
- ) {
564
- this.addSRFindings({ findings, ruleSamples, rule });
565
- } else {
566
- for (const sample of ruleSamples) {
567
- findings.push({
568
- rule,
569
- ruleId: id,
570
- sample,
571
- status: sample.getStatus()
572
- });
573
- }
1152
+ // no need to call add findings if no samples
1153
+ if (ruleSamples.size === 0) {
1154
+ continue;
574
1155
  }
1156
+
1157
+ // only support SR format now; previously there was logic to handle node
1158
+ // analysis differently than SR analysis. agent-lib mimics SR, so both
1159
+ // should be the same now.
1160
+ this.addFindings({ findings, ruleSamples, rule, speedracer });
575
1161
  }
576
1162
 
577
1163
  return findings;
578
1164
  }
1165
+
1166
+ // worth-watching filter. this is located here so agent-lib isn't exposed to the
1167
+ // sample aggregator any more than necessary (agentLibBit is exposed).
1168
+ //
1169
+ // returns true if the finding should be reported as a probe, else false
1170
+ wwFilter(finding) {
1171
+ const { agentLibBit } = finding.rule;
1172
+ const { _type, _value: input } = finding.sample.input;
1173
+ const type = this.agentLib.InputType[_type];
1174
+
1175
+ const alFinding = this.agentLib.scoreAtom(input, type, agentLibBit);
1176
+ if (!alFinding) {
1177
+ return false;
1178
+ }
1179
+ if (alFinding.length > 1) {
1180
+ logger.debug(`scoreAtom() returned ${alFinding.length} findings`);
1181
+ }
1182
+ return alFinding[0].score >= 90;
1183
+ }
579
1184
  }
580
1185
 
581
1186
  module.exports = ProtectService;