dd-trace 4.47.1 → 4.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/LICENSE-3rdparty.csv +1 -1
  2. package/ext/types.d.ts +1 -0
  3. package/ext/types.js +1 -0
  4. package/index.d.ts +361 -0
  5. package/package.json +18 -13
  6. package/packages/datadog-code-origin/index.js +38 -0
  7. package/packages/datadog-core/index.js +2 -2
  8. package/packages/datadog-core/src/utils/src/parse-tags.js +33 -0
  9. package/packages/datadog-esbuild/index.js +4 -2
  10. package/packages/datadog-instrumentations/src/amqplib.js +65 -5
  11. package/packages/datadog-instrumentations/src/avsc.js +37 -0
  12. package/packages/datadog-instrumentations/src/azure-functions.js +48 -0
  13. package/packages/datadog-instrumentations/src/child_process.js +144 -27
  14. package/packages/datadog-instrumentations/src/express.js +37 -4
  15. package/packages/datadog-instrumentations/src/fastify.js +12 -1
  16. package/packages/datadog-instrumentations/src/fs.js +27 -7
  17. package/packages/datadog-instrumentations/src/helpers/hooks.js +6 -0
  18. package/packages/datadog-instrumentations/src/helpers/register.js +9 -0
  19. package/packages/datadog-instrumentations/src/jest.js +2 -1
  20. package/packages/datadog-instrumentations/src/kafkajs.js +123 -63
  21. package/packages/datadog-instrumentations/src/mocha/common.js +1 -1
  22. package/packages/datadog-instrumentations/src/mocha/utils.js +2 -2
  23. package/packages/datadog-instrumentations/src/multer.js +37 -0
  24. package/packages/datadog-instrumentations/src/mysql2.js +220 -1
  25. package/packages/datadog-instrumentations/src/openai.js +2 -2
  26. package/packages/datadog-instrumentations/src/protobufjs.js +127 -0
  27. package/packages/datadog-instrumentations/src/url.js +84 -0
  28. package/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js +7 -4
  29. package/packages/datadog-instrumentations/src/winston.js +22 -0
  30. package/packages/datadog-plugin-amqplib/src/consumer.js +4 -4
  31. package/packages/datadog-plugin-avsc/src/index.js +9 -0
  32. package/packages/datadog-plugin-avsc/src/schema_iterator.js +169 -0
  33. package/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js +1 -0
  34. package/packages/datadog-plugin-aws-sdk/src/services/kinesis.js +1 -0
  35. package/packages/datadog-plugin-aws-sdk/src/services/s3.js +1 -0
  36. package/packages/datadog-plugin-aws-sdk/src/services/sqs.js +1 -0
  37. package/packages/datadog-plugin-azure-functions/src/index.js +77 -0
  38. package/packages/datadog-plugin-fastify/src/code_origin.js +31 -0
  39. package/packages/datadog-plugin-fastify/src/index.js +10 -12
  40. package/packages/datadog-plugin-fastify/src/tracing.js +19 -0
  41. package/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js +8 -1
  42. package/packages/datadog-plugin-google-cloud-pubsub/src/producer.js +8 -0
  43. package/packages/datadog-plugin-grpc/src/client.js +3 -0
  44. package/packages/datadog-plugin-grpc/src/server.js +3 -0
  45. package/packages/datadog-plugin-kafkajs/src/batch-consumer.js +6 -3
  46. package/packages/datadog-plugin-kafkajs/src/consumer.js +8 -4
  47. package/packages/datadog-plugin-kafkajs/src/producer.js +10 -4
  48. package/packages/datadog-plugin-mocha/src/index.js +4 -1
  49. package/packages/datadog-plugin-openai/src/index.js +9 -1015
  50. package/packages/datadog-plugin-openai/src/tracing.js +1023 -0
  51. package/packages/datadog-plugin-protobufjs/src/index.js +14 -0
  52. package/packages/datadog-plugin-protobufjs/src/schema_iterator.js +180 -0
  53. package/packages/dd-trace/src/appsec/addresses.js +8 -1
  54. package/packages/dd-trace/src/appsec/channels.js +7 -1
  55. package/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js +13 -1
  56. package/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js +8 -1
  57. package/packages/dd-trace/src/appsec/iast/iast-plugin.js +1 -1
  58. package/packages/dd-trace/src/appsec/iast/index.js +3 -0
  59. package/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js +1 -0
  60. package/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js +55 -7
  61. package/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js +15 -0
  62. package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js +4 -2
  63. package/packages/dd-trace/src/appsec/index.js +61 -43
  64. package/packages/dd-trace/src/appsec/rasp/command_injection.js +49 -0
  65. package/packages/dd-trace/src/appsec/rasp/fs-plugin.js +99 -0
  66. package/packages/dd-trace/src/appsec/rasp/index.js +27 -10
  67. package/packages/dd-trace/src/appsec/rasp/lfi.js +112 -0
  68. package/packages/dd-trace/src/appsec/rasp/sql_injection.js +24 -4
  69. package/packages/dd-trace/src/appsec/rasp/ssrf.js +4 -3
  70. package/packages/dd-trace/src/appsec/rasp/utils.js +4 -2
  71. package/packages/dd-trace/src/appsec/recommended.json +3 -7
  72. package/packages/dd-trace/src/appsec/remote_config/capabilities.js +6 -1
  73. package/packages/dd-trace/src/appsec/remote_config/index.js +10 -0
  74. package/packages/dd-trace/src/appsec/reporter.js +17 -9
  75. package/packages/dd-trace/src/appsec/sdk/track_event.js +10 -3
  76. package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js +1 -1
  77. package/packages/dd-trace/src/appsec/waf/waf_manager.js +4 -0
  78. package/packages/dd-trace/src/azure_metadata.js +120 -0
  79. package/packages/dd-trace/src/ci-visibility/dynamic-instrumentation/index.js +97 -0
  80. package/packages/dd-trace/src/ci-visibility/dynamic-instrumentation/worker/index.js +90 -0
  81. package/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js +2 -14
  82. package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js +19 -1
  83. package/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js +53 -0
  84. package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js +8 -1
  85. package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +43 -0
  86. package/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js +53 -0
  87. package/packages/dd-trace/src/config.js +86 -6
  88. package/packages/dd-trace/src/constants.js +3 -1
  89. package/packages/dd-trace/src/datastreams/pathway.js +1 -0
  90. package/packages/dd-trace/src/datastreams/schemas/schema_builder.js +25 -17
  91. package/packages/dd-trace/src/debugger/devtools_client/config.js +2 -0
  92. package/packages/dd-trace/src/debugger/devtools_client/index.js +52 -5
  93. package/packages/dd-trace/src/debugger/devtools_client/remote_config.js +4 -4
  94. package/packages/dd-trace/src/debugger/devtools_client/send.js +29 -2
  95. package/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js +187 -0
  96. package/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js +40 -0
  97. package/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js +252 -0
  98. package/packages/dd-trace/src/debugger/devtools_client/snapshot/symbols.js +6 -0
  99. package/packages/dd-trace/src/debugger/devtools_client/state.js +19 -4
  100. package/packages/dd-trace/src/debugger/index.js +10 -3
  101. package/packages/dd-trace/src/exporters/common/request.js +8 -34
  102. package/packages/dd-trace/src/exporters/common/url-to-http-options-polyfill.js +31 -0
  103. package/packages/dd-trace/src/llmobs/constants/tags.js +34 -0
  104. package/packages/dd-trace/src/llmobs/constants/text.js +6 -0
  105. package/packages/dd-trace/src/llmobs/constants/writers.js +13 -0
  106. package/packages/dd-trace/src/llmobs/index.js +103 -0
  107. package/packages/dd-trace/src/llmobs/noop.js +82 -0
  108. package/packages/dd-trace/src/llmobs/plugins/base.js +65 -0
  109. package/packages/dd-trace/src/llmobs/plugins/openai.js +205 -0
  110. package/packages/dd-trace/src/llmobs/sdk.js +377 -0
  111. package/packages/dd-trace/src/llmobs/span_processor.js +195 -0
  112. package/packages/dd-trace/src/llmobs/storage.js +7 -0
  113. package/packages/dd-trace/src/llmobs/tagger.js +322 -0
  114. package/packages/dd-trace/src/llmobs/util.js +176 -0
  115. package/packages/dd-trace/src/llmobs/writers/base.js +111 -0
  116. package/packages/dd-trace/src/llmobs/writers/evaluations.js +29 -0
  117. package/packages/dd-trace/src/llmobs/writers/spans/agentProxy.js +23 -0
  118. package/packages/dd-trace/src/llmobs/writers/spans/agentless.js +17 -0
  119. package/packages/dd-trace/src/llmobs/writers/spans/base.js +49 -0
  120. package/packages/dd-trace/src/noop/proxy.js +3 -0
  121. package/packages/dd-trace/src/noop/span.js +3 -0
  122. package/packages/dd-trace/src/opentelemetry/span.js +1 -1
  123. package/packages/dd-trace/src/opentelemetry/tracer.js +1 -0
  124. package/packages/dd-trace/src/opentracing/propagation/text_map.js +73 -12
  125. package/packages/dd-trace/src/opentracing/span.js +12 -0
  126. package/packages/dd-trace/src/opentracing/tracer.js +8 -1
  127. package/packages/dd-trace/src/payload-tagging/config/aws.json +71 -3
  128. package/packages/dd-trace/src/payload-tagging/index.js +1 -1
  129. package/packages/dd-trace/src/payload-tagging/jsonpath-plus.js +2094 -0
  130. package/packages/dd-trace/src/plugin_manager.js +4 -2
  131. package/packages/dd-trace/src/plugins/ci_plugin.js +2 -0
  132. package/packages/dd-trace/src/plugins/index.js +3 -0
  133. package/packages/dd-trace/src/plugins/log_plugin.js +1 -1
  134. package/packages/dd-trace/src/plugins/outbound.js +9 -0
  135. package/packages/dd-trace/src/plugins/schema.js +35 -0
  136. package/packages/dd-trace/src/plugins/util/ci.js +23 -1
  137. package/packages/dd-trace/src/plugins/util/serverless.js +7 -0
  138. package/packages/dd-trace/src/plugins/util/stacktrace.js +94 -0
  139. package/packages/dd-trace/src/plugins/util/tags.js +7 -0
  140. package/packages/dd-trace/src/plugins/util/test.js +20 -22
  141. package/packages/dd-trace/src/plugins/util/web.js +6 -4
  142. package/packages/dd-trace/src/priority_sampler.js +16 -0
  143. package/packages/dd-trace/src/profiling/config.js +3 -1
  144. package/packages/dd-trace/src/profiling/exporters/agent.js +7 -5
  145. package/packages/dd-trace/src/profiling/profiler.js +24 -14
  146. package/packages/dd-trace/src/profiling/profilers/events.js +3 -3
  147. package/packages/dd-trace/src/profiling/profilers/wall.js +95 -66
  148. package/packages/dd-trace/src/proxy.js +20 -1
  149. package/packages/dd-trace/src/service-naming/schemas/v0/index.js +2 -1
  150. package/packages/dd-trace/src/service-naming/schemas/v0/serverless.js +12 -0
  151. package/packages/dd-trace/src/service-naming/schemas/v1/index.js +2 -1
  152. package/packages/dd-trace/src/service-naming/schemas/v1/serverless.js +12 -0
  153. package/packages/dd-trace/src/span_processor.js +5 -0
  154. package/packages/dd-trace/src/telemetry/index.js +11 -1
  155. package/packages/datadog-core/src/storage/async_resource.js +0 -108
  156. package/packages/datadog-core/src/storage/index.js +0 -5
@@ -0,0 +1,14 @@
1
+ const SchemaPlugin = require('../../dd-trace/src/plugins/schema')
2
+ const SchemaExtractor = require('./schema_iterator')
3
+
4
+ class ProtobufjsPlugin extends SchemaPlugin {
5
+ static get id () {
6
+ return 'protobufjs'
7
+ }
8
+
9
+ static get schemaExtractor () {
10
+ return SchemaExtractor
11
+ }
12
+ }
13
+
14
+ module.exports = ProtobufjsPlugin
@@ -0,0 +1,180 @@
1
+ const PROTOBUF = 'protobuf'
2
+ const {
3
+ SCHEMA_DEFINITION,
4
+ SCHEMA_ID,
5
+ SCHEMA_NAME,
6
+ SCHEMA_OPERATION,
7
+ SCHEMA_WEIGHT,
8
+ SCHEMA_TYPE
9
+ } = require('../../dd-trace/src/constants')
10
+ const log = require('../../dd-trace/src/log')
11
+ const {
12
+ SchemaBuilder
13
+ } = require('../../dd-trace/src/datastreams/schemas/schema_builder')
14
+
15
+ class SchemaExtractor {
16
+ constructor (schema) {
17
+ this.schema = schema
18
+ }
19
+
20
+ static getTypeAndFormat (type) {
21
+ const typeFormatMapping = {
22
+ int32: ['integer', 'int32'],
23
+ int64: ['integer', 'int64'],
24
+ uint32: ['integer', 'uint32'],
25
+ uint64: ['integer', 'uint64'],
26
+ sint32: ['integer', 'sint32'],
27
+ sint64: ['integer', 'sint64'],
28
+ fixed32: ['integer', 'fixed32'],
29
+ fixed64: ['integer', 'fixed64'],
30
+ sfixed32: ['integer', 'sfixed32'],
31
+ sfixed64: ['integer', 'sfixed64'],
32
+ float: ['number', 'float'],
33
+ double: ['number', 'double'],
34
+ bool: ['boolean', null],
35
+ string: ['string', null],
36
+ bytes: ['string', 'byte'],
37
+ Enum: ['enum', null],
38
+ Type: ['type', null],
39
+ map: ['map', null],
40
+ repeated: ['array', null]
41
+ }
42
+
43
+ return typeFormatMapping[type] || ['string', null]
44
+ }
45
+
46
+ static extractProperty (field, schemaName, fieldName, builder, depth) {
47
+ let array = false
48
+ let description
49
+ let ref
50
+ let enumValues
51
+
52
+ const resolvedType = field.resolvedType ? field.resolvedType.constructor.name : field.type
53
+
54
+ const isRepeatedField = field.rule === 'repeated'
55
+
56
+ let typeFormat = this.getTypeAndFormat(isRepeatedField ? 'repeated' : resolvedType)
57
+ let type = typeFormat[0]
58
+ let format = typeFormat[1]
59
+
60
+ if (type === 'array') {
61
+ array = true
62
+ typeFormat = this.getTypeAndFormat(resolvedType)
63
+ type = typeFormat[0]
64
+ format = typeFormat[1]
65
+ }
66
+
67
+ if (type === 'type') {
68
+ format = null
69
+ ref = `#/components/schemas/${removeLeadingPeriod(field.resolvedType.fullName)}`
70
+ // keep a reference to the original builder iterator since when we recurse this reference will get reset to
71
+ // deeper schemas
72
+ const originalSchemaExtractor = builder.iterator
73
+ if (!this.extractSchema(field.resolvedType, builder, depth, this)) {
74
+ return false
75
+ }
76
+ type = 'object'
77
+ builder.iterator = originalSchemaExtractor
78
+ } else if (type === 'enum') {
79
+ enumValues = []
80
+ let i = 0
81
+ while (field.resolvedType.valuesById[i]) {
82
+ enumValues.push(field.resolvedType.valuesById[i])
83
+ i += 1
84
+ }
85
+ }
86
+ return builder.addProperty(schemaName, fieldName, array, type, description, ref, format, enumValues)
87
+ }
88
+
89
+ static extractSchema (schema, builder, depth, extractor) {
90
+ depth += 1
91
+ const schemaName = removeLeadingPeriod(schema.resolvedType ? schema.resolvedType.fullName : schema.fullName)
92
+ if (extractor) {
93
+ // if we already have a defined extractor, this is a nested schema. create a new extractor for the nested
94
+ // schema, ensure it is added to our schema builder's cache, and replace the builders iterator with our
95
+ // nested schema iterator / extractor. Once complete, add the new schema to our builder's schemas.
96
+ const nestedSchemaExtractor = new SchemaExtractor(schema)
97
+ builder.iterator = nestedSchemaExtractor
98
+ const nestedSchema = SchemaBuilder.getSchema(schemaName, nestedSchemaExtractor, builder)
99
+ for (const nestedSubSchemaName in nestedSchema.components.schemas) {
100
+ if (nestedSchema.components.schemas.hasOwnProperty(nestedSubSchemaName)) {
101
+ builder.schema.components.schemas[nestedSubSchemaName] = nestedSchema.components.schemas[nestedSubSchemaName]
102
+ }
103
+ }
104
+ return true
105
+ } else {
106
+ if (!builder.shouldExtractSchema(schemaName, depth)) {
107
+ return false
108
+ }
109
+ for (const field of schema.fieldsArray) {
110
+ if (!this.extractProperty(field, schemaName, field.name, builder, depth)) {
111
+ log.warn(`DSM: Unable to extract field with name: ${field.name} from Avro schema with name: ${schemaName}`)
112
+ }
113
+ }
114
+ return true
115
+ }
116
+ }
117
+
118
+ static extractSchemas (descriptor, dataStreamsProcessor) {
119
+ const schemaName = removeLeadingPeriod(
120
+ descriptor.resolvedType ? descriptor.resolvedType.fullName : descriptor.fullName
121
+ )
122
+ return dataStreamsProcessor.getSchema(schemaName, new SchemaExtractor(descriptor))
123
+ }
124
+
125
+ iterateOverSchema (builder) {
126
+ this.constructor.extractSchema(this.schema, builder, 0)
127
+ }
128
+
129
+ static attachSchemaOnSpan (args, span, operation, tracer) {
130
+ const { messageClass } = args
131
+ const descriptor = messageClass.$type ?? messageClass
132
+
133
+ if (!descriptor || !span) {
134
+ return
135
+ }
136
+
137
+ if (span.context()._tags[SCHEMA_TYPE] && operation === 'serialization') {
138
+ // we have already added a schema to this span, this call is an encode of nested schema types
139
+ return
140
+ }
141
+
142
+ span.setTag(SCHEMA_TYPE, PROTOBUF)
143
+ span.setTag(SCHEMA_NAME, removeLeadingPeriod(descriptor.fullName))
144
+ span.setTag(SCHEMA_OPERATION, operation)
145
+
146
+ if (!tracer._dataStreamsProcessor.canSampleSchema(operation)) {
147
+ return
148
+ }
149
+
150
+ // if the span is unsampled, do not sample the schema
151
+ if (!tracer._prioritySampler.isSampled(span)) {
152
+ return
153
+ }
154
+
155
+ const weight = tracer._dataStreamsProcessor.trySampleSchema(operation)
156
+ if (weight === 0) {
157
+ return
158
+ }
159
+
160
+ const schemaData = SchemaBuilder.getSchemaDefinition(
161
+ this.extractSchemas(descriptor, tracer._dataStreamsProcessor)
162
+ )
163
+
164
+ span.setTag(SCHEMA_DEFINITION, schemaData.definition)
165
+ span.setTag(SCHEMA_WEIGHT, weight)
166
+ span.setTag(SCHEMA_ID, schemaData.id)
167
+ }
168
+ }
169
+
170
+ function removeLeadingPeriod (str) {
171
+ // Check if the first character is a period
172
+ if (str.charAt(0) === '.') {
173
+ // Remove the first character
174
+ return str.slice(1)
175
+ }
176
+ // Return the original string if the first character is not a period
177
+ return str
178
+ }
179
+
180
+ module.exports = SchemaExtractor
@@ -23,6 +23,13 @@ module.exports = {
23
23
  WAF_CONTEXT_PROCESSOR: 'waf.context.processor',
24
24
 
25
25
  HTTP_OUTGOING_URL: 'server.io.net.url',
26
+ FS_OPERATION_PATH: 'server.io.fs.file',
27
+
26
28
  DB_STATEMENT: 'server.db.statement',
27
- DB_SYSTEM: 'server.db.system'
29
+ DB_SYSTEM: 'server.db.system',
30
+
31
+ SHELL_COMMAND: 'server.sys.shell.cmd',
32
+
33
+ LOGIN_SUCCESS: 'server.business_logic.users.login.success',
34
+ LOGIN_FAILURE: 'server.business_logic.users.login.failure'
28
35
  }
@@ -6,6 +6,7 @@ const dc = require('dc-polyfill')
6
6
  module.exports = {
7
7
  bodyParser: dc.channel('datadog:body-parser:read:finish'),
8
8
  cookieParser: dc.channel('datadog:cookie-parser:read:finish'),
9
+ multerParser: dc.channel('datadog:multer:read:finish'),
9
10
  startGraphqlResolve: dc.channel('datadog:graphql:resolver:start'),
10
11
  graphqlMiddlewareChannel: dc.tracingChannel('datadog:apollo:middleware'),
11
12
  apolloChannel: dc.tracingChannel('datadog:apollo:request'),
@@ -17,6 +18,7 @@ module.exports = {
17
18
  setCookieChannel: dc.channel('datadog:iast:set-cookie'),
18
19
  nextBodyParsed: dc.channel('apm:next:body-parsed'),
19
20
  nextQueryParsed: dc.channel('apm:next:query-parsed'),
21
+ expressProcessParams: dc.channel('datadog:express:process_params:start'),
20
22
  responseBody: dc.channel('datadog:express:response:json:start'),
21
23
  responseWriteHead: dc.channel('apm:http:server:response:writeHead:start'),
22
24
  httpClientRequestStart: dc.channel('apm:http:client:request:start'),
@@ -24,5 +26,9 @@ module.exports = {
24
26
  setUncaughtExceptionCaptureCallbackStart: dc.channel('datadog:process:setUncaughtExceptionCaptureCallback:start'),
25
27
  pgQueryStart: dc.channel('apm:pg:query:start'),
26
28
  pgPoolQueryStart: dc.channel('datadog:pg:pool:query:start'),
27
- wafRunFinished: dc.channel('datadog:waf:run:finish')
29
+ mysql2OuterQueryStart: dc.channel('datadog:mysql2:outerquery:start'),
30
+ wafRunFinished: dc.channel('datadog:waf:run:finish'),
31
+ fsOperationStart: dc.channel('apm:fs:operation:start'),
32
+ expressMiddlewareError: dc.channel('apm:express:middleware:error'),
33
+ childProcessExecutionTracingChannel: dc.tracingChannel('datadog:child_process:execution')
28
34
  }
@@ -2,6 +2,7 @@
2
2
 
3
3
  const Analyzer = require('./vulnerability-analyzer')
4
4
  const { getNodeModulesPaths } = require('../path-line')
5
+ const iastLog = require('../iast-log')
5
6
 
6
7
  const EXCLUDED_PATHS = getNodeModulesPaths('express/lib/response.js')
7
8
 
@@ -11,7 +12,14 @@ class CookieAnalyzer extends Analyzer {
11
12
  this.propertyToBeSafe = propertyToBeSafe.toLowerCase()
12
13
  }
13
14
 
14
- onConfigure () {
15
+ onConfigure (config) {
16
+ try {
17
+ this.cookieFilterRegExp = new RegExp(config.iast.cookieFilterPattern)
18
+ } catch {
19
+ iastLog.error('Invalid regex in cookieFilterPattern')
20
+ this.cookieFilterRegExp = /.{32,}/
21
+ }
22
+
15
23
  this.addSub(
16
24
  { channelName: 'datadog:iast:set-cookie', moduleName: 'http' },
17
25
  (cookieInfo) => this.analyze(cookieInfo)
@@ -28,6 +36,10 @@ class CookieAnalyzer extends Analyzer {
28
36
  }
29
37
 
30
38
  _createHashSource (type, evidence, location) {
39
+ if (typeof evidence.value === 'string' && evidence.value.match(this.cookieFilterRegExp)) {
40
+ return 'FILTERED_' + this._type
41
+ }
42
+
31
43
  return `${type}:${evidence.value}`
32
44
  }
33
45
 
@@ -29,7 +29,14 @@ class PathTraversalAnalyzer extends InjectionAnalyzer {
29
29
 
30
30
  onConfigure () {
31
31
  this.addSub('apm:fs:operation:start', (obj) => {
32
- if (ignoredOperations.includes(obj.operation)) return
32
+ const store = storage.getStore()
33
+ const outOfReqOrChild = !store?.fs?.root
34
+
35
+ // we could filter out all the nested fs.operations based on store.fs.root
36
+ // but if we spect a store in the context to be present we are going to exclude
37
+ // all out_of_the_request fs.operations
38
+ // AppsecFsPlugin must be enabled
39
+ if (ignoredOperations.includes(obj.operation) || outOfReqOrChild) return
33
40
 
34
41
  const pathArguments = []
35
42
  if (obj.dest) {
@@ -127,7 +127,7 @@ class IastPlugin extends Plugin {
127
127
  config = { enabled: config }
128
128
  }
129
129
  if (config.enabled && !this.configured) {
130
- this.onConfigure()
130
+ this.onConfigure(config.tracerConfig)
131
131
  this.configured = true
132
132
  }
133
133
 
@@ -14,6 +14,7 @@ const {
14
14
  } = require('./taint-tracking')
15
15
  const { IAST_ENABLED_TAG_KEY } = require('./tags')
16
16
  const iastTelemetry = require('./telemetry')
17
+ const { enable: enableFsPlugin, disable: disableFsPlugin, IAST_MODULE } = require('../rasp/fs-plugin')
17
18
 
18
19
  // TODO Change to `apm:http:server:request:[start|close]` when the subscription
19
20
  // order of the callbacks can be enforce
@@ -27,6 +28,7 @@ function enable (config, _tracer) {
27
28
  if (isEnabled) return
28
29
 
29
30
  iastTelemetry.configure(config, config.iast?.telemetryVerbosity)
31
+ enableFsPlugin(IAST_MODULE)
30
32
  enableAllAnalyzers(config)
31
33
  enableTaintTracking(config.iast, iastTelemetry.verbosity)
32
34
  requestStart.subscribe(onIncomingHttpRequestStart)
@@ -44,6 +46,7 @@ function disable () {
44
46
  isEnabled = false
45
47
 
46
48
  iastTelemetry.stop()
49
+ disableFsPlugin(IAST_MODULE)
47
50
  disableAllAnalyzers()
48
51
  disableTaintTracking()
49
52
  overheadController.finishGlobalContext()
@@ -12,6 +12,7 @@ const csiMethods = [
12
12
  { src: 'substring' },
13
13
  { src: 'toLowerCase', dst: 'stringCase' },
14
14
  { src: 'toUpperCase', dst: 'stringCase' },
15
+ { src: 'tplOperator', operator: true },
15
16
  { src: 'trim' },
16
17
  { src: 'trimEnd' },
17
18
  { src: 'trimStart', dst: 'trim' },
@@ -23,18 +23,26 @@ class TaintTrackingPlugin extends SourceIastPlugin {
23
23
  constructor () {
24
24
  super()
25
25
  this._type = 'taint-tracking'
26
+ this._taintedURLs = new WeakMap()
26
27
  }
27
28
 
28
29
  onConfigure () {
30
+ const onRequestBody = ({ req }) => {
31
+ const iastContext = getIastContext(storage.getStore())
32
+ if (iastContext && iastContext.body !== req.body) {
33
+ this._taintTrackingHandler(HTTP_REQUEST_BODY, req, 'body', iastContext)
34
+ iastContext.body = req.body
35
+ }
36
+ }
37
+
29
38
  this.addSub(
30
39
  { channelName: 'datadog:body-parser:read:finish', tag: HTTP_REQUEST_BODY },
31
- ({ req }) => {
32
- const iastContext = getIastContext(storage.getStore())
33
- if (iastContext && iastContext.body !== req.body) {
34
- this._taintTrackingHandler(HTTP_REQUEST_BODY, req, 'body', iastContext)
35
- iastContext.body = req.body
36
- }
37
- }
40
+ onRequestBody
41
+ )
42
+
43
+ this.addSub(
44
+ { channelName: 'datadog:multer:read:finish', tag: HTTP_REQUEST_BODY },
45
+ onRequestBody
38
46
  )
39
47
 
40
48
  this.addSub(
@@ -81,6 +89,46 @@ class TaintTrackingPlugin extends SourceIastPlugin {
81
89
  }
82
90
  )
83
91
 
92
+ const urlResultTaintedProperties = ['host', 'origin', 'hostname']
93
+ this.addSub(
94
+ { channelName: 'datadog:url:parse:finish' },
95
+ ({ input, base, parsed, isURL }) => {
96
+ const iastContext = getIastContext(storage.getStore())
97
+ let ranges
98
+
99
+ if (base) {
100
+ ranges = getRanges(iastContext, base)
101
+ } else {
102
+ ranges = getRanges(iastContext, input)
103
+ }
104
+
105
+ if (ranges?.length) {
106
+ if (isURL) {
107
+ this._taintedURLs.set(parsed, ranges[0])
108
+ } else {
109
+ urlResultTaintedProperties.forEach(param => {
110
+ this._taintTrackingHandler(ranges[0].iinfo.type, parsed, param, iastContext)
111
+ })
112
+ }
113
+ }
114
+ }
115
+ )
116
+
117
+ this.addSub(
118
+ { channelName: 'datadog:url:getter:finish' },
119
+ (context) => {
120
+ if (!urlResultTaintedProperties.includes(context.property)) return
121
+
122
+ const origRange = this._taintedURLs.get(context.urlObject)
123
+ if (!origRange) return
124
+
125
+ const iastContext = getIastContext(storage.getStore())
126
+ if (!iastContext) return
127
+
128
+ context.result =
129
+ newTaintedString(iastContext, context.result, origRange.iinfo.parameterName, origRange.iinfo.type)
130
+ })
131
+
84
132
  // this is a special case to increment INSTRUMENTED_SOURCE metric for header
85
133
  this.addInstrumentedSource('http', [HTTP_REQUEST_HEADER_VALUE, HTTP_REQUEST_HEADER_NAME])
86
134
  }
@@ -29,6 +29,7 @@ const TaintTrackingNoop = {
29
29
  substr: noop,
30
30
  substring: noop,
31
31
  stringCase: noop,
32
+ tplOperator: noop,
32
33
  trim: noop,
33
34
  trimEnd: noop
34
35
  }
@@ -117,6 +118,20 @@ function csiMethodsOverrides (getContext) {
117
118
  return res
118
119
  },
119
120
 
121
+ tplOperator: function (res, ...rest) {
122
+ try {
123
+ const iastContext = getContext()
124
+ const transactionId = getTransactionId(iastContext)
125
+ if (transactionId) {
126
+ return TaintedUtils.concat(transactionId, res, ...rest)
127
+ }
128
+ } catch (e) {
129
+ iastLog.error('Error invoking CSI tplOperator')
130
+ .errorAndPublish(e)
131
+ }
132
+ return res
133
+ },
134
+
120
135
  stringCase: getCsiFn(
121
136
  (transactionId, res, target) => TaintedUtils.stringCase(transactionId, res, target),
122
137
  getContext,
@@ -1,10 +1,11 @@
1
1
  'use strict'
2
2
 
3
- const { MANUAL_KEEP } = require('../../../../../ext/tags')
4
3
  const LRU = require('lru-cache')
5
4
  const vulnerabilitiesFormatter = require('./vulnerabilities-formatter')
6
5
  const { IAST_ENABLED_TAG_KEY, IAST_JSON_TAG_KEY } = require('./tags')
7
6
  const standalone = require('../standalone')
7
+ const { SAMPLING_MECHANISM_APPSEC } = require('../../constants')
8
+ const { keepTrace } = require('../../priority_sampler')
8
9
 
9
10
  const VULNERABILITIES_KEY = 'vulnerabilities'
10
11
  const VULNERABILITY_HASHES_MAX_SIZE = 1000
@@ -56,9 +57,10 @@ function sendVulnerabilities (vulnerabilities, rootSpan) {
56
57
  const tags = {}
57
58
  // TODO: Store this outside of the span and set the tag in the exporter.
58
59
  tags[IAST_JSON_TAG_KEY] = JSON.stringify(jsonToSend)
59
- tags[MANUAL_KEEP] = 'true'
60
60
  span.addTags(tags)
61
61
 
62
+ keepTrace(span, SAMPLING_MECHANISM_APPSEC)
63
+
62
64
  standalone.sample(span)
63
65
 
64
66
  if (!rootSpan) span.finish()
@@ -6,12 +6,14 @@ const remoteConfig = require('./remote_config')
6
6
  const {
7
7
  bodyParser,
8
8
  cookieParser,
9
+ multerParser,
9
10
  incomingHttpRequestStart,
10
11
  incomingHttpRequestEnd,
11
12
  passportVerify,
12
13
  queryParser,
13
14
  nextBodyParsed,
14
15
  nextQueryParsed,
16
+ expressProcessParams,
15
17
  responseBody,
16
18
  responseWriteHead,
17
19
  responseSetHeader
@@ -30,6 +32,8 @@ const { storage } = require('../../../datadog-core')
30
32
  const graphql = require('./graphql')
31
33
  const rasp = require('./rasp')
32
34
 
35
+ const responseAnalyzedSet = new WeakSet()
36
+
33
37
  let isEnabled = false
34
38
  let config
35
39
 
@@ -54,13 +58,15 @@ function enable (_config) {
54
58
 
55
59
  apiSecuritySampler.configure(_config.appsec)
56
60
 
61
+ bodyParser.subscribe(onRequestBodyParsed)
62
+ multerParser.subscribe(onRequestBodyParsed)
63
+ cookieParser.subscribe(onRequestCookieParser)
57
64
  incomingHttpRequestStart.subscribe(incomingHttpStartTranslator)
58
65
  incomingHttpRequestEnd.subscribe(incomingHttpEndTranslator)
59
- bodyParser.subscribe(onRequestBodyParsed)
66
+ queryParser.subscribe(onRequestQueryParsed)
60
67
  nextBodyParsed.subscribe(onRequestBodyParsed)
61
68
  nextQueryParsed.subscribe(onRequestQueryParsed)
62
- queryParser.subscribe(onRequestQueryParsed)
63
- cookieParser.subscribe(onRequestCookieParser)
69
+ expressProcessParams.subscribe(onRequestProcessParams)
64
70
  responseBody.subscribe(onResponseBody)
65
71
  responseWriteHead.subscribe(onResponseWriteHead)
66
72
  responseSetHeader.subscribe(onResponseSetHeader)
@@ -79,6 +85,41 @@ function enable (_config) {
79
85
  }
80
86
  }
81
87
 
88
+ function onRequestBodyParsed ({ req, res, body, abortController }) {
89
+ if (body === undefined || body === null) return
90
+
91
+ if (!req) {
92
+ const store = storage.getStore()
93
+ req = store?.req
94
+ }
95
+
96
+ const rootSpan = web.root(req)
97
+ if (!rootSpan) return
98
+
99
+ const results = waf.run({
100
+ persistent: {
101
+ [addresses.HTTP_INCOMING_BODY]: body
102
+ }
103
+ }, req)
104
+
105
+ handleResults(results, req, res, rootSpan, abortController)
106
+ }
107
+
108
+ function onRequestCookieParser ({ req, res, abortController, cookies }) {
109
+ if (!cookies || typeof cookies !== 'object') return
110
+
111
+ const rootSpan = web.root(req)
112
+ if (!rootSpan) return
113
+
114
+ const results = waf.run({
115
+ persistent: {
116
+ [addresses.HTTP_INCOMING_COOKIES]: cookies
117
+ }
118
+ }, req)
119
+
120
+ handleResults(results, req, res, rootSpan, abortController)
121
+ }
122
+
82
123
  function incomingHttpStartTranslator ({ req, res, abortController }) {
83
124
  const rootSpan = web.root(req)
84
125
  if (!rootSpan) return
@@ -122,11 +163,6 @@ function incomingHttpEndTranslator ({ req, res }) {
122
163
  persistent[addresses.HTTP_INCOMING_BODY] = req.body
123
164
  }
124
165
 
125
- // TODO: temporary express instrumentation, will use express plugin later
126
- if (req.params !== null && typeof req.params === 'object') {
127
- persistent[addresses.HTTP_INCOMING_PARAMS] = req.params
128
- }
129
-
130
166
  // we need to keep this to support other cookie parsers
131
167
  if (req.cookies !== null && typeof req.cookies === 'object') {
132
168
  persistent[addresses.HTTP_INCOMING_COOKIES] = req.cookies
@@ -145,24 +181,16 @@ function incomingHttpEndTranslator ({ req, res }) {
145
181
  Reporter.finishRequest(req, res)
146
182
  }
147
183
 
148
- function onRequestBodyParsed ({ req, res, body, abortController }) {
149
- if (body === undefined || body === null) return
184
+ function onPassportVerify ({ credentials, user }) {
185
+ const store = storage.getStore()
186
+ const rootSpan = store?.req && web.root(store.req)
150
187
 
151
- if (!req) {
152
- const store = storage.getStore()
153
- req = store?.req
188
+ if (!rootSpan) {
189
+ log.warn('No rootSpan found in onPassportVerify')
190
+ return
154
191
  }
155
192
 
156
- const rootSpan = web.root(req)
157
- if (!rootSpan) return
158
-
159
- const results = waf.run({
160
- persistent: {
161
- [addresses.HTTP_INCOMING_BODY]: body
162
- }
163
- }, req)
164
-
165
- handleResults(results, req, res, rootSpan, abortController)
193
+ passportTrackEvent(credentials, user, rootSpan, config.appsec.eventTracking.mode)
166
194
  }
167
195
 
168
196
  function onRequestQueryParsed ({ req, res, query, abortController }) {
@@ -185,15 +213,15 @@ function onRequestQueryParsed ({ req, res, query, abortController }) {
185
213
  handleResults(results, req, res, rootSpan, abortController)
186
214
  }
187
215
 
188
- function onRequestCookieParser ({ req, res, abortController, cookies }) {
189
- if (!cookies || typeof cookies !== 'object') return
190
-
216
+ function onRequestProcessParams ({ req, res, abortController, params }) {
191
217
  const rootSpan = web.root(req)
192
218
  if (!rootSpan) return
193
219
 
220
+ if (!params || typeof params !== 'object' || !Object.keys(params).length) return
221
+
194
222
  const results = waf.run({
195
223
  persistent: {
196
- [addresses.HTTP_INCOMING_COOKIES]: cookies
224
+ [addresses.HTTP_INCOMING_PARAMS]: params
197
225
  }
198
226
  }, req)
199
227
 
@@ -212,20 +240,6 @@ function onResponseBody ({ req, body }) {
212
240
  }, req)
213
241
  }
214
242
 
215
- function onPassportVerify ({ credentials, user }) {
216
- const store = storage.getStore()
217
- const rootSpan = store?.req && web.root(store.req)
218
-
219
- if (!rootSpan) {
220
- log.warn('No rootSpan found in onPassportVerify')
221
- return
222
- }
223
-
224
- passportTrackEvent(credentials, user, rootSpan, config.appsec.eventTracking.mode)
225
- }
226
-
227
- const responseAnalyzedSet = new WeakSet()
228
-
229
243
  function onResponseWriteHead ({ req, res, abortController, statusCode, responseHeaders }) {
230
244
  // avoid "write after end" error
231
245
  if (isBlocked(res)) {
@@ -287,12 +301,16 @@ function disable () {
287
301
 
288
302
  // Channel#unsubscribe() is undefined for non active channels
289
303
  if (bodyParser.hasSubscribers) bodyParser.unsubscribe(onRequestBodyParsed)
304
+ if (multerParser.hasSubscribers) multerParser.unsubscribe(onRequestBodyParsed)
305
+ if (cookieParser.hasSubscribers) cookieParser.unsubscribe(onRequestCookieParser)
290
306
  if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(incomingHttpStartTranslator)
291
307
  if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(incomingHttpEndTranslator)
308
+ if (passportVerify.hasSubscribers) passportVerify.unsubscribe(onPassportVerify)
292
309
  if (queryParser.hasSubscribers) queryParser.unsubscribe(onRequestQueryParsed)
293
- if (cookieParser.hasSubscribers) cookieParser.unsubscribe(onRequestCookieParser)
310
+ if (nextBodyParsed.hasSubscribers) nextBodyParsed.unsubscribe(onRequestBodyParsed)
311
+ if (nextQueryParsed.hasSubscribers) nextQueryParsed.unsubscribe(onRequestQueryParsed)
312
+ if (expressProcessParams.hasSubscribers) expressProcessParams.unsubscribe(onRequestProcessParams)
294
313
  if (responseBody.hasSubscribers) responseBody.unsubscribe(onResponseBody)
295
- if (passportVerify.hasSubscribers) passportVerify.unsubscribe(onPassportVerify)
296
314
  if (responseWriteHead.hasSubscribers) responseWriteHead.unsubscribe(onResponseWriteHead)
297
315
  if (responseSetHeader.hasSubscribers) responseSetHeader.unsubscribe(onResponseSetHeader)
298
316
  }