@datadog/datadog-ci-plugin-synthetics 3.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +1114 -0
  3. package/package.json +63 -0
package/LICENSE ADDED
@@ -0,0 +1,201 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright 2020 Datadog, Inc.
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
package/README.md ADDED
@@ -0,0 +1,1114 @@
1
+ <!-- partial <div class="alert alert-info">This page is about configuring Continuous Testing tests for your Continuous Integration (CI) and Continuous Delivery (CD) pipelines. If you want to bring your CI/CD metrics and data into Datadog dashboards, see the <a href="https://docs.datadoghq.com/continuous_integration/" target="_blank">CI Visibility</a> section.</div> partial -->
2
+
3
+ ## Overview
4
+
5
+ Use the [`@datadog-ci` NPM package][1] to run Continuous Testing tests directly within your CI/CD pipeline. You can automatically halt a build, block a deployment, and roll back a deployment when a Synthetic test detects a regression.
6
+
7
+ ## Setup
8
+
9
+ ### Install the package
10
+
11
+ <!-- xxx tabs xxx -->
12
+ <!-- xxx tab "NPM" xxx -->
13
+
14
+ Install the package through NPM:
15
+
16
+ ```bash
17
+ npm install --save-dev @datadog/datadog-ci
18
+ ```
19
+
20
+ <!-- xxz tab xxx -->
21
+ <!-- xxx tab "Yarn" xxx -->
22
+
23
+ Install the package through Yarn:
24
+
25
+ ```bash
26
+ yarn add --dev @datadog/datadog-ci
27
+ ```
28
+
29
+ <!-- xxz tab xxx -->
30
+ <!-- xxz tabs xxx -->
31
+
32
+ ### Setup the client
33
+
34
+ To setup the client, your Datadog API and application keys need to be configured. These keys can be defined in three different ways:
35
+
36
+ 1. Defined in a [global configuration file](#global-configuration-file):
37
+
38
+ ```json
39
+ {
40
+ "apiKey": "<API_KEY>",
41
+ "appKey": "<APPLICATION_KEY>",
42
+ }
43
+ ```
44
+
45
+ 2. Defined as environment variables:
46
+
47
+ ```bash
48
+ export DD_API_KEY="<API_KEY>"
49
+ export DD_APP_KEY="<APPLICATION_KEY>"
50
+ ```
51
+
52
+ 3. Passed to the CLI when running your tests:
53
+
54
+ ```bash
55
+ yarn datadog-ci synthetics run-tests --apiKey "<API_KEY>" --appKey "<APPLICATION_KEY>"
56
+ ```
57
+
58
+ ### Global configuration file
59
+
60
+ Using a global configuration file (Global Config) is one of the ways to configure datadog-ci. To do so, create a JSON configuration file on your system. Specify the path to the file using the `--config` flag or configure it through the `DATADOG_SYNTHETICS_CONFIG_PATH` environment variable [when launching your tests](#run-tests-command) or [uploading a new application](#upload-application-command). If you don't specify a file path, Datadog looks for a file with the default filename of `datadog-ci.json`.
61
+
62
+ See each command's list of configurations below for the list of advanced options in the global configuration file relevant to each [run-tests command](#run-tests-command) and [upload-application command](#upload-application-command). For an example configuration file, see this [`global-config-complete-example.json` file][9].
63
+
64
+ Example:
65
+
66
+ ```jsonc
67
+ {
68
+ "apiKey": "<API_KEY>",
69
+ "appKey": "<APPLICATION_KEY>",
70
+ "batchTimeout": 1800000,
71
+ "datadogSite": "datadoghq.com",
72
+ "defaultTestOverrides": {
73
+ "allowInsecureCertificates": true,
74
+ "basicAuth": {"username": "test", "password": "test"},
75
+ "body": "{\"fakeContent\":true}",
76
+ "bodyType": "application/json",
77
+ "cookies": "name1=value1;name2=value2;",
78
+ "setCookies": "name1=value1 \n name2=value2; Domain=example.com \n name3=value3; Secure; HttpOnly",
79
+ "defaultStepTimeout": 15,
80
+ "deviceIds": ["chrome.laptop_large"],
81
+ "executionRule": "skipped",
82
+ "followRedirects": true,
83
+ "headers": {"NEW_HEADER": "NEW VALUE"},
84
+ "locations": ["aws:us-east-1"],
85
+ "mobileApplicationVersion": "01234567-8888-9999-abcd-efffffffffff",
86
+ "mobileApplicationVersionFilePath": "path/to/application.apk",
87
+ "resourceUrlSubstitutionRegexes": ["(https://www.)(.*)|$1staging-$2"],
88
+ "retry": {"count": 2, "interval": 300},
89
+ "startUrl": "{{URL}}?static_hash={{STATIC_HASH}}",
90
+ "startUrlSubstitutionRegex": "s/(https://www.)(.*)/$1extra-$2/",
91
+ "testTimeout": 300,
92
+ "variables": {"NEW_VARIABLE": "NEW VARIABLE"}
93
+ },
94
+ "failOnCriticalErrors": true,
95
+ "failOnMissingTests": true,
96
+ "failOnTimeout": true,
97
+ "files": ["{,!(node_modules)/**/}*.synthetics.json"],
98
+ "proxy": {
99
+ "auth": {
100
+ "username": "login",
101
+ "password": "pwd"
102
+ },
103
+ "host": "127.0.0.1",
104
+ "port": 3128,
105
+ "protocol": "http"
106
+ },
107
+ "subdomain": "subdomainname",
108
+ "tunnel": true
109
+ }
110
+ ```
111
+
112
+ ### Environment Variables
113
+
114
+ In addition to the global configuration file, you can configure all properties using environment variables. If a property is defined in both the global configuration file and as an environment variable, the environment variable takes precedence.
115
+
116
+ Example:
117
+
118
+ ```bash
119
+ export DATADOG_SITE=datadoghq.com
120
+ ```
121
+
122
+ ### Command line options
123
+
124
+ The CLI provides another way to set options and configure the behavior of datadog-ci. These options will override the global configuration file and environment variables.
125
+
126
+ Example:
127
+
128
+ ```bash
129
+ yarn datadog-ci synthetics run-tests --public-id pub-lic-id1
130
+ ```
131
+
132
+ The priority of the 3 forms of configuration is as follows:
133
+
134
+ ```yml
135
+ Global Config < Environment variables < CLI parameters
136
+ ```
137
+
138
+ ### Using datadog-ci as a library
139
+
140
+ You can also use the `datadog-ci` package as a library in your Node.js application to trigger tests. To do so, import the package from the Synthetics `run-tests` command and call the `executeWithDetails()` function.
141
+
142
+ ``` javascript
143
+ import { synthetics } from '@datadog/datadog-ci'
144
+
145
+ const { results, summary } = await synthetics.executeTests(...)
146
+ ```
147
+
148
+ ### Use a proxy
149
+
150
+ You can configure a proxy to be used for outgoing connections to Datadog. To do this, use the `proxy` key of the global configuration file or the `HTTPS_PROXY` environment variable.
151
+
152
+ **Note**: This is the only exception where the global configuration file takes precedence over the environment variable. There is no option to set this through the CLI.
153
+
154
+ As the [`proxy-agent` library][2] is used to configure the proxy, the supported protocols include `http`, `https`, `socks`, `socks4`, `socks4a`, `socks5`, `socks5h`, `pac+data`, `pac+file`, `pac+ftp`, `pac+http`, and `pac+https`. The `proxy` key of the global configuration file is passed to a new `proxy-agent` instance, which means the same configuration for the library is supported.
155
+
156
+ To use a proxy, you need to first set the CA certificate so datadog-ci trusts your proxy. You can do this by setting the `NODE_EXTRA_CA_CERTS` environment variable to the path of your CA certificate. Otherwise, you might get a `unable to verify the first certificate` error.
157
+
158
+ ```bash
159
+ export NODE_EXTRA_CA_CERTS=/path/to/your-ca-cert.pem
160
+ ```
161
+
162
+ When using the global configuration, `host` and `port` keys are mandatory arguments and the `protocol` key defaults to `http` if not defined.
163
+
164
+ Example:
165
+
166
+ ```jsonc
167
+ {
168
+ // ...
169
+ "proxy": {
170
+ "auth": {
171
+ "username": "login",
172
+ "password": "pwd"
173
+ },
174
+ "host": "127.0.0.1",
175
+ "port": 3128,
176
+ "protocol": "http"
177
+ },
178
+ // ...
179
+ }
180
+ ```
181
+
182
+ The format used for the `HTTPS_PROXY` environment variable is `<protocol>://<username>:<password>@<host>:<port>`, as described by the [proxy-from-env][13] library that [`proxy-agent` library][2] uses for parsing env variables.
183
+ The `HTTPS_PROXY` variable is used instead of the `HTTP_PROXY` one, because the Datadog API uses the HTTPS protocol.
184
+
185
+ Example:
186
+
187
+ ```bash
188
+ export HTTPS_PROXY=http://login:pwd@127.0.0.1:3128
189
+ ```
190
+
191
+ If you want to confirm that a proxy is being used, you can set the `DEBUG` environment variable to `proxy-agent` like this:
192
+
193
+ ```bash
194
+ DEBUG=proxy-agent yarn datadog-ci synthetics run-tests
195
+ ```
196
+
197
+ ## Run Tests Command
198
+
199
+ You can decide to have the CLI auto-discover all your `**/*.synthetics.json` Synthetic tests (see [test files](#test-files)) or specify the tests you want to run using the `-p,--public-id` flag.
200
+
201
+ <!-- xxx tabs xxx -->
202
+ <!-- xxx tab "NPM" xxx -->
203
+
204
+ Run tests by executing the CLI through **NPM**:
205
+
206
+ Add the following to your `package.json`:
207
+
208
+ ```json
209
+ {
210
+ "scripts": {
211
+ "datadog-ci-synthetics": "datadog-ci synthetics run-tests"
212
+ }
213
+ }
214
+ ```
215
+
216
+ Then, run:
217
+
218
+ ```bash
219
+ npm run datadog-ci-synthetics
220
+ ```
221
+
222
+ **Note**: If you are launching your tests with a custom filename for the [global configuration file](#global-configuration-file), append the command associated to your `datadog-ci-synthetics` script with `--config <CUSTOM_PATH_TO_GLOBAL_CONFIG_FILE>`.
223
+
224
+ <!-- xxz tab xxx -->
225
+ <!-- xxx tab "Yarn" xxx -->
226
+
227
+ Run tests by executing the CLI through **Yarn**:
228
+
229
+ The `run-tests` sub-command accepts the `--public-id` (or shorthand `-p`) argument to trigger only the specified test. It can be set multiple times to run multiple tests:
230
+
231
+ ```bash
232
+ yarn datadog-ci synthetics run-tests --public-id aaa-aaa-aaa --public-id bbb-bbb-bbb
233
+ ```
234
+
235
+ You can also specify a specific version of a test using the format `<public-id>@<version>`:
236
+
237
+ ```bash
238
+ yarn datadog-ci synthetics run-tests --public-id aaa-aaa-aaa@2 --public-id bbb-bbb-bbb@4
239
+ ```
240
+
241
+ It is also possible to trigger tests corresponding to a search query by using the `--search` (or shorthand `-s`) argument. With this option, the overrides defined in your [global configuration file](#global-configuration-file) apply to all tests discovered with the search query.
242
+
243
+ ```bash
244
+ yarn datadog-ci synthetics run-tests -s 'tag:e2e-tests'
245
+ ```
246
+
247
+ You can use `--files` (shorthand `-f`) to override the default glob pattern (which would match all `**/*.synthetics.json` files).
248
+
249
+ ```bash
250
+ yarn datadog-ci synthetics run-tests -f ./component-1/**/*.synthetics.json -f ./component-2/**/*.synthetics.json
251
+ ```
252
+
253
+ **Note**: If you are launching your tests with a custom filename for the [global configuration file](#global-configuration-file), append the command associated to your `datadog-ci-synthetics` script with `--config <CUSTOM_PATH_TO_GLOBAL_CONFIG_FILE>`.
254
+
255
+ <!-- xxz tab xxx -->
256
+ <!-- xxz tabs xxx -->
257
+
258
+ ### List of Configurations
259
+
260
+ <!--
261
+ When updating any of these, don't forget to update the Google Sheets document and relevant CI integrations:
262
+ https://docs.google.com/spreadsheets/d/1VB8ntED7hz2McIwp7NaHADVt16nFUuNnKERBl78tldQ/edit?usp=sharing
263
+
264
+ For more information, see https://datadoghq.atlassian.net/wiki/x/LwBfyQ
265
+ -->
266
+
267
+ #### `apiKey` (Required)
268
+
269
+ Your Datadog API key. This key is [created in your Datadog organization][15] and should be stored as a secret.
270
+
271
+ **Configuration options**
272
+
273
+ * Global Config: `"apiKey": "<API_KEY>"`
274
+ * ENV variable: `DD_API_KEY="<API_KEY>"`
275
+ * CLI param: `--apiKey "<API_KEY>"`
276
+
277
+ #### `appKey` (Required)
278
+
279
+ Your Datadog application key. This key is [created in your Datadog organization][15] and should be stored as a secret.
280
+
281
+ **Configuration options**
282
+
283
+ * Global Config: `"appKey": "<APPLICATION_KEY>"`
284
+ * ENV variable: `DD_APP_KEY="<APPLICATION_KEY>"`
285
+ * CLI param: `--appKey "<APPLICATION_KEY>"`
286
+
287
+ #### `batchTimeout`
288
+
289
+ The duration in milliseconds after which the CI batch fails as timed out. This does not affect the outcome of a test run that already started.
290
+
291
+ **Configuration options**
292
+
293
+ * Default: `1800000` (30 minutes)
294
+ * Global Config: `"batchTimeout": 1800000`
295
+ * ENV variable: `DATADOG_SYNTHETICS_BATCH_TIMEOUT=1800000`
296
+ * CLI param: `--batchTimeout 1800000`
297
+
298
+ #### `configPath`
299
+
300
+ The path to the [global configuration file](#global-configuration-file) that configures datadog-ci.
301
+
302
+ **Configuration options**
303
+
304
+ * Default: `datadog-ci.json`
305
+ * Global Config: N/A
306
+ * ENV variable: `DATADOG_SYNTHETICS_CONFIG_PATH=global-config.json`
307
+ * CLI param: `--config global-config.json`
308
+
309
+ #### `datadogSite`
310
+
311
+ Your Datadog site. The possible values are listed [in this table][16].
312
+
313
+ <!-- partial Set it to {{< region-param key="dd_site" code="true" >}} (ensure the correct SITE is selected on the right). partial -->
314
+
315
+ **Configuration options**
316
+
317
+ * Default: `datadoghq.com`
318
+ * Global Config: `"datadogSite": "datadoghq.com"`
319
+ * ENV variable: `DATADOG_SITE=datadoghq.com`
320
+ * CLI param: `--datadogSite datadoghq.com`
321
+
322
+ #### `defaultTestOverrides`
323
+
324
+ Overrides for Synthetic tests applied to all tests.
325
+
326
+ **Configuration options**
327
+
328
+ * Global Config: See [test overrides](#test-overrides)
329
+ * ENV variable: all variables follow the `DATADOG_SYNTHETICS_OVERRIDE_...` pattern
330
+ * CLI param: all CLI params use the `--override option=value` pattern
331
+
332
+ #### `failOnCriticalErrors`
333
+
334
+ Fail the CI job if a critical error that is typically transient occurs, such as rate limits, authentication failures, or Datadog infrastructure issues.
335
+
336
+ **Configuration options**
337
+
338
+ * Default: `false`
339
+ * Global Config: `"failOnCriticalErrors": true`
340
+ * ENV variable: `DATADOG_SYNTHETICS_FAIL_ON_CRITICAL_ERRORS=true`
341
+ * CLI param: `--failOnCriticalErrors` / `--no-failOnCriticalErrors`
342
+
343
+ #### `failOnMissingTests`
344
+
345
+ Fail the CI job if the list of tests to run is empty or if some explicitly listed tests are missing.
346
+
347
+ **Configuration options**
348
+
349
+ * Default: `false`
350
+ * Global Config: `"failOnMissingTests": true`
351
+ * ENV variable: `DATADOG_SYNTHETICS_FAIL_ON_MISSING_TESTS=true`
352
+ * CLI param: `--failOnMissingTests` / `--no-failOnMissingTests`
353
+
354
+ #### `failOnTimeout`
355
+
356
+ Fail the CI job if the CI batch fails as timed out.
357
+
358
+ **Configuration options**
359
+
360
+ * Default: `true`
361
+ * Global Config: `"failOnTimeout": true`
362
+ * ENV variable: `DATADOG_SYNTHETICS_FAIL_ON_TIMEOUT=true`
363
+ * CLI param: `--failOnTimeout` / `--no-failOnTimeout`
364
+
365
+ #### `files`
366
+
367
+ Glob patterns to detect Synthetic [test configuration files](#test-files).
368
+
369
+ **Configuration options**
370
+
371
+ * Default: `["{,!(node_modules)/**/}*.synthetics.json"]`
372
+ * Global Config: `"files": ["{,!(node_modules)/**/}*.synthetics.json"]`
373
+ * ENV variable: `DATADOG_SYNTHETICS_FILES="{,!(node_modules)/**/}*.synthetics.json"`
374
+ * CLI param: `-f "{,!(node_modules)/**/}*.synthetics.json"` / `--files "{,!(node_modules)/**/}*.synthetics.json"`
375
+
376
+ #### `jUnitReport`
377
+
378
+ The filename for a JUnit report if you want to generate one.
379
+
380
+ **Configuration options**
381
+
382
+ * Default: None
383
+ * Global Config: `"jUnitReport": "e2e-test-junit.xml"`
384
+ * ENV variable: `DATADOG_SYNTHETICS_JUNIT_REPORT="e2e-test-junit.xml"`
385
+ * CLI param:`-j "e2e-test-junit.xml"` / `--jUnitReport "e2e-test-junit.xml"`
386
+
387
+ #### `mobileApplicationVersionFilePath`
388
+
389
+ Override the mobile application version for [Synthetic mobile application tests][18] with a local or recently built application.
390
+
391
+ **Configuration options**
392
+
393
+ * Global Config: `"mobileApplicationVersionFilePath": "path/to/application.apk"`
394
+ * ENV variable: Not Available
395
+ * CLI param: `--mobileApp "path/to/application.apk"` / `--mobileApplicationVersionFilePath "path/to/application.apk"`
396
+
397
+ #### `proxy`
398
+
399
+ The proxy to be used for outgoing connections to Datadog. `host` and `port` keys are mandatory arguments, the `protocol` key defaults to `http`. Supported values for the `protocol` key are `http`, `https`, `socks`, `socks4`, `socks4a`, `socks5`, `socks5h`, `pac+data`, `pac+file`, `pac+ftp`, `pac+http`, and `pac+https`. The library used to configure the proxy is the [proxy-agent][2] library.
400
+
401
+ **Configuration options**
402
+
403
+ * Global Config: See [Use a proxy](#use-a-proxy) for an example.
404
+ * ENV variable: `HTTPS_PROXY=http://login:pwd@127.0.0.1:3128`
405
+ * CLI param: N/A
406
+
407
+ #### `publicIds`
408
+
409
+ Public IDs of Synthetic tests to run. If no value is provided, tests are discovered in Synthetic [test configuration files](#test-files).
410
+
411
+ You can specify a specific version of a test using the format `<public-id>@<version>`. For example, `abc-def-ghi@123` runs version 123 of the test with public ID `abc-def-ghi`. If no version is specified, the latest version of the test is used.
412
+
413
+ **Configuration options**
414
+
415
+ * Default: None
416
+ * Global Config: `"publicIds": ["abc-def-ghi", "123-456-789"]`
417
+ * ENV variable: `DATADOG_SYNTHETICS_PUBLIC_IDS="abc-def-ghi;123-456-789"`
418
+ * CLI param: `-p "abc-def-ghi" --public-id "123-456-789"`
419
+
420
+ #### `selectiveRerun`
421
+
422
+ Whether to only rerun failed tests. If a test has already passed for a given commit, it will not be rerun in subsequent CI batches. By default, your [organization's default setting][17] is used. Set it to `false` to force full runs when your configuration enables it by default.
423
+
424
+ **Configuration options**
425
+
426
+ * Default: `false`
427
+ * Global Config: `"selectiveRerun": true`
428
+ * ENV variable: `DATADOG_SYNTHETICS_SELECTIVE_RERUN=true`
429
+ * CLI param: `--selectiveRerun` / `--no-selectiveRerun`
430
+
431
+ #### `subdomain`
432
+
433
+ The custom subdomain to access your Datadog organization. If your URL is `myorg.datadoghq.com`, the custom subdomain is `myorg`.
434
+
435
+ **Configuration options**
436
+
437
+ * Default: `app`
438
+ * Global Config: `"subdomain": "myorg"`
439
+ * ENV variable: `DATADOG_SUBDOMAIN="myorg"`
440
+ * CLI param: `--subdomain "myorg"`
441
+
442
+ #### `testSearchQuery`
443
+
444
+ Use a [search query][14] to select which Synthetic tests to run. Use the [Synthetic Tests list page's search bar][5] to craft your query, then copy and paste it.
445
+
446
+ In the command line, the query should be inside single quotes. Here is an example with a facet, a `value` tag, and a `<KEY>:<VALUE>` tag:
447
+
448
+ ```
449
+ datadog-ci synthetics run-tests --search 'team:unicorn tag:e2e-tests tag:"managedBy:terraform"'
450
+ ```
451
+
452
+ **Configuration options**
453
+
454
+ * Default: None
455
+ * Global Config: `"testSearchQuery": "tag:e2e-tests"`
456
+ * ENV variable: `DATADOG_SYNTHETICS_TEST_SEARCH_QUERY="tag:e2e-tests"`
457
+ * CLI param: `-s "tag:e2e-tests"` / `--search "tag:e2e-tests"`
458
+
459
+ #### `tunnel`
460
+
461
+ Use the [Continuous Testing tunnel](https://docs.datadoghq.com/continuous_testing/environments/proxy_firewall_vpn#what-is-the-testing-tunnel) to launch tests against internal environments.
462
+
463
+ For more information, see [Using Local and Staging Environments](#using-local-and-staging-environments).
464
+
465
+ To troubleshoot tests failing due to the tunnel, you can enable debug logs with `DEBUG=synthetics:tunnel`.
466
+
467
+ **Configuration options**
468
+
469
+ * Default: `false`
470
+ * Global Config: `"tunnel": true`
471
+ * ENV variable: `DATADOG_SYNTHETICS_TUNNEL=true`
472
+ * CLI param: `-t` / `--tunnel` / `--no-tunnel`
473
+
474
+ ### Test overrides
475
+
476
+ <!--
477
+ When updating any of these, don't forget to update the Google Sheets document and relevant CI integrations:
478
+ https://docs.google.com/spreadsheets/d/1VB8ntED7hz2McIwp7NaHADVt16nFUuNnKERBl78tldQ/edit?usp=sharing
479
+
480
+ For more information, see https://datadoghq.atlassian.net/wiki/x/LwBfyQ
481
+ -->
482
+
483
+ All test overrides are optional and allow overriding the test configuration that is stored in Datadog.
484
+
485
+ These overrides can either be applied to all tests with `defaultTestOverrides` in the [global configuration file](#global-configuration-file), or to some specific tests with `testOverrides` in a [test configuration file](#test-files).
486
+
487
+ These options can also be set with environment variables starting with `DATADOG_SYNTHETICS_OVERRIDE_...` or with the `--override` CLI parameter following this pattern: `--override option=value`.
488
+
489
+ #### `allowInsecureCertificates` (Boolean)
490
+
491
+ Override the certificate checks in Synthetic API and Browser tests.
492
+
493
+ **Configuration options**
494
+
495
+ * Global/Test Config: `"allowInsecureCertificates": true`
496
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_ALLOW_INSECURE_CERTIFICATES=true`
497
+ * CLI param: `--override allowInsecureCertificates=true`
498
+
499
+ #### `basicAuth` (Object)
500
+
501
+ Override the credentials for basic authentication.
502
+
503
+ * `username` (String): The username for basic authentication.
504
+ * `password` (String): The password for basic authentication.
505
+
506
+ **Configuration options**
507
+
508
+ * Global/Test Config: `"basicAuth": {"username": "test_username", "password": "test_password"}`
509
+ * ENV variable:
510
+ * `DATADOG_SYNTHETICS_OVERRIDE_BASIC_AUTH_USERNAME=test_username`
511
+ * `DATADOG_SYNTHETICS_OVERRIDE_BASIC_AUTH_PASSWORD=test_password`
512
+ * CLI param:
513
+ * `--override basicAuth.username=test_username`
514
+ * `--override basicAuth.password=test_password`
515
+
516
+ #### `body` (String)
517
+
518
+ Override the data to send in API tests.
519
+
520
+ **Configuration options**
521
+
522
+ * Global/Test Config: `"body": "{\"fakeContent\":true}"`
523
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_BODY={"fakeContent":true}`
524
+ * CLI param: `--override body={"fakeContent":true}`
525
+
526
+ #### `bodyType` (String)
527
+
528
+ Override the content type for the data to send in API tests.
529
+
530
+ **Configuration options**
531
+
532
+ * Global/Test Config: `"bodyType": "application/json"`
533
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_BODY_TYPE=application/json`
534
+ * CLI param: `--override bodyType=application/json`
535
+
536
+ #### `cookies` (String or object)
537
+
538
+ Override the cookies for API and browser tests.
539
+
540
+ * If this is a string, it is used to replace the original cookies.
541
+ * If this is an object, the format must be `{append?: boolean, value: string}`, and depending on the value of `append`, it is appended or replaces the original cookies.
542
+
543
+ **Configuration options**
544
+
545
+ * Global/Test Config: `"cookies": "name1=value1;name2=value2"` (equivalent to `"append": false`) or `"cookies": {"append": true, "value": "name1=value1;name2=value2"}`
546
+ * ENV variable:
547
+ * `DATADOG_SYNTHETICS_OVERRIDE_COOKIES="name1=value1;name2=value2"`
548
+ * `DATADOG_SYNTHETICS_OVERRIDE_COOKIES_APPEND=true`
549
+ * CLI param:
550
+ * `--override cookies="name1=value1;name2=value2"`
551
+ * `--override cookies.append=true`
552
+
553
+ #### `setCookies` (String or object)
554
+
555
+ Override the `Set-Cookie` headers in browser tests only.
556
+
557
+ * If this is a string, it is used to replace the original `Set-Cookie` headers.
558
+ * If this is an object, the format must be `{append?: boolean, value: string}`, and depending on the value of `append`, it is appended or replaces the original `Set-Cookie` headers.
559
+
560
+ **Configuration options**
561
+
562
+ * Global/Test Config: `"setCookies": "name1=value1 \n name2=value2; Domain=example.com \n name3=value3; Secure; HttpOnly"` (equivalent to `"append": false`) or `"setCookies": {"append": true, "value": "setCookies": "name1=value1 \n name2=value2; Domain=example.com \n name3=value3; Secure; HttpOnly"}`
563
+ * ENV variable:
564
+ * `DATADOG_SYNTHETICS_OVERRIDE_SET_COOKIES="name1=value1;name2=value2"`
565
+ * `DATADOG_SYNTHETICS_OVERRIDE_SET_COOKIES_APPEND=true`
566
+ * CLI param:
567
+ * `--override setCookies="setCookies": "name1=value1 \n name2=value2; Domain=example.com \n name3=value3; Secure; HttpOnly"`
568
+ * `--override setCookies.append=true`
569
+
570
+ #### `defaultStepTimeout` (Number)
571
+
572
+ Override the maximum duration of steps in seconds for browser tests. This does not override individually set step timeouts.
573
+
574
+ **Configuration options**
575
+
576
+ * Global/Test Config: `"defaultStepTimeout": 15`
577
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_DEFAULT_STEP_TIMEOUT=15`
578
+ * CLI param: `--override defaultStepTimeout=15`
579
+
580
+ #### `deviceIds` (Array)
581
+
582
+ Override the list of devices on which to run the Synthetic tests.
583
+
584
+ **Configuration options**
585
+
586
+ * Global/Test Config: `"deviceIds": ["chrome.laptop_large", "firefox.tablet"]`
587
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_DEVICE_IDS="chrome.laptop_large;firefox.tablet"`
588
+ * CLI param: `--override deviceIds="chrome.laptop_large;firefox.tablet"`
589
+
590
+ #### `executionRule` (String)
591
+
592
+ Override the execution rule for Synthetic tests.
593
+
594
+ The execution rule for the test defines the behavior of the CI batch in case of a failing test. It accepts one of the following values:
595
+
596
+ * `blocking`: A failed test causes the CI batch to fail.
597
+ * `non_blocking`: A failed test does not cause the CI batch to fail.
598
+ * `skipped`: The test is not run at all.
599
+
600
+ **Configuration options**
601
+
602
+ * Global/Test Config: `"executionRule": "skipped"`
603
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_EXECUTION_RULE=skipped`
604
+ * CLI param: `--override executionRule=skipped`
605
+
606
+ #### `followRedirects` (Boolean)
607
+
608
+ Override whether or not to follow HTTP redirections in API tests.
609
+
610
+ **Configuration options**
611
+
612
+ * Global/Test Config: `"followRedirects": true`
613
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_FOLLOW_REDIRECTS=true`
614
+ * CLI param: `--override followRedirects=true`
615
+
616
+ #### `headers` (Object)
617
+
618
+ Override the headers in the API and browser tests.
619
+
620
+ This object specifies the headers to be replaced in the test. It should have keys representing the names of the headers to be replaced, and values indicating the new header values.
621
+
622
+ **Configuration options**
623
+
624
+ * Global/Test Config: `"headers": {"NEW_HEADER_1": "NEW VALUE 1", "NEW_HEADER_2": "NEW VALUE 2"}`
625
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_HEADERS='{"NEW_HEADER_1":"NEW VALUE 1", "NEW_HEADER_2":"NEW VALUE 2"}'` (**Note**: This must be valid JSON)
626
+ * CLI param:
627
+ * `--override headers.NEW_HEADER_1="NEW VALUE 1"`
628
+ * `--override headers.NEW_HEADER_2="NEW VALUE 2"`
629
+
630
+ #### `locations` (Array)
631
+
632
+ Override the list of locations to run the test from. The possible values are listed [in this API response][12].
633
+
634
+ **Configuration options**
635
+
636
+ * Global/Test Config: `"locations": ["aws:us-east-1", "gcp:europe-west3"]`
637
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_LOCATIONS="aws:us-east-1;gcp:europe-west3"`
638
+ * CLI param: `--override locations="aws:us-east-1;gcp:europe-west3"`
639
+
640
+ #### `mobileApplicationVersion` (String)
641
+
642
+ Override the mobile application version for Synthetic mobile application tests. The version must be uploaded and available within Datadog.
643
+
644
+ **Configuration options**
645
+
646
+ * Global/Test Config: `"mobileApplicationVersion": "01234567-8888-9999-abcd-efffffffffff"`
647
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_MOBILE_APPLICATION_VERSION=01234567-8888-9999-abcd-efffffffffff`
648
+ * CLI param: `--mobileApplicationVersion=01234567-8888-9999-abcd-efffffffffff`
649
+
650
+ #### `mobileApplicationVersionFilePath` (String)
651
+
652
+ Override the application version for Synthetic mobile application tests.
653
+
654
+ **Configuration options**
655
+
656
+ * Global/Test Config: `"mobileApplicationVersionFilePath": "path/to/application.apk"`
657
+ * ENV variable: Not Available
658
+ * CLI param: `--mobileApplicationVersionFilePath=path/to/application.apk`
659
+
660
+ #### `resourceUrlSubstitutionRegexes` (Array)
661
+
662
+ An array of regex patterns to modify resource URLs in the test. This can be useful for dynamically changing resource URLs during test execution.
663
+
664
+ Each regex pattern should be in the format:
665
+
666
+ - **`your_regex|your_substitution`**: The pipe-based syntax, to avoid any conflicts with / characters in URLs.
667
+ - For example, `https://example.com(.*)|http://subdomain.example.com$1` to transform `https://example.com/resource` to `http://subdomain.example.com/resource`.
668
+ - **`s/your_regex/your_substitution/modifiers`**: The slash syntax, which supports modifiers.
669
+ - For example, `s/(https://www.)(.*)/$1staging-$2/` to transform `https://www.example.com/resource` into `https://www.staging-example.com/resource`.
670
+
671
+ **Configuration options**
672
+
673
+ * Global/Test Config: `"resourceUrlSubstitutionRegexes": ["(https://www.)(.*)|$1staging-$2"]`
674
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_RESOURCE_URL_SUBSTITUTION_REGEXES='(https://www.)(.*)|$1staging-$2'`
675
+ * CLI param: `--override resourceUrlSubstitutionRegexes='(https://www.)(.*)|$1staging-$2'`
676
+
677
+ #### `retry` (Object)
678
+
679
+ Override the retry policy for the test.
680
+
681
+ This object has the two following independent attributes:
682
+ * `count` (Integer): The number of attempts to perform in case of test failure.
683
+ * `interval` (Integer): The interval between attempts in milliseconds.
684
+
685
+ **Configuration options**
686
+
687
+ * Global/Test Config: `"retry": {"count": 2, "interval": 300}`
688
+ * ENV variable:
689
+ * `DATADOG_SYNTHETICS_OVERRIDE_RETRY_COUNT=2`
690
+ * `DATADOG_SYNTHETICS_OVERRIDE_RETRY_INTERVAL=300`
691
+ * CLI param:
692
+ * `--override retry.count=2`
693
+ * `--override retry.interval=300`
694
+
695
+ #### `startUrl` (String)
696
+
697
+ Override the start URL for API and browser tests.
698
+
699
+ Local and [global variables][11] specified in the URL (for example, `{{ URL }}`) are replaced when the test is run.
700
+
701
+ You can combine this with the `variables` override to override both the start URL and the variable values. For example:
702
+
703
+ ```bash
704
+ --override startUrl="{{ URL }}?static_hash={{ STATIC_HASH }}" --override variables.STATIC_HASH=abcdef
705
+ ```
706
+
707
+ **Configuration options**
708
+
709
+ * Global/Test Config: `"startUrl": "{{ URL }}?static_hash={{ STATIC_HASH }}"`
710
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_START_URL="{{ URL }}?static_hash={{ STATIC_HASH }}"`
711
+ * CLI param: `--override startUrl="{{ URL }}?static_hash={{ STATIC_HASH }}"`
712
+
713
+ #### `startUrlSubstitutionRegex` (String)
714
+
715
+ A regex to modify the starting URL of browser and HTTP tests, whether it comes from the original test or the `startUrl` override.
716
+
717
+ If the URL contains variables, this regex applies after the interpolation of the variables.
718
+
719
+ There are two possible formats:
720
+
721
+ - **`your_regex|your_substitution`**: The pipe-based syntax, to avoid any conflicts with `/` characters in URLs.
722
+ - For example, `https://example.com(.*)|http://subdomain.example.com$1` to transform `https://example.com/test` to `http://subdomain.example.com/test`.
723
+ - **`s/your_regex/your_substitution/modifiers`**: The slash syntax, which supports modifiers.
724
+ - For example, `s/(https://www.)(.*)/$1extra-$2/` to transform `https://www.example.com` into `https://www.extra-example.com`.
725
+
726
+ **Configuration options**
727
+
728
+ * Global/Test Config: `"startUrlSubstitutionRegex": "(https://www.)(.*)|$1extra-$2"`
729
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_START_URL_SUBSTITUTION_REGEX='(https://www.)(.*)|$1extra-$2'`
730
+ * CLI param: `--override startUrlSubstitutionRegex='(https://www.)(.*)|$1extra-$2'`
731
+
732
+ #### `testTimeout` (Number)
733
+
734
+ Override the maximum duration in seconds for browser tests.
735
+
736
+ **Configuration options**
737
+
738
+ * Global/Test Config: `"testTimeout": 300`
739
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_TEST_TIMEOUT=300`
740
+ * CLI param: `--override testTimeout=300`
741
+
742
+ #### `variables` (Object)
743
+
744
+ Override existing or inject new local and [global variables][11] in Synthetic tests.
745
+
746
+ This object should include keys corresponding to the names of the variables to be replaced, and values representing the new values for these variables.
747
+
748
+ **Configuration options**
749
+
750
+ * Global/Test Config: `"variables": {"NEW_VARIABLE_1": "NEW VARIABLE 1", "NEW_VARIABLE_2": "NEW VARIABLE 2"}`
751
+ * ENV variable: `DATADOG_SYNTHETICS_OVERRIDE_VARIABLES='{"NEW_VARIABLE_1":"NEW VARIABLE 1", "NEW_VARIABLE_2":"NEW VARIABLE 2"}'` (**Note**: This must be valid JSON)
752
+ * CLI param:
753
+ * `--override variables.NEW_VARIABLE_1="NEW VARIABLE 1"`
754
+ * `--override variables.NEW_VARIABLE_2="NEW VARIABLE 2"`
755
+
756
+ ### Configure a start URL
757
+
758
+ To configure which URL your test starts on, provide a `startUrl` to your test object. Build your own starting URL with any part of your test's original starting URL and include local and [global variables][11].
759
+
760
+ ### Configure a custom subdomain
761
+
762
+ If the organization uses a custom subdomain to access Datadog, this needs to be set in the `DATADOG_SUBDOMAIN` environment variable or in the global configuration file under the `subdomain` key in order to properly display the test results URL.
763
+
764
+ For example, if the URL used to access Datadog is `myorg.datadoghq.com`, set the environment variable to `myorg`:
765
+
766
+ ```bash
767
+ export DATADOG_SUBDOMAIN="myorg"
768
+ ```
769
+
770
+ ### Configure custom locations
771
+
772
+ You can use `DATADOG_SYNTHETICS_OVERRIDE_LOCATIONS` to override the locations where your tests run. Locations should be separated with a semicolon (`;`). The configuration in [test files](#test-files) takes precedence over other overrides.
773
+
774
+ ```bash
775
+ export DATADOG_SYNTHETICS_OVERRIDE_LOCATIONS="aws:us-east-1;aws:us-east-2"
776
+ ```
777
+
778
+ ### Test files
779
+
780
+ Test configuration files (Test Config) let you customize individual tests or set up multiple runs of the same test with different settings, beyond what you can do with other configuration methods.
781
+
782
+ You can find a list of all these options in the [test overrides](#test-overrides) section.
783
+
784
+ These files take precedence over global configuration files, environment variables, and CLI parameters. The priority order including test configurations is as follows:
785
+
786
+ ``` yml
787
+ Global Config < Environment variables < CLI parameters < Test Config
788
+ ```
789
+
790
+ To determine which tests to run, one or more of those options may be passed to `datadog-ci`:
791
+ - The [`files` option](#files)
792
+ - The [`publicIds` option](#publicids)
793
+ - The [`testSearchQuery` option](#testsearchquery)
794
+
795
+ If none of these options is passed, `datadog-ci` auto-discovers test configuration files with the `{,!(node_modules)/**/}*.synthetics.json` glob pattern (every file ending with `.synthetics.json`, except for those in the `node_modules` folder).
796
+
797
+ **Note**: The file search starts from the current working directory, so it may be slow if the command is run from a large directory, like a home folder. If file search command is too slow, consider:
798
+ - Using the above options to specify the tests (this will disable the file search),
799
+ - Or refining the glob pattern with the [`files` option](#files).
800
+ - For example, by using `*` instead of `**` or by adding a specific folder to the pattern.
801
+
802
+ The `<TEST_PUBLIC_ID>` can be either the identifier of the test found in the URL of a test details page (for example, for `https://app.datadoghq.com/synthetics/details/abc-def-ghi`, it would be `abc-def-ghi`) or the full URL to the details page (for example, directly `https://app.datadoghq.com/synthetics/details/abc-def-ghi`).
803
+
804
+ Example:
805
+
806
+ ```jsonc
807
+ // myTest.synthetics.json
808
+ {
809
+ "tests": [
810
+ {
811
+ "id": "<TEST_PUBLIC_ID_1>",
812
+ "testOverrides": {
813
+ "allowInsecureCertificates": true,
814
+ "basicAuth": {"username": "test", "password": "test"},
815
+ "body": "{\"fakeContent\":true}",
816
+ "bodyType": "application/json",
817
+ "cookies": "name1=value1;name2=value2;",
818
+ "defaultStepTimeout": 15,
819
+ "deviceIds": ["chrome.laptop_large"],
820
+ "executionRule": "skipped",
821
+ "followRedirects": true,
822
+ "headers": {"NEW_HEADER": "NEW VALUE"},
823
+ "locations": ["aws:us-east-1"],
824
+ "mobileApplicationVersion": "01234567-8888-9999-abcd-efffffffffff",
825
+ "mobileApplicationVersionFilePath": "path/to/application.apk",
826
+ "resourceUrlSubstitutionRegexes": ["(https://www.)(.*)|$1staging-$2"],
827
+ "retry": {"count": 2, "interval": 300},
828
+ "startUrl": "{{URL}}?static_hash={{STATIC_HASH}}",
829
+ "startUrlSubstitutionRegex": "s/(https://www.)(.*)/$1extra-$2/",
830
+ "testTimeout": 300,
831
+ "variables": {"MY_VARIABLE": "new title"}
832
+ }
833
+ },
834
+ {
835
+ "id": "<TEST_PUBLIC_ID_2>",
836
+ "testOverrides": {
837
+ "allowInsecureCertificates": true,
838
+ // ...
839
+ "variables": {"MY_VARIABLE": "new title"}
840
+ }
841
+ }
842
+ ]
843
+ }
844
+ ```
845
+
846
+ ## Upload Application Command
847
+
848
+ This command uploads a new version to an **existing** mobile application.
849
+
850
+ ### List of Configurations
851
+
852
+ <!--
853
+ When updating any of these, don't forget to update the Google Sheets document and relevant CI integrations:
854
+ https://docs.google.com/spreadsheets/d/1VB8ntED7hz2McIwp7NaHADVt16nFUuNnKERBl78tldQ/edit?usp=sharing
855
+
856
+ For more information, see https://datadoghq.atlassian.net/wiki/x/LwBfyQ
857
+ -->
858
+
859
+ #### `apiKey` (Required)
860
+
861
+ Your Datadog API key. This key is [created in your Datadog organization][15] and should be stored as a secret.
862
+
863
+ **Configuration options**
864
+
865
+ * Global Config: `"apiKey": "<API_KEY>"`
866
+ * ENV variable: `DD_API_KEY="<API_KEY>"`
867
+ * CLI param: `--apiKey "<API_KEY>"`
868
+
869
+ #### `appKey` (Required)
870
+
871
+ Your Datadog application key. This key is [created in your Datadog organization][15] and should be stored as a secret.
872
+
873
+ **Configuration options**
874
+
875
+ * Global Config: `"appKey": "<APPLICATION_KEY>"`
876
+ * ENV variable: `DD_APP_KEY="<APPLICATION_KEY>"`
877
+ * CLI param: `--appKey "<APPLICATION_KEY>"`
878
+
879
+ #### `configPath`
880
+
881
+ The path to the [global configuration file](#global-configuration-file) that configures datadog-ci.
882
+
883
+ **Configuration options**
884
+
885
+ * Default: `datadog-ci.json`
886
+ * Global Config: N/A
887
+ * ENV variable: `DATADOG_SYNTHETICS_CONFIG_PATH=global-config.json`
888
+ * CLI param: `--config global-config.json`
889
+
890
+ #### `datadogSite`
891
+
892
+ Your Datadog site. The possible values are listed [in this table][16].
893
+
894
+ <!-- partial Set it to {{< region-param key="dd_site" code="true" >}} (ensure the correct SITE is selected on the right). partial -->
895
+
896
+ **Configuration options**
897
+
898
+ * Default: `datadoghq.com`
899
+ * Global Config: `"datadogSite": "datadoghq.com"`
900
+ * ENV variable: `DATADOG_SITE=datadoghq.com`
901
+ * CLI param: `--datadogSite datadoghq.com`
902
+
903
+ #### `latest`
904
+
905
+ Mark the new version as `latest`. Any tests that run on the latest version will use this version on their next run.
906
+
907
+ **Configuration options**
908
+
909
+ * Default: `false`
910
+ * Global Config: `"latest": true`
911
+ * ENV variable: `DATADOG_SYNTHETICS_LATEST=true`
912
+ * CLI param: `--latest` / `--no-latest`
913
+
914
+ #### `mobileApplicationId` (Required)
915
+
916
+ The ID of the application you want to upload the new version to.
917
+
918
+ **Configuration options**
919
+
920
+ * Global Config: `"mobileApplicationId": "123-123-123"`
921
+ * ENV variable: `DATADOG_SYNTHETICS_MOBILE_APPLICATION_ID=123-123-123`
922
+ * CLI param: `--mobileApplicationId 123-123-123`
923
+
924
+ #### `mobileApplicationVersionFilePath` (Required)
925
+
926
+ The path to the new version of your mobile application (`.apk` or `.ipa`).
927
+
928
+ **Configuration options**
929
+
930
+ * Global Config: `"mobileApplicationVersionFilePath": example/test.apk`
931
+ * ENV variable: Not Available
932
+ * CLI param: `--mobileApplicationVersionFilePath example/test.apk`
933
+
934
+ #### `proxy`
935
+
936
+ The proxy to be used for outgoing connections to Datadog. `host` and `port` keys are mandatory arguments, the `protocol` key defaults to `http`. Supported values for the `protocol` key are `http`, `https`, `socks`, `socks4`, `socks4a`, `socks5`, `socks5h`, `pac+data`, `pac+file`, `pac+ftp`, `pac+http`, and `pac+https`. The library used to configure the proxy is the [proxy-agent][2] library.
937
+
938
+ **Configuration options**
939
+
940
+ * Global Config: See [Use a proxy](#use-a-proxy) for an example.
941
+ * ENV variable: N/A
942
+ * CLI param: N/A
943
+
944
+ #### `versionName` (Required)
945
+
946
+ The name of the new version. It has to be unique.
947
+
948
+ **Configuration options**
949
+
950
+ * Global Config: `"versionName": "example"`
951
+ * ENV variable: `DATADOG_SYNTHETICS_VERSION_NAME=example`
952
+ * CLI param: `--versionName example`
953
+
954
+ Example:
955
+
956
+ ```bash
957
+ datadog-ci synthetics upload-application \
958
+ --mobileApplicationId '123-123-123' \
959
+ --mobileApplicationVersionFilePath example/test.apk \
960
+ --versionName 'example 1.0' \
961
+ --latest
962
+ ```
963
+
964
+ ### Using the global configuration file
965
+
966
+ You can also pass these options in a configuration file:
967
+
968
+ ```json
969
+ {
970
+ "apiKey": "<API_KEY>",
971
+ "appKey": "<APPLICATION_KEY>",
972
+ "mobileApplicationVersionFilePath": "example_path/example_app.apk",
973
+ "mobileApplicationId": "example-abc",
974
+ "versionName": "example",
975
+ "latest": true
976
+ }
977
+ ```
978
+
979
+ These options can also be added to the same global configuration file used for the run-tests command.
980
+
981
+ Pass this config file to the command with the `--config` flag:
982
+
983
+ ```bash
984
+ datadog-ci synthetics upload-application --config global-config.json
985
+ ```
986
+
987
+ The default file name for the [global configuration file](#global-configuration-file) is `datadog-ci.json`. If you use this name for your global configuration file, you may omit the `--config` flag.
988
+
989
+ ## Using local and staging environments
990
+
991
+ You can combine variable overrides with [Local and Staging Environments][3] to run tests within your development environment. This connection ensures that all test requests sent through the CLI are automatically routed through the `datadog-ci` client.
992
+
993
+ This allows you to run tests with end-to-end encryption at every stage of your software development lifecycle, from pre-production environments to your production system.
994
+
995
+ ## End-to-end testing process
996
+
997
+ To verify the Synthetics command works as expected, trigger a test run and verify it returns 0:
998
+
999
+ ```bash
1000
+ export DD_API_KEY='<API_KEY>'
1001
+ export DD_APP_KEY='<APPLICATION_KEY>'
1002
+
1003
+ yarn datadog-ci synthetics run-tests --public-id abc-def-ghi
1004
+ ```
1005
+
1006
+ Successful output should look like this:
1007
+
1008
+ ```bash
1009
+ [abc-def-ghi] Trigger test "Check on testing.website"
1010
+ [abc-def-ghi] Waiting results for "Check on testing.website"
1011
+
1012
+
1013
+ === REPORT ===
1014
+ Took 11546ms
1015
+
1016
+ ✓ [abc-def-ghi] | Check on testing.website
1017
+ ✓ location: Frankfurt (AWS)
1018
+ ⎋ total duration: 28.9 ms - result url: https://app.datadoghq.com/synthetics/details/abc-def-ghi?resultId=123456789123456789
1019
+ ✓ GET - https://testing.website
1020
+ ```
1021
+
1022
+ ### Reporters
1023
+
1024
+ Two reporters are supported out-of-the-box:
1025
+
1026
+ 1. `stdout`
1027
+ 2. JUnit
1028
+
1029
+ To enable the JUnit report, specify a filename for your JUnit report with `-j/--jUnitReport`.
1030
+
1031
+ ```bash
1032
+ yarn datadog-ci synthetics run-tests -s 'tag:e2e-tests' --config global-config.json --jUnitReport junit-report.xml
1033
+ ```
1034
+
1035
+ Reporters can hook themselves into the `MainReporter` of the command.
1036
+
1037
+ ### Available hooks
1038
+
1039
+ | Hook name | Parameters | Description |
1040
+ | :--------------- | :---------------------------------------------------------------------------------------------- | :-------------------------------------------------------------- |
1041
+ | `log` | `(log: string)` | Called for logging. |
1042
+ | `error` | `(error: string)` | Called whenever an error occurs. |
1043
+ | `initErrors` | `(errors: string[])` | Called whenever an error occurs during the tests parsing phase. |
1044
+ | `testTrigger` | `(test: Test, testId: string, executionRule: ExecutionRule, testOverrides: UserConfigOverride)` | Called when a test is triggered. |
1045
+ | `testWait` | `(test: Test)` | Called when a test is waiting to receive its results. |
1046
+ | `testsWait` | `(tests: Test[], baseUrl: string, batchId: string, skippedCount?: number)` | Called when all tests are waiting to receive their results. |
1047
+ | `resultReceived` | `(result: ResultInBatch)` | Called when a result is received. |
1048
+ | `resultEnd` | `(result: Result, baseUrl: string)` | Called for each result at the end of all results. |
1049
+ | `reportStart` | `(timings: {startTime: number})` | Called at the start of the report. |
1050
+ | `runEnd` | `(summary: Summary, baseUrl: string, orgSettings?: SyntheticsOrgSettings)` | Called at the end of the run. |
1051
+
1052
+ ## View test results
1053
+
1054
+ You can see results for CI batches by clicking on a batch in the [Synthetic Monitoring & Testing Results Explorer][4] or clicking on a test on the [Synthetic Tests list page][5].
1055
+
1056
+ You can also see the outcome of test executions directly in your CI as your tests are being executed. To identify what caused a test to fail, look at the execution logs and search for causes of the failed assertion.
1057
+
1058
+ ```bash
1059
+ yarn datadog-ci synthetics run-tests --config global-config.json
1060
+ yarn run v1.22.4
1061
+ $ /Users/demo.user/go/src/github.com/Datadog/tmp/test/testDemo/node_modules/.bin/datadog-ci synthetics run-tests --config global-config.json
1062
+ Finding files matching /Users/demo.user/go/src/github.com/Datadog/tmp/test/testDemo/{,!(node_modules)/**/}*.synthetics.json
1063
+
1064
+ Got test files:
1065
+ - user.synthetics.json
1066
+
1067
+ [2cj-h3c-39x] Trigger test "Test CI connection"
1068
+ [2cj-h3c-39x] Waiting results for "Test CI connection"
1069
+
1070
+ === REPORT ===
1071
+ Took 2242ms
1072
+
1073
+ x [2cj-h3c-39x] | Test CI connection
1074
+ * location: 30019
1075
+ ⎋ total duration: 32.6 ms - result url: https://app.datadoghq.com/synthetics/details/2cj-h3c-39x?resultId=122140688175981634
1076
+ x GET - https://www.datadoghq.com
1077
+ [INCORRECT_ASSUMPTION] - [{"index":1,"operator":"is","property":"content-type","type":"header","target":"text/html","valid":false,"actual":"text/html"; charset=utf-8"}]
1078
+ error Command failed with exit code 1.
1079
+ info Visit https://yarnpkg.com/en/docs/cli/run for documentation about this command.
1080
+ ```
1081
+
1082
+ ## Further reading
1083
+
1084
+ Additional helpful documentation, links, and articles:
1085
+
1086
+ * [Use Datadog's GitHub Action to add continuous testing to your workflows][6]
1087
+ * [Learn about Continuous Testing and CI/CD][7]
1088
+ * [Learn about Mobile Application Testing][10]
1089
+ * [Learn about the Synthetic Monitoring & Testing Results Explorer][8]
1090
+ * [Learn about Testing Local and Staging Environments][3]
1091
+
1092
+ [1]: https://www.npmjs.com/package/@datadog/datadog-ci
1093
+ [2]: https://github.com/TooTallNate/proxy-agents/tree/main/packages/proxy-agent
1094
+ [3]: https://docs.datadoghq.com/continuous_testing/environments/
1095
+ [4]: https://app.datadoghq.com/synthetics/explorer/
1096
+ [5]: https://app.datadoghq.com/synthetics/tests
1097
+ [6]: https://www.datadoghq.com/blog/datadog-github-action-synthetics-ci-visibility/
1098
+ [7]: https://docs.datadoghq.com/continuous_testing/cicd_integrations/
1099
+ [8]: https://docs.datadoghq.com/continuous_testing/explorer/
1100
+ [9]: https://github.com/DataDog/datadog-ci/blob/master/packages/plugin-synthetics/src/examples/global-config-complete-example.json
1101
+ [10]: https://docs.datadoghq.com/mobile_app_testing/
1102
+ [11]: https://docs.datadoghq.com/synthetics/platform/settings/?tab=specifyvalue#global-variables
1103
+ [12]: https://app.datadoghq.com/api/v1/synthetics/locations?only_public=true
1104
+ [13]: https://www.npmjs.com/package/proxy-from-env#external-resources
1105
+ [14]: https://docs.datadoghq.com/synthetics/explore/#search
1106
+ [15]: https://docs.datadoghq.com/account_management/api-app-keys/
1107
+ [16]: https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site
1108
+ [17]: https://app.datadoghq.com/synthetics/settings/continuous-testing
1109
+ [18]: https://docs.datadoghq.com/synthetics/mobile_app_testing/
1110
+
1111
+ <!--
1112
+ This page is single-sourced:
1113
+ https://github.com/DataDog/documentation/blob/7007931530baf7da59310e7224a26dc9a71c53c5/local/bin/py/build/configurations/pull_config_preview.yaml#L315
1114
+ -->
package/package.json ADDED
@@ -0,0 +1,63 @@
1
+ {
2
+ "name": "@datadog/datadog-ci-plugin-synthetics",
3
+ "version": "3.21.0",
4
+ "license": "Apache-2.0",
5
+ "homepage": "https://github.com/DataDog/datadog-ci/tree/master/packages/plugin-synthetics",
6
+ "repository": {
7
+ "type": "git",
8
+ "url": "https://github.com/DataDog/datadog-ci.git",
9
+ "directory": "packages/plugin-synthetics"
10
+ },
11
+ "exports": {
12
+ "./package.json": "./package.json",
13
+ ".": {
14
+ "development": "./src/index.ts",
15
+ "default": "./dist/index.js"
16
+ },
17
+ "./commands/*": {
18
+ "development": "./src/commands/*.ts",
19
+ "default": "./dist/commands/*.js"
20
+ }
21
+ },
22
+ "files": [
23
+ "dist/**/*",
24
+ "README",
25
+ "LICENSE"
26
+ ],
27
+ "publishConfig": {
28
+ "access": "public"
29
+ },
30
+ "scripts": {
31
+ "build": "yarn package:clean; yarn package:build",
32
+ "lint": "yarn package:lint",
33
+ "prepack": "yarn package:clean-dist"
34
+ },
35
+ "peerDependencies": {
36
+ "@datadog/datadog-ci": "3.21.0"
37
+ },
38
+ "dependencies": {
39
+ "@datadog/datadog-ci-base": "3.21.0",
40
+ "axios": "^1.12.1",
41
+ "chalk": "3.0.0",
42
+ "debug": "^4.4.1",
43
+ "deep-extend": "0.6.0",
44
+ "fast-levenshtein": "^3.0.0",
45
+ "get-value": "^4.0.1",
46
+ "ora": "5.4.1",
47
+ "set-value": "^4.1.0",
48
+ "ssh2": "^1.16.0",
49
+ "sshpk": "1.16.1",
50
+ "upath": "^2.0.1",
51
+ "ws": "^7.5.10",
52
+ "xml2js": "0.5.0",
53
+ "yamux-js": "0.1.2"
54
+ },
55
+ "devDependencies": {
56
+ "@types/deep-extend": "0.4.31",
57
+ "@types/set-value": "^4.0.3",
58
+ "@types/ssh2": "1.15.5",
59
+ "@types/sshpk": "1.10.5",
60
+ "@types/ws": "7.2.9",
61
+ "@types/xml2js": "0.4.9"
62
+ }
63
+ }