@ppdostudio/ppdo-cli 1.0.0-beta.0 → 1.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -54
- package/LICENSE +114 -114
- package/README.md +381 -331
- package/THIRD-PARTY-NOTICES +1 -0
- package/dist/bin.js +2 -2
- package/dist/chunks/{ado-NLCWVIVQ.js → ado-OTSVKTOE.js} +6 -6
- package/dist/chunks/{api-YKKPVMTX.js → api-IRGGH5DJ.js} +1 -1
- package/dist/chunks/{api-T5SWGOFX.js → api-TZJXGWWQ.js} +1 -1
- package/dist/chunks/auth-7CDWOYJE.js +6 -0
- package/dist/chunks/azCli-UEDQHRLP.js +2 -0
- package/dist/chunks/batch-LKWCBKAB.js +2 -0
- package/dist/chunks/{build-XDCWMEEL.js → build-PBPXO3GJ.js} +3 -3
- package/dist/chunks/bulk-NY5XX752.js +2 -0
- package/dist/chunks/chunk-2IUXH534.js +2 -0
- package/dist/chunks/chunk-3L7657R3.js +5 -0
- package/dist/chunks/{chunk-5KZDCOFA.js → chunk-46KBBWEH.js} +1 -1
- package/dist/chunks/chunk-6NBCHK6X.js +2 -0
- package/dist/chunks/chunk-6VLFO5FJ.js +7 -0
- package/dist/chunks/chunk-B3TJ3RJN.js +2 -0
- package/dist/chunks/chunk-BG2OIOSN.js +2 -0
- package/dist/chunks/chunk-BWOEIEFY.js +6 -0
- package/dist/chunks/chunk-ELSYHAYC.js +2 -0
- package/dist/chunks/chunk-FOEBNCSD.js +3 -0
- package/dist/chunks/{chunk-XL7E7LX5.js → chunk-GLU4X7PT.js} +2 -2
- package/dist/chunks/chunk-GQLXEQVQ.js +4 -0
- package/dist/chunks/chunk-GY6BUIDP.js +2 -0
- package/dist/chunks/chunk-H4OY4EFC.js +3 -0
- package/dist/chunks/chunk-HJQJ77F2.js +3 -0
- package/dist/chunks/chunk-I7MHI5CU.js +151 -0
- package/dist/chunks/chunk-ITP35SAY.js +2 -0
- package/dist/chunks/{chunk-J777TCSE.js → chunk-J67P6QFC.js} +2 -2
- package/dist/chunks/{chunk-NNHEYYF5.js → chunk-KBZI5UDL.js} +1 -1
- package/dist/chunks/chunk-LB7WNZVK.js +5 -0
- package/dist/chunks/chunk-MBNUMAAZ.js +2 -0
- package/dist/chunks/chunk-MNGA7IHQ.js +4 -0
- package/dist/chunks/chunk-P4IFCLVF.js +2 -0
- package/dist/chunks/chunk-Q2RD4TRK.js +2 -0
- package/dist/chunks/chunk-S6NK6GX2.js +5 -0
- package/dist/chunks/chunk-SWXCK5C2.js +2 -0
- package/dist/chunks/chunk-T4IH34D7.js +2 -0
- package/dist/chunks/{chunk-HLE2WURE.js → chunk-TADCW6ZI.js} +1 -1
- package/dist/chunks/{chunk-UYDBLVAM.js → chunk-TCLZ4XM6.js} +1 -1
- package/dist/chunks/chunk-VC7ADQ6E.js +2 -0
- package/dist/chunks/{chunk-GT52BRS2.js → chunk-XUHLW6B2.js} +18 -18
- package/dist/chunks/{commands-GHYN6GZT.js → commands-EOUU353W.js} +1 -1
- package/dist/chunks/component-JCIJQELN.js +2 -0
- package/dist/chunks/config-WV3NZGTI.js +2 -0
- package/dist/chunks/connection-L6CUFUPJ.js +2 -0
- package/dist/chunks/connref-IT6K56UT.js +2 -0
- package/dist/chunks/convert-ILMJ2HHN.js +5 -0
- package/dist/chunks/customizations-AX7ZSXCK.js +2 -0
- package/dist/chunks/data-UWOV3LBO.js +5 -0
- package/dist/chunks/dependency-V4ZJIWQL.js +3 -0
- package/dist/chunks/{discovery-XXVLVHD6.js → discovery-PSGWFMJH.js} +1 -1
- package/dist/chunks/{dv-3OESFBNW.js → dv-SD5LTNZJ.js} +109 -48
- package/dist/chunks/{env-MZPI3UA2.js → env-6RZCHQUC.js} +2 -2
- package/dist/chunks/environment-IBLJHUAL.js +2 -0
- package/dist/chunks/{environments-WMBLSAQF.js → environments-3P2I4QUC.js} +1 -1
- package/dist/chunks/envvar-KJ2TUZGJ.js +2 -0
- package/dist/chunks/fanout-2WQYFUWL.js +2 -0
- package/dist/chunks/{fetchxml-ONTNTA6Z.js → fetchxml-M6EOF7N6.js} +1 -1
- package/dist/chunks/functions-NILMLS4R.js +2 -0
- package/dist/chunks/{graph-JI23YQB5.js → graph-NO2OT2TL.js} +1 -1
- package/dist/chunks/{group-DKEPSVRV.js → group-ISSJ4H2A.js} +2 -2
- package/dist/chunks/history-JPMLQDJI.js +2 -0
- package/dist/chunks/identity-SBQNHSBF.js +2 -0
- package/dist/chunks/init-X2VEK4T2.js +2 -0
- package/dist/chunks/jobs-XC7EYPBP.js +2 -0
- package/dist/chunks/lifecycle-2F7MF6EP.js +2 -0
- package/dist/chunks/{main-ZVIKC3VA.js → main-J3O5YMIY.js} +2 -2
- package/dist/chunks/metadata-NZFQIAAG.js +2 -0
- package/dist/chunks/metadataCreate-XSHK32TR.js +2 -0
- package/dist/chunks/msalUser-KC75VDVV.js +2 -0
- package/dist/chunks/names-N5JRXO33.js +2 -0
- package/dist/chunks/{pack-2UDE25IY.js → pack-3OYGV36R.js} +1 -1
- package/dist/chunks/pat-PCMOZYM5.js +2 -0
- package/dist/chunks/{pipelines-O3OYNCHT.js → pipelines-OCHSIQ2E.js} +1 -1
- package/dist/chunks/pluginRegister-LEVZNDZN.js +4 -0
- package/dist/chunks/{plugins-EUDJ7QIS.js → plugins-TEME2R77.js} +1 -1
- package/dist/chunks/{policies-KVPQVUHS.js → policies-EWCANNBE.js} +1 -1
- package/dist/chunks/{pools-7H2L4QJC.js → pools-54KMEL5D.js} +1 -1
- package/dist/chunks/process-KR7LIXZB.js +2 -0
- package/dist/chunks/{projects-BHG2B4IX.js → projects-LPVDFLPC.js} +1 -1
- package/dist/chunks/protect-X2J7LU7X.js +2 -0
- package/dist/chunks/provider-D7DMMAS6.js +2 -0
- package/dist/chunks/{prs-NOXIZ7AT.js → prs-C4EKGJWA.js} +1 -1
- package/dist/chunks/records-C4RLQXFK.js +2 -0
- package/dist/chunks/{run-BXFENCN6.js → run-27MYTWHY.js} +1 -1
- package/dist/chunks/run-46PEJS3S.js +2 -0
- package/dist/chunks/run-7L6W2YWJ.js +3 -0
- package/dist/chunks/run-CD3ZOB5V.js +4 -0
- package/dist/chunks/run-EUNN6OZO.js +2 -0
- package/dist/chunks/{run-JYU5BTWN.js → run-JDNEQMJC.js} +1 -1
- package/dist/chunks/{run-MGLIIM7Z.js → run-T3IIJ7SK.js} +1 -1
- package/dist/chunks/run-ZHIDOKQH.js +2 -0
- package/dist/chunks/{runtime-VOF7XYXW.js → runtime-FJJPTBEE.js} +1 -1
- package/dist/chunks/security-SBIAK6QD.js +2 -0
- package/dist/chunks/{service-V2TQEEID.js → service-4N3YP4Z7.js} +1 -1
- package/dist/chunks/{service-4PIAYFYF.js → service-HASXLUGE.js} +1 -1
- package/dist/chunks/services-C6G4XRXR.js +2 -0
- package/dist/chunks/{skills-GD6Z6P3R.js → skills-FW4F3EIW.js} +1 -1
- package/dist/chunks/solution-CNBNIS7Z.js +2 -0
- package/dist/chunks/solutionExport-Y524HMGN.js +2 -0
- package/dist/chunks/{solutionfiles-P223QW2J.js → solutionfiles-2TP3Y2RW.js} +1 -1
- package/dist/chunks/spn-YSYMDTOI.js +2 -0
- package/dist/chunks/tokenCache-TQFT6LAD.js +2 -0
- package/dist/chunks/{unpack-XZLTBUGC.js → unpack-HUH7OJMP.js} +1 -1
- package/dist/chunks/{users-KFAPPGF4.js → users-Q7BJKBPP.js} +1 -1
- package/dist/chunks/{vargroups-33EZAHGZ.js → vargroups-GNZBODVJ.js} +1 -1
- package/dist/chunks/{whoami-ULK5UMZV.js → whoami-6G6UFTNS.js} +1 -1
- package/dist/chunks/{workitems-YGES6KKN.js → workitems-2IOGD3EL.js} +1 -1
- package/package.json +61 -57
- package/schema/ppdo-cli.schema.json +16 -4
- package/schema/reference-data.schema.json +192 -0
- package/skills/ppdo-cli/SKILL.md +2 -1
- package/skills/ppdo-cli-data/SKILL.md +669 -0
- package/skills/ppdo-cli-dataverse/SKILL.md +6 -0
- package/skills/ppdo-cli-plugins/PluginRegistration.cs +229 -0
- package/skills/ppdo-cli-plugins/SKILL.md +95 -0
- package/skills/ppdo-cli-plugins/reference.md +72 -0
- package/dist/chunks/auth-IP7AOWSF.js +0 -6
- package/dist/chunks/azCli-T67ZMIC3.js +0 -2
- package/dist/chunks/batch-GCXLGQET.js +0 -4
- package/dist/chunks/bulk-5QO343HO.js +0 -2
- package/dist/chunks/chunk-3C3GSN3J.js +0 -2
- package/dist/chunks/chunk-4S5ETLH4.js +0 -2
- package/dist/chunks/chunk-5SBBTR3Y.js +0 -2
- package/dist/chunks/chunk-5UOZCXQF.js +0 -5
- package/dist/chunks/chunk-BWZ6HZXM.js +0 -2
- package/dist/chunks/chunk-JGLUQBCK.js +0 -2
- package/dist/chunks/chunk-MME53RWW.js +0 -4
- package/dist/chunks/chunk-OP6EXLWQ.js +0 -5
- package/dist/chunks/chunk-PDFXH652.js +0 -2
- package/dist/chunks/chunk-QE4DELWW.js +0 -2
- package/dist/chunks/chunk-RHQP7X6H.js +0 -3
- package/dist/chunks/chunk-UKKSN4F7.js +0 -3
- package/dist/chunks/chunk-UWFDJKOJ.js +0 -2
- package/dist/chunks/chunk-WXCLWAE2.js +0 -2
- package/dist/chunks/chunk-X7VWUI6N.js +0 -2
- package/dist/chunks/component-PAASRIK6.js +0 -2
- package/dist/chunks/config-JN3I5EII.js +0 -2
- package/dist/chunks/connection-IWFELK5W.js +0 -2
- package/dist/chunks/connref-ZDY7JYK5.js +0 -2
- package/dist/chunks/customizations-IE2KO275.js +0 -7
- package/dist/chunks/dependency-EFDSIKSN.js +0 -3
- package/dist/chunks/environment-NUQHU74A.js +0 -2
- package/dist/chunks/envvar-YN35IDWJ.js +0 -2
- package/dist/chunks/fanout-DUP5CD5X.js +0 -2
- package/dist/chunks/functions-K5TYRB2C.js +0 -2
- package/dist/chunks/history-SNSJRV6B.js +0 -2
- package/dist/chunks/identity-OW4WCDOA.js +0 -2
- package/dist/chunks/jobs-PXNJFWWF.js +0 -2
- package/dist/chunks/lifecycle-7RMWV26J.js +0 -2
- package/dist/chunks/metadata-74WGGOV5.js +0 -2
- package/dist/chunks/metadataCreate-W3NDI5FB.js +0 -2
- package/dist/chunks/msalUser-JRNX5HPO.js +0 -2
- package/dist/chunks/names-SFYZQY6Q.js +0 -2
- package/dist/chunks/process-G7LCTGKS.js +0 -2
- package/dist/chunks/protect-WZO3MAHF.js +0 -2
- package/dist/chunks/provider-VXRAQRN2.js +0 -2
- package/dist/chunks/records-TN7ZKUJ7.js +0 -2
- package/dist/chunks/run-F6H4QCWR.js +0 -2
- package/dist/chunks/run-HBQCSGYY.js +0 -3
- package/dist/chunks/run-MHM3GTGH.js +0 -2
- package/dist/chunks/run-SRQPP6N3.js +0 -3
- package/dist/chunks/run-ZNDSC7LR.js +0 -2
- package/dist/chunks/security-FLEAZ4EA.js +0 -2
- package/dist/chunks/services-B6V4VVX5.js +0 -2
- package/dist/chunks/solution-HF57MP27.js +0 -2
- package/dist/chunks/solutionExport-TLGHMUM5.js +0 -2
- package/dist/chunks/spn-AIUWML5R.js +0 -2
- package/dist/chunks/tokenCache-3LMJRUTI.js +0 -2
|
@@ -0,0 +1,669 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: ppdo-cli-data
|
|
3
|
+
description: Keep Dataverse configuration data (reference data) in source control as YAML and move it between environments with ppdo-cli dv data - export, import (plan, check, overlays, files, many-to-many links), delete (the delete modes and the deletion log, with a confirmation), convert from Configuration Migration Tool (CMT) / pac data files, and init schema files. Covers the reference-data folder, every schema option, the data file formats (values, lookups, nulls, file and image columns, layouts), how the import plans and writes, errors, and pipeline use. Use for any task with reference data, configuration data, data.xml, ReferenceData or reference-data folders.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Configuration data with ppdo-cli: `dv data`
|
|
7
|
+
|
|
8
|
+
Load the ppdo-cli skill first (accounts, modes, exit codes, output). For other Dataverse work, load ppdo-cli-dataverse.
|
|
9
|
+
|
|
10
|
+
Configuration data (reference data) is the records that a solution needs in each environment: categories, codes, settings, templates. `dv data` keeps them as YAML in source control and makes a target environment equal to the files. It replaces `pac data export`/`import` and the Configuration Migration Tool (CMT) files, and it is 60 to 200 times faster.
|
|
11
|
+
|
|
12
|
+
- **The export** reads the source (development) environment and writes the data files. Only files whose bytes change are written, so one changed value is one changed line in a pull request.
|
|
13
|
+
- **The import** makes a plan (it reads the target and compares each record), then sends only the differences: creates, updates, files and links. It never deletes. An import with nothing different sends no write. A stopped import can run again: the plan comes from the target each time, and there is no state file.
|
|
14
|
+
- **The delete** (`dv data delete`) is a separate command, for safety: it deletes the records and removes the links that the delete modes of the schema give, after a person confirms (or `--yes` in a pipeline).
|
|
15
|
+
- **A record is its GUID.** The primary key is the only match key: each record has the same GUID in each environment (the import creates a record with the GUID of the file), and each lookup is the GUID of its target. A GUID that is different in one environment (a currency, a team, a business unit) goes in an overlay.
|
|
16
|
+
|
|
17
|
+
## Commands
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
ppdo-cli dv data export <folder> [--only <name>[,<name>...]] [--check] [--parallel <n>] [--log-file <file>]
|
|
21
|
+
ppdo-cli dv data import <folder> [--only <name>[,<name>...]] [--check] [--overlay <name> | --no-overlay]
|
|
22
|
+
[--continue-on-error] [--batch-size <n>] [--parallel <n>]
|
|
23
|
+
[--bypass-plugins sync|async|all] [--bypass-step <step-id>[,...]] [--limit n] [--all] [--log-file <file>]
|
|
24
|
+
ppdo-cli dv data delete <folder> [--only <name>[,<name>...]] [--check] [--yes] [--batch-size <n>] [--parallel <n>]
|
|
25
|
+
[--bypass-plugins sync|async|all] [--bypass-step <step-id>[,...]] [--log-file <file>] [--delete-log-file <file>]
|
|
26
|
+
ppdo-cli dv data convert <source> <folder> [--cmt-schema <file>] [--force]
|
|
27
|
+
ppdo-cli dv data init <folder> --table <name>[,<name>...]
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
| Command | Class | Connects | What it does |
|
|
31
|
+
|---|---|---|---|
|
|
32
|
+
| `export` | R | Source environment | Writes `<folder>/data` from the environment. Never changes the environment. |
|
|
33
|
+
| `import` | W | Target environment | Plans, then writes the differences (never a delete). `--check` sends reads only (class R). |
|
|
34
|
+
| `delete` | W | Target environment | Deletes, deactivates and removes links as the delete modes give, after a confirmation. A record delete or link removal needs mode `full`. `--check` sends reads only. |
|
|
35
|
+
| `convert` | R | The source environment of the CMT files (metadata reads only) | CMT `data.xml` + CMT schema to a new folder (schema and data), with the metadata names. |
|
|
36
|
+
| `init` | R | Any environment with the tables | Writes a first schema file for each named table. |
|
|
37
|
+
|
|
38
|
+
All five take the usual `dv` options (`--env`, `--client-id`, `--tenant`, `--mode`, ...). `export`, `import`, `delete`, `convert` and `init` refuse `--group` and `--env` with more than one environment: one folder has one source, one run has one target. `<folder>` is the root folder, usually `<Solution>/reference-data`. The schema is always `<folder>/schema` and the data always `<folder>/data`.
|
|
39
|
+
|
|
40
|
+
### Flags
|
|
41
|
+
|
|
42
|
+
| Flag | Command | Meaning |
|
|
43
|
+
|---|---|---|
|
|
44
|
+
| `--only <a,b>` | export, import | Only these tables and relationships (logical names / relationship schema names). Other files stay as they are. An import lookup to a table not in the list must find its target in the environment. |
|
|
45
|
+
| `--check` | export | Write nothing. Exit 6 `out_of_date` if a file would change. |
|
|
46
|
+
| `--check` | import | Show the plan, write nothing. Exit 6 `changes_pending` if the plan has a create, update or link. Works in a `read-only` group. |
|
|
47
|
+
| `--check` | delete | List each delete, deactivate and link removal, write nothing. Exit 6 `changes_pending` if there is one. Works in a `read-only` group. |
|
|
48
|
+
| `--yes` | delete | Delete with no confirmation. For a pipeline, where no person can type at a terminal: review `--check` first. Without it, the command lists the deletes on stderr and a person types `delete`; with no terminal, exit 7. |
|
|
49
|
+
| `--parallel <n>` | export | File downloads at the same time (file and image columns), 1 to 16, default 8. |
|
|
50
|
+
| `--parallel <n>` | import, delete | Requests at the same time, 1 to 16, default 8. |
|
|
51
|
+
| `--batch-size <n>` | import, delete | Records in one `CreateMultiple`/`UpdateMultiple` (delete: one deactivate `UpdateMultiple`), 1 to 1000, default 100, for each table that has no `batchSize` in its schema. |
|
|
52
|
+
| `--overlay <name>` | import | Put `data/overlays/<name>` on top of the base. Exit 2 `overlay_not_found` if the folder is missing. |
|
|
53
|
+
| `--no-overlay` | import | Use the base only. With both flags, the last one wins. |
|
|
54
|
+
| `--continue-on-error` | import | After a failure, go on with each record that does not depend on a failed record. Without it the import stops after the set of requests that had the failure. Exit 1 in both cases. |
|
|
55
|
+
| `--bypass-plugins`, `--bypass-step` | import, delete | For every table of the run, in place of `bypass` of the schema. Needs mode `full`. |
|
|
56
|
+
| `--limit n`, `--all` | import | Entries in each `changes` list (default 100). Counts are always complete. (The delete always lists each change.) |
|
|
57
|
+
| `--log-file <file>` | export, import, delete | Write the progress lines, each retry, each failure, the JSON result and the error to this file. Folders are made. A file that cannot be written is exit 2. |
|
|
58
|
+
| `--delete-log-file <file>` | delete | Write each record that this run deleted or deactivated, and each link that it removed, to this file as JSON (also after a failure). Not written with `--check`. |
|
|
59
|
+
| `--cmt-schema <file>` | convert | The CMT schema (`schema.xml` or `data_schema.xml`). |
|
|
60
|
+
| `--force` | convert | Replace files in `<folder>`. Without it: exit 6 `folder_not_empty`. |
|
|
61
|
+
| `--table <a,b>` | init | Required. Logical names of the tables. |
|
|
62
|
+
|
|
63
|
+
`--verbose` (`-v`) writes a progress line on stderr for each step as it happens (`[dv data import] 12.3s create wave 2: ...`), plus the HTTP requests. stdout stays the JSON result only. `--log-file` works with or without `--verbose`; its lines have an ISO time, the seconds from the start, and end with `end (peak memory N MB)`.
|
|
64
|
+
|
|
65
|
+
## The folder
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
<Solution>/
|
|
69
|
+
reference-data/ <folder> (not the CMT folder "ReferenceData")
|
|
70
|
+
schema/ a person writes these (init/convert make the first version)
|
|
71
|
+
contoso_category.yaml one file for each table, named <logical name>.yaml
|
|
72
|
+
contoso_item.yaml
|
|
73
|
+
contoso_tag.yaml
|
|
74
|
+
contoso_template.yaml
|
|
75
|
+
data/ export writes tables, relationships, files
|
|
76
|
+
tables/
|
|
77
|
+
contoso_category.yaml one data file for each table (layout table)
|
|
78
|
+
contoso_item.yaml
|
|
79
|
+
contoso_tag.yaml
|
|
80
|
+
contoso_template.yaml layout record: the header only
|
|
81
|
+
contoso_template/
|
|
82
|
+
0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f.yaml layout record: one file for each record, named by its GUID
|
|
83
|
+
relationships/
|
|
84
|
+
contoso_item_tag.yaml one file for each many-to-many relationship
|
|
85
|
+
files/
|
|
86
|
+
contoso_template/ bytes of file and image columns: <GUID>.<column>.<ext>
|
|
87
|
+
0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f.contoso_attachment.pdf
|
|
88
|
+
0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f.contoso_logo.png
|
|
89
|
+
overlays/ a person writes these; export never touches them
|
|
90
|
+
production/
|
|
91
|
+
tables/
|
|
92
|
+
contoso_item.yaml only the values that differ in that environment
|
|
93
|
+
.deleted/ the deletion log: export adds entries; nothing changes or removes them
|
|
94
|
+
tables/
|
|
95
|
+
contoso_item.yaml records deleted in the source (tables with a delete mode)
|
|
96
|
+
relationships/
|
|
97
|
+
contoso_item_tag.yaml links removed in the source (relationships with a delete mode)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
- A table has `schema/<table>.yaml` and `data/tables/<table>.yaml` with the same name.
|
|
101
|
+
- To add a table, add a schema file and export. To remove a table, delete its schema file: the next export removes its data file, record folder and `data/files/<table>` and reports it.
|
|
102
|
+
- The export removes any file or folder in `data/tables`, `data/relationships` or `data/files` that the schema does not name, and reports it. It never writes in `schema` or `data/overlays`.
|
|
103
|
+
- All YAML is UTF-8 with no BOM and LF line ends. Add this to the repository's `.gitattributes`:
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
**/reference-data/**/*.yaml text eol=lf
|
|
107
|
+
**/reference-data/data/files/** binary
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Schema files
|
|
111
|
+
|
|
112
|
+
```yaml
|
|
113
|
+
# yaml-language-server: $schema=https://raw.githubusercontent.com/PPDOStudio/ppdo-cli-release/main/schema/reference-data.schema.json
|
|
114
|
+
version: 1
|
|
115
|
+
table: contoso_item
|
|
116
|
+
columns:
|
|
117
|
+
- contoso_code
|
|
118
|
+
- contoso_name
|
|
119
|
+
- contoso_categoryid
|
|
120
|
+
- name: contoso_supplierid
|
|
121
|
+
missingLookup: skip
|
|
122
|
+
nullValues: ignore
|
|
123
|
+
- contoso_type
|
|
124
|
+
- contoso_price
|
|
125
|
+
- transactioncurrencyid
|
|
126
|
+
- name: contoso_notes
|
|
127
|
+
write: create
|
|
128
|
+
- statecode
|
|
129
|
+
- statuscode
|
|
130
|
+
filter: contoso_type ne 471100003
|
|
131
|
+
delete:
|
|
132
|
+
mode: manifest
|
|
133
|
+
bypass:
|
|
134
|
+
plugins: sync
|
|
135
|
+
flows: true
|
|
136
|
+
batchSize: 100
|
|
137
|
+
relationships:
|
|
138
|
+
- name: contoso_item_tag
|
|
139
|
+
delete:
|
|
140
|
+
mode: mirror
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The smallest schema file (nothing is deleted):
|
|
144
|
+
|
|
145
|
+
```yaml
|
|
146
|
+
version: 1
|
|
147
|
+
table: contoso_tag
|
|
148
|
+
columns:
|
|
149
|
+
- contoso_name
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
| Property | Meaning | Default |
|
|
153
|
+
|---|---|---|
|
|
154
|
+
| `version` | Always `1`. | required |
|
|
155
|
+
| `table` | Logical name; must equal the file name. | required |
|
|
156
|
+
| `match` | The match key. Only the primary key column is supported in this version: leave it out, or write `[<primary key>]`. Other kinds can come later. | the primary key (GUID) |
|
|
157
|
+
| `columns` | The columns to export and import, in file order. An entry is a logical name, or a map `name:` + column options. Not the primary key (each record has it); each column once. An alternate key column (a code) is an ordinary column. | required |
|
|
158
|
+
| `layout` | `table` (one file) or `record` (one file for each record). | `table` |
|
|
159
|
+
| `filter` | OData `$filter`: the scope. | all records |
|
|
160
|
+
| `fetchXmlFilter` | FetchXML `<filter>`/`<link-entity>` elements, or a full CMT `<fetch>` document (the elements of its `<entity>` are used). Not with `filter`. | |
|
|
161
|
+
| `write` | `upsert` or `create` (create missing records, never update). | `upsert` |
|
|
162
|
+
| `nullValues` | `clear`: a `null` in the file removes the target value. `ignore`: a `null` changes nothing. | `clear` |
|
|
163
|
+
| `missingLookup` | `error`: stop the plan if a lookup target is not found. `skip`: leave that column out of the write, with a warning. | `error` |
|
|
164
|
+
| `delete.mode` | `off`, `manifest` or `mirror` (see Deletes). | `off` |
|
|
165
|
+
| `delete.action` | `delete` or `deactivate` (only for a table with `statecode`). | `delete` |
|
|
166
|
+
| `bypass.plugins` | `none`, `sync`, `async`, `all`. Needs mode `full`. | `none` |
|
|
167
|
+
| `bypass.steps` | Plug-in step IDs to skip. Needs mode `full`. | |
|
|
168
|
+
| `bypass.flows` | `true`: do not start cloud flows. | `false` |
|
|
169
|
+
| `batchSize` | Records in one bulk request, 1 to 1000. | `--batch-size` (100) |
|
|
170
|
+
| `files.name` | File name pattern for the file, image and base64 columns: the path in `data/files/<table>` (see File and image columns). | `{$id}.{$column}.{$ext}` |
|
|
171
|
+
| column `file` | File name pattern for one file, image or base64 column; wins over `files.name`. | the table's |
|
|
172
|
+
| `relationships[].name` | Schema name of a many-to-many relationship (`SchemaName` in the metadata). Write it in one of its two tables' schema files only; both tables need schema files. | required |
|
|
173
|
+
| `relationships[].delete.mode` | `off` (only add links), `manifest` (also remove the links of the deletion log) or `mirror` (also remove links not in the file). | `off` |
|
|
174
|
+
|
|
175
|
+
Column options (in a `- name: ...` entry):
|
|
176
|
+
|
|
177
|
+
| Option | Meaning |
|
|
178
|
+
|---|---|
|
|
179
|
+
| `write: create` | Write the column only when the record is created. A start value that each environment can change. Never compared for an existing record. |
|
|
180
|
+
| `nullValues: clear` / `ignore` | For this column, in place of the table's value. |
|
|
181
|
+
| `missingLookup: error` / `skip` | For this lookup column, in place of the table's value. |
|
|
182
|
+
|
|
183
|
+
No setting is for all tables, and no flag turns on deletes: each schema file gives all the rules for its table.
|
|
184
|
+
|
|
185
|
+
What the schema check refuses (exit 2 `schema_invalid`, or exit 4 `not_found` when the environment lacks the table, column or relationship): a file name that is not `<table>.yaml`; unknown properties or words; `filter` with `fetchXmlFilter`; a column twice; the primary key in `columns`; a `match` that is not the primary key; a relationship in two files or to a table with no schema file; a column that the metadata says cannot be written in a create or an update (`createdon`, `modifiedon`, `versionnumber`, calculated and rollup columns) unless it is a file or image column; an activity party column; `delete.action: deactivate` on a table with no `statecode`; the properties of earlier issues, each with its reason: `reference` (reference tables are removed), `delete.max` (removed), the modes `flag` and `log` (now `manifest`). A column that can be written only on create is handled as `write: create`. Owner and business unit columns are ordinary lookups.
|
|
186
|
+
|
|
187
|
+
## Data files
|
|
188
|
+
|
|
189
|
+
### Layout `table`: `data/tables/<table>.yaml`
|
|
190
|
+
|
|
191
|
+
```yaml
|
|
192
|
+
table: contoso_item
|
|
193
|
+
match:
|
|
194
|
+
- contoso_itemid
|
|
195
|
+
columns:
|
|
196
|
+
- contoso_code
|
|
197
|
+
- contoso_name
|
|
198
|
+
- contoso_categoryid
|
|
199
|
+
- contoso_supplierid
|
|
200
|
+
- contoso_type
|
|
201
|
+
- contoso_price
|
|
202
|
+
- transactioncurrencyid
|
|
203
|
+
- contoso_validfrom
|
|
204
|
+
- contoso_reviewedon
|
|
205
|
+
- contoso_isfeatured
|
|
206
|
+
- contoso_notes
|
|
207
|
+
- statecode
|
|
208
|
+
- statuscode
|
|
209
|
+
filter: contoso_type ne 471100003
|
|
210
|
+
records:
|
|
211
|
+
- contoso_itemid: 0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f
|
|
212
|
+
contoso_code: ITEM-00042
|
|
213
|
+
contoso_name: Compact Valve 00042
|
|
214
|
+
contoso_categoryid: 4a1b7c2d-8e9f-4a0b-9c1d-2e3f4a5b6c7d # Fasteners
|
|
215
|
+
contoso_supplierid: 7c9e6679-7425-40de-944b-e07fc1f90ae7 # Fabrikam Supplies
|
|
216
|
+
contoso_type: 471100001 # Premium
|
|
217
|
+
contoso_price: 1845.08
|
|
218
|
+
transactioncurrencyid: 9b8a7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d # New Zealand Dollar
|
|
219
|
+
contoso_validfrom: "2025-05-20"
|
|
220
|
+
contoso_reviewedon: "2025-08-01T23:18:57Z"
|
|
221
|
+
contoso_isfeatured: false
|
|
222
|
+
contoso_notes: |-
|
|
223
|
+
Line one for item 42.
|
|
224
|
+
Line two: handle with care.
|
|
225
|
+
statecode: 0 # Active
|
|
226
|
+
statuscode: 1 # Active
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
- **Header**: `table`, `match`, `columns` always; `layout: record` only for that layout; `filter` or `fetchXmlFilter` only if the schema has one. It is the schema after defaults (`match` is always the primary key column), names only. If the header and the schema differ (match, columns, filter, layout), the import stops with `schema_mismatch`: export again after each schema change.
|
|
230
|
+
- **Records** are sorted by GUID, never by name or code, so a record does not move when a value changes. Columns: the primary key (the GUID) first, then `columns` order. A changed code is one changed line.
|
|
231
|
+
- **Every column has a line**: no value is written `null`.
|
|
232
|
+
- **Only records that exist.** A deleted record is not in the data file; the deletion log keeps it. A record with `$deleted` (the old inline flag) is `data_invalid`.
|
|
233
|
+
|
|
234
|
+
### Layout `record`: one file for each record
|
|
235
|
+
|
|
236
|
+
The schema has `layout: record`. `data/tables/<table>.yaml` holds the header (with `layout: record`) and no `records`. Each record is `data/tables/<table>/<identifier>.yaml`, with its columns at the top level:
|
|
237
|
+
|
|
238
|
+
```yaml
|
|
239
|
+
contoso_templateid: 0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f
|
|
240
|
+
contoso_code: WELCOME
|
|
241
|
+
contoso_name: Welcome message
|
|
242
|
+
contoso_subject: Welcome to Contoso
|
|
243
|
+
contoso_body: |-
|
|
244
|
+
Hello,
|
|
245
|
+
|
|
246
|
+
Your account is ready.
|
|
247
|
+
contoso_attachment: { $file: 0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f.contoso_attachment.pdf, name: Getting started.pdf }
|
|
248
|
+
contoso_logo: { $file: 0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f.contoso_logo.png }
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
- The import identifies a record by the GUID inside the file, not by the file name: a person can add a record under any `.yaml` name, and the next export renames it. Two files with the same GUID: `data_invalid`.
|
|
252
|
+
- A deleted record loses its file; the deletion log keeps it.
|
|
253
|
+
- The export rewrites only changed record files and removes `.yaml` files that are not a record of the scope.
|
|
254
|
+
- Use `record` for few large records (templates, long texts, files): each changed record is its own file in a pull request. Use `table` (default) for many small records: some thousand files make git slow (5,000 record files export in about 40 s, against 8 to 14 s for one file).
|
|
255
|
+
- Changing the layout needs a new export (the import stops with `schema_mismatch`); the export removes the old layout's files.
|
|
256
|
+
|
|
257
|
+
### The identifier (record file names and `data/files` names)
|
|
258
|
+
|
|
259
|
+
The identifier of a record is its GUID in lower case: `0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f`. It is the same in each environment and on each machine. (The rules of design section 3.12, for a text that is not safe in a file name, stay for other kinds of match key later.)
|
|
260
|
+
|
|
261
|
+
### Values
|
|
262
|
+
|
|
263
|
+
| Column type | In the file |
|
|
264
|
+
|---|---|
|
|
265
|
+
| Text | Plain, or a double-quoted JSON string when needed (see YAML rules). |
|
|
266
|
+
| Multi-line text | A `\|-` block. The comparison ignores CRLF against LF. |
|
|
267
|
+
| Whole number | `805` |
|
|
268
|
+
| Decimal, float, money | `1845.08`: the shortest exact decimal; digits are kept, never rounded through a float. |
|
|
269
|
+
| Yes/no | `true`, `false` |
|
|
270
|
+
| Choice, state, status | `471100001 # Premium` (the number is the value; the comment is the label, ignored on read). |
|
|
271
|
+
| Choices (multi-select) | `[471100000, 471100002] # Red, Blue`, ascending. An empty list is no value. |
|
|
272
|
+
| Date only | `"2025-05-20"` |
|
|
273
|
+
| Date and time | `"2025-08-01T23:18:57Z"`: UTC, to the second. |
|
|
274
|
+
| Unique identifier | `7c9e6679-7425-40de-944b-e07fc1f90ae7` (lower case, no quotes). |
|
|
275
|
+
| Lookup | See Lookups. |
|
|
276
|
+
| File column | `{ $file: <identifier>.<column>.<ext>, name: <file name in Dataverse> }` |
|
|
277
|
+
| Image column | `{ $file: <identifier>.<column>.<ext> }` |
|
|
278
|
+
| No value | `null` |
|
|
279
|
+
|
|
280
|
+
The import also accepts other valid forms when a person edits by hand: a quoted number (`"12.50"`), a number in a text column, a CMT date and time (`2025-06-15T00:00:00.0000000`, also for a date-only column), choices in any order or repeated. A wrong type (text in a number column, `yes` in a yes/no column, an invalid date or GUID) is `data_invalid`. The next export writes the canonical form.
|
|
281
|
+
|
|
282
|
+
"Different" in the plan: text after CRLF becomes LF (case matters); numbers as decimals (`12.50` = `12.5`); a date by its date; a date and time to the second; a lookup by the target GUID; multi-select as a set; `null` = empty text. A column with no line in a record is not compared.
|
|
283
|
+
|
|
284
|
+
### Lookups
|
|
285
|
+
|
|
286
|
+
A lookup is always the GUID of its target, with the target's name as a comment (the comment is ignored on read):
|
|
287
|
+
|
|
288
|
+
| The lookup | In the file | Example |
|
|
289
|
+
|---|---|---|
|
|
290
|
+
| One target table | The GUID | `contoso_categoryid: 4a1b7c2d-8e9f-4a0b-9c1d-2e3f4a5b6c7d # Fasteners` |
|
|
291
|
+
| More than one target table (owner, customer, regarding) | A map with `$table` and the primary key | `ownerid: { $table: team, teamid: 0b8e5c5e-3c56-4f0e-9d3a-6a8f4f1d2b11 } # Records Team` |
|
|
292
|
+
|
|
293
|
+
- A self-reference and a lookup to a table that is not in the schema (a currency, a team, an account) are the same: the GUID.
|
|
294
|
+
- A code or a map of other columns in a lookup is an error: there are no reference tables.
|
|
295
|
+
- A target whose GUID is different in an environment (a currency, a team, a business unit): put the GUID of that environment in an overlay record. Records that the import creates have the GUID of the file, so a target in the data has the same GUID everywhere.
|
|
296
|
+
- The import resolves each lookup to the GUID in the target: a record of the data (in the target, or created in this run), or a read by GUID (25 for each request). Not found: `missingLookup: error` stops the plan with `unresolved_lookup` (each record and column listed); `skip` leaves the column out of that record's write (a new record gets no value, an existing one keeps its value) and warns.
|
|
297
|
+
- A target that is in the file but not yet in the environment waits for its create (see the import order).
|
|
298
|
+
|
|
299
|
+
### Many-to-many relationships: `data/relationships/<name>.yaml`
|
|
300
|
+
|
|
301
|
+
```yaml
|
|
302
|
+
relationship: contoso_item_tag
|
|
303
|
+
from: contoso_item
|
|
304
|
+
to: contoso_tag
|
|
305
|
+
links:
|
|
306
|
+
- from: 0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f # Compact Valve 00042
|
|
307
|
+
to:
|
|
308
|
+
- 1cb74a92-6a9a-48ba-a9b1-98577681c593 # Fragile
|
|
309
|
+
- 9cef176a-f0bb-464b-a3aa-3fc7d60e688c # Outdoor
|
|
310
|
+
- from: 6e5d4c3b-2a19-4f08-8e7d-6c5b4a392817 # Solar Relay 00044
|
|
311
|
+
to:
|
|
312
|
+
- 9cef176a-f0bb-464b-a3aa-3fc7d60e688c # Outdoor
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
- `from` is `Entity1LogicalName` of the metadata, `to` the second table. Each value is a GUID (a lookup value). Sorted by `from`, then `to`. A record with no links is not listed.
|
|
316
|
+
- The file is the full set of links for each `from` record in the data files: with `delete.mode: mirror` the import removes a link of a listed or unlisted (zero links) `from` record that the file lacks. Links of an unmanaged `from` record are never changed. With `manifest` it removes only the links of the deletion log. With `off` (default) links are only added.
|
|
317
|
+
- `from`/`to` the other way round is `data_invalid`.
|
|
318
|
+
|
|
319
|
+
### File and image columns
|
|
320
|
+
|
|
321
|
+
- Put them in `columns` like any column; the type comes from the metadata.
|
|
322
|
+
- The bytes live in `data/files/<table>/`, by default as `<GUID>.<column>.<ext>` (a file name pattern can change the path; see below). The extension: for a file column, the Dataverse file name's extension (lower case, at most 10 letters/digits, else none); for an image, from the bytes (`png`, `jpg`, `gif`, `bmp`, else `bin`).
|
|
323
|
+
- `name` (file columns only) is the file name a user sees in Dataverse. Renaming in Dataverse changes one line; replacing the file changes the binary only.
|
|
324
|
+
- To change a file, replace the bytes in `data/files/<table>` under the same name; the next import uploads it. A record that names a missing file: `file_missing` (exit 2).
|
|
325
|
+
- At most 40 MB a file (`file_too_large`, exit 2); the export warns above 10 MB because git keeps each version.
|
|
326
|
+
- Image columns export the full image (`?size=full`) when the column stores one, else the 144x144 thumbnail. The comparison is stable either way.
|
|
327
|
+
- Base64 text columns: `annotation.documentbody` and `activitymimeattachment.body` (the attachment of a note or an email) also go to `data/files/<table>/<GUID>.<column>.<ext>`, as `{ $file: ... }` with no `name`; the extension comes from the record's `filename` column. The import reads the file and sends base64 text in the create or update, so no download or upload request; the plan shows `{ size }`. Base64 text in the YAML (older folders) is still accepted, and `convert` writes the files too. No overlay value for them.
|
|
328
|
+
- A file is compared by name and size, then by downloading and comparing the bytes when they match (images are always downloaded). This costs one download for each file on each export and import.
|
|
329
|
+
- Import: a file upload is phase 3 (`PATCH` with `x-ms-file-name`; a non-ASCII name uses the block upload messages, so it is kept exactly). A file removal is `DELETE` on the column: an update (class V), allowed in `read-write`, not affected by the delete mode; the import does it. Images go as base64 in `UpdateMultiple` in phase 2 (at most 10 records with an image for each request); an image is never in a create.
|
|
330
|
+
- Not in overlays, and `convert` does not convert them (the first export saves the files).
|
|
331
|
+
|
|
332
|
+
File name patterns: `files.name` (table) or `file` (one column) set the path of each file:
|
|
333
|
+
|
|
334
|
+
```yaml
|
|
335
|
+
files:
|
|
336
|
+
name: "{objectid.name}/{$filename}" # data/files/annotation/<target name>/<file name>
|
|
337
|
+
columns:
|
|
338
|
+
- name: contoso_attachment
|
|
339
|
+
file: "{$id}/{$filename}" # this column only
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
| Token | Value |
|
|
343
|
+
|---|---|
|
|
344
|
+
| `{$id}` | The identifier of the record (its GUID, section 3.12). |
|
|
345
|
+
| `{$column}` | The logical name of the file column. |
|
|
346
|
+
| `{$filename}` | The file name in Dataverse: `name` of a file column, or `filename` of the record for a base64 column. An image has none. |
|
|
347
|
+
| `{$stem}`, `{$ext}` | That file name without its extension, and the extension (for an image: from its first bytes). |
|
|
348
|
+
| `{<column>}` | The value of a column of the schema, as text: a choice gives its number, a lookup its GUID. |
|
|
349
|
+
| `{<lookup column>.name}` | The name of the target of a lookup column of the schema. |
|
|
350
|
+
|
|
351
|
+
- `/` makes a folder; a `/` inside a value becomes `_`. Unsafe characters (`< > : " \ | ? *`, control characters, a trailing dot or space) become `_`, Windows reserved names get a `_` prefix, an empty value is `_`, and each name is cut to 100 characters (the extension is kept).
|
|
352
|
+
- The extension is added when the last name does not already end with it (so `{$stem}` and images get one).
|
|
353
|
+
- Only the export applies the pattern; the import reads the path in `$file` (relative, with folders; `..`, `\` and absolute paths are refused). Changing a value or the pattern moves the file on the next export (git shows a rename); identity is the GUID, so the environment does not change.
|
|
354
|
+
- Folder names that differ only in case get one spelling (the first in sort order), because Windows and macOS see one folder.
|
|
355
|
+
- Paths must be unique in the table (case ignored): otherwise the export stops with `file_name_conflict` (exit 2) before writing. Include `{$id}` to be safe; without it, a later record can collide.
|
|
356
|
+
- The schema check refuses an unknown token (for example `{$name}`: it is `{$filename}`), a column not in `columns`, `.name` on a column that is not a lookup, and `file` on a column that is not a file, image or base64 column.
|
|
357
|
+
|
|
358
|
+
### YAML rules
|
|
359
|
+
|
|
360
|
+
The export writes a fixed subset, so the same data always gives the same bytes: two-space indent; no tabs, document markers, anchors, aliases, tags or merge keys. A text is plain only if it matches `^[A-Za-z_][A-Za-z0-9_ ./()+-]*$`, does not end with a space and is not `true/false/null/yes/no/on/off/y/n/~` (any case); a GUID is plain; everything else is a double-quoted JSON string (so a date, a number in a text column and `no` always have quotes). Text with line breaks is a `|-` block when each line can be (no trailing space, no control characters), else one quoted string. A comment is ` # label` (line breaks become spaces). One-line maps only for lookups and file values.
|
|
361
|
+
|
|
362
|
+
The import reads YAML 1.2 (core schema), so any valid hand edit and comments work. It refuses anchors, aliases, tags and duplicate keys.
|
|
363
|
+
|
|
364
|
+
## Null values and clearing a value
|
|
365
|
+
|
|
366
|
+
| In the record | Meaning |
|
|
367
|
+
|---|---|
|
|
368
|
+
| `column: null` | No value. The export writes this for each empty column. |
|
|
369
|
+
| `column: ""` | No value (Dataverse stores empty text as null). Exported as `null`. |
|
|
370
|
+
| No line for the column | The record does not give the column: not compared, not written, not in a create. The plan warns with the count. Only from hand edits and `convert`; the next export adds the line. |
|
|
371
|
+
|
|
372
|
+
| Case | `nullValues: clear` (default) | `nullValues: ignore` |
|
|
373
|
+
|---|---|---|
|
|
374
|
+
| File `null`, target has a value | Update that removes it. | Nothing; counted in `nulls_ignored`. |
|
|
375
|
+
| File `null`, target empty | Nothing. | Nothing. |
|
|
376
|
+
| New record, file `null` | The create sends the column as `null`, so Dataverse gives no default. A `null` lookup is never sent in a create. | The column is left out; Dataverse gives its default. |
|
|
377
|
+
| Overlay `null` | No value in that environment. | Nothing. |
|
|
378
|
+
|
|
379
|
+
Always: a record must have its GUID (`data_invalid`); `statecode`/`statuscode` cannot be `null` (`data_invalid`); a `write: create` column or a `write: create` table never clears an existing value; a column not in `columns` is never read or written. A lookup is cleared by sending its navigation property as `null`; a file by `DELETE` on the column; an image by `null` in the update.
|
|
380
|
+
|
|
381
|
+
To keep a value that an environment sets itself: `nullValues: ignore` (table or column), `write: create` for the column, an overlay value, or leave the column out of the schema.
|
|
382
|
+
|
|
383
|
+
## Overlays: values for one environment
|
|
384
|
+
|
|
385
|
+
`data/overlays/<name>/tables/<table>.yaml` holds only what differs in that environment. `<name>` matches `^[a-z0-9][a-z0-9-]*$` and has no tie to ppdo-cli environment names; one overlay can serve several environments.
|
|
386
|
+
|
|
387
|
+
```yaml
|
|
388
|
+
table: contoso_item
|
|
389
|
+
overlay: production
|
|
390
|
+
records:
|
|
391
|
+
- contoso_itemid: 0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f # Compact Valve 00042
|
|
392
|
+
contoso_supplierid: 3f2504e0-4f89-41d3-9a0c-0305e82c3301 # Fabrikam Supplies in production
|
|
393
|
+
transactioncurrencyid: 5a4b3c2d-1e0f-4a9b-8c7d-6e5f4a3b2c1d # NZD in production
|
|
394
|
+
contoso_price: 1999
|
|
395
|
+
- contoso_itemid: 6e5d4c3b-2a19-4f08-8e7d-6c5b4a392817
|
|
396
|
+
contoso_validfrom: null
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
- A record is named by its GUID (a name comment helps a reader). `import --overlay production` replaces the listed columns of those base records, then plans from the result. A lookup to a record whose GUID is different in that environment (a currency, a team) is an overlay value. Column options (`write`, `missingLookup`, `nullValues`) apply as for base values. An update entry whose value came from the overlay has `"overlay": true`.
|
|
400
|
+
- **Required flag**: if `data/overlays` has any overlay, the import needs `--overlay <name>` or `--no-overlay`, else exit 2 `overlay_required` (lists the overlays). This stops a production pipeline from writing development values because a flag was missing.
|
|
401
|
+
- Exit 2 `overlay_invalid` (each problem with file and key) for: a record with no GUID, or a GUID not in the base file; a column not in `columns`; `$deleted`; a file or image column value; two records with one key; a file for a table not in the schema, or `table`/`overlay` not equal to the file and folder names.
|
|
402
|
+
- An overlay record for a record of the deletion log gives a warning and is ignored.
|
|
403
|
+
- Overlays cannot add or remove records or log a delete, have no relationship files and no file values, and are in source control (no secrets). One overlay file for each table, for either layout.
|
|
404
|
+
- The export always writes the base (run it only against the source environment). When it removes a base record that an overlay names, it warns with the overlay file.
|
|
405
|
+
|
|
406
|
+
## Deletes
|
|
407
|
+
|
|
408
|
+
| `delete.mode` | The export | `dv data delete` deletes |
|
|
409
|
+
|---|---|---|
|
|
410
|
+
| `off` (default) | Removes a deleted record from the data file; writes no log. | Nothing. The log is not read. |
|
|
411
|
+
| `manifest` | Removes it from the data file and adds it to the deletion log. | Each record of the deletion log (reason `manifest`). |
|
|
412
|
+
| `mirror` | The same as `manifest`. | Each record of the log (reason `manifest`), and each record in the scope that is not in the data file (reason `mirror`). |
|
|
413
|
+
|
|
414
|
+
A relationship has the same three modes for its links.
|
|
415
|
+
|
|
416
|
+
### The deletion log: `data/.deleted`
|
|
417
|
+
|
|
418
|
+
`data/.deleted/tables/<table>.yaml`:
|
|
419
|
+
|
|
420
|
+
```yaml
|
|
421
|
+
table: contoso_item
|
|
422
|
+
records:
|
|
423
|
+
- id: 0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f
|
|
424
|
+
name: Basic Panel 00051
|
|
425
|
+
deleted: "2026-10-02"
|
|
426
|
+
- id: 8c7b6a59-4837-4261-9f0e-dcba98765432
|
|
427
|
+
name: Rapid Sensor 00043
|
|
428
|
+
deleted: "2026-11-16"
|
|
429
|
+
```
|
|
430
|
+
|
|
431
|
+
`data/.deleted/relationships/<name>.yaml`:
|
|
432
|
+
|
|
433
|
+
```yaml
|
|
434
|
+
relationship: contoso_item_tag
|
|
435
|
+
from: contoso_item
|
|
436
|
+
to: contoso_tag
|
|
437
|
+
links:
|
|
438
|
+
- from: 0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f
|
|
439
|
+
to: 1cb74a92-6a9a-48ba-a9b1-98577681c593
|
|
440
|
+
removed: "2026-11-16"
|
|
441
|
+
```
|
|
442
|
+
|
|
443
|
+
- **The export writes it, only for a table or relationship with `manifest` or `mirror`.** A record entry has `id` (the GUID), `name` (the primary name, for review) and `deleted` (the UTC date the export found the record gone from the source). A link entry has the two GUIDs and `removed`. The import uses the GUID only: a target record with the same code and another GUID is not deleted.
|
|
444
|
+
- **It only grows.** The export adds entries; nothing changes or removes an entry or a log file, even if the record comes back, the table leaves the schema or its mode goes back to `off`. There is no prune step. Git keeps the history.
|
|
445
|
+
- **A record that comes back** with the same GUID (for example from the recycle bin) is in the data file again; its entry stays. The data file wins: the import creates or updates it and does not delete it, with a warning in the export and the plan.
|
|
446
|
+
- **Not logged:** a record that still exists in the source but left the scope (the filter or a value changed). The export removes it from the data file with a warning. Links that went with a deleted record get no entry of their own.
|
|
447
|
+
- **Deletes start in the source environment.** The import never writes the log.
|
|
448
|
+
- Entries are sorted by the date (`deleted`, `removed`), then by GUID, so new deletes go at the end. They carry no comments, so the same log always gives the same bytes. `export --check` exits 6 when only the log would change.
|
|
449
|
+
- **A broken log file stops the export** (exit 2 `data_invalid`), so an entry is never lost. A log in the form before issue 12 (match keys, `restored`) is also `data_invalid`.
|
|
450
|
+
|
|
451
|
+
### What `dv data delete` does
|
|
452
|
+
|
|
453
|
+
The import never deletes a record, never removes a link and never does the deactivate of `delete.action: deactivate`. (It does set the state that a data file gives: see the last item of this list.) When the delete modes give such work, the import warns: "... the import never deletes: run ppdo-cli dv data delete". `dv data delete <folder>` makes the same plan (the schema, the data files, the deletion log, the target scope; `schema_mismatch` and `data_invalid` apply) but for deletes only. It reads the base: it needs no `--overlay` and resolves no lookup, so a lookup target that a target environment lacks does not stop it.
|
|
454
|
+
|
|
455
|
+
1. It lists each delete on stderr: for each table, the count and the first 20 records (GUID, name, reason); for each relationship, the count of link removals.
|
|
456
|
+
2. A person types `delete` to confirm. Any other answer: exit 7, nothing deleted. With no person at a terminal and no `--yes`: exit 7, with the hint `--yes`.
|
|
457
|
+
3. `--yes` gives the confirmation (for a pipeline). Review `--check` first.
|
|
458
|
+
4. It removes the links, then deletes or deactivates the records, child records first.
|
|
459
|
+
|
|
460
|
+
| In the files | In the target | `off` | `manifest` | `mirror` |
|
|
461
|
+
|---|---|---|---|---|
|
|
462
|
+
| A record of the log | In the scope | Nothing | Delete | Delete |
|
|
463
|
+
| A record of the log | Exists, out of scope | Nothing | Nothing (warning) | Nothing (warning) |
|
|
464
|
+
| A record of the log | Not there | Nothing | Nothing (done) | Nothing |
|
|
465
|
+
| No record | In the scope | Unmanaged | Unmanaged (warning) | Delete |
|
|
466
|
+
|
|
467
|
+
- A record of the log is never created or updated. A record in the data file and in the log: the data file wins, with a warning. The same for a link in the relationship file and in the log.
|
|
468
|
+
- `delete.action: deactivate` sets `statecode` 1 and the default inactive `statuscode` (an `UpdateMultiple`, `read-write` is enough). An already inactive record needs nothing; the output says `deactivate`. Use it for records that business records point to.
|
|
469
|
+
- **The import still deactivates and activates records that are in the data file.** `statecode` and `statuscode` are ordinary columns: a record whose data file says `statecode: 1 # Inactive` while the target has it active gets an update that deactivates it (and the other way round). That is an update of the import (mode `read-write`, no delete mode needed), not a delete. To deactivate a record everywhere, deactivate it in the development environment and export.
|
|
470
|
+
- **A delete or link removal that Dataverse refuses** (a restrict relationship, a record in use) does not stop `dv data delete`: the other deletes still run, the refused one is in `failures` (`"action": "delete"`, `"deactivate"` or `"unlink"`), and the exit is 1. Because the log never changes, each run tries it again until the target is fixed (remove what blocks it, or use `delete.action: deactivate`).
|
|
471
|
+
- **Protections**: a separate command with a confirmation (the import never deletes); off until a schema sets a mode; only tables of the schema; only records in the target scope; `schema_mismatch` blocks a schema change without a new export (so a changed filter cannot delete by mistake); a record delete or link removal is a `DELETE` request and needs mode `full` (exit 7 before the first write); `--check` lists every delete with GUID, name and reason (`manifest`/`mirror`); a warning names a cascade-delete relationship to real child data (not the platform bookkeeping tables each table has); deletes run last; overlays cannot delete. There is no delete maximum: with `mirror`, an empty or cut-off data file deletes each record in the scope, so read `--check` first.
|
|
472
|
+
- `dv data delete --delete-log-file <file>` keeps a JSON record of what one run deleted: `folder`, `environment`, `time`, `records[]` (`table`, `id`, `name`, `action`, `reason`) and `links[]` (`relationship`, `from`, `to`, `reason`). Publish it as a pipeline artifact.
|
|
473
|
+
- **Every environment gets each delete**, whenever it gets the release: entries stay forever, and an entry whose record is already gone costs one read.
|
|
474
|
+
|
|
475
|
+
## How the import works
|
|
476
|
+
|
|
477
|
+
**The plan** (always first; `--check` stops after it):
|
|
478
|
+
|
|
479
|
+
1. Read the schema and the metadata (one request for each table, plus typed metadata for lookups, choices, state/status, dates, images), and check the schema against it.
|
|
480
|
+
2. Read and check the data files (`data_invalid`, `schema_mismatch`); warn about records with missing column lines and data files with no schema file (ignored). Apply the overlay.
|
|
481
|
+
3. Read the target scope of each table (4 tables at a time; a scope of 5,000+ rows is read as 16 primary-key ranges, 8 at a time). Index it by GUID.
|
|
482
|
+
4. For a table with a filter only: find file records outside the scope by their GUID (25 for each request). Such a record is updated, never created twice.
|
|
483
|
+
5. Resolve each lookup (GUIDs, 25 for each request).
|
|
484
|
+
6. Compare each record: `create`, `update` (only the changed columns), `unchanged`, `delete`/`deactivate`, `unmanaged`. Compare the files. Compare the links.
|
|
485
|
+
7. Safety: the mode (`blocked`, exit 7). With `--check` in a lower mode, a warning names what the real run would need.
|
|
486
|
+
|
|
487
|
+
**The writes** of the import, in four phases; each completes before the next (`dv data delete` does phase 4 for link removals only, then the deletes):
|
|
488
|
+
|
|
489
|
+
1. **Creates in waves.** A record is created when each of its lookup targets exists. Each wave sends `CreateMultiple` for each table in sets of `batchSize`, in parallel. A self-referencing tree goes one level for each wave (a chain of 100 records is 100 waves). When nothing can go, the remaining records hold a circle of lookups: in each circle (strongly connected set) only the lookups inside the circle are taken out, and set again by an update in phase 2. The create sends the file's GUID, so the record has the same GUID everywhere.
|
|
490
|
+
2. **Updates**: `UpdateMultiple` with the changed columns, plus the circle lookups and the images.
|
|
491
|
+
3. **Files**: uploads and removals of file columns.
|
|
492
|
+
4. **Links**: associate/disassociate (`$ref`) in `$batch` requests. Adding a link that exists or removing one that is gone is harmless.
|
|
493
|
+
|
|
494
|
+
Only changed records get a request, so unchanged records keep their `modifiedon` and run no plug-ins or flows. Plug-ins and flows run for written records unless `bypass` (or `--bypass-plugins`/`--bypass-step`) says otherwise. Duplicate detection does not run. Do not run two imports into one environment at the same time.
|
|
495
|
+
|
|
496
|
+
**Failures and retries:**
|
|
497
|
+
|
|
498
|
+
| Failure | What the import does |
|
|
499
|
+
|---|---|
|
|
500
|
+
| 429 throttling | Waits for `Retry-After` and sends again (counted in the log as retries). |
|
|
501
|
+
| SQL 2812 or 2766: a table made or changed by a recent solution import is not ready for the bulk messages | Waits 5, 10, ... 40 s and sends again, up to 8 times (about 3 minutes), then one request for each record. |
|
|
502
|
+
| SQL deadlock (1205) | Sends again up to 5 times after 0.5 to 2 s. |
|
|
503
|
+
| Network error, time-out, 5xx | Reads the records of the request again by GUID and sends only what is still different (no duplicates when a write went through but the answer was lost). |
|
|
504
|
+
| 4xx (an invalid record; a bulk request is all-or-nothing) | Sends the same records again in one `$batch` with continue-on-error, one request for each record: the good ones are written, each bad one goes to `failures` with its status, code and message. |
|
|
505
|
+
|
|
506
|
+
Records that depend on a failed record are not sent (`"action": "skip"`, `"error": "depends on <table> <GUID>, which failed"`). A failed upload is `"action": "upload"`. A failed create, update or upload stops the import (unless `--continue-on-error`); a failed delete, deactivate or link removal does not. The output always gives what was written, what failed and why, and what was not done; exit 1 `rows_failed`. Correct the data and run the same import again: it sends only what remains.
|
|
507
|
+
|
|
508
|
+
## Output
|
|
509
|
+
|
|
510
|
+
`export`:
|
|
511
|
+
|
|
512
|
+
```json
|
|
513
|
+
{
|
|
514
|
+
"folder": "reference-data", "checked_only": false,
|
|
515
|
+
"tables": [
|
|
516
|
+
{ "table": "contoso_item", "file": "data/tables/contoso_item.yaml", "records": 4999, "added": 0, "changed": 1, "removed": 1, "logged": 1, "written": true },
|
|
517
|
+
{ "table": "contoso_template", "file": "data/tables/contoso_template.yaml", "layout": "record", "folder": "data/tables/contoso_template", "records": 2, "added": 0, "changed": 0, "removed": 0, "logged": 0, "files": 2, "files_changed": 0, "written": false }
|
|
518
|
+
],
|
|
519
|
+
"relationships": [ { "name": "contoso_item_tag", "file": "data/relationships/contoso_item_tag.yaml", "links": 3001, "added": 1, "removed": 0, "written": true } ],
|
|
520
|
+
"warnings": []
|
|
521
|
+
}
|
|
522
|
+
```
|
|
523
|
+
|
|
524
|
+
`removed` is records taken out of the data file; `logged` is records added to the deletion log; a relationship has `logged` for links. `written` is true if a data or log file changed (with `--check`: is out of date).
|
|
525
|
+
|
|
526
|
+
`import` (with `--check`, `checked_only` is true and the counts are the plan; otherwise they are what was done):
|
|
527
|
+
|
|
528
|
+
```json
|
|
529
|
+
{
|
|
530
|
+
"folder": "reference-data", "overlay": null, "checked_only": true,
|
|
531
|
+
"summary": { "create": 1, "update": 1, "link": 1, "unchanged": 5260, "unmanaged": 2, "skipped": 0, "failed": 0 },
|
|
532
|
+
"tables": [
|
|
533
|
+
{ "table": "contoso_item", "create": 1, "update": 1, "unchanged": 4998, "unmanaged": 2, "skipped": 0, "failed": 0,
|
|
534
|
+
"changes": [
|
|
535
|
+
{ "action": "create", "key": { "contoso_itemid": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d" }, "name": "Alpine Module 05001" },
|
|
536
|
+
{ "action": "update", "key": { "contoso_itemid": "0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f" }, "name": "Compact Valve 00042", "id": "0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f",
|
|
537
|
+
"columns": { "contoso_price": { "from": 1845.08, "to": 199.5 } } }
|
|
538
|
+
] }
|
|
539
|
+
],
|
|
540
|
+
"relationships": [ { "name": "contoso_item_tag", "link": 1, "unchanged": 3000, "skipped": 0, "failed": 0,
|
|
541
|
+
"changes": [ { "action": "link", "from": "6e5d4c3b-2a19-4f08-8e7d-6c5b4a392817", "to": "1cb74a92-6a9a-48ba-a9b1-98577681c593" } ] } ],
|
|
542
|
+
"failures": [],
|
|
543
|
+
"warnings": [ "contoso_item: 2 records are in the scope in the environment and are not in the data file (unmanaged)" ]
|
|
544
|
+
}
|
|
545
|
+
```
|
|
546
|
+
|
|
547
|
+
- A table entry has `nulls_ignored` when `nullValues: ignore` kept target values.
|
|
548
|
+
- A file change: `"contoso_attachment": { "from": { "name": "Getting started.pdf", "size": 48211 }, "to": { "name": "Getting started.pdf", "size": 51734 } }` (images: `size` only).
|
|
549
|
+
- A text over 200 characters in `from`/`to` shows its first 200 characters and `... (N characters)`.
|
|
550
|
+
- `skipped`: records that depend on a failure.
|
|
551
|
+
|
|
552
|
+
`delete` (with `--check`, `checked_only` is true and the counts are the plan; each change is always listed):
|
|
553
|
+
|
|
554
|
+
```json
|
|
555
|
+
{
|
|
556
|
+
"folder": "reference-data", "checked_only": false,
|
|
557
|
+
"summary": { "delete": 1, "deactivate": 0, "unlink": 1, "failed": 0 },
|
|
558
|
+
"tables": [ { "table": "contoso_item", "mode": "manifest", "delete": 1, "deactivate": 0, "failed": 0,
|
|
559
|
+
"changes": [ { "action": "delete", "key": { "contoso_itemid": "8c7b6a59-4837-4261-9f0e-dcba98765432" }, "name": "Rapid Sensor 00043", "id": "8c7b6a59-4837-4261-9f0e-dcba98765432", "reason": "manifest" } ] } ],
|
|
560
|
+
"relationships": [ { "name": "contoso_item_tag", "mode": "mirror", "unlink": 1, "failed": 0,
|
|
561
|
+
"changes": [ { "action": "unlink", "from": "0d3f2c1e-5b7a-4c1d-9e2f-3a4b5c6d7e8f", "to": "1cb74a92-6a9a-48ba-a9b1-98577681c593", "reason": "mirror" } ] } ],
|
|
562
|
+
"failures": [],
|
|
563
|
+
"warnings": []
|
|
564
|
+
}
|
|
565
|
+
```
|
|
566
|
+
|
|
567
|
+
- `reason`: `manifest` (the record or link is in the deletion log) or `mirror` (unmanaged, with the mode `mirror`).
|
|
568
|
+
- A failure: `{ "table": "contoso_item", "action": "create", "key": { "contoso_itemid": "f1e2d3c4-b5a6-4978-8a9b-0c1d2e3f4a5b" }, "name": "Basic Panel 05002", "status": 400, "code": "0x8004431a", "error": "A validation error occurred. ..." }`.
|
|
569
|
+
|
|
570
|
+
`convert`: `source`, `data`, `cmt_schema`, `folder`, `tables[]` (`table`, `schema`, `file`, `records`, `columns`), `relationships[]` (`name`, `file`, `links`), `warnings`. `init`: `folder`, `added[]` (`table`, `file`, `match`, `columns`), `existing[]`, `lookups[]` (lookup targets with no schema file, and a note).
|
|
571
|
+
|
|
572
|
+
## Errors
|
|
573
|
+
|
|
574
|
+
| Exit | Code | When | What to do |
|
|
575
|
+
|---|---|---|---|
|
|
576
|
+
| 1 | `rows_failed` | Records failed; see `failures`. | Correct the data or the target, run again. |
|
|
577
|
+
| 1 | `unresolved_lookup` | A lookup target is not in the target (no write was sent), or a lookup is not a GUID. | Import the target table too, give the GUID of that environment in an overlay, or `missingLookup: skip`. |
|
|
578
|
+
| 1 | `schema_mismatch` | A data file header differs from its schema file. | Export again from the source, commit, then import. |
|
|
579
|
+
| 2 | `schema_invalid` | A schema file is not valid (or no schema folder/file). | Fix it; `init` makes a first file. |
|
|
580
|
+
| 2 | `data_invalid` | A data file is not valid YAML, is missing, has a record with no GUID or two records with one GUID, a `null` state/status, or a wrong value type; or a deletion log is not valid. | Fix the file, or export again. |
|
|
581
|
+
| 2 | `overlay_required`, `overlay_not_found`, `overlay_invalid` | See Overlays. | Give `--overlay <name>` or `--no-overlay`; fix the overlay file. |
|
|
582
|
+
| 2 | `file_missing`, `file_too_large` | A named file is not in `data/files/<table>`, or is over 40 MB. | Export again, or add the file. |
|
|
583
|
+
| 2 | `file_name_conflict` | The file name pattern gives one path to two files of a table (export). | Add `{$id}` to the pattern, or use columns with unique values. |
|
|
584
|
+
| 2 | `cmt_schema_missing` | `convert` found no CMT schema. | Give `--cmt-schema`. |
|
|
585
|
+
| 2 | `usage` | A flag out of range. A log file that cannot be written is also exit 2. | |
|
|
586
|
+
| 3 | | No token. | Ask the user to sign in (ppdo-cli skill). |
|
|
587
|
+
| 4 | `not_found` | The environment lacks a table, column or relationship of the schema (`convert`: of the CMT schema, before any file is written). | Import the solution first, or fix the schema; for `convert`, use the environment the CMT files came from. |
|
|
588
|
+
| 5 | | Throttling that retries did not clear. | Run again later, or lower `--parallel`. |
|
|
589
|
+
| 6 | `out_of_date`, `changes_pending`, `folder_not_empty` | `--check` found a difference; `convert` target has files. | Expected in a check; run without `--check` (or `--force`). |
|
|
590
|
+
| 7 | `blocked` | The plan needs mode `full` (record deletes, link removals, a bypass), or `dv data delete` has no confirmation (no terminal and no `--yes`, or a person did not type `delete`). | Use a `full` group; confirm, or give `--yes` after a review of `--check`. |
|
|
591
|
+
| 8 | | The identity lacks a privilege. | |
|
|
592
|
+
|
|
593
|
+
## Convert from CMT (`dv data convert`)
|
|
594
|
+
|
|
595
|
+
`<source>` is a `data.xml`, a folder with `data.xml` or `Extracted/data.xml` (the CMT `ReferenceData` folder), or a `.zip` from `pac data export` or the CMT. The CMT schema is `--cmt-schema`, else `data_schema.xml` next to `data.xml`, else `schema.xml` in the source folder (a warning if both differ).
|
|
596
|
+
|
|
597
|
+
`convert` needs an environment (`--env`, usually the development environment the CMT files came from; exit 2 without one). It sends reads only, so a `read-only` group works, and reads the metadata of each CMT table so that the converted folder exports on the first run with no hand edits. A table, column or relationship of the CMT schema that the environment lacks is exit 4 (`not_found`), before any file is written.
|
|
598
|
+
|
|
599
|
+
| CMT | New format |
|
|
600
|
+
|---|---|
|
|
601
|
+
| `<entity>` in the schema | `schema/<table>.yaml` with `nullValues: ignore` and no delete mode, so the import, like the CMT import, never removes a value or a record. |
|
|
602
|
+
| `primaryKey="true"` field | No `match`: each record has the GUID of its CMT `id`. |
|
|
603
|
+
| Other fields | `columns`, in schema order. A column the metadata says cannot be written (calculated, rollup, Power Fx formula) or an activity party column is left out, with a warning naming it (CMT exports it but cannot import it). |
|
|
604
|
+
| Entity `<filter>` (a full `<fetch>` document) | `fetchXmlFilter` with its `<filter>` elements, one per line, indented two spaces a level; replace it with an OData `filter` (e.g. `statecode eq 0`) for readability. |
|
|
605
|
+
| `disableplugins="true"` | `bypass.plugins: all` and a warning (needs mode `full`). |
|
|
606
|
+
| `<relationship manyToMany="true">` (its `name` is the intersect table) | The relationship with that `IntersectEntityName` in the metadata: its `SchemaName` in `relationships`, the file `data/relationships/<schemaname lower>.yaml`, `from`/`to` in metadata order (the CMT pairs turn round when needed). A relationship whose other table is not in the CMT schema (e.g. `account`) is left out, with a warning. |
|
|
607
|
+
| `<record>` | A record; text decoded twice (CMT escapes twice). |
|
|
608
|
+
| `number`, `decimal`, `float`, `money` | A number with no trailing zeros. |
|
|
609
|
+
| `datetime` | A quoted date and time (accepted for date-only columns). |
|
|
610
|
+
| `optionsetvalue`, `state`, `status` | The number (the first export adds the label). |
|
|
611
|
+
| `entityreference` | The GUID with the name as comment; a map with `$table` if the lookup has several targets. |
|
|
612
|
+
| `<m2mrelationship>` | A `from` entry with its `to` list. |
|
|
613
|
+
| A missing field in a record | No line (not changed by the import). |
|
|
614
|
+
| File and image columns | Added to the schema with a warning; not converted. |
|
|
615
|
+
| `annotation.documentbody`, `activitymimeattachment.body` | The bytes are written to `data/files/<table>`; the record has `{ $file }`. |
|
|
616
|
+
|
|
617
|
+
Then: `export --check` against the source (exit 6 is expected: labels, `null` lines, date forms), `export`, review the diff, `import --check` against a test environment (expect few changes; `nulls_ignored` shows values the target has and the source does not). Keep `ReferenceData` until the pipeline imports the new format, and do not let both imports write the same tables (set `DeployData` false or delete `ReferenceData`). Later: add delete modes and overlays (the GUIDs of lookup targets that differ in an environment), export again, and remove `nullValues: ignore` once `import --check` shows the `"to": null` updates are wanted.
|
|
618
|
+
|
|
619
|
+
## Workflows
|
|
620
|
+
|
|
621
|
+
Start a new folder (development environment):
|
|
622
|
+
|
|
623
|
+
```
|
|
624
|
+
ppdo-cli dv data init MySolution/reference-data --table contoso_category,contoso_item,contoso_tag
|
|
625
|
+
ppdo-cli dv data export MySolution/reference-data
|
|
626
|
+
```
|
|
627
|
+
|
|
628
|
+
Then edit the schema files (remove columns that are not configuration; add delete modes, filters, layouts, relationships) and export again. For each lookup that `init` listed in `lookups` whose target has another GUID in a target environment (a currency, a team), add an overlay value. Commit `schema` and `data` together.
|
|
629
|
+
|
|
630
|
+
Change data: change the records in the development environment, `export`, review `git diff`, commit. Or edit the YAML by hand and import it to development first.
|
|
631
|
+
|
|
632
|
+
Deploy (check first, then apply):
|
|
633
|
+
|
|
634
|
+
```
|
|
635
|
+
ppdo-cli dv data import MySolution/reference-data --env test --overlay test --check
|
|
636
|
+
ppdo-cli dv data import MySolution/reference-data --env test --overlay test --log-file logs/reference-data.log
|
|
637
|
+
```
|
|
638
|
+
|
|
639
|
+
Pipeline step, as the deployment's service principal, with the secret only in `PPDEVOPS_CLIENT_SECRET` of the step (no ppdo-cli setup or token cache needed):
|
|
640
|
+
|
|
641
|
+
```
|
|
642
|
+
ppdo-cli dv data import MySolution/reference-data --env https://contoso-test.crm.dynamics.com --client-id <client-id> --tenant <tenant-id> --mode read-write --overlay test --log-file $(Build.ArtifactStagingDirectory)/reference-data.log
|
|
643
|
+
```
|
|
644
|
+
|
|
645
|
+
The import never deletes, so `--mode read-write` is enough for it unless it uses a bypass. Deletes are a separate step after the import, with `--yes` (no person can confirm in a pipeline), mode `full` when it deletes records or removes links, and `--delete-log-file` to keep what was deleted as an artifact:
|
|
646
|
+
|
|
647
|
+
```
|
|
648
|
+
ppdo-cli dv data delete MySolution/reference-data --env https://contoso-test.crm.dynamics.com --client-id <client-id> --tenant <tenant-id> --mode full --yes --delete-log-file $(Build.ArtifactStagingDirectory)/deleted.json
|
|
649
|
+
```
|
|
650
|
+
|
|
651
|
+
Run `dv data delete --check` in the pull request or an earlier stage, so a person sees what the pipeline will delete. Fail the step on any exit code other than 0. Needs Node.js 22.12+, not `pac` or Windows. Run the import after the solution import, so the tables and columns exist; right after it, a new or changed table may need up to about 3 minutes before it accepts bulk requests (the import waits).
|
|
652
|
+
|
|
653
|
+
Check that source control is current (exit 6 if not):
|
|
654
|
+
|
|
655
|
+
```
|
|
656
|
+
ppdo-cli dv data export MySolution/reference-data --check
|
|
657
|
+
```
|
|
658
|
+
|
|
659
|
+
Delete configuration data: delete the record in the development environment, `export` (the record leaves the data file and enters `data/.deleted`), review and merge, and `dv data delete` deletes it in each target with mode `manifest` or `mirror` (needs mode `full`, or `read-write` with `delete.action: deactivate`).
|
|
660
|
+
|
|
661
|
+
## Notes for agents
|
|
662
|
+
|
|
663
|
+
- Run `import --check` and show the user the summary and the warnings before a real import. Say which environment and overlay you will use.
|
|
664
|
+
- Run `dv data delete --check` and show the user each delete before a delete. Never give `--yes` without the user's approval for that run: the confirmation is the user's.
|
|
665
|
+
- Never export from a target environment into the folder: the export writes the base and would replace the source values with that environment's values. Use overlays for differences.
|
|
666
|
+
- After a schema change, export from the source before any import (`schema_mismatch` otherwise).
|
|
667
|
+
- `unmanaged` records are not an error: they exist in the target scope and not in the file. They are deleted only with `mirror`.
|
|
668
|
+
- Do not add a delete mode, `mirror` or a bypass without the user's approval; a delete needs mode `full`, and there is no delete maximum.
|
|
669
|
+
- Scale: 50,000 records with 20,000 links import into empty tables in about 6 minutes; an import with nothing different reads only (1 to 2 minutes for 50,000 records, 13 to 17 s for 5,000). Use `--verbose` or `--log-file` to watch a large run.
|