dsh-d365-kit 0.34.1 → 0.34.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +318 -308
- package/CHANGELOG.zh-CN.md +317 -307
- package/INSTALL.md +94 -94
- package/INSTALL.zh-CN.md +93 -93
- package/LICENSE +21 -21
- package/README.md +336 -327
- package/README.zh-CN.md +336 -327
- package/client.js +1818 -1818
- package/cordis.patch.yml +44 -44
- package/docs/en/architecture.md +172 -172
- package/docs/en/checks.md +184 -184
- package/docs/en/installation.md +151 -151
- package/docs/en/permissions.md +161 -161
- package/docs/en/scenarios.md +202 -202
- package/docs/en/tools.md +665 -665
- package/docs/en/troubleshooting.md +131 -131
- package/docs/zh/architecture.md +172 -172
- package/docs/zh/checks.md +184 -184
- package/docs/zh/installation.md +151 -151
- package/docs/zh/permissions.md +161 -161
- package/docs/zh/scenarios.md +202 -202
- package/docs/zh/tools.md +665 -665
- package/docs/zh/troubleshooting.md +131 -131
- package/examples/prompts.en.md +206 -206
- package/examples/prompts.zh-CN.md +206 -206
- package/examples/waivers.example.json +20 -20
- package/icon.svg +6 -6
- package/index.js +139 -139
- package/lib/auth.js +181 -181
- package/lib/client.js +496 -496
- package/lib/code-check.js +549 -549
- package/lib/config.js +323 -323
- package/lib/contract-history.js +49 -49
- package/lib/contract.js +583 -583
- package/lib/csharp-check.js +493 -493
- package/lib/data-migrate.js +1318 -1318
- package/lib/data-policy.js +78 -78
- package/lib/error-codes.js +80 -80
- package/lib/gitlab.js +161 -161
- package/lib/health.js +205 -205
- package/lib/migrate.js +732 -732
- package/lib/preflight.js +116 -116
- package/lib/refs.js +191 -191
- package/lib/resources.js +320 -320
- package/lib/retention-defaults.js +17 -17
- package/lib/retention.js +384 -384
- package/lib/runtime.js +232 -232
- package/lib/settings-http.js +512 -512
- package/lib/settings-store.js +371 -371
- package/lib/snapshot.js +149 -149
- package/lib/tool-code-check.js +133 -133
- package/lib/tool-configure.js +169 -169
- package/lib/tool-contract.js +376 -376
- package/lib/tool-data-sync.js +559 -559
- package/lib/tool-data-write.js +203 -203
- package/lib/tool-data.js +282 -282
- package/lib/tool-diff.js +484 -484
- package/lib/tool-environments.js +106 -106
- package/lib/tool-flows.js +540 -540
- package/lib/tool-health.js +127 -127
- package/lib/tool-layers.js +497 -497
- package/lib/tool-metadata.js +417 -417
- package/lib/tool-migrate.js +102 -102
- package/lib/tool-refs.js +206 -206
- package/lib/tool-reports.js +299 -299
- package/lib/tool-sync.js +430 -430
- package/lib/tool-write.js +563 -563
- package/lib/waivers.js +83 -83
- package/lib/write-actions.js +1374 -1374
- package/lib/write-policy.js +199 -199
- package/lib/write-store.js +167 -167
- package/locale/en.json +68 -68
- package/locale/zh.json +68 -68
- package/package.json +2 -2
- package/skills/d365-troubleshooting/SKILL.md +115 -115
package/lib/data-migrate.js
CHANGED
|
@@ -1,1318 +1,1318 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Business-data migration between two environments.
|
|
3
|
-
*
|
|
4
|
-
* Two phases, like metadata writes: `plan` only reads and produces a report
|
|
5
|
-
* that states volume, fields, lookup resolution and write options; `execute`
|
|
6
|
-
* writes, and only after a person approved the confirmation card.
|
|
7
|
-
*
|
|
8
|
-
* Identity rules (the product owner's decisions, in order):
|
|
9
|
-
* - mirror is the default: the source GUID becomes the target GUID;
|
|
10
|
-
* - `keepGuid: false` is allowed when asked for; then lookups that point
|
|
11
|
-
* inside the batch are remapped to the ids the target assigned;
|
|
12
|
-
* - a lookup keeps its GUID when the target has that record, otherwise the
|
|
13
|
-
* referenced record is searched by primary name — never invented;
|
|
14
|
-
* - an existing target record with the same GUID is reported and left alone
|
|
15
|
-
* unless the caller explicitly asks for `conflict: overwrite`.
|
|
16
|
-
*
|
|
17
|
-
* The scope is never guessed: a caller must pass `fetchXml`, `filter`, `ids`,
|
|
18
|
-
* or a `since` window. The tool answers with instructions otherwise.
|
|
19
|
-
*/
|
|
20
|
-
import { randomUUID } from 'node:crypto'
|
|
21
|
-
import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'
|
|
22
|
-
import { join } from 'node:path'
|
|
23
|
-
import { D365Error } from './errors.js'
|
|
24
|
-
import { dataRoot } from './snapshot.js'
|
|
25
|
-
import { makeClient, writeArtifact } from './runtime.js'
|
|
26
|
-
import { getPagedWithSelectFallback } from './client.js'
|
|
27
|
-
import { odataString } from './resources.js'
|
|
28
|
-
|
|
29
|
-
const LOOKUP_TYPES = new Set(['Lookup', 'Customer', 'Owner'])
|
|
30
|
-
const SIMPLE_TYPES = new Set([
|
|
31
|
-
'String', 'Memo', 'Integer', 'BigInt', 'Decimal', 'Double', 'Money',
|
|
32
|
-
'Boolean', 'DateTime', 'Picklist', 'State', 'Status', 'Uniqueidentifier',
|
|
33
|
-
'MultiSelectPicklist',
|
|
34
|
-
])
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Attributes that describe bookkeeping rather than the record. `createdon` is
|
|
38
|
-
* carried through `overriddencreatedon`, `modifiedon` cannot be written, and
|
|
39
|
-
* the image columns are not record data.
|
|
40
|
-
*/
|
|
41
|
-
const SYSTEM_ATTRIBUTES = new Set([
|
|
42
|
-
'createdby', 'createdonbehalfby', 'modifiedby', 'modifiedonbehalfby',
|
|
43
|
-
'owningbusinessunit', 'owninguser', 'owningteam',
|
|
44
|
-
'versionnumber', 'importsequencenumber',
|
|
45
|
-
'timezoneruleversionnumber', 'utcconversiontimezonecode',
|
|
46
|
-
'processid', 'stageid', 'traversedpath',
|
|
47
|
-
'entityimage', 'entityimageid', 'entityimage_timestamp', 'entityimage_url',
|
|
48
|
-
'createdon', 'modifiedon', 'overriddencreatedon',
|
|
49
|
-
])
|
|
50
|
-
|
|
51
|
-
const ID = /^[0-9A-Za-z-]{6,64}$/
|
|
52
|
-
const MAX_RECORDS = 5_000
|
|
53
|
-
|
|
54
|
-
export function plansDir(config) {
|
|
55
|
-
return join(dataRoot(config), 'data-sync', 'plans')
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
export function runsDir(config) {
|
|
59
|
-
return join(dataRoot(config), 'data-sync', 'runs')
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
function planPath(config, id) {
|
|
63
|
-
return ID.test(String(id ?? '')) ? join(plansDir(config), `${id}.json`) : undefined
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
export async function saveMigrationPlan(config, plan) {
|
|
67
|
-
const record = {
|
|
68
|
-
...plan,
|
|
69
|
-
id: plan.id ?? randomUUID().slice(0, 12),
|
|
70
|
-
createdAt: new Date().toISOString(),
|
|
71
|
-
status: 'pending',
|
|
72
|
-
}
|
|
73
|
-
const dir = plansDir(config)
|
|
74
|
-
await mkdir(dir, { recursive: true })
|
|
75
|
-
await writeFile(join(dir, `${record.id}.json`), `${JSON.stringify(record, null, 2)}\n`, 'utf8')
|
|
76
|
-
return record
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
export async function readMigrationPlan(config, id) {
|
|
80
|
-
const path = planPath(config, id)
|
|
81
|
-
if (!path) return undefined
|
|
82
|
-
try {
|
|
83
|
-
return JSON.parse(await readFile(path, 'utf8'))
|
|
84
|
-
} catch {
|
|
85
|
-
return undefined
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
export async function deleteMigrationPlan(config, id) {
|
|
90
|
-
const path = planPath(config, id)
|
|
91
|
-
if (!path) return false
|
|
92
|
-
try {
|
|
93
|
-
await rm(path)
|
|
94
|
-
return true
|
|
95
|
-
} catch {
|
|
96
|
-
return false
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
export async function listMigrationPlans(config) {
|
|
101
|
-
const dir = plansDir(config)
|
|
102
|
-
let names = []
|
|
103
|
-
try {
|
|
104
|
-
names = await readdir(dir)
|
|
105
|
-
} catch {
|
|
106
|
-
return []
|
|
107
|
-
}
|
|
108
|
-
const plans = []
|
|
109
|
-
for (const name of names.filter((item) => item.endsWith('.json'))) {
|
|
110
|
-
const plan = await readMigrationPlan(config, name.replace(/\.json$/, ''))
|
|
111
|
-
if (plan) plans.push(plan)
|
|
112
|
-
}
|
|
113
|
-
return plans.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
export async function setMigrationPlanStatus(config, id, status, extra = {}) {
|
|
117
|
-
const plan = await readMigrationPlan(config, id)
|
|
118
|
-
if (!plan) return undefined
|
|
119
|
-
const next = { ...plan, ...extra, status, updatedAt: new Date().toISOString() }
|
|
120
|
-
await writeFile(planPath(config, id), `${JSON.stringify(next, null, 2)}\n`, 'utf8')
|
|
121
|
-
return next
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
export async function readMigrationRuns(config, limit = 20) {
|
|
125
|
-
const dir = runsDir(config)
|
|
126
|
-
let names = []
|
|
127
|
-
try {
|
|
128
|
-
names = await readdir(dir)
|
|
129
|
-
} catch {
|
|
130
|
-
return []
|
|
131
|
-
}
|
|
132
|
-
const runs = []
|
|
133
|
-
for (const name of names.filter((item) => item.endsWith('.json')).sort().reverse().slice(0, limit)) {
|
|
134
|
-
try {
|
|
135
|
-
runs.push(JSON.parse(await readFile(join(dir, name), 'utf8')))
|
|
136
|
-
} catch {
|
|
137
|
-
// A half-written run file is skipped rather than failing the listing.
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
return runs
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
// ---------------------------------------------------------------- metadata --
|
|
144
|
-
|
|
145
|
-
/** Chunk an array; OData `or` chains and request bodies both need a bound. */
|
|
146
|
-
function chunk(items, size) {
|
|
147
|
-
const out = []
|
|
148
|
-
for (let index = 0; index < items.length; index += size) out.push(items.slice(index, index + size))
|
|
149
|
-
return out
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
function envOf(config, key, label) {
|
|
153
|
-
const env = config.environments[key]
|
|
154
|
-
if (!env) {
|
|
155
|
-
const available = Object.keys(config.environments).join(', ')
|
|
156
|
-
throw new D365Error(`${label}环境不存在:${key || '(未指定)'}`, {
|
|
157
|
-
code: 'unknown-environment',
|
|
158
|
-
hint: `可用:${available}`,
|
|
159
|
-
})
|
|
160
|
-
}
|
|
161
|
-
return env
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
/**
|
|
165
|
-
* Read one entity's set name, primary id and primary name attribute.
|
|
166
|
-
*
|
|
167
|
-
* Accepts either the logical name (`account`) or the set name (`accounts`),
|
|
168
|
-
* because both spellings appear in user requests.
|
|
169
|
-
*/
|
|
170
|
-
export async function readEntityInfo(client, entity, signal) {
|
|
171
|
-
const candidate = String(entity ?? '').trim()
|
|
172
|
-
if (!candidate) throw new D365Error('entity 不能为空', { code: 'missing-entity' })
|
|
173
|
-
const select = 'LogicalName,EntitySetName,PrimaryIdAttribute,PrimaryNameAttribute'
|
|
174
|
-
|
|
175
|
-
const attempts = [
|
|
176
|
-
`LogicalName eq '${odataString(candidate)}' or EntitySetName eq '${odataString(candidate)}'`,
|
|
177
|
-
`LogicalName eq '${odataString(candidate)}'`,
|
|
178
|
-
]
|
|
179
|
-
for (const filter of attempts) {
|
|
180
|
-
try {
|
|
181
|
-
const page = await getPagedWithSelectFallback(client, 'EntityDefinitions', { $select: select, $filter: filter }, 2, signal)
|
|
182
|
-
const row = page?.page?.records?.[0]
|
|
183
|
-
if (row) {
|
|
184
|
-
return {
|
|
185
|
-
logical: row.LogicalName,
|
|
186
|
-
set: row.EntitySetName ?? `${row.LogicalName}s`,
|
|
187
|
-
idField: row.PrimaryIdAttribute ?? `${row.LogicalName}id`,
|
|
188
|
-
nameField: row.PrimaryNameAttribute,
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
} catch (error) {
|
|
192
|
-
// A rejected filter shape is worth one narrower retry; a 401 or a timeout
|
|
193
|
-
// is not, and must surface as itself.
|
|
194
|
-
if (error?.status !== 400) throw error
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
throw new D365Error(`实体不存在:${candidate}`, {
|
|
198
|
-
code: 'unknown-entity',
|
|
199
|
-
hint: '传实体逻辑名(account)或集合名(accounts);数据库里没有这张表的话,先确认它随解决方案导入到了该环境。',
|
|
200
|
-
})
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
/** Read the attribute catalogue; `$top` is rejected by this endpoint. */
|
|
204
|
-
async function readAttributes(client, logical, signal) {
|
|
205
|
-
const page = await getPagedWithSelectFallback(client, `EntityDefinitions(LogicalName='${odataString(logical)}')/Attributes`, {
|
|
206
|
-
__noTop: true,
|
|
207
|
-
$select: 'LogicalName,AttributeType,IsValidForCreate,IsValidForRead',
|
|
208
|
-
}, Infinity, signal)
|
|
209
|
-
const targets = await readLookupTargets(client, logical, signal)
|
|
210
|
-
const navigation = await readNavigationNames(client, logical, signal)
|
|
211
|
-
return (page?.page?.records ?? []).map((row) => ({
|
|
212
|
-
name: row.LogicalName,
|
|
213
|
-
type: row.AttributeType,
|
|
214
|
-
writable: row.IsValidForCreate !== false,
|
|
215
|
-
targets: targets.get(row.LogicalName) ?? [],
|
|
216
|
-
navigation: navigation.get(String(row.LogicalName).toLowerCase()) ?? null,
|
|
217
|
-
}))
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
/**
|
|
221
|
-
* The name a lookup is bound by is the *navigation property* name, which is not
|
|
222
|
-
* always the attribute's logical name: a custom relationship can keep
|
|
223
|
-
* `contoso_approverid` as the column while the navigation property is
|
|
224
|
-
* `contoso_ApproverId`. Binding the logical name then fails with 0x80048d19
|
|
225
|
-
* ("An undeclared property ... which only has property annotations in the
|
|
226
|
-
* payload"). One cast request per entity maps logical name -> navigation name.
|
|
227
|
-
*/
|
|
228
|
-
async function readNavigationNames(client, logical, signal) {
|
|
229
|
-
const map = new Map()
|
|
230
|
-
try {
|
|
231
|
-
const page = await getPagedWithSelectFallback(
|
|
232
|
-
client,
|
|
233
|
-
'RelationshipDefinitions/Microsoft.Dynamics.CRM.OneToManyRelationshipMetadata',
|
|
234
|
-
{
|
|
235
|
-
__noTop: true,
|
|
236
|
-
$select: 'ReferencingAttribute,ReferencingEntityNavigationPropertyName',
|
|
237
|
-
$filter: `ReferencingEntity eq '${odataString(logical)}'`,
|
|
238
|
-
},
|
|
239
|
-
Infinity,
|
|
240
|
-
signal,
|
|
241
|
-
)
|
|
242
|
-
for (const row of page?.page?.records ?? []) {
|
|
243
|
-
const attribute = String(row.ReferencingAttribute ?? '').toLowerCase()
|
|
244
|
-
const navigation = row.ReferencingEntityNavigationPropertyName
|
|
245
|
-
if (attribute && navigation) map.set(attribute, String(navigation))
|
|
246
|
-
}
|
|
247
|
-
} catch (error) {
|
|
248
|
-
// Relationship metadata can be gated on some orgs; without it the logical
|
|
249
|
-
// name is still the common case, so this stays a soft failure.
|
|
250
|
-
if (error?.status === 401 || error?.status === 403) throw error
|
|
251
|
-
}
|
|
252
|
-
return map
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
/**
|
|
256
|
-
* Lookup targets live on `LookupAttributeMetadata`, not on the base
|
|
257
|
-
* `AttributeMetadata` the collection returns: selecting `Targets` there fails
|
|
258
|
-
* with 0x80060888 ("Could not find a property named 'Targets'"). One cast
|
|
259
|
-
* request per entity gets the whole map, so each lookup knows which table(s) it
|
|
260
|
-
* may point at.
|
|
261
|
-
*/
|
|
262
|
-
async function readLookupTargets(client, logical, signal) {
|
|
263
|
-
const map = new Map()
|
|
264
|
-
try {
|
|
265
|
-
const page = await getPagedWithSelectFallback(
|
|
266
|
-
client,
|
|
267
|
-
`EntityDefinitions(LogicalName='${odataString(logical)}')/Attributes/Microsoft.Dynamics.CRM.LookupAttributeMetadata`,
|
|
268
|
-
{ __noTop: true, $select: 'LogicalName,Targets' },
|
|
269
|
-
Infinity,
|
|
270
|
-
signal,
|
|
271
|
-
)
|
|
272
|
-
for (const row of page?.page?.records ?? []) {
|
|
273
|
-
if (row.LogicalName && Array.isArray(row.Targets) && row.Targets.length) map.set(row.LogicalName, row.Targets)
|
|
274
|
-
}
|
|
275
|
-
} catch (error) {
|
|
276
|
-
// A shape this org will not serve is not fatal: lookups still resolve by
|
|
277
|
-
// GUID, and the polymorphic case says so instead of guessing a table.
|
|
278
|
-
if (error?.status === 401 || error?.status === 403) throw error
|
|
279
|
-
}
|
|
280
|
-
return map
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
/**
|
|
284
|
-
* Which source columns travel. Explicit `fields` win; otherwise every writable
|
|
285
|
-
* simple or lookup column except bookkeeping.
|
|
286
|
-
*/
|
|
287
|
-
export function chooseFields(attributes, options = {}) {
|
|
288
|
-
const byName = new Map(attributes.map((attribute) => [attribute.name, attribute]))
|
|
289
|
-
const requested = Array.isArray(options.fields) ? options.fields.map((item) => String(item).trim()).filter(Boolean) : []
|
|
290
|
-
|
|
291
|
-
if (requested.length) {
|
|
292
|
-
const unknown = requested.filter((name) => !byName.has(name))
|
|
293
|
-
if (unknown.length) {
|
|
294
|
-
throw new D365Error(`字段不存在:${unknown.join(', ')}`, {
|
|
295
|
-
code: 'unknown-attribute',
|
|
296
|
-
hint: `${options.entity ?? '该表'} 的可写字段里没有这些名字;先用 d365_metadata resource=attributes 核对逻辑名。`,
|
|
297
|
-
})
|
|
298
|
-
}
|
|
299
|
-
const blocked = requested.filter((name) => {
|
|
300
|
-
const attribute = byName.get(name)
|
|
301
|
-
return !attribute.writable || SYSTEM_ATTRIBUTES.has(name) || name === 'ownerid'
|
|
302
|
-
})
|
|
303
|
-
if (blocked.length) {
|
|
304
|
-
throw new D365Error(`字段不可写:${blocked.join(', ')}`, {
|
|
305
|
-
code: 'readonly-attribute',
|
|
306
|
-
hint: 'createdon/modifiedon/versionnumber 这类字段由平台维护;创建时间会被自动带到 overriddencreatedon。',
|
|
307
|
-
})
|
|
308
|
-
}
|
|
309
|
-
// includeState governs statecode/statuscode for the default field set, so an
|
|
310
|
-
// explicit list must obey it too: otherwise the plan shows them, the preview
|
|
311
|
-
// promises them, and buildBody strips them at write time.
|
|
312
|
-
const explicit = options.includeState === true
|
|
313
|
-
? requested
|
|
314
|
-
: requested.filter((name) => name !== 'statecode' && name !== 'statuscode')
|
|
315
|
-
return explicit.map((name) => toPlanField(byName.get(name)))
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
const includeState = options.includeState === true
|
|
319
|
-
return attributes
|
|
320
|
-
.filter((attribute) => {
|
|
321
|
-
if (SYSTEM_ATTRIBUTES.has(attribute.name)) return false
|
|
322
|
-
if (attribute.name === 'ownerid') return false
|
|
323
|
-
if (attribute.name === 'statecode' || attribute.name === 'statuscode') return includeState
|
|
324
|
-
if (!attribute.writable) return false
|
|
325
|
-
return SIMPLE_TYPES.has(attribute.type) || LOOKUP_TYPES.has(attribute.type)
|
|
326
|
-
})
|
|
327
|
-
.map(toPlanField)
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
function toPlanField(attribute) {
|
|
331
|
-
return {
|
|
332
|
-
name: attribute.name,
|
|
333
|
-
type: attribute.type,
|
|
334
|
-
lookup: LOOKUP_TYPES.has(attribute.type),
|
|
335
|
-
targets: attribute.targets ?? [],
|
|
336
|
-
navigation: attribute.navigation ?? attribute.name,
|
|
337
|
-
navigationKnown: Boolean(attribute.navigation),
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
// ------------------------------------------------------------------- scope --
|
|
342
|
-
|
|
343
|
-
/**
|
|
344
|
-
* Turn the caller's scope into one OData filter, defaulting to the whole table.
|
|
345
|
-
*
|
|
346
|
-
* "Sync today's accounts" is a date window on createdon, "the ones I fixed" is
|
|
347
|
-
* a filter, and a hand-written fetchXml is already exact. With no condition at
|
|
348
|
-
* all the whole table is the documented default; the plan reports the volume so
|
|
349
|
-
* a big table gets a recommendation to narrow it rather than a silent guess.
|
|
350
|
-
*/
|
|
351
|
-
export function describeScope(options = {}) {
|
|
352
|
-
const fetchXml = typeof options.fetchXml === 'string' ? options.fetchXml.trim() : ''
|
|
353
|
-
const filter = typeof options.filter === 'string' ? options.filter.trim() : ''
|
|
354
|
-
const ids = Array.isArray(options.ids) ? options.ids.map((item) => String(item).trim()).filter(Boolean) : []
|
|
355
|
-
const since = options.since === undefined || options.since === null || options.since === '' ? '' : String(options.since).trim()
|
|
356
|
-
const mode = options.mode ?? 'new'
|
|
357
|
-
|
|
358
|
-
if (fetchXml) {
|
|
359
|
-
const entity = /<entity[^>]*\bname\s*=\s*["']([^"']+)["']/i.exec(fetchXml)?.[1]
|
|
360
|
-
return { kind: 'fetchXml', fetchXml, filter: '', ids: [], since: '', mode, description: `fetchXml${entity ? `(${entity})` : ''}` }
|
|
361
|
-
}
|
|
362
|
-
if (ids.length) {
|
|
363
|
-
return { kind: 'ids', fetchXml: '', filter: '', ids, since: '', mode, description: `${ids.length} 个指定 GUID` }
|
|
364
|
-
}
|
|
365
|
-
if (since) {
|
|
366
|
-
const date = new Date(since)
|
|
367
|
-
if (Number.isNaN(date.getTime())) {
|
|
368
|
-
throw new D365Error(`since 不是合法时间:${since}`, {
|
|
369
|
-
code: 'bad-since',
|
|
370
|
-
hint: '用 ISO 时间,例如 2026-11-05T00:00:00+08:00;不带时区会被当作 UTC。',
|
|
371
|
-
})
|
|
372
|
-
}
|
|
373
|
-
const field = mode === 'modified' ? 'modifiedon' : 'createdon'
|
|
374
|
-
const iso = date.toISOString()
|
|
375
|
-
const composed = filter ? `(${filter}) and ${field} ge ${iso}` : `${field} ge ${iso}`
|
|
376
|
-
return {
|
|
377
|
-
kind: 'filter',
|
|
378
|
-
fetchXml: '',
|
|
379
|
-
filter: composed,
|
|
380
|
-
ids: [],
|
|
381
|
-
since: iso,
|
|
382
|
-
mode,
|
|
383
|
-
description: `${field} ge ${iso}${filter ? ` 且 ${filter}` : ''}`,
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
if (filter) {
|
|
387
|
-
return { kind: 'filter', fetchXml: '', filter, ids: [], since: '', mode, description: filter }
|
|
388
|
-
}
|
|
389
|
-
if (mode === 'all') {
|
|
390
|
-
// "Everything" is a scope the caller can state on purpose; the record cap
|
|
391
|
-
// then decides how much of it the plan actually covers, and the plan says so.
|
|
392
|
-
return {
|
|
393
|
-
kind: 'all',
|
|
394
|
-
fetchXml: '',
|
|
395
|
-
filter: '',
|
|
396
|
-
ids: [],
|
|
397
|
-
since: '',
|
|
398
|
-
mode,
|
|
399
|
-
description: '全部记录(没有时间条件,只受 maxRecords 限制)',
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
// No condition at all: the documented default is the whole table. The plan
|
|
403
|
-
// then reports the real volume and suggests a narrower scope when it is big,
|
|
404
|
-
// instead of refusing or silently guessing what "today's" means.
|
|
405
|
-
return {
|
|
406
|
-
kind: 'all',
|
|
407
|
-
fetchXml: '',
|
|
408
|
-
filter: '',
|
|
409
|
-
ids: [],
|
|
410
|
-
since: '',
|
|
411
|
-
mode: 'all',
|
|
412
|
-
implicit: true,
|
|
413
|
-
description: '全部记录(未指定条件,默认整表;受 maxRecords 限制)',
|
|
414
|
-
}
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
/** Read the source rows for one scope. @returns `{ rows, hasMore }`. */
|
|
418
|
-
async function readRows(client, info, scope, fields, maxRecords, signal) {
|
|
419
|
-
const lookupSelect = fields.filter((field) => field.lookup).map((field) => `_${field.name}_value`)
|
|
420
|
-
const plainSelect = fields.filter((field) => !field.lookup).map((field) => field.name)
|
|
421
|
-
const select = [...new Set([
|
|
422
|
-
info.idField,
|
|
423
|
-
...plainSelect,
|
|
424
|
-
...lookupSelect,
|
|
425
|
-
'createdon',
|
|
426
|
-
'statecode',
|
|
427
|
-
'statuscode',
|
|
428
|
-
'_ownerid_value',
|
|
429
|
-
])].join(',')
|
|
430
|
-
|
|
431
|
-
if (scope.kind === 'fetchXml') {
|
|
432
|
-
const page = await client.getPaged(info.set, { fetchXml: scope.fetchXml }, maxRecords, signal)
|
|
433
|
-
return { rows: page.records ?? [], hasMore: page.hasMore, select: null }
|
|
434
|
-
}
|
|
435
|
-
|
|
436
|
-
if (scope.kind === 'ids') {
|
|
437
|
-
const rows = []
|
|
438
|
-
let hasMore = false
|
|
439
|
-
for (const group of chunk(scope.ids, 20)) {
|
|
440
|
-
if (rows.length >= maxRecords) {
|
|
441
|
-
// The id list itself is longer than the read cap: say so instead of
|
|
442
|
-
// reporting a complete read.
|
|
443
|
-
hasMore = true
|
|
444
|
-
break
|
|
445
|
-
}
|
|
446
|
-
const filter = group.map((id) => `${info.idField} eq ${/^[0-9a-f-]{36}$/i.test(id) ? id : `'${odataString(id)}'`}`).join(' or ')
|
|
447
|
-
const page = await client.getPaged(info.set, { $select: select, $filter: `(${filter})` }, maxRecords, signal)
|
|
448
|
-
rows.push(...page.records.slice(0, maxRecords - rows.length))
|
|
449
|
-
if (rows.length >= maxRecords && scope.ids.length > rows.length) hasMore = true
|
|
450
|
-
}
|
|
451
|
-
return { rows, hasMore, select }
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
const params = { $select: select, $orderby: `${info.idField} asc` }
|
|
455
|
-
if (scope.filter) params.$filter = scope.filter
|
|
456
|
-
const page = await client.getPaged(info.set, params, maxRecords, signal)
|
|
457
|
-
return { rows: page.records, hasMore: page.hasMore, select }
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
// ------------------------------------------------------- lookup resolution --
|
|
461
|
-
|
|
462
|
-
function singleKeyed(client, path, params, signal) {
|
|
463
|
-
return client.requestJson(client.buildUrl(path, params), { signal })
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
async function tryKeyed(client, path, params, signal) {
|
|
467
|
-
try {
|
|
468
|
-
return await singleKeyed(client, path, params, signal)
|
|
469
|
-
} catch (error) {
|
|
470
|
-
if (error?.status === 404) return undefined
|
|
471
|
-
throw error
|
|
472
|
-
}
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
/**
|
|
476
|
-
* The primary-name attribute of a referenced entity, resolved per environment
|
|
477
|
-
* (a customised set name differs between orgs).
|
|
478
|
-
*/
|
|
479
|
-
async function entityInfoFor(client, logical, cache, signal) {
|
|
480
|
-
const key = String(logical).toLowerCase()
|
|
481
|
-
if (cache.entities.has(key)) return cache.entities.get(key)
|
|
482
|
-
const info = await readEntityInfo(client, logical, signal).catch(() => null)
|
|
483
|
-
cache.entities.set(key, info)
|
|
484
|
-
return info
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
/** Names for a set of ids, chunked; missing rows are simply absent. */
|
|
488
|
-
async function readNames(client, info, field, ids, signal) {
|
|
489
|
-
const names = new Map()
|
|
490
|
-
if (!info?.nameField) return names
|
|
491
|
-
for (const group of chunk(ids, 20)) {
|
|
492
|
-
const filter = group.map((id) => `${info.idField} eq ${id}`).join(' or ')
|
|
493
|
-
const page = await client.getPaged(info.set, { $select: `${info.idField},${info.nameField}`, $filter: `(${filter})` }, group.length + 1, signal)
|
|
494
|
-
for (const row of page.records ?? []) names.set(String(row[info.idField]).toLowerCase(), row[info.nameField])
|
|
495
|
-
}
|
|
496
|
-
return names
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
/** Which of these ids already exist in the target. */
|
|
500
|
-
async function existingIds(client, info, ids, signal) {
|
|
501
|
-
const found = new Set()
|
|
502
|
-
for (const group of chunk(ids, 20)) {
|
|
503
|
-
const filter = group.map((id) => `${info.idField} eq ${id}`).join(' or ')
|
|
504
|
-
const page = await client.getPaged(info.set, { $select: info.idField, $filter: `(${filter})` }, group.length + 1, signal)
|
|
505
|
-
for (const row of page.records ?? []) found.add(String(row[info.idField]).toLowerCase())
|
|
506
|
-
}
|
|
507
|
-
return found
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
/** Exact primary-name match in the target; `>=2` hits are ambiguous, not a match. */
|
|
511
|
-
async function findByName(client, info, name, signal) {
|
|
512
|
-
if (!info?.nameField || name === undefined || name === null || String(name).trim() === '') return { matches: [] }
|
|
513
|
-
const page = await client.getPaged(info.set, {
|
|
514
|
-
$select: `${info.idField},${info.nameField}`,
|
|
515
|
-
$filter: `${info.nameField} eq '${odataString(String(name))}'`,
|
|
516
|
-
}, 3, signal)
|
|
517
|
-
return { matches: page.records ?? [] }
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
/**
|
|
521
|
-
* Which entity actually holds each id in the source.
|
|
522
|
-
*
|
|
523
|
-
* `customerid` and `ownerid` point at more than one table, and matching a
|
|
524
|
-
* contact's name against accounts because the account lookup happened to be
|
|
525
|
-
* tried first would silently attach the wrong record. The owning table is
|
|
526
|
-
* therefore read from the source, not guessed from the attribute's Targets.
|
|
527
|
-
*/
|
|
528
|
-
async function typeOfIds(client, candidates, ids, signal) {
|
|
529
|
-
const found = new Map()
|
|
530
|
-
const pending = () => ids.filter((id) => !found.has(id.toLowerCase()))
|
|
531
|
-
for (const logical of candidates) {
|
|
532
|
-
if (!pending().length) break
|
|
533
|
-
const info = await readEntityInfo(client, logical, signal).catch(() => null)
|
|
534
|
-
if (!info) continue
|
|
535
|
-
for (const group of chunk(pending(), 20)) {
|
|
536
|
-
const filter = group.map((id) => `${info.idField} eq ${id}`).join(' or ')
|
|
537
|
-
const page = await client.getPaged(info.set, { $select: info.idField, $filter: `(${filter})` }, group.length + 1, signal)
|
|
538
|
-
for (const row of page.records ?? []) found.set(String(row[info.idField]).toLowerCase(), logical)
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
return found
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
/**
|
|
545
|
-
* Resolve every lookup value of every row against the target.
|
|
546
|
-
*
|
|
547
|
-
* Resolution order is fixed: same GUID, then primary-name match, then missing.
|
|
548
|
-
* Nothing is written here; the plan only records what would happen.
|
|
549
|
-
*/
|
|
550
|
-
async function resolveLookups(sourceClient, targetClient, sourceCache, targetCache, rows, fields, options, signal) {
|
|
551
|
-
const lookups = fields.filter((field) => field.lookup)
|
|
552
|
-
/** key `${rowId}|${attribute}` -> resolution, plus `*|attr|id` for the shared lookups */
|
|
553
|
-
const results = new Map()
|
|
554
|
-
const summary = []
|
|
555
|
-
|
|
556
|
-
for (const field of lookups) {
|
|
557
|
-
const values = new Map()
|
|
558
|
-
for (const row of rows) {
|
|
559
|
-
const value = row[`_${field.name}_value`]
|
|
560
|
-
if (value) values.set(String(row[sourceCache.info.idField]), String(value))
|
|
561
|
-
}
|
|
562
|
-
const entry = {
|
|
563
|
-
attribute: field.name,
|
|
564
|
-
navigation: field.navigation,
|
|
565
|
-
targets: field.targets,
|
|
566
|
-
filled: values.size,
|
|
567
|
-
sameGuid: 0,
|
|
568
|
-
byName: 0,
|
|
569
|
-
missing: 0,
|
|
570
|
-
ambiguous: 0,
|
|
571
|
-
empty: rows.length - values.size,
|
|
572
|
-
missingSamples: [],
|
|
573
|
-
}
|
|
574
|
-
if (!values.size) {
|
|
575
|
-
summary.push(entry)
|
|
576
|
-
continue
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
const distinct = [...new Set(values.values())]
|
|
580
|
-
const single = field.targets.length === 1 ? field.targets[0] : null
|
|
581
|
-
const types = single
|
|
582
|
-
? new Map(distinct.map((id) => [id.toLowerCase(), single]))
|
|
583
|
-
: await typeOfIds(sourceClient, field.targets, distinct, signal)
|
|
584
|
-
|
|
585
|
-
// Group by the table that owns the value, so each id is resolved once.
|
|
586
|
-
const groups = new Map()
|
|
587
|
-
for (const id of distinct) {
|
|
588
|
-
const logical = types.get(id.toLowerCase())
|
|
589
|
-
if (!logical) {
|
|
590
|
-
const reason = field.targets.length
|
|
591
|
-
? '源环境里找不到这条被引用记录(可能已被删除)'
|
|
592
|
-
: '元数据没有给出这个 lookup 的目标表,无法判断它指向哪张表'
|
|
593
|
-
results.set(`*|${field.name}|${id}`, { how: 'missing', sourceId: id, reason })
|
|
594
|
-
continue
|
|
595
|
-
}
|
|
596
|
-
if (!groups.has(logical)) groups.set(logical, [])
|
|
597
|
-
groups.get(logical).push(id)
|
|
598
|
-
}
|
|
599
|
-
|
|
600
|
-
for (const [logical, ids] of groups) {
|
|
601
|
-
const sourceInfo = await entityInfoFor(sourceClient, logical, sourceCache, signal)
|
|
602
|
-
const targetInfo = await entityInfoFor(targetClient, logical, targetCache, signal)
|
|
603
|
-
if (!sourceInfo || !targetInfo) {
|
|
604
|
-
for (const id of ids) {
|
|
605
|
-
results.set(`*|${field.name}|${id}`, { how: 'missing', sourceId: id, reason: `目标环境没有 ${logical} 这张表` })
|
|
606
|
-
}
|
|
607
|
-
continue
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
const names = options.resolveLookups === false ? new Map() : await readNames(sourceClient, sourceInfo, sourceInfo.idField, ids, signal)
|
|
611
|
-
const present = await existingIds(targetClient, targetInfo, ids, signal)
|
|
612
|
-
|
|
613
|
-
for (const id of ids) {
|
|
614
|
-
if (present.has(id.toLowerCase())) {
|
|
615
|
-
// Same GUID in both orgs: a mirror-copy can carry the value as-is.
|
|
616
|
-
results.set(`*|${field.name}|${id}`, { how: 'same-guid', targetId: id, targetSet: targetInfo.set, sourceId: id, name: names.get(id.toLowerCase()) })
|
|
617
|
-
continue
|
|
618
|
-
}
|
|
619
|
-
const name = names.get(id.toLowerCase())
|
|
620
|
-
if (!name || options.resolveLookups === false) {
|
|
621
|
-
results.set(`*|${field.name}|${id}`, {
|
|
622
|
-
how: 'missing',
|
|
623
|
-
sourceId: id,
|
|
624
|
-
name,
|
|
625
|
-
reason: options.resolveLookups === false ? '本次关闭了 lookup 名称解析(includeLookups=false)' : '源环境读不到该引用的名称',
|
|
626
|
-
})
|
|
627
|
-
continue
|
|
628
|
-
}
|
|
629
|
-
const found = await findByName(targetClient, targetInfo, name, signal)
|
|
630
|
-
if (found.matches.length === 1) {
|
|
631
|
-
results.set(`*|${field.name}|${id}`, {
|
|
632
|
-
how: 'by-name',
|
|
633
|
-
targetId: String(found.matches[0][targetInfo.idField]),
|
|
634
|
-
targetSet: targetInfo.set,
|
|
635
|
-
sourceId: id,
|
|
636
|
-
name,
|
|
637
|
-
})
|
|
638
|
-
} else if (found.matches.length > 1) {
|
|
639
|
-
results.set(`*|${field.name}|${id}`, { how: 'ambiguous', sourceId: id, name, reason: `目标里同名 ${logical} 有 ${found.matches.length} 条` })
|
|
640
|
-
} else {
|
|
641
|
-
results.set(`*|${field.name}|${id}`, { how: 'missing', sourceId: id, name, reason: `目标里按名称找不到对应的 ${logical}` })
|
|
642
|
-
}
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
for (const row of rows) {
|
|
647
|
-
const rowId = String(row[sourceCache.info.idField])
|
|
648
|
-
const value = values.get(rowId)
|
|
649
|
-
if (!value) {
|
|
650
|
-
results.set(`${rowId}|${field.name}`, { how: 'empty' })
|
|
651
|
-
continue
|
|
652
|
-
}
|
|
653
|
-
const resolution = results.get(`*|${field.name}|${value}`) ?? { how: 'missing', sourceId: value, reason: '未能解析' }
|
|
654
|
-
results.set(`${rowId}|${field.name}`, resolution)
|
|
655
|
-
if (resolution.how === 'same-guid') entry.sameGuid += 1
|
|
656
|
-
else if (resolution.how === 'by-name') entry.byName += 1
|
|
657
|
-
else if (resolution.how === 'ambiguous') {
|
|
658
|
-
entry.ambiguous += 1
|
|
659
|
-
if (entry.missingSamples.length < 5) entry.missingSamples.push({ sourceId: value, name: resolution.name, reason: resolution.reason })
|
|
660
|
-
} else if (resolution.how === 'missing') {
|
|
661
|
-
entry.missing += 1
|
|
662
|
-
if (entry.missingSamples.length < 5) entry.missingSamples.push({ sourceId: value, name: resolution.name, reason: resolution.reason })
|
|
663
|
-
}
|
|
664
|
-
}
|
|
665
|
-
summary.push(entry)
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
return { results, summary }
|
|
669
|
-
}
|
|
670
|
-
|
|
671
|
-
// ------------------------------------------------------------------ owners --
|
|
672
|
-
|
|
673
|
-
/** Who owned the record in the source, and who should own it in the target. */
|
|
674
|
-
async function resolveOwners(sourceClient, targetClient, rows, options, signal) {
|
|
675
|
-
const ownerIds = [...new Set(rows.map((row) => row._ownerid_value).filter(Boolean).map(String))]
|
|
676
|
-
const map = new Map()
|
|
677
|
-
let mapped = 0
|
|
678
|
-
let fallback = 0
|
|
679
|
-
|
|
680
|
-
for (const id of ownerIds) {
|
|
681
|
-
const user = await tryKeyed(sourceClient, `systemusers(${id})`, { $select: 'domainname,internalemailaddress,fullname' }, signal)
|
|
682
|
-
const team = user ? undefined : await tryKeyed(sourceClient, `teams(${id})`, { $select: 'name,emailaddress' }, signal)
|
|
683
|
-
if (!user && !team) {
|
|
684
|
-
map.set(id, { how: 'unknown', sourceId: id })
|
|
685
|
-
continue
|
|
686
|
-
}
|
|
687
|
-
if (options.ownerMode === 'keep') {
|
|
688
|
-
map.set(id, { how: 'keep', sourceId: id, targetId: id, set: user ? 'systemusers' : 'teams' })
|
|
689
|
-
continue
|
|
690
|
-
}
|
|
691
|
-
if (options.ownerMode === 'caller') {
|
|
692
|
-
map.set(id, { how: 'caller', sourceId: id })
|
|
693
|
-
fallback += 1
|
|
694
|
-
continue
|
|
695
|
-
}
|
|
696
|
-
const key = user ? (user.domainname ?? user.internalemailaddress) : (team.emailaddress ?? team.name)
|
|
697
|
-
let hit
|
|
698
|
-
if (user && key) {
|
|
699
|
-
const page = await targetClient.getPaged('systemusers', {
|
|
700
|
-
$select: 'systemuserid,domainname',
|
|
701
|
-
$filter: `domainname eq '${odataString(String(key))}' or internalemailaddress eq '${odataString(String(key))}'`,
|
|
702
|
-
}, 2, signal)
|
|
703
|
-
hit = page.records?.[0]
|
|
704
|
-
if (hit) map.set(id, { how: 'mapped', sourceId: id, targetId: hit.systemuserid, set: 'systemusers', key })
|
|
705
|
-
} else if (team && key) {
|
|
706
|
-
const page = await targetClient.getPaged('teams', {
|
|
707
|
-
$select: 'teamid,name',
|
|
708
|
-
$filter: `name eq '${odataString(String(key))}'`,
|
|
709
|
-
}, 2, signal)
|
|
710
|
-
hit = page.records?.[0]
|
|
711
|
-
if (hit) map.set(id, { how: 'mapped', sourceId: id, targetId: hit.teamid, set: 'teams', key })
|
|
712
|
-
}
|
|
713
|
-
if (hit) {
|
|
714
|
-
mapped += 1
|
|
715
|
-
} else {
|
|
716
|
-
map.set(id, { how: 'fallback', sourceId: id, key: key ?? id })
|
|
717
|
-
fallback += 1
|
|
718
|
-
}
|
|
719
|
-
}
|
|
720
|
-
|
|
721
|
-
return { map, distinct: ownerIds.length, mapped, fallback }
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
// -------------------------------------------------------------------- plan --
|
|
725
|
-
|
|
726
|
-
function numberArg(value, fallback, min, max) {
|
|
727
|
-
const number = Number(value)
|
|
728
|
-
if (!Number.isFinite(number)) return fallback
|
|
729
|
-
return Math.max(min, Math.min(max, number))
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
/**
|
|
733
|
-
* Build a migration plan. Reads both environments; writes nothing.
|
|
734
|
-
*/
|
|
735
|
-
export async function buildPlan(config, args = {}, signal) {
|
|
736
|
-
const sourceKey = String(args.source ?? '').trim()
|
|
737
|
-
const targetKey = String(args.target ?? '').trim()
|
|
738
|
-
if (!sourceKey || !targetKey) {
|
|
739
|
-
throw new D365Error('需要 source 和 target', {
|
|
740
|
-
code: 'missing-environment',
|
|
741
|
-
hint: `例如 source="dev" target="sit";可用:${Object.keys(config.environments).join(', ')}`,
|
|
742
|
-
})
|
|
743
|
-
}
|
|
744
|
-
if (sourceKey === targetKey) {
|
|
745
|
-
throw new D365Error('source 和 target 是同一个环境', { code: 'same-environment', hint: '迁移的两个环境必须不同。' })
|
|
746
|
-
}
|
|
747
|
-
const sourceEnv = envOf(config, sourceKey, '源')
|
|
748
|
-
const targetEnv = envOf(config, targetKey, '目标')
|
|
749
|
-
const sourceClient = makeClient(sourceEnv, config)
|
|
750
|
-
const targetClient = makeClient(targetEnv, config)
|
|
751
|
-
|
|
752
|
-
const scope = describeScope(args)
|
|
753
|
-
const maxRecords = numberArg(args.maxRecords, 500, 1, MAX_RECORDS)
|
|
754
|
-
const options = {
|
|
755
|
-
keepGuid: true,
|
|
756
|
-
resolveLookups: args.includeLookups !== false,
|
|
757
|
-
ownerMode: ['map', 'keep', 'caller'].includes(args.ownerMode) ? args.ownerMode : 'map',
|
|
758
|
-
conflict: ['report', 'skip', 'overwrite'].includes(args.conflict) ? args.conflict : 'report',
|
|
759
|
-
onMissingLookup: ['report', 'null', 'fail'].includes(args.onMissingLookup) ? args.onMissingLookup : 'report',
|
|
760
|
-
bypassPlugins: args.bypassPlugins !== false,
|
|
761
|
-
preserveCreatedOn: args.preserveCreatedOn !== false,
|
|
762
|
-
suppressDuplicateDetection: args.suppressDuplicateDetection !== false,
|
|
763
|
-
includeState: args.includeState === true,
|
|
764
|
-
concurrency: numberArg(args.concurrency, 4, 1, 8),
|
|
765
|
-
batchSize: numberArg(args.batchSize, 50, 10, 500),
|
|
766
|
-
}
|
|
767
|
-
|
|
768
|
-
const sourceCache = { info: await readEntityInfo(sourceClient, args.entity, signal), entities: new Map() }
|
|
769
|
-
const targetCache = { info: await readEntityInfo(targetClient, args.entity, signal), entities: new Map() }
|
|
770
|
-
sourceCache.info.attributes = await readAttributes(sourceClient, sourceCache.info.logical, signal)
|
|
771
|
-
|
|
772
|
-
const fields = chooseFields(sourceCache.info.attributes, {
|
|
773
|
-
fields: args.fields,
|
|
774
|
-
includeState: options.includeState,
|
|
775
|
-
entity: sourceCache.info.logical,
|
|
776
|
-
})
|
|
777
|
-
if (!fields.length) {
|
|
778
|
-
throw new D365Error('没有可迁移的字段', { code: 'no-fields', hint: '该表除系统字段外没有可写列;用 fields 显式指定要看/要搬的列。' })
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
// Keeping the source GUID is not a separate toggle to guess at: it follows the
|
|
782
|
-
// field list. A field list that carries the primary key mirrors the record
|
|
783
|
-
// (same GUID); one that leaves it out lets the target assign a new id, and the
|
|
784
|
-
// references inside the batch are remapped. An explicit keepGuid still wins.
|
|
785
|
-
const explicitKeepGuid = typeof args.keepGuid === 'boolean' ? args.keepGuid : undefined
|
|
786
|
-
const carriesId = fields.some((field) => field.name === sourceCache.info.idField)
|
|
787
|
-
options.keepGuid = explicitKeepGuid ?? carriesId
|
|
788
|
-
options.keepGuidBecause = explicitKeepGuid !== undefined
|
|
789
|
-
? 'explicit'
|
|
790
|
-
: carriesId ? 'id-in-fields' : 'no-id-in-fields'
|
|
791
|
-
|
|
792
|
-
const { rows, hasMore, select } = await readRows(sourceClient, sourceCache.info, scope, fields, maxRecords, signal)
|
|
793
|
-
// `hasMore` is reported, never silently dropped: the cap is a guard rail, and
|
|
794
|
-
// the plan says how much it read so a truncated scope cannot look complete.
|
|
795
|
-
const warnings = []
|
|
796
|
-
if (scope.implicit) {
|
|
797
|
-
warnings.push(`没有给迁移条件,按全部记录预演;读到 ${rows.length} 条${hasMore ? '(已到上限,还有更多)' : ''}。`
|
|
798
|
-
+ '数据量大时建议用 since / filter / ids / fetchXml 缩小范围。')
|
|
799
|
-
}
|
|
800
|
-
if (hasMore) {
|
|
801
|
-
warnings.push(`只读了前 ${rows.length} 条(上限 ${maxRecords}),其余未纳入本次计划;要么调大 maxRecords,要么加条件分批迁。`)
|
|
802
|
-
}
|
|
803
|
-
// A lookup is bound through its navigation property name; if relationship
|
|
804
|
-
// metadata did not answer, the logical name is only a guess, and the failure
|
|
805
|
-
// would otherwise show up at write time as 0x80048d19 "undeclared property".
|
|
806
|
-
const unknownNavigation = fields.filter((field) => field.lookup && !field.navigationKnown)
|
|
807
|
-
if (unknownNavigation.length) {
|
|
808
|
-
const sample = unknownNavigation.slice(0, 5).map((field) => field.name).join('、')
|
|
809
|
-
warnings.push(`有 ${unknownNavigation.length} 个 lookup 没查到导航属性名(${sample}${unknownNavigation.length > 5 ? ' 等' : ''}),写入时按逻辑名绑定;若目标返回 0x80048d19「undeclared property」,先确认字段名再缩小 fields 重出计划。`)
|
|
810
|
-
}
|
|
811
|
-
const ids = rows.map((row) => String(row[sourceCache.info.idField]))
|
|
812
|
-
const existing = options.keepGuid ? await existingIds(targetClient, targetCache.info, ids, signal) : new Set()
|
|
813
|
-
const { results, summary: lookupSummary } = await resolveLookups(
|
|
814
|
-
sourceClient, targetClient, sourceCache, targetCache, rows, fields, options, signal,
|
|
815
|
-
)
|
|
816
|
-
const owners = await resolveOwners(sourceClient, targetClient, rows, options, signal)
|
|
817
|
-
|
|
818
|
-
const records = rows.map((row) => {
|
|
819
|
-
const id = String(row[sourceCache.info.idField])
|
|
820
|
-
const values = {}
|
|
821
|
-
const lookups = []
|
|
822
|
-
for (const field of fields) {
|
|
823
|
-
if (field.lookup) {
|
|
824
|
-
const resolution = results.get(`${id}|${field.name}`) ?? { how: 'empty' }
|
|
825
|
-
lookups.push({ attribute: field.name, navigation: field.navigation, ...resolution })
|
|
826
|
-
if (resolution.how === 'same-guid' || resolution.how === 'by-name') values[field.name] = resolution.targetId
|
|
827
|
-
else if (resolution.how === 'empty') values[field.name] = null
|
|
828
|
-
else if (options.onMissingLookup === 'null') values[field.name] = null
|
|
829
|
-
continue
|
|
830
|
-
}
|
|
831
|
-
if (row[field.name] !== undefined) values[field.name] = row[field.name]
|
|
832
|
-
}
|
|
833
|
-
const owner = row._ownerid_value ? owners.map.get(String(row._ownerid_value)) : undefined
|
|
834
|
-
return {
|
|
835
|
-
id,
|
|
836
|
-
action: existing.has(id.toLowerCase()) ? 'conflict' : 'create',
|
|
837
|
-
name: sourceCache.info.nameField ? row[sourceCache.info.nameField] : undefined,
|
|
838
|
-
createdon: row.createdon,
|
|
839
|
-
statecode: row.statecode,
|
|
840
|
-
statuscode: row.statuscode,
|
|
841
|
-
values,
|
|
842
|
-
lookups,
|
|
843
|
-
owner,
|
|
844
|
-
// Whether a record has an unresolved reference is a property of the
|
|
845
|
-
// record, not of the policy: the `fail` guard later counts these, and
|
|
846
|
-
// making it policy-dependent meant `fail` could never see them.
|
|
847
|
-
blocked: lookups.some((lookup) => lookup.how === 'missing' || lookup.how === 'ambiguous'),
|
|
848
|
-
}
|
|
849
|
-
})
|
|
850
|
-
|
|
851
|
-
const conflicts = records.filter((record) => record.action === 'conflict').length
|
|
852
|
-
const blocked = records.filter((record) => record.blocked).length
|
|
853
|
-
const missing = lookupSummary.reduce((sum, entry) => sum + entry.missing + entry.ambiguous, 0)
|
|
854
|
-
|
|
855
|
-
return {
|
|
856
|
-
kind: 'data-sync',
|
|
857
|
-
source: { key: sourceKey, url: sourceEnv.url, entity: sourceCache.info.logical, set: sourceCache.info.set },
|
|
858
|
-
target: { key: targetKey, url: targetEnv.url, entity: targetCache.info.logical, set: targetCache.info.set },
|
|
859
|
-
scope: { ...scope, description: scope.description, select },
|
|
860
|
-
options,
|
|
861
|
-
fields,
|
|
862
|
-
idField: sourceCache.info.idField,
|
|
863
|
-
limits: { maxRecords, read: rows.length, hasMore },
|
|
864
|
-
warnings,
|
|
865
|
-
lookupSummary,
|
|
866
|
-
owners: { distinct: owners.distinct, mapped: owners.mapped, fallback: owners.fallback },
|
|
867
|
-
counts: {
|
|
868
|
-
read: rows.length,
|
|
869
|
-
create: records.filter((record) => record.action === 'create').length,
|
|
870
|
-
conflict: conflicts,
|
|
871
|
-
blocked,
|
|
872
|
-
missingLookups: missing,
|
|
873
|
-
},
|
|
874
|
-
records,
|
|
875
|
-
}
|
|
876
|
-
}
|
|
877
|
-
|
|
878
|
-
// ----------------------------------------------------------------- execute --
|
|
879
|
-
|
|
880
|
-
/** Run `worker` over `items` with a bounded number of in-flight requests. */
|
|
881
|
-
async function mapLimit(items, limit, worker) {
|
|
882
|
-
const results = new Array(items.length)
|
|
883
|
-
let index = 0
|
|
884
|
-
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
885
|
-
while (index < items.length) {
|
|
886
|
-
const current = index
|
|
887
|
-
index += 1
|
|
888
|
-
results[current] = await worker(items[current], current)
|
|
889
|
-
}
|
|
890
|
-
})
|
|
891
|
-
await Promise.all(runners)
|
|
892
|
-
return results
|
|
893
|
-
}
|
|
894
|
-
|
|
895
|
-
/** Which lookups point at another record in the same batch (only matters without GUIDs). */
|
|
896
|
-
function dependencyOrder(records) {
|
|
897
|
-
const bySource = new Map(records.map((record) => [record.id.toLowerCase(), record]))
|
|
898
|
-
const edges = new Map()
|
|
899
|
-
for (const record of records) {
|
|
900
|
-
const needs = new Set()
|
|
901
|
-
for (const lookup of record.lookups) {
|
|
902
|
-
if (lookup.how === 'same-guid' && bySource.has(String(lookup.sourceId).toLowerCase())) needs.add(String(lookup.sourceId).toLowerCase())
|
|
903
|
-
}
|
|
904
|
-
edges.set(record.id.toLowerCase(), needs)
|
|
905
|
-
}
|
|
906
|
-
const ordered = []
|
|
907
|
-
const temporary = new Set()
|
|
908
|
-
const permanent = new Set()
|
|
909
|
-
const visit = (id) => {
|
|
910
|
-
if (permanent.has(id)) return
|
|
911
|
-
if (temporary.has(id)) return // cycle: broken by deferring the lookup, handled later
|
|
912
|
-
temporary.add(id)
|
|
913
|
-
for (const dependency of edges.get(id) ?? []) visit(dependency)
|
|
914
|
-
temporary.delete(id)
|
|
915
|
-
permanent.add(id)
|
|
916
|
-
ordered.push(bySource.get(id))
|
|
917
|
-
}
|
|
918
|
-
for (const record of records) visit(record.id.toLowerCase())
|
|
919
|
-
return ordered
|
|
920
|
-
}
|
|
921
|
-
|
|
922
|
-
function writeHeaders(options) {
|
|
923
|
-
const headers = { Prefer: 'return=representation' }
|
|
924
|
-
if (options.suppressDuplicateDetection) headers['MSCRM.SuppressDuplicateDetection'] = 'true'
|
|
925
|
-
if (options.bypassPlugins) headers['MSCRM.BypassCustomPluginExecution'] = 'true'
|
|
926
|
-
return headers
|
|
927
|
-
}
|
|
928
|
-
|
|
929
|
-
/**
|
|
930
|
-
* Point a lookup at a record.
|
|
931
|
-
*
|
|
932
|
-
* Dataverse's Web API treats a lookup as a navigation property, not a column: a
|
|
933
|
-
* bare GUID (`"contoso_bg": "<guid>"`) is rejected with 0x80048d19 "a 'PrimitiveValue'
|
|
934
|
-
* node with non-null value was found ... a 'PrimitiveValue' node with null value
|
|
935
|
-
* was expected". The value has to travel as `"contoso_bg@odata.bind": "/set(id)"`.
|
|
936
|
-
*
|
|
937
|
-
* The annotation is keyed by the navigation property name, which only usually
|
|
938
|
-
* equals the attribute's logical name — see `readNavigationNames`.
|
|
939
|
-
*/
|
|
940
|
-
function bindLookup(body, lookup, set, id) {
|
|
941
|
-
const attribute = lookup?.attribute
|
|
942
|
-
const navigation = lookup?.navigation || attribute
|
|
943
|
-
if (attribute) delete body[attribute]
|
|
944
|
-
if (navigation && navigation !== attribute) delete body[navigation]
|
|
945
|
-
if (!navigation) return false
|
|
946
|
-
delete body[`${navigation}@odata.bind`]
|
|
947
|
-
if (!set) return false
|
|
948
|
-
body[`${navigation}@odata.bind`] = `/${set}(${id})`
|
|
949
|
-
return true
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
function buildBody(record, info, options) {
|
|
953
|
-
const body = { ...record.values }
|
|
954
|
-
// The primary key only travels through the keepGuid decision: a source GUID
|
|
955
|
-
// left in `values` would make the target reject the create (or worse, collide
|
|
956
|
-
// with a record copied earlier).
|
|
957
|
-
delete body[info.idField]
|
|
958
|
-
if (options.keepGuid) body[info.idField] = record.id
|
|
959
|
-
if (options.preserveCreatedOn && record.createdon) body.overriddencreatedon = record.createdon
|
|
960
|
-
if (options.includeState === false) {
|
|
961
|
-
delete body.statecode
|
|
962
|
-
delete body.statuscode
|
|
963
|
-
}
|
|
964
|
-
// A null lookup stays a literal null (the API accepts that for a navigation
|
|
965
|
-
// property); a resolved one becomes an @odata.bind annotation.
|
|
966
|
-
for (const lookup of record.lookups ?? []) {
|
|
967
|
-
const value = body[lookup.attribute]
|
|
968
|
-
if (value !== null && value !== undefined) bindLookup(body, lookup, lookup.targetSet, value)
|
|
969
|
-
}
|
|
970
|
-
const owner = record.owner
|
|
971
|
-
if (owner && options.ownerMode !== 'caller') {
|
|
972
|
-
if (owner.how === 'mapped' || owner.how === 'keep') {
|
|
973
|
-
bindLookup(body, { attribute: 'ownerid', navigation: 'ownerid' }, owner.set ?? 'systemusers', owner.targetId)
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
return body
|
|
977
|
-
}
|
|
978
|
-
|
|
979
|
-
/**
|
|
980
|
-
* Execute an approved plan.
|
|
981
|
-
*
|
|
982
|
-
* `hooks.onRecord` is called after each record settles so a caller can record
|
|
983
|
-
* progress; the run file is the durable copy.
|
|
984
|
-
*/
|
|
985
|
-
export async function executeMigration(config, plan, options = {}, signal, hooks = {}) {
|
|
986
|
-
const planOptions = { ...plan.options, ...options }
|
|
987
|
-
const targetEnv = envOf(config, plan.target.key, '目标')
|
|
988
|
-
// `targetClient` lets a test drive the write path without an org behind it.
|
|
989
|
-
const targetClient = hooks.targetClient ?? makeClient(targetEnv, config)
|
|
990
|
-
const targetInfo = await readEntityInfo(targetClient, plan.target.entity, signal)
|
|
991
|
-
|
|
992
|
-
// A plan written before navigation names were resolved would bind lookups by
|
|
993
|
-
// their logical name — the exact failure 0x80048d19 this change fixes. Make
|
|
994
|
-
// the user re-preview instead of repeating it silently.
|
|
995
|
-
const staleFields = (plan.fields ?? []).filter((field) => field.lookup && !field.navigation)
|
|
996
|
-
if (staleFields.length) {
|
|
997
|
-
throw new D365Error('这份计划是旧版本生成的,缺少 lookup 的导航属性名', {
|
|
998
|
-
code: 'stale-plan',
|
|
999
|
-
hint: '重新跑一次 plan 再 execute;旧计划会按逻辑名绑定 lookup,目标环境可能返回 0x80048d19「undeclared property」。',
|
|
1000
|
-
})
|
|
1001
|
-
}
|
|
1002
|
-
if (planOptions.onMissingLookup === 'fail') {
|
|
1003
|
-
const blocked = plan.records.filter((record) => record.blocked)
|
|
1004
|
-
if (blocked.length) {
|
|
1005
|
-
throw new D365Error(`有 ${blocked.length} 条记录的引用在目标环境找不到`, {
|
|
1006
|
-
code: 'missing-lookup',
|
|
1007
|
-
hint: '把 onMissingLookup 改成 report(跳过这些记录)或 null(清空这些引用),或先在目标环境补齐被引用记录。',
|
|
1008
|
-
})
|
|
1009
|
-
}
|
|
1010
|
-
}
|
|
1011
|
-
|
|
1012
|
-
const runId = `run-${plan.id}-${randomUUID().slice(0, 6)}`
|
|
1013
|
-
const runFile = join(runsDir(config), `${new Date().toISOString().replace(/[:.]/g, '-')}-${runId}.json`)
|
|
1014
|
-
await mkdir(runsDir(config), { recursive: true })
|
|
1015
|
-
|
|
1016
|
-
const writable = plan.records.filter((record) => {
|
|
1017
|
-
if (record.action === 'conflict' && planOptions.conflict !== 'overwrite') return false
|
|
1018
|
-
if (record.blocked && planOptions.onMissingLookup === 'report') return false
|
|
1019
|
-
return true
|
|
1020
|
-
})
|
|
1021
|
-
const skipped = plan.records.length - writable.length
|
|
1022
|
-
|
|
1023
|
-
// Without GUIDs the target assigns ids, so lookups inside the batch must be
|
|
1024
|
-
// remapped after the create. Ordering keeps referenced records first.
|
|
1025
|
-
const ordered = planOptions.keepGuid ? writable : dependencyOrder(writable)
|
|
1026
|
-
const idMap = new Map()
|
|
1027
|
-
|
|
1028
|
-
const results = await mapLimit(ordered, planOptions.concurrency ?? 4, async (record) => {
|
|
1029
|
-
const body = buildBody(record, targetInfo, planOptions)
|
|
1030
|
-
// A lookup that points at a batch member whose new id is not known yet is
|
|
1031
|
-
// sent later, in a second PATCH pass.
|
|
1032
|
-
const deferred = []
|
|
1033
|
-
if (!planOptions.keepGuid) {
|
|
1034
|
-
for (const lookup of record.lookups) {
|
|
1035
|
-
if (lookup.how !== 'same-guid') continue
|
|
1036
|
-
const mapped = idMap.get(String(lookup.sourceId).toLowerCase())
|
|
1037
|
-
if (mapped) {
|
|
1038
|
-
bindLookup(body, lookup, lookup.targetSet, mapped)
|
|
1039
|
-
} else {
|
|
1040
|
-
delete body[lookup.attribute]
|
|
1041
|
-
delete body[`${lookup.navigation ?? lookup.attribute}@odata.bind`]
|
|
1042
|
-
deferred.push(lookup.attribute)
|
|
1043
|
-
}
|
|
1044
|
-
}
|
|
1045
|
-
}
|
|
1046
|
-
try {
|
|
1047
|
-
if (record.action === 'conflict') {
|
|
1048
|
-
await targetClient.sendJson(`${targetInfo.set}(${record.id})`, {
|
|
1049
|
-
method: 'PATCH',
|
|
1050
|
-
body,
|
|
1051
|
-
headers: { ...writeHeaders(planOptions), 'If-Match': '*' },
|
|
1052
|
-
}, signal)
|
|
1053
|
-
return { id: record.id, action: 'updated', deferred }
|
|
1054
|
-
}
|
|
1055
|
-
const created = await targetClient.sendJson(targetInfo.set, {
|
|
1056
|
-
method: 'POST',
|
|
1057
|
-
body,
|
|
1058
|
-
headers: writeHeaders(planOptions),
|
|
1059
|
-
}, signal)
|
|
1060
|
-
const newId = created?.[targetInfo.idField] ?? record.id
|
|
1061
|
-
idMap.set(record.id.toLowerCase(), String(newId))
|
|
1062
|
-
return { id: record.id, targetId: String(newId), action: 'created', deferred, values: body }
|
|
1063
|
-
} catch (error) {
|
|
1064
|
-
return { id: record.id, action: 'failed', error: String(error?.message ?? error), code: error?.code, status: error?.status }
|
|
1065
|
-
}
|
|
1066
|
-
})
|
|
1067
|
-
|
|
1068
|
-
// Second pass: lookups that pointed at records created later in this run.
|
|
1069
|
-
const deferredRecords = ordered
|
|
1070
|
-
.map((record, index) => ({ record, result: results[index] }))
|
|
1071
|
-
.filter((entry) => entry.result?.action !== 'failed' && entry.result?.deferred?.length)
|
|
1072
|
-
for (const entry of deferredRecords) {
|
|
1073
|
-
const body = {}
|
|
1074
|
-
for (const attribute of entry.result.deferred) {
|
|
1075
|
-
const lookup = entry.record.lookups.find((item) => item.attribute === attribute)
|
|
1076
|
-
const mapped = lookup ? idMap.get(String(lookup.sourceId).toLowerCase()) : undefined
|
|
1077
|
-
if (mapped) bindLookup(body, lookup, lookup.targetSet, mapped)
|
|
1078
|
-
}
|
|
1079
|
-
if (!Object.keys(body).length) continue
|
|
1080
|
-
try {
|
|
1081
|
-
await targetClient.sendJson(`${targetInfo.set}(${entry.result.targetId ?? entry.record.id})`, {
|
|
1082
|
-
method: 'PATCH',
|
|
1083
|
-
body,
|
|
1084
|
-
headers: { ...writeHeaders(planOptions), 'If-Match': '*' },
|
|
1085
|
-
}, signal)
|
|
1086
|
-
} catch (error) {
|
|
1087
|
-
// A failed back-reference write must not be swallowed: the record exists
|
|
1088
|
-
// but points at nothing, which is exactly the state the caller asked to
|
|
1089
|
-
// avoid. Count it as a failed row instead.
|
|
1090
|
-
entry.result.action = 'failed'
|
|
1091
|
-
entry.result.error = `引用回填失败:${String(error?.message ?? error)}`
|
|
1092
|
-
entry.result.code = error?.code
|
|
1093
|
-
entry.result.status = error?.status
|
|
1094
|
-
}
|
|
1095
|
-
}
|
|
1096
|
-
|
|
1097
|
-
const created = results.filter((item) => item.action === 'created')
|
|
1098
|
-
const updated = results.filter((item) => item.action === 'updated')
|
|
1099
|
-
const failed = results.filter((item) => item.action === 'failed')
|
|
1100
|
-
const run = {
|
|
1101
|
-
kind: 'data-sync-run',
|
|
1102
|
-
runId,
|
|
1103
|
-
planId: plan.id,
|
|
1104
|
-
at: new Date().toISOString(),
|
|
1105
|
-
source: plan.source,
|
|
1106
|
-
target: plan.target,
|
|
1107
|
-
options: planOptions,
|
|
1108
|
-
counts: { planned: plan.records.length, written: created.length + updated.length, skipped, failed: failed.length },
|
|
1109
|
-
created: created.map((item) => ({ sourceId: item.id, targetId: item.targetId })),
|
|
1110
|
-
updated: updated.map((item) => item.id),
|
|
1111
|
-
failures: failed,
|
|
1112
|
-
skippedRecords: plan.records
|
|
1113
|
-
.filter((record) => !writable.includes(record))
|
|
1114
|
-
.map((record) => ({
|
|
1115
|
-
id: record.id,
|
|
1116
|
-
reason: record.action === 'conflict' ? '目标已存在同 GUID,按 conflict=report 跳过' : '引用缺失,按 onMissingLookup=report 跳过',
|
|
1117
|
-
})),
|
|
1118
|
-
}
|
|
1119
|
-
await writeFile(runFile, `${JSON.stringify(run, null, 2)}\n`, 'utf8')
|
|
1120
|
-
hooks.onRecord?.(run)
|
|
1121
|
-
run.file = runFile
|
|
1122
|
-
return run
|
|
1123
|
-
}
|
|
1124
|
-
|
|
1125
|
-
/** Write the plan itself as an artifact, so the full record list survives. */
|
|
1126
|
-
export async function writePlanArtifact(config, plan) {
|
|
1127
|
-
return writeArtifact(config, `data-sync-${plan.source.key}-${plan.target.key}`, plan)
|
|
1128
|
-
}
|
|
1129
|
-
|
|
1130
|
-
/* ------------------------------------------------------------------------ *
|
|
1131
|
-
* Update mode: change fields on many existing records in one environment.
|
|
1132
|
-
*
|
|
1133
|
-
* The 377-row case is the reason this exists. `d365_data_write` is
|
|
1134
|
-
* deliberately one record at a time, and 377 confirmation cards is not a
|
|
1135
|
-
* workflow. This reuses the migration plan/confirm/run shape but never
|
|
1136
|
-
* creates records: every id must already exist, and a missing one is
|
|
1137
|
-
* reported instead of turned into an upsert.
|
|
1138
|
-
* ------------------------------------------------------------------------ */
|
|
1139
|
-
|
|
1140
|
-
const MAX_UPDATE_RECORDS = 1000
|
|
1141
|
-
|
|
1142
|
-
/**
|
|
1143
|
-
* Build a bulk-update plan for one environment.
|
|
1144
|
-
*
|
|
1145
|
-
* Only scalar columns are accepted in v1: a lookup value in a PATCH needs the
|
|
1146
|
-
* same navigation-property resolution the migration engine does, and doing it
|
|
1147
|
-
* half-way would produce writes that look right in the plan and fail on the
|
|
1148
|
-
* wire. Lookups point at `d365_data_write` for now.
|
|
1149
|
-
*/
|
|
1150
|
-
export async function buildUpdatePlan(config, args = {}, signal) {
|
|
1151
|
-
const envKey = String(args.environment ?? '').trim()
|
|
1152
|
-
if (!envKey) {
|
|
1153
|
-
throw new D365Error('批量更新需要在 environment 里指定环境', {
|
|
1154
|
-
code: 'missing-environment',
|
|
1155
|
-
hint: '批量更新只改一个环境里的记录;跨环境复制用 plan 模式。',
|
|
1156
|
-
})
|
|
1157
|
-
}
|
|
1158
|
-
const target = config.environments[envKey]
|
|
1159
|
-
if (!target) {
|
|
1160
|
-
throw new D365Error(`环境不存在:${envKey}`, {
|
|
1161
|
-
code: 'unknown-environment',
|
|
1162
|
-
hint: `可用:${Object.keys(config.environments).join(', ')}`,
|
|
1163
|
-
})
|
|
1164
|
-
}
|
|
1165
|
-
const entity = String(args.entity ?? '').trim()
|
|
1166
|
-
if (!entity) throw new D365Error('批量更新需要 entity', { code: 'missing-entity' })
|
|
1167
|
-
|
|
1168
|
-
const ids = [...new Set((args.ids ?? []).map((id) => String(id).trim()).filter(Boolean))]
|
|
1169
|
-
if (!ids.length) {
|
|
1170
|
-
throw new D365Error('批量更新需要显式 ids 清单', {
|
|
1171
|
-
code: 'ids-required',
|
|
1172
|
-
hint: '批量更新不接受 filter 或 fetchXml:改错一批记录没有撤销键。先用 d365_data action=query 查出 id,再把这些 id 传进来。',
|
|
1173
|
-
})
|
|
1174
|
-
}
|
|
1175
|
-
if (ids.length > MAX_UPDATE_RECORDS) {
|
|
1176
|
-
throw new D365Error(`一次最多更新 ${MAX_UPDATE_RECORDS} 条,收到 ${ids.length} 条`, {
|
|
1177
|
-
code: 'too-many-records',
|
|
1178
|
-
hint: '分批执行,每批都重新 plan 并确认一次。',
|
|
1179
|
-
})
|
|
1180
|
-
}
|
|
1181
|
-
const badId = ids.find((id) => !/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/.test(id))
|
|
1182
|
-
if (badId) throw new D365Error(`不是合法 GUID:${badId}`, { code: 'bad-id' })
|
|
1183
|
-
|
|
1184
|
-
const rawValues = args.values && typeof args.values === 'object' ? args.values : {}
|
|
1185
|
-
const columns = Object.keys(rawValues)
|
|
1186
|
-
if (!columns.length) throw new D365Error('批量更新需要 values', { code: 'missing-values' })
|
|
1187
|
-
|
|
1188
|
-
const client = makeClient(target, config)
|
|
1189
|
-
const info = await readEntityInfo(client, entity, signal)
|
|
1190
|
-
const attributes = await readAttributes(client, info.logical, signal)
|
|
1191
|
-
const byName = new Map(attributes.map((attribute) => [attribute.name.toLowerCase(), attribute]))
|
|
1192
|
-
|
|
1193
|
-
const fields = []
|
|
1194
|
-
const normalized = {}
|
|
1195
|
-
for (const column of columns) {
|
|
1196
|
-
const attribute = byName.get(column.toLowerCase())
|
|
1197
|
-
if (!attribute) {
|
|
1198
|
-
throw new D365Error(`实体 ${info.logical} 没有字段 ${column}`, {
|
|
1199
|
-
code: 'unknown-attribute',
|
|
1200
|
-
hint: '先用 d365_metadata resource=attributes 确认字段逻辑名。',
|
|
1201
|
-
})
|
|
1202
|
-
}
|
|
1203
|
-
if (!attribute.writable) {
|
|
1204
|
-
throw new D365Error(`字段 ${column} 不可写`, { code: 'readonly-attribute', hint: '它是一个计算列、汇总列或系统维护的列。' })
|
|
1205
|
-
}
|
|
1206
|
-
if (attribute.lookup) {
|
|
1207
|
-
throw new D365Error(`批量更新暂不支持 lookup 字段 ${column}`, {
|
|
1208
|
-
code: 'lookup-unsupported',
|
|
1209
|
-
hint: 'lookup 需要按导航属性名绑定,用 d365_data_write 单条改更可靠;标量字段可以用批量更新。',
|
|
1210
|
-
})
|
|
1211
|
-
}
|
|
1212
|
-
fields.push({ name: attribute.name, type: attribute.type })
|
|
1213
|
-
// Key the write by the canonical logical name. Dataverse is not
|
|
1214
|
-
// case-sensitive, but the plan's own preview and run record look values up
|
|
1215
|
-
// by `fields[].name`, so a payload spelled differently would print
|
|
1216
|
-
// `undefined` and send a key the plan never showed.
|
|
1217
|
-
normalized[attribute.name] = rawValues[column]
|
|
1218
|
-
}
|
|
1219
|
-
const values = normalized
|
|
1220
|
-
|
|
1221
|
-
// Read what the records look like now: the plan must show before → after,
|
|
1222
|
-
// not just the value the caller wants to write.
|
|
1223
|
-
const names = await readNames(client, info, info.nameField, ids, signal).catch(() => new Map())
|
|
1224
|
-
const existing = new Set()
|
|
1225
|
-
const before = []
|
|
1226
|
-
for (const group of chunk(ids, 20)) {
|
|
1227
|
-
const filter = group.map((id) => `${info.idField} eq ${id}`).join(' or ')
|
|
1228
|
-
const page = await client.getPaged(info.set, {
|
|
1229
|
-
$select: [info.idField, info.nameField, ...fields.map((field) => field.name)].filter(Boolean).join(','),
|
|
1230
|
-
$filter: `(${filter})`,
|
|
1231
|
-
}, group.length + 1, signal)
|
|
1232
|
-
for (const row of page.records ?? []) {
|
|
1233
|
-
const id = String(row[info.idField]).toLowerCase()
|
|
1234
|
-
existing.add(id)
|
|
1235
|
-
before.push({
|
|
1236
|
-
id,
|
|
1237
|
-
name: row[info.nameField ?? ''] ?? names.get(id) ?? null,
|
|
1238
|
-
current: Object.fromEntries(fields.map((field) => [field.name, row[field.name] ?? null])),
|
|
1239
|
-
})
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
|
|
1243
|
-
const missing = ids.filter((id) => !existing.has(id.toLowerCase()))
|
|
1244
|
-
const plan = {
|
|
1245
|
-
kind: 'data-update',
|
|
1246
|
-
capability: 'data.write',
|
|
1247
|
-
environment: envKey,
|
|
1248
|
-
url: target.url,
|
|
1249
|
-
entity: info.logical,
|
|
1250
|
-
set: info.set,
|
|
1251
|
-
idField: info.idField,
|
|
1252
|
-
fields,
|
|
1253
|
-
values,
|
|
1254
|
-
ids,
|
|
1255
|
-
before,
|
|
1256
|
-
missing,
|
|
1257
|
-
options: {
|
|
1258
|
-
bypassPlugins: args.bypassPlugins !== false,
|
|
1259
|
-
suppressDuplicateDetection: args.suppressDuplicateDetection !== false,
|
|
1260
|
-
concurrency: Number.isFinite(args.concurrency) ? Math.max(1, Math.min(8, Number(args.concurrency))) : 4,
|
|
1261
|
-
},
|
|
1262
|
-
counts: { planned: ids.length, existing: existing.size, missing: missing.length },
|
|
1263
|
-
rationale: args.rationale,
|
|
1264
|
-
createdAt: new Date().toISOString(),
|
|
1265
|
-
status: 'pending',
|
|
1266
|
-
}
|
|
1267
|
-
plan.id = `${Date.now().toString(36)}-${randomUUID().slice(0, 5)}`
|
|
1268
|
-
return plan
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
|
-
/** Apply a bulk-update plan, one PATCH per record, bounded concurrency. */
|
|
1272
|
-
export async function executeUpdate(config, plan, options = {}, signal, hooks = {}) {
|
|
1273
|
-
const target = config.environments[plan.environment]
|
|
1274
|
-
if (!target) throw new D365Error(`环境不存在:${plan.environment}`, { code: 'unknown-environment' })
|
|
1275
|
-
const client = hooks.targetClient ?? makeClient(target, config)
|
|
1276
|
-
const planOptions = { ...plan.options, ...options }
|
|
1277
|
-
const runId = `run-${plan.id}-${randomUUID().slice(0, 6)}`
|
|
1278
|
-
const runFile = join(runsDir(config), `${new Date().toISOString().replace(/[:.]/g, '-')}-${runId}.json`)
|
|
1279
|
-
|
|
1280
|
-
const applicable = plan.before.map((row) => row.id)
|
|
1281
|
-
const results = await mapLimit(applicable, planOptions.concurrency ?? 4, async (id) => {
|
|
1282
|
-
const body = { ...plan.values }
|
|
1283
|
-
try {
|
|
1284
|
-
await client.sendJson(`${plan.set}(${id})`, {
|
|
1285
|
-
method: 'PATCH',
|
|
1286
|
-
body,
|
|
1287
|
-
headers: { ...writeHeaders(planOptions), 'If-Match': '*' },
|
|
1288
|
-
}, signal)
|
|
1289
|
-
return { id, ok: true }
|
|
1290
|
-
} catch (error) {
|
|
1291
|
-
return { id, ok: false, error: String(error?.message ?? error) }
|
|
1292
|
-
}
|
|
1293
|
-
})
|
|
1294
|
-
|
|
1295
|
-
const updated = results.filter((item) => item.ok)
|
|
1296
|
-
const failed = results.filter((item) => !item.ok)
|
|
1297
|
-
const run = {
|
|
1298
|
-
kind: 'data-update-run',
|
|
1299
|
-
runId,
|
|
1300
|
-
planId: plan.id,
|
|
1301
|
-
at: new Date().toISOString(),
|
|
1302
|
-
environment: plan.environment,
|
|
1303
|
-
entity: plan.entity,
|
|
1304
|
-
values: plan.values,
|
|
1305
|
-
options: planOptions,
|
|
1306
|
-
counts: { planned: plan.ids.length, applied: applicable.length, updated: updated.length, failed: failed.length, missing: plan.missing.length },
|
|
1307
|
-
updated: updated.map((item) => item.id),
|
|
1308
|
-
failures: failed,
|
|
1309
|
-
missing: plan.missing,
|
|
1310
|
-
}
|
|
1311
|
-
// The very first bulk update would otherwise PATCH every record and then fail
|
|
1312
|
-
// to write its own run file, leaving the plan marked failed with no record.
|
|
1313
|
-
await mkdir(runsDir(config), { recursive: true })
|
|
1314
|
-
await writeFile(runFile, `${JSON.stringify(run, null, 2)}\n`, 'utf8')
|
|
1315
|
-
hooks.onRecord?.(run)
|
|
1316
|
-
run.file = runFile
|
|
1317
|
-
return run
|
|
1318
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Business-data migration between two environments.
|
|
3
|
+
*
|
|
4
|
+
* Two phases, like metadata writes: `plan` only reads and produces a report
|
|
5
|
+
* that states volume, fields, lookup resolution and write options; `execute`
|
|
6
|
+
* writes, and only after a person approved the confirmation card.
|
|
7
|
+
*
|
|
8
|
+
* Identity rules (the product owner's decisions, in order):
|
|
9
|
+
* - mirror is the default: the source GUID becomes the target GUID;
|
|
10
|
+
* - `keepGuid: false` is allowed when asked for; then lookups that point
|
|
11
|
+
* inside the batch are remapped to the ids the target assigned;
|
|
12
|
+
* - a lookup keeps its GUID when the target has that record, otherwise the
|
|
13
|
+
* referenced record is searched by primary name — never invented;
|
|
14
|
+
* - an existing target record with the same GUID is reported and left alone
|
|
15
|
+
* unless the caller explicitly asks for `conflict: overwrite`.
|
|
16
|
+
*
|
|
17
|
+
* The scope is never guessed: a caller must pass `fetchXml`, `filter`, `ids`,
|
|
18
|
+
* or a `since` window. The tool answers with instructions otherwise.
|
|
19
|
+
*/
|
|
20
|
+
import { randomUUID } from 'node:crypto'
|
|
21
|
+
import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'
|
|
22
|
+
import { join } from 'node:path'
|
|
23
|
+
import { D365Error } from './errors.js'
|
|
24
|
+
import { dataRoot } from './snapshot.js'
|
|
25
|
+
import { makeClient, writeArtifact } from './runtime.js'
|
|
26
|
+
import { getPagedWithSelectFallback } from './client.js'
|
|
27
|
+
import { odataString } from './resources.js'
|
|
28
|
+
|
|
29
|
+
const LOOKUP_TYPES = new Set(['Lookup', 'Customer', 'Owner'])
|
|
30
|
+
const SIMPLE_TYPES = new Set([
|
|
31
|
+
'String', 'Memo', 'Integer', 'BigInt', 'Decimal', 'Double', 'Money',
|
|
32
|
+
'Boolean', 'DateTime', 'Picklist', 'State', 'Status', 'Uniqueidentifier',
|
|
33
|
+
'MultiSelectPicklist',
|
|
34
|
+
])
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Attributes that describe bookkeeping rather than the record. `createdon` is
|
|
38
|
+
* carried through `overriddencreatedon`, `modifiedon` cannot be written, and
|
|
39
|
+
* the image columns are not record data.
|
|
40
|
+
*/
|
|
41
|
+
const SYSTEM_ATTRIBUTES = new Set([
|
|
42
|
+
'createdby', 'createdonbehalfby', 'modifiedby', 'modifiedonbehalfby',
|
|
43
|
+
'owningbusinessunit', 'owninguser', 'owningteam',
|
|
44
|
+
'versionnumber', 'importsequencenumber',
|
|
45
|
+
'timezoneruleversionnumber', 'utcconversiontimezonecode',
|
|
46
|
+
'processid', 'stageid', 'traversedpath',
|
|
47
|
+
'entityimage', 'entityimageid', 'entityimage_timestamp', 'entityimage_url',
|
|
48
|
+
'createdon', 'modifiedon', 'overriddencreatedon',
|
|
49
|
+
])
|
|
50
|
+
|
|
51
|
+
const ID = /^[0-9A-Za-z-]{6,64}$/
|
|
52
|
+
const MAX_RECORDS = 5_000
|
|
53
|
+
|
|
54
|
+
export function plansDir(config) {
|
|
55
|
+
return join(dataRoot(config), 'data-sync', 'plans')
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function runsDir(config) {
|
|
59
|
+
return join(dataRoot(config), 'data-sync', 'runs')
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function planPath(config, id) {
|
|
63
|
+
return ID.test(String(id ?? '')) ? join(plansDir(config), `${id}.json`) : undefined
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export async function saveMigrationPlan(config, plan) {
|
|
67
|
+
const record = {
|
|
68
|
+
...plan,
|
|
69
|
+
id: plan.id ?? randomUUID().slice(0, 12),
|
|
70
|
+
createdAt: new Date().toISOString(),
|
|
71
|
+
status: 'pending',
|
|
72
|
+
}
|
|
73
|
+
const dir = plansDir(config)
|
|
74
|
+
await mkdir(dir, { recursive: true })
|
|
75
|
+
await writeFile(join(dir, `${record.id}.json`), `${JSON.stringify(record, null, 2)}\n`, 'utf8')
|
|
76
|
+
return record
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export async function readMigrationPlan(config, id) {
|
|
80
|
+
const path = planPath(config, id)
|
|
81
|
+
if (!path) return undefined
|
|
82
|
+
try {
|
|
83
|
+
return JSON.parse(await readFile(path, 'utf8'))
|
|
84
|
+
} catch {
|
|
85
|
+
return undefined
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export async function deleteMigrationPlan(config, id) {
|
|
90
|
+
const path = planPath(config, id)
|
|
91
|
+
if (!path) return false
|
|
92
|
+
try {
|
|
93
|
+
await rm(path)
|
|
94
|
+
return true
|
|
95
|
+
} catch {
|
|
96
|
+
return false
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export async function listMigrationPlans(config) {
|
|
101
|
+
const dir = plansDir(config)
|
|
102
|
+
let names = []
|
|
103
|
+
try {
|
|
104
|
+
names = await readdir(dir)
|
|
105
|
+
} catch {
|
|
106
|
+
return []
|
|
107
|
+
}
|
|
108
|
+
const plans = []
|
|
109
|
+
for (const name of names.filter((item) => item.endsWith('.json'))) {
|
|
110
|
+
const plan = await readMigrationPlan(config, name.replace(/\.json$/, ''))
|
|
111
|
+
if (plan) plans.push(plan)
|
|
112
|
+
}
|
|
113
|
+
return plans.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export async function setMigrationPlanStatus(config, id, status, extra = {}) {
|
|
117
|
+
const plan = await readMigrationPlan(config, id)
|
|
118
|
+
if (!plan) return undefined
|
|
119
|
+
const next = { ...plan, ...extra, status, updatedAt: new Date().toISOString() }
|
|
120
|
+
await writeFile(planPath(config, id), `${JSON.stringify(next, null, 2)}\n`, 'utf8')
|
|
121
|
+
return next
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export async function readMigrationRuns(config, limit = 20) {
|
|
125
|
+
const dir = runsDir(config)
|
|
126
|
+
let names = []
|
|
127
|
+
try {
|
|
128
|
+
names = await readdir(dir)
|
|
129
|
+
} catch {
|
|
130
|
+
return []
|
|
131
|
+
}
|
|
132
|
+
const runs = []
|
|
133
|
+
for (const name of names.filter((item) => item.endsWith('.json')).sort().reverse().slice(0, limit)) {
|
|
134
|
+
try {
|
|
135
|
+
runs.push(JSON.parse(await readFile(join(dir, name), 'utf8')))
|
|
136
|
+
} catch {
|
|
137
|
+
// A half-written run file is skipped rather than failing the listing.
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return runs
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// ---------------------------------------------------------------- metadata --
|
|
144
|
+
|
|
145
|
+
/** Chunk an array; OData `or` chains and request bodies both need a bound. */
|
|
146
|
+
function chunk(items, size) {
|
|
147
|
+
const out = []
|
|
148
|
+
for (let index = 0; index < items.length; index += size) out.push(items.slice(index, index + size))
|
|
149
|
+
return out
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function envOf(config, key, label) {
|
|
153
|
+
const env = config.environments[key]
|
|
154
|
+
if (!env) {
|
|
155
|
+
const available = Object.keys(config.environments).join(', ')
|
|
156
|
+
throw new D365Error(`${label}环境不存在:${key || '(未指定)'}`, {
|
|
157
|
+
code: 'unknown-environment',
|
|
158
|
+
hint: `可用:${available}`,
|
|
159
|
+
})
|
|
160
|
+
}
|
|
161
|
+
return env
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Read one entity's set name, primary id and primary name attribute.
|
|
166
|
+
*
|
|
167
|
+
* Accepts either the logical name (`account`) or the set name (`accounts`),
|
|
168
|
+
* because both spellings appear in user requests.
|
|
169
|
+
*/
|
|
170
|
+
export async function readEntityInfo(client, entity, signal) {
|
|
171
|
+
const candidate = String(entity ?? '').trim()
|
|
172
|
+
if (!candidate) throw new D365Error('entity 不能为空', { code: 'missing-entity' })
|
|
173
|
+
const select = 'LogicalName,EntitySetName,PrimaryIdAttribute,PrimaryNameAttribute'
|
|
174
|
+
|
|
175
|
+
const attempts = [
|
|
176
|
+
`LogicalName eq '${odataString(candidate)}' or EntitySetName eq '${odataString(candidate)}'`,
|
|
177
|
+
`LogicalName eq '${odataString(candidate)}'`,
|
|
178
|
+
]
|
|
179
|
+
for (const filter of attempts) {
|
|
180
|
+
try {
|
|
181
|
+
const page = await getPagedWithSelectFallback(client, 'EntityDefinitions', { $select: select, $filter: filter }, 2, signal)
|
|
182
|
+
const row = page?.page?.records?.[0]
|
|
183
|
+
if (row) {
|
|
184
|
+
return {
|
|
185
|
+
logical: row.LogicalName,
|
|
186
|
+
set: row.EntitySetName ?? `${row.LogicalName}s`,
|
|
187
|
+
idField: row.PrimaryIdAttribute ?? `${row.LogicalName}id`,
|
|
188
|
+
nameField: row.PrimaryNameAttribute,
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
} catch (error) {
|
|
192
|
+
// A rejected filter shape is worth one narrower retry; a 401 or a timeout
|
|
193
|
+
// is not, and must surface as itself.
|
|
194
|
+
if (error?.status !== 400) throw error
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
throw new D365Error(`实体不存在:${candidate}`, {
|
|
198
|
+
code: 'unknown-entity',
|
|
199
|
+
hint: '传实体逻辑名(account)或集合名(accounts);数据库里没有这张表的话,先确认它随解决方案导入到了该环境。',
|
|
200
|
+
})
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Read the attribute catalogue; `$top` is rejected by this endpoint. */
|
|
204
|
+
async function readAttributes(client, logical, signal) {
|
|
205
|
+
const page = await getPagedWithSelectFallback(client, `EntityDefinitions(LogicalName='${odataString(logical)}')/Attributes`, {
|
|
206
|
+
__noTop: true,
|
|
207
|
+
$select: 'LogicalName,AttributeType,IsValidForCreate,IsValidForRead',
|
|
208
|
+
}, Infinity, signal)
|
|
209
|
+
const targets = await readLookupTargets(client, logical, signal)
|
|
210
|
+
const navigation = await readNavigationNames(client, logical, signal)
|
|
211
|
+
return (page?.page?.records ?? []).map((row) => ({
|
|
212
|
+
name: row.LogicalName,
|
|
213
|
+
type: row.AttributeType,
|
|
214
|
+
writable: row.IsValidForCreate !== false,
|
|
215
|
+
targets: targets.get(row.LogicalName) ?? [],
|
|
216
|
+
navigation: navigation.get(String(row.LogicalName).toLowerCase()) ?? null,
|
|
217
|
+
}))
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* The name a lookup is bound by is the *navigation property* name, which is not
|
|
222
|
+
* always the attribute's logical name: a custom relationship can keep
|
|
223
|
+
* `contoso_approverid` as the column while the navigation property is
|
|
224
|
+
* `contoso_ApproverId`. Binding the logical name then fails with 0x80048d19
|
|
225
|
+
* ("An undeclared property ... which only has property annotations in the
|
|
226
|
+
* payload"). One cast request per entity maps logical name -> navigation name.
|
|
227
|
+
*/
|
|
228
|
+
async function readNavigationNames(client, logical, signal) {
|
|
229
|
+
const map = new Map()
|
|
230
|
+
try {
|
|
231
|
+
const page = await getPagedWithSelectFallback(
|
|
232
|
+
client,
|
|
233
|
+
'RelationshipDefinitions/Microsoft.Dynamics.CRM.OneToManyRelationshipMetadata',
|
|
234
|
+
{
|
|
235
|
+
__noTop: true,
|
|
236
|
+
$select: 'ReferencingAttribute,ReferencingEntityNavigationPropertyName',
|
|
237
|
+
$filter: `ReferencingEntity eq '${odataString(logical)}'`,
|
|
238
|
+
},
|
|
239
|
+
Infinity,
|
|
240
|
+
signal,
|
|
241
|
+
)
|
|
242
|
+
for (const row of page?.page?.records ?? []) {
|
|
243
|
+
const attribute = String(row.ReferencingAttribute ?? '').toLowerCase()
|
|
244
|
+
const navigation = row.ReferencingEntityNavigationPropertyName
|
|
245
|
+
if (attribute && navigation) map.set(attribute, String(navigation))
|
|
246
|
+
}
|
|
247
|
+
} catch (error) {
|
|
248
|
+
// Relationship metadata can be gated on some orgs; without it the logical
|
|
249
|
+
// name is still the common case, so this stays a soft failure.
|
|
250
|
+
if (error?.status === 401 || error?.status === 403) throw error
|
|
251
|
+
}
|
|
252
|
+
return map
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Lookup targets live on `LookupAttributeMetadata`, not on the base
|
|
257
|
+
* `AttributeMetadata` the collection returns: selecting `Targets` there fails
|
|
258
|
+
* with 0x80060888 ("Could not find a property named 'Targets'"). One cast
|
|
259
|
+
* request per entity gets the whole map, so each lookup knows which table(s) it
|
|
260
|
+
* may point at.
|
|
261
|
+
*/
|
|
262
|
+
async function readLookupTargets(client, logical, signal) {
|
|
263
|
+
const map = new Map()
|
|
264
|
+
try {
|
|
265
|
+
const page = await getPagedWithSelectFallback(
|
|
266
|
+
client,
|
|
267
|
+
`EntityDefinitions(LogicalName='${odataString(logical)}')/Attributes/Microsoft.Dynamics.CRM.LookupAttributeMetadata`,
|
|
268
|
+
{ __noTop: true, $select: 'LogicalName,Targets' },
|
|
269
|
+
Infinity,
|
|
270
|
+
signal,
|
|
271
|
+
)
|
|
272
|
+
for (const row of page?.page?.records ?? []) {
|
|
273
|
+
if (row.LogicalName && Array.isArray(row.Targets) && row.Targets.length) map.set(row.LogicalName, row.Targets)
|
|
274
|
+
}
|
|
275
|
+
} catch (error) {
|
|
276
|
+
// A shape this org will not serve is not fatal: lookups still resolve by
|
|
277
|
+
// GUID, and the polymorphic case says so instead of guessing a table.
|
|
278
|
+
if (error?.status === 401 || error?.status === 403) throw error
|
|
279
|
+
}
|
|
280
|
+
return map
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Which source columns travel. Explicit `fields` win; otherwise every writable
|
|
285
|
+
* simple or lookup column except bookkeeping.
|
|
286
|
+
*/
|
|
287
|
+
export function chooseFields(attributes, options = {}) {
|
|
288
|
+
const byName = new Map(attributes.map((attribute) => [attribute.name, attribute]))
|
|
289
|
+
const requested = Array.isArray(options.fields) ? options.fields.map((item) => String(item).trim()).filter(Boolean) : []
|
|
290
|
+
|
|
291
|
+
if (requested.length) {
|
|
292
|
+
const unknown = requested.filter((name) => !byName.has(name))
|
|
293
|
+
if (unknown.length) {
|
|
294
|
+
throw new D365Error(`字段不存在:${unknown.join(', ')}`, {
|
|
295
|
+
code: 'unknown-attribute',
|
|
296
|
+
hint: `${options.entity ?? '该表'} 的可写字段里没有这些名字;先用 d365_metadata resource=attributes 核对逻辑名。`,
|
|
297
|
+
})
|
|
298
|
+
}
|
|
299
|
+
const blocked = requested.filter((name) => {
|
|
300
|
+
const attribute = byName.get(name)
|
|
301
|
+
return !attribute.writable || SYSTEM_ATTRIBUTES.has(name) || name === 'ownerid'
|
|
302
|
+
})
|
|
303
|
+
if (blocked.length) {
|
|
304
|
+
throw new D365Error(`字段不可写:${blocked.join(', ')}`, {
|
|
305
|
+
code: 'readonly-attribute',
|
|
306
|
+
hint: 'createdon/modifiedon/versionnumber 这类字段由平台维护;创建时间会被自动带到 overriddencreatedon。',
|
|
307
|
+
})
|
|
308
|
+
}
|
|
309
|
+
// includeState governs statecode/statuscode for the default field set, so an
|
|
310
|
+
// explicit list must obey it too: otherwise the plan shows them, the preview
|
|
311
|
+
// promises them, and buildBody strips them at write time.
|
|
312
|
+
const explicit = options.includeState === true
|
|
313
|
+
? requested
|
|
314
|
+
: requested.filter((name) => name !== 'statecode' && name !== 'statuscode')
|
|
315
|
+
return explicit.map((name) => toPlanField(byName.get(name)))
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const includeState = options.includeState === true
|
|
319
|
+
return attributes
|
|
320
|
+
.filter((attribute) => {
|
|
321
|
+
if (SYSTEM_ATTRIBUTES.has(attribute.name)) return false
|
|
322
|
+
if (attribute.name === 'ownerid') return false
|
|
323
|
+
if (attribute.name === 'statecode' || attribute.name === 'statuscode') return includeState
|
|
324
|
+
if (!attribute.writable) return false
|
|
325
|
+
return SIMPLE_TYPES.has(attribute.type) || LOOKUP_TYPES.has(attribute.type)
|
|
326
|
+
})
|
|
327
|
+
.map(toPlanField)
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
function toPlanField(attribute) {
|
|
331
|
+
return {
|
|
332
|
+
name: attribute.name,
|
|
333
|
+
type: attribute.type,
|
|
334
|
+
lookup: LOOKUP_TYPES.has(attribute.type),
|
|
335
|
+
targets: attribute.targets ?? [],
|
|
336
|
+
navigation: attribute.navigation ?? attribute.name,
|
|
337
|
+
navigationKnown: Boolean(attribute.navigation),
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// ------------------------------------------------------------------- scope --
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* Turn the caller's scope into one OData filter, defaulting to the whole table.
|
|
345
|
+
*
|
|
346
|
+
* "Sync today's accounts" is a date window on createdon, "the ones I fixed" is
|
|
347
|
+
* a filter, and a hand-written fetchXml is already exact. With no condition at
|
|
348
|
+
* all the whole table is the documented default; the plan reports the volume so
|
|
349
|
+
* a big table gets a recommendation to narrow it rather than a silent guess.
|
|
350
|
+
*/
|
|
351
|
+
export function describeScope(options = {}) {
|
|
352
|
+
const fetchXml = typeof options.fetchXml === 'string' ? options.fetchXml.trim() : ''
|
|
353
|
+
const filter = typeof options.filter === 'string' ? options.filter.trim() : ''
|
|
354
|
+
const ids = Array.isArray(options.ids) ? options.ids.map((item) => String(item).trim()).filter(Boolean) : []
|
|
355
|
+
const since = options.since === undefined || options.since === null || options.since === '' ? '' : String(options.since).trim()
|
|
356
|
+
const mode = options.mode ?? 'new'
|
|
357
|
+
|
|
358
|
+
if (fetchXml) {
|
|
359
|
+
const entity = /<entity[^>]*\bname\s*=\s*["']([^"']+)["']/i.exec(fetchXml)?.[1]
|
|
360
|
+
return { kind: 'fetchXml', fetchXml, filter: '', ids: [], since: '', mode, description: `fetchXml${entity ? `(${entity})` : ''}` }
|
|
361
|
+
}
|
|
362
|
+
if (ids.length) {
|
|
363
|
+
return { kind: 'ids', fetchXml: '', filter: '', ids, since: '', mode, description: `${ids.length} 个指定 GUID` }
|
|
364
|
+
}
|
|
365
|
+
if (since) {
|
|
366
|
+
const date = new Date(since)
|
|
367
|
+
if (Number.isNaN(date.getTime())) {
|
|
368
|
+
throw new D365Error(`since 不是合法时间:${since}`, {
|
|
369
|
+
code: 'bad-since',
|
|
370
|
+
hint: '用 ISO 时间,例如 2026-11-05T00:00:00+08:00;不带时区会被当作 UTC。',
|
|
371
|
+
})
|
|
372
|
+
}
|
|
373
|
+
const field = mode === 'modified' ? 'modifiedon' : 'createdon'
|
|
374
|
+
const iso = date.toISOString()
|
|
375
|
+
const composed = filter ? `(${filter}) and ${field} ge ${iso}` : `${field} ge ${iso}`
|
|
376
|
+
return {
|
|
377
|
+
kind: 'filter',
|
|
378
|
+
fetchXml: '',
|
|
379
|
+
filter: composed,
|
|
380
|
+
ids: [],
|
|
381
|
+
since: iso,
|
|
382
|
+
mode,
|
|
383
|
+
description: `${field} ge ${iso}${filter ? ` 且 ${filter}` : ''}`,
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
if (filter) {
|
|
387
|
+
return { kind: 'filter', fetchXml: '', filter, ids: [], since: '', mode, description: filter }
|
|
388
|
+
}
|
|
389
|
+
if (mode === 'all') {
|
|
390
|
+
// "Everything" is a scope the caller can state on purpose; the record cap
|
|
391
|
+
// then decides how much of it the plan actually covers, and the plan says so.
|
|
392
|
+
return {
|
|
393
|
+
kind: 'all',
|
|
394
|
+
fetchXml: '',
|
|
395
|
+
filter: '',
|
|
396
|
+
ids: [],
|
|
397
|
+
since: '',
|
|
398
|
+
mode,
|
|
399
|
+
description: '全部记录(没有时间条件,只受 maxRecords 限制)',
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
// No condition at all: the documented default is the whole table. The plan
|
|
403
|
+
// then reports the real volume and suggests a narrower scope when it is big,
|
|
404
|
+
// instead of refusing or silently guessing what "today's" means.
|
|
405
|
+
return {
|
|
406
|
+
kind: 'all',
|
|
407
|
+
fetchXml: '',
|
|
408
|
+
filter: '',
|
|
409
|
+
ids: [],
|
|
410
|
+
since: '',
|
|
411
|
+
mode: 'all',
|
|
412
|
+
implicit: true,
|
|
413
|
+
description: '全部记录(未指定条件,默认整表;受 maxRecords 限制)',
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/** Read the source rows for one scope. @returns `{ rows, hasMore }`. */
|
|
418
|
+
async function readRows(client, info, scope, fields, maxRecords, signal) {
|
|
419
|
+
const lookupSelect = fields.filter((field) => field.lookup).map((field) => `_${field.name}_value`)
|
|
420
|
+
const plainSelect = fields.filter((field) => !field.lookup).map((field) => field.name)
|
|
421
|
+
const select = [...new Set([
|
|
422
|
+
info.idField,
|
|
423
|
+
...plainSelect,
|
|
424
|
+
...lookupSelect,
|
|
425
|
+
'createdon',
|
|
426
|
+
'statecode',
|
|
427
|
+
'statuscode',
|
|
428
|
+
'_ownerid_value',
|
|
429
|
+
])].join(',')
|
|
430
|
+
|
|
431
|
+
if (scope.kind === 'fetchXml') {
|
|
432
|
+
const page = await client.getPaged(info.set, { fetchXml: scope.fetchXml }, maxRecords, signal)
|
|
433
|
+
return { rows: page.records ?? [], hasMore: page.hasMore, select: null }
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
if (scope.kind === 'ids') {
|
|
437
|
+
const rows = []
|
|
438
|
+
let hasMore = false
|
|
439
|
+
for (const group of chunk(scope.ids, 20)) {
|
|
440
|
+
if (rows.length >= maxRecords) {
|
|
441
|
+
// The id list itself is longer than the read cap: say so instead of
|
|
442
|
+
// reporting a complete read.
|
|
443
|
+
hasMore = true
|
|
444
|
+
break
|
|
445
|
+
}
|
|
446
|
+
const filter = group.map((id) => `${info.idField} eq ${/^[0-9a-f-]{36}$/i.test(id) ? id : `'${odataString(id)}'`}`).join(' or ')
|
|
447
|
+
const page = await client.getPaged(info.set, { $select: select, $filter: `(${filter})` }, maxRecords, signal)
|
|
448
|
+
rows.push(...page.records.slice(0, maxRecords - rows.length))
|
|
449
|
+
if (rows.length >= maxRecords && scope.ids.length > rows.length) hasMore = true
|
|
450
|
+
}
|
|
451
|
+
return { rows, hasMore, select }
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
const params = { $select: select, $orderby: `${info.idField} asc` }
|
|
455
|
+
if (scope.filter) params.$filter = scope.filter
|
|
456
|
+
const page = await client.getPaged(info.set, params, maxRecords, signal)
|
|
457
|
+
return { rows: page.records, hasMore: page.hasMore, select }
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// ------------------------------------------------------- lookup resolution --
|
|
461
|
+
|
|
462
|
+
function singleKeyed(client, path, params, signal) {
|
|
463
|
+
return client.requestJson(client.buildUrl(path, params), { signal })
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
async function tryKeyed(client, path, params, signal) {
|
|
467
|
+
try {
|
|
468
|
+
return await singleKeyed(client, path, params, signal)
|
|
469
|
+
} catch (error) {
|
|
470
|
+
if (error?.status === 404) return undefined
|
|
471
|
+
throw error
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* The primary-name attribute of a referenced entity, resolved per environment
|
|
477
|
+
* (a customised set name differs between orgs).
|
|
478
|
+
*/
|
|
479
|
+
async function entityInfoFor(client, logical, cache, signal) {
|
|
480
|
+
const key = String(logical).toLowerCase()
|
|
481
|
+
if (cache.entities.has(key)) return cache.entities.get(key)
|
|
482
|
+
const info = await readEntityInfo(client, logical, signal).catch(() => null)
|
|
483
|
+
cache.entities.set(key, info)
|
|
484
|
+
return info
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/** Names for a set of ids, chunked; missing rows are simply absent. */
|
|
488
|
+
async function readNames(client, info, field, ids, signal) {
|
|
489
|
+
const names = new Map()
|
|
490
|
+
if (!info?.nameField) return names
|
|
491
|
+
for (const group of chunk(ids, 20)) {
|
|
492
|
+
const filter = group.map((id) => `${info.idField} eq ${id}`).join(' or ')
|
|
493
|
+
const page = await client.getPaged(info.set, { $select: `${info.idField},${info.nameField}`, $filter: `(${filter})` }, group.length + 1, signal)
|
|
494
|
+
for (const row of page.records ?? []) names.set(String(row[info.idField]).toLowerCase(), row[info.nameField])
|
|
495
|
+
}
|
|
496
|
+
return names
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/** Which of these ids already exist in the target. */
|
|
500
|
+
async function existingIds(client, info, ids, signal) {
|
|
501
|
+
const found = new Set()
|
|
502
|
+
for (const group of chunk(ids, 20)) {
|
|
503
|
+
const filter = group.map((id) => `${info.idField} eq ${id}`).join(' or ')
|
|
504
|
+
const page = await client.getPaged(info.set, { $select: info.idField, $filter: `(${filter})` }, group.length + 1, signal)
|
|
505
|
+
for (const row of page.records ?? []) found.add(String(row[info.idField]).toLowerCase())
|
|
506
|
+
}
|
|
507
|
+
return found
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/** Exact primary-name match in the target; `>=2` hits are ambiguous, not a match. */
|
|
511
|
+
async function findByName(client, info, name, signal) {
|
|
512
|
+
if (!info?.nameField || name === undefined || name === null || String(name).trim() === '') return { matches: [] }
|
|
513
|
+
const page = await client.getPaged(info.set, {
|
|
514
|
+
$select: `${info.idField},${info.nameField}`,
|
|
515
|
+
$filter: `${info.nameField} eq '${odataString(String(name))}'`,
|
|
516
|
+
}, 3, signal)
|
|
517
|
+
return { matches: page.records ?? [] }
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Which entity actually holds each id in the source.
|
|
522
|
+
*
|
|
523
|
+
* `customerid` and `ownerid` point at more than one table, and matching a
|
|
524
|
+
* contact's name against accounts because the account lookup happened to be
|
|
525
|
+
* tried first would silently attach the wrong record. The owning table is
|
|
526
|
+
* therefore read from the source, not guessed from the attribute's Targets.
|
|
527
|
+
*/
|
|
528
|
+
async function typeOfIds(client, candidates, ids, signal) {
|
|
529
|
+
const found = new Map()
|
|
530
|
+
const pending = () => ids.filter((id) => !found.has(id.toLowerCase()))
|
|
531
|
+
for (const logical of candidates) {
|
|
532
|
+
if (!pending().length) break
|
|
533
|
+
const info = await readEntityInfo(client, logical, signal).catch(() => null)
|
|
534
|
+
if (!info) continue
|
|
535
|
+
for (const group of chunk(pending(), 20)) {
|
|
536
|
+
const filter = group.map((id) => `${info.idField} eq ${id}`).join(' or ')
|
|
537
|
+
const page = await client.getPaged(info.set, { $select: info.idField, $filter: `(${filter})` }, group.length + 1, signal)
|
|
538
|
+
for (const row of page.records ?? []) found.set(String(row[info.idField]).toLowerCase(), logical)
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
return found
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Resolve every lookup value of every row against the target.
|
|
546
|
+
*
|
|
547
|
+
* Resolution order is fixed: same GUID, then primary-name match, then missing.
|
|
548
|
+
* Nothing is written here; the plan only records what would happen.
|
|
549
|
+
*/
|
|
550
|
+
async function resolveLookups(sourceClient, targetClient, sourceCache, targetCache, rows, fields, options, signal) {
|
|
551
|
+
const lookups = fields.filter((field) => field.lookup)
|
|
552
|
+
/** key `${rowId}|${attribute}` -> resolution, plus `*|attr|id` for the shared lookups */
|
|
553
|
+
const results = new Map()
|
|
554
|
+
const summary = []
|
|
555
|
+
|
|
556
|
+
for (const field of lookups) {
|
|
557
|
+
const values = new Map()
|
|
558
|
+
for (const row of rows) {
|
|
559
|
+
const value = row[`_${field.name}_value`]
|
|
560
|
+
if (value) values.set(String(row[sourceCache.info.idField]), String(value))
|
|
561
|
+
}
|
|
562
|
+
const entry = {
|
|
563
|
+
attribute: field.name,
|
|
564
|
+
navigation: field.navigation,
|
|
565
|
+
targets: field.targets,
|
|
566
|
+
filled: values.size,
|
|
567
|
+
sameGuid: 0,
|
|
568
|
+
byName: 0,
|
|
569
|
+
missing: 0,
|
|
570
|
+
ambiguous: 0,
|
|
571
|
+
empty: rows.length - values.size,
|
|
572
|
+
missingSamples: [],
|
|
573
|
+
}
|
|
574
|
+
if (!values.size) {
|
|
575
|
+
summary.push(entry)
|
|
576
|
+
continue
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
const distinct = [...new Set(values.values())]
|
|
580
|
+
const single = field.targets.length === 1 ? field.targets[0] : null
|
|
581
|
+
const types = single
|
|
582
|
+
? new Map(distinct.map((id) => [id.toLowerCase(), single]))
|
|
583
|
+
: await typeOfIds(sourceClient, field.targets, distinct, signal)
|
|
584
|
+
|
|
585
|
+
// Group by the table that owns the value, so each id is resolved once.
|
|
586
|
+
const groups = new Map()
|
|
587
|
+
for (const id of distinct) {
|
|
588
|
+
const logical = types.get(id.toLowerCase())
|
|
589
|
+
if (!logical) {
|
|
590
|
+
const reason = field.targets.length
|
|
591
|
+
? '源环境里找不到这条被引用记录(可能已被删除)'
|
|
592
|
+
: '元数据没有给出这个 lookup 的目标表,无法判断它指向哪张表'
|
|
593
|
+
results.set(`*|${field.name}|${id}`, { how: 'missing', sourceId: id, reason })
|
|
594
|
+
continue
|
|
595
|
+
}
|
|
596
|
+
if (!groups.has(logical)) groups.set(logical, [])
|
|
597
|
+
groups.get(logical).push(id)
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
for (const [logical, ids] of groups) {
|
|
601
|
+
const sourceInfo = await entityInfoFor(sourceClient, logical, sourceCache, signal)
|
|
602
|
+
const targetInfo = await entityInfoFor(targetClient, logical, targetCache, signal)
|
|
603
|
+
if (!sourceInfo || !targetInfo) {
|
|
604
|
+
for (const id of ids) {
|
|
605
|
+
results.set(`*|${field.name}|${id}`, { how: 'missing', sourceId: id, reason: `目标环境没有 ${logical} 这张表` })
|
|
606
|
+
}
|
|
607
|
+
continue
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
const names = options.resolveLookups === false ? new Map() : await readNames(sourceClient, sourceInfo, sourceInfo.idField, ids, signal)
|
|
611
|
+
const present = await existingIds(targetClient, targetInfo, ids, signal)
|
|
612
|
+
|
|
613
|
+
for (const id of ids) {
|
|
614
|
+
if (present.has(id.toLowerCase())) {
|
|
615
|
+
// Same GUID in both orgs: a mirror-copy can carry the value as-is.
|
|
616
|
+
results.set(`*|${field.name}|${id}`, { how: 'same-guid', targetId: id, targetSet: targetInfo.set, sourceId: id, name: names.get(id.toLowerCase()) })
|
|
617
|
+
continue
|
|
618
|
+
}
|
|
619
|
+
const name = names.get(id.toLowerCase())
|
|
620
|
+
if (!name || options.resolveLookups === false) {
|
|
621
|
+
results.set(`*|${field.name}|${id}`, {
|
|
622
|
+
how: 'missing',
|
|
623
|
+
sourceId: id,
|
|
624
|
+
name,
|
|
625
|
+
reason: options.resolveLookups === false ? '本次关闭了 lookup 名称解析(includeLookups=false)' : '源环境读不到该引用的名称',
|
|
626
|
+
})
|
|
627
|
+
continue
|
|
628
|
+
}
|
|
629
|
+
const found = await findByName(targetClient, targetInfo, name, signal)
|
|
630
|
+
if (found.matches.length === 1) {
|
|
631
|
+
results.set(`*|${field.name}|${id}`, {
|
|
632
|
+
how: 'by-name',
|
|
633
|
+
targetId: String(found.matches[0][targetInfo.idField]),
|
|
634
|
+
targetSet: targetInfo.set,
|
|
635
|
+
sourceId: id,
|
|
636
|
+
name,
|
|
637
|
+
})
|
|
638
|
+
} else if (found.matches.length > 1) {
|
|
639
|
+
results.set(`*|${field.name}|${id}`, { how: 'ambiguous', sourceId: id, name, reason: `目标里同名 ${logical} 有 ${found.matches.length} 条` })
|
|
640
|
+
} else {
|
|
641
|
+
results.set(`*|${field.name}|${id}`, { how: 'missing', sourceId: id, name, reason: `目标里按名称找不到对应的 ${logical}` })
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
for (const row of rows) {
|
|
647
|
+
const rowId = String(row[sourceCache.info.idField])
|
|
648
|
+
const value = values.get(rowId)
|
|
649
|
+
if (!value) {
|
|
650
|
+
results.set(`${rowId}|${field.name}`, { how: 'empty' })
|
|
651
|
+
continue
|
|
652
|
+
}
|
|
653
|
+
const resolution = results.get(`*|${field.name}|${value}`) ?? { how: 'missing', sourceId: value, reason: '未能解析' }
|
|
654
|
+
results.set(`${rowId}|${field.name}`, resolution)
|
|
655
|
+
if (resolution.how === 'same-guid') entry.sameGuid += 1
|
|
656
|
+
else if (resolution.how === 'by-name') entry.byName += 1
|
|
657
|
+
else if (resolution.how === 'ambiguous') {
|
|
658
|
+
entry.ambiguous += 1
|
|
659
|
+
if (entry.missingSamples.length < 5) entry.missingSamples.push({ sourceId: value, name: resolution.name, reason: resolution.reason })
|
|
660
|
+
} else if (resolution.how === 'missing') {
|
|
661
|
+
entry.missing += 1
|
|
662
|
+
if (entry.missingSamples.length < 5) entry.missingSamples.push({ sourceId: value, name: resolution.name, reason: resolution.reason })
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
summary.push(entry)
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
return { results, summary }
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
// ------------------------------------------------------------------ owners --
|
|
672
|
+
|
|
673
|
+
/** Who owned the record in the source, and who should own it in the target. */
|
|
674
|
+
async function resolveOwners(sourceClient, targetClient, rows, options, signal) {
|
|
675
|
+
const ownerIds = [...new Set(rows.map((row) => row._ownerid_value).filter(Boolean).map(String))]
|
|
676
|
+
const map = new Map()
|
|
677
|
+
let mapped = 0
|
|
678
|
+
let fallback = 0
|
|
679
|
+
|
|
680
|
+
for (const id of ownerIds) {
|
|
681
|
+
const user = await tryKeyed(sourceClient, `systemusers(${id})`, { $select: 'domainname,internalemailaddress,fullname' }, signal)
|
|
682
|
+
const team = user ? undefined : await tryKeyed(sourceClient, `teams(${id})`, { $select: 'name,emailaddress' }, signal)
|
|
683
|
+
if (!user && !team) {
|
|
684
|
+
map.set(id, { how: 'unknown', sourceId: id })
|
|
685
|
+
continue
|
|
686
|
+
}
|
|
687
|
+
if (options.ownerMode === 'keep') {
|
|
688
|
+
map.set(id, { how: 'keep', sourceId: id, targetId: id, set: user ? 'systemusers' : 'teams' })
|
|
689
|
+
continue
|
|
690
|
+
}
|
|
691
|
+
if (options.ownerMode === 'caller') {
|
|
692
|
+
map.set(id, { how: 'caller', sourceId: id })
|
|
693
|
+
fallback += 1
|
|
694
|
+
continue
|
|
695
|
+
}
|
|
696
|
+
const key = user ? (user.domainname ?? user.internalemailaddress) : (team.emailaddress ?? team.name)
|
|
697
|
+
let hit
|
|
698
|
+
if (user && key) {
|
|
699
|
+
const page = await targetClient.getPaged('systemusers', {
|
|
700
|
+
$select: 'systemuserid,domainname',
|
|
701
|
+
$filter: `domainname eq '${odataString(String(key))}' or internalemailaddress eq '${odataString(String(key))}'`,
|
|
702
|
+
}, 2, signal)
|
|
703
|
+
hit = page.records?.[0]
|
|
704
|
+
if (hit) map.set(id, { how: 'mapped', sourceId: id, targetId: hit.systemuserid, set: 'systemusers', key })
|
|
705
|
+
} else if (team && key) {
|
|
706
|
+
const page = await targetClient.getPaged('teams', {
|
|
707
|
+
$select: 'teamid,name',
|
|
708
|
+
$filter: `name eq '${odataString(String(key))}'`,
|
|
709
|
+
}, 2, signal)
|
|
710
|
+
hit = page.records?.[0]
|
|
711
|
+
if (hit) map.set(id, { how: 'mapped', sourceId: id, targetId: hit.teamid, set: 'teams', key })
|
|
712
|
+
}
|
|
713
|
+
if (hit) {
|
|
714
|
+
mapped += 1
|
|
715
|
+
} else {
|
|
716
|
+
map.set(id, { how: 'fallback', sourceId: id, key: key ?? id })
|
|
717
|
+
fallback += 1
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
return { map, distinct: ownerIds.length, mapped, fallback }
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
// -------------------------------------------------------------------- plan --
|
|
725
|
+
|
|
726
|
+
function numberArg(value, fallback, min, max) {
|
|
727
|
+
const number = Number(value)
|
|
728
|
+
if (!Number.isFinite(number)) return fallback
|
|
729
|
+
return Math.max(min, Math.min(max, number))
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/**
|
|
733
|
+
* Build a migration plan. Reads both environments; writes nothing.
|
|
734
|
+
*/
|
|
735
|
+
export async function buildPlan(config, args = {}, signal) {
|
|
736
|
+
const sourceKey = String(args.source ?? '').trim()
|
|
737
|
+
const targetKey = String(args.target ?? '').trim()
|
|
738
|
+
if (!sourceKey || !targetKey) {
|
|
739
|
+
throw new D365Error('需要 source 和 target', {
|
|
740
|
+
code: 'missing-environment',
|
|
741
|
+
hint: `例如 source="dev" target="sit";可用:${Object.keys(config.environments).join(', ')}`,
|
|
742
|
+
})
|
|
743
|
+
}
|
|
744
|
+
if (sourceKey === targetKey) {
|
|
745
|
+
throw new D365Error('source 和 target 是同一个环境', { code: 'same-environment', hint: '迁移的两个环境必须不同。' })
|
|
746
|
+
}
|
|
747
|
+
const sourceEnv = envOf(config, sourceKey, '源')
|
|
748
|
+
const targetEnv = envOf(config, targetKey, '目标')
|
|
749
|
+
const sourceClient = makeClient(sourceEnv, config)
|
|
750
|
+
const targetClient = makeClient(targetEnv, config)
|
|
751
|
+
|
|
752
|
+
const scope = describeScope(args)
|
|
753
|
+
const maxRecords = numberArg(args.maxRecords, 500, 1, MAX_RECORDS)
|
|
754
|
+
const options = {
|
|
755
|
+
keepGuid: true,
|
|
756
|
+
resolveLookups: args.includeLookups !== false,
|
|
757
|
+
ownerMode: ['map', 'keep', 'caller'].includes(args.ownerMode) ? args.ownerMode : 'map',
|
|
758
|
+
conflict: ['report', 'skip', 'overwrite'].includes(args.conflict) ? args.conflict : 'report',
|
|
759
|
+
onMissingLookup: ['report', 'null', 'fail'].includes(args.onMissingLookup) ? args.onMissingLookup : 'report',
|
|
760
|
+
bypassPlugins: args.bypassPlugins !== false,
|
|
761
|
+
preserveCreatedOn: args.preserveCreatedOn !== false,
|
|
762
|
+
suppressDuplicateDetection: args.suppressDuplicateDetection !== false,
|
|
763
|
+
includeState: args.includeState === true,
|
|
764
|
+
concurrency: numberArg(args.concurrency, 4, 1, 8),
|
|
765
|
+
batchSize: numberArg(args.batchSize, 50, 10, 500),
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
const sourceCache = { info: await readEntityInfo(sourceClient, args.entity, signal), entities: new Map() }
|
|
769
|
+
const targetCache = { info: await readEntityInfo(targetClient, args.entity, signal), entities: new Map() }
|
|
770
|
+
sourceCache.info.attributes = await readAttributes(sourceClient, sourceCache.info.logical, signal)
|
|
771
|
+
|
|
772
|
+
const fields = chooseFields(sourceCache.info.attributes, {
|
|
773
|
+
fields: args.fields,
|
|
774
|
+
includeState: options.includeState,
|
|
775
|
+
entity: sourceCache.info.logical,
|
|
776
|
+
})
|
|
777
|
+
if (!fields.length) {
|
|
778
|
+
throw new D365Error('没有可迁移的字段', { code: 'no-fields', hint: '该表除系统字段外没有可写列;用 fields 显式指定要看/要搬的列。' })
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
// Keeping the source GUID is not a separate toggle to guess at: it follows the
|
|
782
|
+
// field list. A field list that carries the primary key mirrors the record
|
|
783
|
+
// (same GUID); one that leaves it out lets the target assign a new id, and the
|
|
784
|
+
// references inside the batch are remapped. An explicit keepGuid still wins.
|
|
785
|
+
const explicitKeepGuid = typeof args.keepGuid === 'boolean' ? args.keepGuid : undefined
|
|
786
|
+
const carriesId = fields.some((field) => field.name === sourceCache.info.idField)
|
|
787
|
+
options.keepGuid = explicitKeepGuid ?? carriesId
|
|
788
|
+
options.keepGuidBecause = explicitKeepGuid !== undefined
|
|
789
|
+
? 'explicit'
|
|
790
|
+
: carriesId ? 'id-in-fields' : 'no-id-in-fields'
|
|
791
|
+
|
|
792
|
+
const { rows, hasMore, select } = await readRows(sourceClient, sourceCache.info, scope, fields, maxRecords, signal)
|
|
793
|
+
// `hasMore` is reported, never silently dropped: the cap is a guard rail, and
|
|
794
|
+
// the plan says how much it read so a truncated scope cannot look complete.
|
|
795
|
+
const warnings = []
|
|
796
|
+
if (scope.implicit) {
|
|
797
|
+
warnings.push(`没有给迁移条件,按全部记录预演;读到 ${rows.length} 条${hasMore ? '(已到上限,还有更多)' : ''}。`
|
|
798
|
+
+ '数据量大时建议用 since / filter / ids / fetchXml 缩小范围。')
|
|
799
|
+
}
|
|
800
|
+
if (hasMore) {
|
|
801
|
+
warnings.push(`只读了前 ${rows.length} 条(上限 ${maxRecords}),其余未纳入本次计划;要么调大 maxRecords,要么加条件分批迁。`)
|
|
802
|
+
}
|
|
803
|
+
// A lookup is bound through its navigation property name; if relationship
|
|
804
|
+
// metadata did not answer, the logical name is only a guess, and the failure
|
|
805
|
+
// would otherwise show up at write time as 0x80048d19 "undeclared property".
|
|
806
|
+
const unknownNavigation = fields.filter((field) => field.lookup && !field.navigationKnown)
|
|
807
|
+
if (unknownNavigation.length) {
|
|
808
|
+
const sample = unknownNavigation.slice(0, 5).map((field) => field.name).join('、')
|
|
809
|
+
warnings.push(`有 ${unknownNavigation.length} 个 lookup 没查到导航属性名(${sample}${unknownNavigation.length > 5 ? ' 等' : ''}),写入时按逻辑名绑定;若目标返回 0x80048d19「undeclared property」,先确认字段名再缩小 fields 重出计划。`)
|
|
810
|
+
}
|
|
811
|
+
const ids = rows.map((row) => String(row[sourceCache.info.idField]))
|
|
812
|
+
const existing = options.keepGuid ? await existingIds(targetClient, targetCache.info, ids, signal) : new Set()
|
|
813
|
+
const { results, summary: lookupSummary } = await resolveLookups(
|
|
814
|
+
sourceClient, targetClient, sourceCache, targetCache, rows, fields, options, signal,
|
|
815
|
+
)
|
|
816
|
+
const owners = await resolveOwners(sourceClient, targetClient, rows, options, signal)
|
|
817
|
+
|
|
818
|
+
const records = rows.map((row) => {
|
|
819
|
+
const id = String(row[sourceCache.info.idField])
|
|
820
|
+
const values = {}
|
|
821
|
+
const lookups = []
|
|
822
|
+
for (const field of fields) {
|
|
823
|
+
if (field.lookup) {
|
|
824
|
+
const resolution = results.get(`${id}|${field.name}`) ?? { how: 'empty' }
|
|
825
|
+
lookups.push({ attribute: field.name, navigation: field.navigation, ...resolution })
|
|
826
|
+
if (resolution.how === 'same-guid' || resolution.how === 'by-name') values[field.name] = resolution.targetId
|
|
827
|
+
else if (resolution.how === 'empty') values[field.name] = null
|
|
828
|
+
else if (options.onMissingLookup === 'null') values[field.name] = null
|
|
829
|
+
continue
|
|
830
|
+
}
|
|
831
|
+
if (row[field.name] !== undefined) values[field.name] = row[field.name]
|
|
832
|
+
}
|
|
833
|
+
const owner = row._ownerid_value ? owners.map.get(String(row._ownerid_value)) : undefined
|
|
834
|
+
return {
|
|
835
|
+
id,
|
|
836
|
+
action: existing.has(id.toLowerCase()) ? 'conflict' : 'create',
|
|
837
|
+
name: sourceCache.info.nameField ? row[sourceCache.info.nameField] : undefined,
|
|
838
|
+
createdon: row.createdon,
|
|
839
|
+
statecode: row.statecode,
|
|
840
|
+
statuscode: row.statuscode,
|
|
841
|
+
values,
|
|
842
|
+
lookups,
|
|
843
|
+
owner,
|
|
844
|
+
// Whether a record has an unresolved reference is a property of the
|
|
845
|
+
// record, not of the policy: the `fail` guard later counts these, and
|
|
846
|
+
// making it policy-dependent meant `fail` could never see them.
|
|
847
|
+
blocked: lookups.some((lookup) => lookup.how === 'missing' || lookup.how === 'ambiguous'),
|
|
848
|
+
}
|
|
849
|
+
})
|
|
850
|
+
|
|
851
|
+
const conflicts = records.filter((record) => record.action === 'conflict').length
|
|
852
|
+
const blocked = records.filter((record) => record.blocked).length
|
|
853
|
+
const missing = lookupSummary.reduce((sum, entry) => sum + entry.missing + entry.ambiguous, 0)
|
|
854
|
+
|
|
855
|
+
return {
|
|
856
|
+
kind: 'data-sync',
|
|
857
|
+
source: { key: sourceKey, url: sourceEnv.url, entity: sourceCache.info.logical, set: sourceCache.info.set },
|
|
858
|
+
target: { key: targetKey, url: targetEnv.url, entity: targetCache.info.logical, set: targetCache.info.set },
|
|
859
|
+
scope: { ...scope, description: scope.description, select },
|
|
860
|
+
options,
|
|
861
|
+
fields,
|
|
862
|
+
idField: sourceCache.info.idField,
|
|
863
|
+
limits: { maxRecords, read: rows.length, hasMore },
|
|
864
|
+
warnings,
|
|
865
|
+
lookupSummary,
|
|
866
|
+
owners: { distinct: owners.distinct, mapped: owners.mapped, fallback: owners.fallback },
|
|
867
|
+
counts: {
|
|
868
|
+
read: rows.length,
|
|
869
|
+
create: records.filter((record) => record.action === 'create').length,
|
|
870
|
+
conflict: conflicts,
|
|
871
|
+
blocked,
|
|
872
|
+
missingLookups: missing,
|
|
873
|
+
},
|
|
874
|
+
records,
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// ----------------------------------------------------------------- execute --
|
|
879
|
+
|
|
880
|
+
/** Run `worker` over `items` with a bounded number of in-flight requests. */
|
|
881
|
+
async function mapLimit(items, limit, worker) {
|
|
882
|
+
const results = new Array(items.length)
|
|
883
|
+
let index = 0
|
|
884
|
+
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
885
|
+
while (index < items.length) {
|
|
886
|
+
const current = index
|
|
887
|
+
index += 1
|
|
888
|
+
results[current] = await worker(items[current], current)
|
|
889
|
+
}
|
|
890
|
+
})
|
|
891
|
+
await Promise.all(runners)
|
|
892
|
+
return results
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/** Which lookups point at another record in the same batch (only matters without GUIDs). */
|
|
896
|
+
function dependencyOrder(records) {
|
|
897
|
+
const bySource = new Map(records.map((record) => [record.id.toLowerCase(), record]))
|
|
898
|
+
const edges = new Map()
|
|
899
|
+
for (const record of records) {
|
|
900
|
+
const needs = new Set()
|
|
901
|
+
for (const lookup of record.lookups) {
|
|
902
|
+
if (lookup.how === 'same-guid' && bySource.has(String(lookup.sourceId).toLowerCase())) needs.add(String(lookup.sourceId).toLowerCase())
|
|
903
|
+
}
|
|
904
|
+
edges.set(record.id.toLowerCase(), needs)
|
|
905
|
+
}
|
|
906
|
+
const ordered = []
|
|
907
|
+
const temporary = new Set()
|
|
908
|
+
const permanent = new Set()
|
|
909
|
+
const visit = (id) => {
|
|
910
|
+
if (permanent.has(id)) return
|
|
911
|
+
if (temporary.has(id)) return // cycle: broken by deferring the lookup, handled later
|
|
912
|
+
temporary.add(id)
|
|
913
|
+
for (const dependency of edges.get(id) ?? []) visit(dependency)
|
|
914
|
+
temporary.delete(id)
|
|
915
|
+
permanent.add(id)
|
|
916
|
+
ordered.push(bySource.get(id))
|
|
917
|
+
}
|
|
918
|
+
for (const record of records) visit(record.id.toLowerCase())
|
|
919
|
+
return ordered
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
function writeHeaders(options) {
|
|
923
|
+
const headers = { Prefer: 'return=representation' }
|
|
924
|
+
if (options.suppressDuplicateDetection) headers['MSCRM.SuppressDuplicateDetection'] = 'true'
|
|
925
|
+
if (options.bypassPlugins) headers['MSCRM.BypassCustomPluginExecution'] = 'true'
|
|
926
|
+
return headers
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
/**
|
|
930
|
+
* Point a lookup at a record.
|
|
931
|
+
*
|
|
932
|
+
* Dataverse's Web API treats a lookup as a navigation property, not a column: a
|
|
933
|
+
* bare GUID (`"contoso_bg": "<guid>"`) is rejected with 0x80048d19 "a 'PrimitiveValue'
|
|
934
|
+
* node with non-null value was found ... a 'PrimitiveValue' node with null value
|
|
935
|
+
* was expected". The value has to travel as `"contoso_bg@odata.bind": "/set(id)"`.
|
|
936
|
+
*
|
|
937
|
+
* The annotation is keyed by the navigation property name, which only usually
|
|
938
|
+
* equals the attribute's logical name — see `readNavigationNames`.
|
|
939
|
+
*/
|
|
940
|
+
function bindLookup(body, lookup, set, id) {
|
|
941
|
+
const attribute = lookup?.attribute
|
|
942
|
+
const navigation = lookup?.navigation || attribute
|
|
943
|
+
if (attribute) delete body[attribute]
|
|
944
|
+
if (navigation && navigation !== attribute) delete body[navigation]
|
|
945
|
+
if (!navigation) return false
|
|
946
|
+
delete body[`${navigation}@odata.bind`]
|
|
947
|
+
if (!set) return false
|
|
948
|
+
body[`${navigation}@odata.bind`] = `/${set}(${id})`
|
|
949
|
+
return true
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
function buildBody(record, info, options) {
|
|
953
|
+
const body = { ...record.values }
|
|
954
|
+
// The primary key only travels through the keepGuid decision: a source GUID
|
|
955
|
+
// left in `values` would make the target reject the create (or worse, collide
|
|
956
|
+
// with a record copied earlier).
|
|
957
|
+
delete body[info.idField]
|
|
958
|
+
if (options.keepGuid) body[info.idField] = record.id
|
|
959
|
+
if (options.preserveCreatedOn && record.createdon) body.overriddencreatedon = record.createdon
|
|
960
|
+
if (options.includeState === false) {
|
|
961
|
+
delete body.statecode
|
|
962
|
+
delete body.statuscode
|
|
963
|
+
}
|
|
964
|
+
// A null lookup stays a literal null (the API accepts that for a navigation
|
|
965
|
+
// property); a resolved one becomes an @odata.bind annotation.
|
|
966
|
+
for (const lookup of record.lookups ?? []) {
|
|
967
|
+
const value = body[lookup.attribute]
|
|
968
|
+
if (value !== null && value !== undefined) bindLookup(body, lookup, lookup.targetSet, value)
|
|
969
|
+
}
|
|
970
|
+
const owner = record.owner
|
|
971
|
+
if (owner && options.ownerMode !== 'caller') {
|
|
972
|
+
if (owner.how === 'mapped' || owner.how === 'keep') {
|
|
973
|
+
bindLookup(body, { attribute: 'ownerid', navigation: 'ownerid' }, owner.set ?? 'systemusers', owner.targetId)
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
return body
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
/**
|
|
980
|
+
* Execute an approved plan.
|
|
981
|
+
*
|
|
982
|
+
* `hooks.onRecord` is called after each record settles so a caller can record
|
|
983
|
+
* progress; the run file is the durable copy.
|
|
984
|
+
*/
|
|
985
|
+
export async function executeMigration(config, plan, options = {}, signal, hooks = {}) {
|
|
986
|
+
const planOptions = { ...plan.options, ...options }
|
|
987
|
+
const targetEnv = envOf(config, plan.target.key, '目标')
|
|
988
|
+
// `targetClient` lets a test drive the write path without an org behind it.
|
|
989
|
+
const targetClient = hooks.targetClient ?? makeClient(targetEnv, config)
|
|
990
|
+
const targetInfo = await readEntityInfo(targetClient, plan.target.entity, signal)
|
|
991
|
+
|
|
992
|
+
// A plan written before navigation names were resolved would bind lookups by
|
|
993
|
+
// their logical name — the exact failure 0x80048d19 this change fixes. Make
|
|
994
|
+
// the user re-preview instead of repeating it silently.
|
|
995
|
+
const staleFields = (plan.fields ?? []).filter((field) => field.lookup && !field.navigation)
|
|
996
|
+
if (staleFields.length) {
|
|
997
|
+
throw new D365Error('这份计划是旧版本生成的,缺少 lookup 的导航属性名', {
|
|
998
|
+
code: 'stale-plan',
|
|
999
|
+
hint: '重新跑一次 plan 再 execute;旧计划会按逻辑名绑定 lookup,目标环境可能返回 0x80048d19「undeclared property」。',
|
|
1000
|
+
})
|
|
1001
|
+
}
|
|
1002
|
+
if (planOptions.onMissingLookup === 'fail') {
|
|
1003
|
+
const blocked = plan.records.filter((record) => record.blocked)
|
|
1004
|
+
if (blocked.length) {
|
|
1005
|
+
throw new D365Error(`有 ${blocked.length} 条记录的引用在目标环境找不到`, {
|
|
1006
|
+
code: 'missing-lookup',
|
|
1007
|
+
hint: '把 onMissingLookup 改成 report(跳过这些记录)或 null(清空这些引用),或先在目标环境补齐被引用记录。',
|
|
1008
|
+
})
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
const runId = `run-${plan.id}-${randomUUID().slice(0, 6)}`
|
|
1013
|
+
const runFile = join(runsDir(config), `${new Date().toISOString().replace(/[:.]/g, '-')}-${runId}.json`)
|
|
1014
|
+
await mkdir(runsDir(config), { recursive: true })
|
|
1015
|
+
|
|
1016
|
+
const writable = plan.records.filter((record) => {
|
|
1017
|
+
if (record.action === 'conflict' && planOptions.conflict !== 'overwrite') return false
|
|
1018
|
+
if (record.blocked && planOptions.onMissingLookup === 'report') return false
|
|
1019
|
+
return true
|
|
1020
|
+
})
|
|
1021
|
+
const skipped = plan.records.length - writable.length
|
|
1022
|
+
|
|
1023
|
+
// Without GUIDs the target assigns ids, so lookups inside the batch must be
|
|
1024
|
+
// remapped after the create. Ordering keeps referenced records first.
|
|
1025
|
+
const ordered = planOptions.keepGuid ? writable : dependencyOrder(writable)
|
|
1026
|
+
const idMap = new Map()
|
|
1027
|
+
|
|
1028
|
+
const results = await mapLimit(ordered, planOptions.concurrency ?? 4, async (record) => {
|
|
1029
|
+
const body = buildBody(record, targetInfo, planOptions)
|
|
1030
|
+
// A lookup that points at a batch member whose new id is not known yet is
|
|
1031
|
+
// sent later, in a second PATCH pass.
|
|
1032
|
+
const deferred = []
|
|
1033
|
+
if (!planOptions.keepGuid) {
|
|
1034
|
+
for (const lookup of record.lookups) {
|
|
1035
|
+
if (lookup.how !== 'same-guid') continue
|
|
1036
|
+
const mapped = idMap.get(String(lookup.sourceId).toLowerCase())
|
|
1037
|
+
if (mapped) {
|
|
1038
|
+
bindLookup(body, lookup, lookup.targetSet, mapped)
|
|
1039
|
+
} else {
|
|
1040
|
+
delete body[lookup.attribute]
|
|
1041
|
+
delete body[`${lookup.navigation ?? lookup.attribute}@odata.bind`]
|
|
1042
|
+
deferred.push(lookup.attribute)
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
try {
|
|
1047
|
+
if (record.action === 'conflict') {
|
|
1048
|
+
await targetClient.sendJson(`${targetInfo.set}(${record.id})`, {
|
|
1049
|
+
method: 'PATCH',
|
|
1050
|
+
body,
|
|
1051
|
+
headers: { ...writeHeaders(planOptions), 'If-Match': '*' },
|
|
1052
|
+
}, signal)
|
|
1053
|
+
return { id: record.id, action: 'updated', deferred }
|
|
1054
|
+
}
|
|
1055
|
+
const created = await targetClient.sendJson(targetInfo.set, {
|
|
1056
|
+
method: 'POST',
|
|
1057
|
+
body,
|
|
1058
|
+
headers: writeHeaders(planOptions),
|
|
1059
|
+
}, signal)
|
|
1060
|
+
const newId = created?.[targetInfo.idField] ?? record.id
|
|
1061
|
+
idMap.set(record.id.toLowerCase(), String(newId))
|
|
1062
|
+
return { id: record.id, targetId: String(newId), action: 'created', deferred, values: body }
|
|
1063
|
+
} catch (error) {
|
|
1064
|
+
return { id: record.id, action: 'failed', error: String(error?.message ?? error), code: error?.code, status: error?.status }
|
|
1065
|
+
}
|
|
1066
|
+
})
|
|
1067
|
+
|
|
1068
|
+
// Second pass: lookups that pointed at records created later in this run.
|
|
1069
|
+
const deferredRecords = ordered
|
|
1070
|
+
.map((record, index) => ({ record, result: results[index] }))
|
|
1071
|
+
.filter((entry) => entry.result?.action !== 'failed' && entry.result?.deferred?.length)
|
|
1072
|
+
for (const entry of deferredRecords) {
|
|
1073
|
+
const body = {}
|
|
1074
|
+
for (const attribute of entry.result.deferred) {
|
|
1075
|
+
const lookup = entry.record.lookups.find((item) => item.attribute === attribute)
|
|
1076
|
+
const mapped = lookup ? idMap.get(String(lookup.sourceId).toLowerCase()) : undefined
|
|
1077
|
+
if (mapped) bindLookup(body, lookup, lookup.targetSet, mapped)
|
|
1078
|
+
}
|
|
1079
|
+
if (!Object.keys(body).length) continue
|
|
1080
|
+
try {
|
|
1081
|
+
await targetClient.sendJson(`${targetInfo.set}(${entry.result.targetId ?? entry.record.id})`, {
|
|
1082
|
+
method: 'PATCH',
|
|
1083
|
+
body,
|
|
1084
|
+
headers: { ...writeHeaders(planOptions), 'If-Match': '*' },
|
|
1085
|
+
}, signal)
|
|
1086
|
+
} catch (error) {
|
|
1087
|
+
// A failed back-reference write must not be swallowed: the record exists
|
|
1088
|
+
// but points at nothing, which is exactly the state the caller asked to
|
|
1089
|
+
// avoid. Count it as a failed row instead.
|
|
1090
|
+
entry.result.action = 'failed'
|
|
1091
|
+
entry.result.error = `引用回填失败:${String(error?.message ?? error)}`
|
|
1092
|
+
entry.result.code = error?.code
|
|
1093
|
+
entry.result.status = error?.status
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
const created = results.filter((item) => item.action === 'created')
|
|
1098
|
+
const updated = results.filter((item) => item.action === 'updated')
|
|
1099
|
+
const failed = results.filter((item) => item.action === 'failed')
|
|
1100
|
+
const run = {
|
|
1101
|
+
kind: 'data-sync-run',
|
|
1102
|
+
runId,
|
|
1103
|
+
planId: plan.id,
|
|
1104
|
+
at: new Date().toISOString(),
|
|
1105
|
+
source: plan.source,
|
|
1106
|
+
target: plan.target,
|
|
1107
|
+
options: planOptions,
|
|
1108
|
+
counts: { planned: plan.records.length, written: created.length + updated.length, skipped, failed: failed.length },
|
|
1109
|
+
created: created.map((item) => ({ sourceId: item.id, targetId: item.targetId })),
|
|
1110
|
+
updated: updated.map((item) => item.id),
|
|
1111
|
+
failures: failed,
|
|
1112
|
+
skippedRecords: plan.records
|
|
1113
|
+
.filter((record) => !writable.includes(record))
|
|
1114
|
+
.map((record) => ({
|
|
1115
|
+
id: record.id,
|
|
1116
|
+
reason: record.action === 'conflict' ? '目标已存在同 GUID,按 conflict=report 跳过' : '引用缺失,按 onMissingLookup=report 跳过',
|
|
1117
|
+
})),
|
|
1118
|
+
}
|
|
1119
|
+
await writeFile(runFile, `${JSON.stringify(run, null, 2)}\n`, 'utf8')
|
|
1120
|
+
hooks.onRecord?.(run)
|
|
1121
|
+
run.file = runFile
|
|
1122
|
+
return run
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
/** Write the plan itself as an artifact, so the full record list survives. */
|
|
1126
|
+
export async function writePlanArtifact(config, plan) {
|
|
1127
|
+
return writeArtifact(config, `data-sync-${plan.source.key}-${plan.target.key}`, plan)
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
/* ------------------------------------------------------------------------ *
|
|
1131
|
+
* Update mode: change fields on many existing records in one environment.
|
|
1132
|
+
*
|
|
1133
|
+
* The 377-row case is the reason this exists. `d365_data_write` is
|
|
1134
|
+
* deliberately one record at a time, and 377 confirmation cards is not a
|
|
1135
|
+
* workflow. This reuses the migration plan/confirm/run shape but never
|
|
1136
|
+
* creates records: every id must already exist, and a missing one is
|
|
1137
|
+
* reported instead of turned into an upsert.
|
|
1138
|
+
* ------------------------------------------------------------------------ */
|
|
1139
|
+
|
|
1140
|
+
const MAX_UPDATE_RECORDS = 1000
|
|
1141
|
+
|
|
1142
|
+
/**
|
|
1143
|
+
* Build a bulk-update plan for one environment.
|
|
1144
|
+
*
|
|
1145
|
+
* Only scalar columns are accepted in v1: a lookup value in a PATCH needs the
|
|
1146
|
+
* same navigation-property resolution the migration engine does, and doing it
|
|
1147
|
+
* half-way would produce writes that look right in the plan and fail on the
|
|
1148
|
+
* wire. Lookups point at `d365_data_write` for now.
|
|
1149
|
+
*/
|
|
1150
|
+
export async function buildUpdatePlan(config, args = {}, signal) {
|
|
1151
|
+
const envKey = String(args.environment ?? '').trim()
|
|
1152
|
+
if (!envKey) {
|
|
1153
|
+
throw new D365Error('批量更新需要在 environment 里指定环境', {
|
|
1154
|
+
code: 'missing-environment',
|
|
1155
|
+
hint: '批量更新只改一个环境里的记录;跨环境复制用 plan 模式。',
|
|
1156
|
+
})
|
|
1157
|
+
}
|
|
1158
|
+
const target = config.environments[envKey]
|
|
1159
|
+
if (!target) {
|
|
1160
|
+
throw new D365Error(`环境不存在:${envKey}`, {
|
|
1161
|
+
code: 'unknown-environment',
|
|
1162
|
+
hint: `可用:${Object.keys(config.environments).join(', ')}`,
|
|
1163
|
+
})
|
|
1164
|
+
}
|
|
1165
|
+
const entity = String(args.entity ?? '').trim()
|
|
1166
|
+
if (!entity) throw new D365Error('批量更新需要 entity', { code: 'missing-entity' })
|
|
1167
|
+
|
|
1168
|
+
const ids = [...new Set((args.ids ?? []).map((id) => String(id).trim()).filter(Boolean))]
|
|
1169
|
+
if (!ids.length) {
|
|
1170
|
+
throw new D365Error('批量更新需要显式 ids 清单', {
|
|
1171
|
+
code: 'ids-required',
|
|
1172
|
+
hint: '批量更新不接受 filter 或 fetchXml:改错一批记录没有撤销键。先用 d365_data action=query 查出 id,再把这些 id 传进来。',
|
|
1173
|
+
})
|
|
1174
|
+
}
|
|
1175
|
+
if (ids.length > MAX_UPDATE_RECORDS) {
|
|
1176
|
+
throw new D365Error(`一次最多更新 ${MAX_UPDATE_RECORDS} 条,收到 ${ids.length} 条`, {
|
|
1177
|
+
code: 'too-many-records',
|
|
1178
|
+
hint: '分批执行,每批都重新 plan 并确认一次。',
|
|
1179
|
+
})
|
|
1180
|
+
}
|
|
1181
|
+
const badId = ids.find((id) => !/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/.test(id))
|
|
1182
|
+
if (badId) throw new D365Error(`不是合法 GUID:${badId}`, { code: 'bad-id' })
|
|
1183
|
+
|
|
1184
|
+
const rawValues = args.values && typeof args.values === 'object' ? args.values : {}
|
|
1185
|
+
const columns = Object.keys(rawValues)
|
|
1186
|
+
if (!columns.length) throw new D365Error('批量更新需要 values', { code: 'missing-values' })
|
|
1187
|
+
|
|
1188
|
+
const client = makeClient(target, config)
|
|
1189
|
+
const info = await readEntityInfo(client, entity, signal)
|
|
1190
|
+
const attributes = await readAttributes(client, info.logical, signal)
|
|
1191
|
+
const byName = new Map(attributes.map((attribute) => [attribute.name.toLowerCase(), attribute]))
|
|
1192
|
+
|
|
1193
|
+
const fields = []
|
|
1194
|
+
const normalized = {}
|
|
1195
|
+
for (const column of columns) {
|
|
1196
|
+
const attribute = byName.get(column.toLowerCase())
|
|
1197
|
+
if (!attribute) {
|
|
1198
|
+
throw new D365Error(`实体 ${info.logical} 没有字段 ${column}`, {
|
|
1199
|
+
code: 'unknown-attribute',
|
|
1200
|
+
hint: '先用 d365_metadata resource=attributes 确认字段逻辑名。',
|
|
1201
|
+
})
|
|
1202
|
+
}
|
|
1203
|
+
if (!attribute.writable) {
|
|
1204
|
+
throw new D365Error(`字段 ${column} 不可写`, { code: 'readonly-attribute', hint: '它是一个计算列、汇总列或系统维护的列。' })
|
|
1205
|
+
}
|
|
1206
|
+
if (attribute.lookup) {
|
|
1207
|
+
throw new D365Error(`批量更新暂不支持 lookup 字段 ${column}`, {
|
|
1208
|
+
code: 'lookup-unsupported',
|
|
1209
|
+
hint: 'lookup 需要按导航属性名绑定,用 d365_data_write 单条改更可靠;标量字段可以用批量更新。',
|
|
1210
|
+
})
|
|
1211
|
+
}
|
|
1212
|
+
fields.push({ name: attribute.name, type: attribute.type })
|
|
1213
|
+
// Key the write by the canonical logical name. Dataverse is not
|
|
1214
|
+
// case-sensitive, but the plan's own preview and run record look values up
|
|
1215
|
+
// by `fields[].name`, so a payload spelled differently would print
|
|
1216
|
+
// `undefined` and send a key the plan never showed.
|
|
1217
|
+
normalized[attribute.name] = rawValues[column]
|
|
1218
|
+
}
|
|
1219
|
+
const values = normalized
|
|
1220
|
+
|
|
1221
|
+
// Read what the records look like now: the plan must show before → after,
|
|
1222
|
+
// not just the value the caller wants to write.
|
|
1223
|
+
const names = await readNames(client, info, info.nameField, ids, signal).catch(() => new Map())
|
|
1224
|
+
const existing = new Set()
|
|
1225
|
+
const before = []
|
|
1226
|
+
for (const group of chunk(ids, 20)) {
|
|
1227
|
+
const filter = group.map((id) => `${info.idField} eq ${id}`).join(' or ')
|
|
1228
|
+
const page = await client.getPaged(info.set, {
|
|
1229
|
+
$select: [info.idField, info.nameField, ...fields.map((field) => field.name)].filter(Boolean).join(','),
|
|
1230
|
+
$filter: `(${filter})`,
|
|
1231
|
+
}, group.length + 1, signal)
|
|
1232
|
+
for (const row of page.records ?? []) {
|
|
1233
|
+
const id = String(row[info.idField]).toLowerCase()
|
|
1234
|
+
existing.add(id)
|
|
1235
|
+
before.push({
|
|
1236
|
+
id,
|
|
1237
|
+
name: row[info.nameField ?? ''] ?? names.get(id) ?? null,
|
|
1238
|
+
current: Object.fromEntries(fields.map((field) => [field.name, row[field.name] ?? null])),
|
|
1239
|
+
})
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
const missing = ids.filter((id) => !existing.has(id.toLowerCase()))
|
|
1244
|
+
const plan = {
|
|
1245
|
+
kind: 'data-update',
|
|
1246
|
+
capability: 'data.write',
|
|
1247
|
+
environment: envKey,
|
|
1248
|
+
url: target.url,
|
|
1249
|
+
entity: info.logical,
|
|
1250
|
+
set: info.set,
|
|
1251
|
+
idField: info.idField,
|
|
1252
|
+
fields,
|
|
1253
|
+
values,
|
|
1254
|
+
ids,
|
|
1255
|
+
before,
|
|
1256
|
+
missing,
|
|
1257
|
+
options: {
|
|
1258
|
+
bypassPlugins: args.bypassPlugins !== false,
|
|
1259
|
+
suppressDuplicateDetection: args.suppressDuplicateDetection !== false,
|
|
1260
|
+
concurrency: Number.isFinite(args.concurrency) ? Math.max(1, Math.min(8, Number(args.concurrency))) : 4,
|
|
1261
|
+
},
|
|
1262
|
+
counts: { planned: ids.length, existing: existing.size, missing: missing.length },
|
|
1263
|
+
rationale: args.rationale,
|
|
1264
|
+
createdAt: new Date().toISOString(),
|
|
1265
|
+
status: 'pending',
|
|
1266
|
+
}
|
|
1267
|
+
plan.id = `${Date.now().toString(36)}-${randomUUID().slice(0, 5)}`
|
|
1268
|
+
return plan
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
/** Apply a bulk-update plan, one PATCH per record, bounded concurrency. */
|
|
1272
|
+
export async function executeUpdate(config, plan, options = {}, signal, hooks = {}) {
|
|
1273
|
+
const target = config.environments[plan.environment]
|
|
1274
|
+
if (!target) throw new D365Error(`环境不存在:${plan.environment}`, { code: 'unknown-environment' })
|
|
1275
|
+
const client = hooks.targetClient ?? makeClient(target, config)
|
|
1276
|
+
const planOptions = { ...plan.options, ...options }
|
|
1277
|
+
const runId = `run-${plan.id}-${randomUUID().slice(0, 6)}`
|
|
1278
|
+
const runFile = join(runsDir(config), `${new Date().toISOString().replace(/[:.]/g, '-')}-${runId}.json`)
|
|
1279
|
+
|
|
1280
|
+
const applicable = plan.before.map((row) => row.id)
|
|
1281
|
+
const results = await mapLimit(applicable, planOptions.concurrency ?? 4, async (id) => {
|
|
1282
|
+
const body = { ...plan.values }
|
|
1283
|
+
try {
|
|
1284
|
+
await client.sendJson(`${plan.set}(${id})`, {
|
|
1285
|
+
method: 'PATCH',
|
|
1286
|
+
body,
|
|
1287
|
+
headers: { ...writeHeaders(planOptions), 'If-Match': '*' },
|
|
1288
|
+
}, signal)
|
|
1289
|
+
return { id, ok: true }
|
|
1290
|
+
} catch (error) {
|
|
1291
|
+
return { id, ok: false, error: String(error?.message ?? error) }
|
|
1292
|
+
}
|
|
1293
|
+
})
|
|
1294
|
+
|
|
1295
|
+
const updated = results.filter((item) => item.ok)
|
|
1296
|
+
const failed = results.filter((item) => !item.ok)
|
|
1297
|
+
const run = {
|
|
1298
|
+
kind: 'data-update-run',
|
|
1299
|
+
runId,
|
|
1300
|
+
planId: plan.id,
|
|
1301
|
+
at: new Date().toISOString(),
|
|
1302
|
+
environment: plan.environment,
|
|
1303
|
+
entity: plan.entity,
|
|
1304
|
+
values: plan.values,
|
|
1305
|
+
options: planOptions,
|
|
1306
|
+
counts: { planned: plan.ids.length, applied: applicable.length, updated: updated.length, failed: failed.length, missing: plan.missing.length },
|
|
1307
|
+
updated: updated.map((item) => item.id),
|
|
1308
|
+
failures: failed,
|
|
1309
|
+
missing: plan.missing,
|
|
1310
|
+
}
|
|
1311
|
+
// The very first bulk update would otherwise PATCH every record and then fail
|
|
1312
|
+
// to write its own run file, leaving the plan marked failed with no record.
|
|
1313
|
+
await mkdir(runsDir(config), { recursive: true })
|
|
1314
|
+
await writeFile(runFile, `${JSON.stringify(run, null, 2)}\n`, 'utf8')
|
|
1315
|
+
hooks.onRecord?.(run)
|
|
1316
|
+
run.file = runFile
|
|
1317
|
+
return run
|
|
1318
|
+
}
|