@tenonhq/dovetail-core 0.0.123 → 0.0.125
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/allScopesCommands.js +2 -1
- package/dist/commander.js +71 -4
- package/dist/initSystem/orchestrator.js +10 -2
- package/dist/pullRecord.js +608 -0
- package/dist/pullRecordCommand.js +161 -0
- package/dist/snClient.js +33 -0
- package/dist/tests/commanderStrict.test.js +41 -0
- package/dist/tests/pullRecord.test.js +609 -0
- package/dist/tests/watchHidden.test.js +50 -0
- package/package.json +1 -1
- package/release-manifest.json +283 -261
|
@@ -0,0 +1,608 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Per-record pull: mirror ONE (or a handful of) ServiceNow records into the
|
|
4
|
+
* repo without a scope-wide refresh.
|
|
5
|
+
*
|
|
6
|
+
* `dove refresh` is a whole-scope operation: it rewrites the scope manifest in
|
|
7
|
+
* full and re-downloads every record in every whitelisted table. Mirroring a
|
|
8
|
+
* handful of records captured on the instance (a Now Assist skill is ~50 records
|
|
9
|
+
* across 17 tables) therefore meant a scoped refresh on a feature branch plus a
|
|
10
|
+
* dozen manual restore-and-splice steps. This module is the narrow alternative:
|
|
11
|
+
*
|
|
12
|
+
* - resolves each record's owning scope from the instance (`sys_scope`) and
|
|
13
|
+
* REFUSES a record outside the target scope before anything is written;
|
|
14
|
+
* - reuses the two read-only sync endpoints refresh already uses (`getManifest`
|
|
15
|
+
* for the record's file list + folder name, `bulkDownload` for content), so
|
|
16
|
+
* the folder, file names, and manifest entry are byte-for-byte what a full
|
|
17
|
+
* refresh would produce for that record;
|
|
18
|
+
* - writes ONLY that record's folder and splices ONLY that record's manifest
|
|
19
|
+
* entry, leaving every other manifest entry byte-identical;
|
|
20
|
+
* - works for a table the server-side manifest skips (no script/html column
|
|
21
|
+
* and no field override): it still writes the record's `metaData.json`.
|
|
22
|
+
*
|
|
23
|
+
* Two phases, like every other Dovetail write: `planPull` is read-only against
|
|
24
|
+
* the instance and the disk, `applyPull` performs the writes from the plan.
|
|
25
|
+
*/
|
|
26
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
27
|
+
if (k2 === undefined) k2 = k;
|
|
28
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
29
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
30
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
31
|
+
}
|
|
32
|
+
Object.defineProperty(o, k2, desc);
|
|
33
|
+
}) : (function(o, m, k, k2) {
|
|
34
|
+
if (k2 === undefined) k2 = k;
|
|
35
|
+
o[k2] = m[k];
|
|
36
|
+
}));
|
|
37
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
38
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
39
|
+
}) : function(o, v) {
|
|
40
|
+
o["default"] = v;
|
|
41
|
+
});
|
|
42
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
43
|
+
var ownKeys = function(o) {
|
|
44
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
45
|
+
var ar = [];
|
|
46
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
47
|
+
return ar;
|
|
48
|
+
};
|
|
49
|
+
return ownKeys(o);
|
|
50
|
+
};
|
|
51
|
+
return function (mod) {
|
|
52
|
+
if (mod && mod.__esModule) return mod;
|
|
53
|
+
var result = {};
|
|
54
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
55
|
+
__setModuleDefault(result, mod);
|
|
56
|
+
return result;
|
|
57
|
+
};
|
|
58
|
+
})();
|
|
59
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
60
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
61
|
+
};
|
|
62
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
63
|
+
exports.TABLE_NAME_RE = exports.SYS_ID_RE = void 0;
|
|
64
|
+
exports.normalizeSysIds = normalizeSysIds;
|
|
65
|
+
exports.assertTableName = assertTableName;
|
|
66
|
+
exports.parseUpdateXmlName = parseUpdateXmlName;
|
|
67
|
+
exports.dedupeTargets = dedupeTargets;
|
|
68
|
+
exports.toManifestFiles = toManifestFiles;
|
|
69
|
+
exports.findRecordBySysId = findRecordBySysId;
|
|
70
|
+
exports.spliceManifestRecord = spliceManifestRecord;
|
|
71
|
+
exports.parseManifestText = parseManifestText;
|
|
72
|
+
exports.serializeManifest = serializeManifest;
|
|
73
|
+
exports.targetsFromUpdateSet = targetsFromUpdateSet;
|
|
74
|
+
exports.planPull = planPull;
|
|
75
|
+
exports.applyPull = applyPull;
|
|
76
|
+
const path_1 = __importDefault(require("path"));
|
|
77
|
+
const fs_1 = require("fs");
|
|
78
|
+
const ConfigManager = __importStar(require("./config"));
|
|
79
|
+
const fUtils = __importStar(require("./FileUtils"));
|
|
80
|
+
const snClient_1 = require("./snClient");
|
|
81
|
+
const appUtils_1 = require("./appUtils");
|
|
82
|
+
const FileLogger_1 = require("./FileLogger");
|
|
83
|
+
exports.SYS_ID_RE = /^[0-9a-f]{32}$/;
|
|
84
|
+
// ServiceNow table names: lowercase, digits, underscores, never leading digit.
|
|
85
|
+
exports.TABLE_NAME_RE = /^[a-z][a-z0-9_]*$/;
|
|
86
|
+
// sys_update_xml.name for a per-record capture is `<table>_<sys_id>`.
|
|
87
|
+
const UPDATE_XML_NAME_RE = /^([a-z][a-z0-9_]*)_([0-9a-f]{32})$/;
|
|
88
|
+
const UPDATE_SET_PAGE_SIZE = 1000;
|
|
89
|
+
// ---------------------------------------------------------------------------
|
|
90
|
+
// Pure helpers (exported for tests)
|
|
91
|
+
// ---------------------------------------------------------------------------
|
|
92
|
+
/**
|
|
93
|
+
* Normalize the sys_id inputs (positional list + `--sys-ids` flag) into one
|
|
94
|
+
* validated, de-duplicated list. Throws on the first malformed value — a typo
|
|
95
|
+
* must not turn into a 404 against the instance halfway through a batch.
|
|
96
|
+
*/
|
|
97
|
+
function normalizeSysIds(raw) {
|
|
98
|
+
if (raw === undefined || raw === null)
|
|
99
|
+
return [];
|
|
100
|
+
const parts = Array.isArray(raw) ? raw : [raw];
|
|
101
|
+
const out = [];
|
|
102
|
+
for (const part of parts) {
|
|
103
|
+
if (part === undefined || part === null)
|
|
104
|
+
continue;
|
|
105
|
+
for (const piece of String(part).split(",")) {
|
|
106
|
+
const id = piece.trim().toLowerCase();
|
|
107
|
+
if (id === "")
|
|
108
|
+
continue;
|
|
109
|
+
if (!exports.SYS_ID_RE.test(id)) {
|
|
110
|
+
throw new Error("'" + piece.trim() + "' is not a sys_id (expected 32 hex characters).");
|
|
111
|
+
}
|
|
112
|
+
if (out.indexOf(id) === -1)
|
|
113
|
+
out.push(id);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return out;
|
|
117
|
+
}
|
|
118
|
+
function assertTableName(table) {
|
|
119
|
+
const name = String(table || "").trim();
|
|
120
|
+
if (!exports.TABLE_NAME_RE.test(name)) {
|
|
121
|
+
throw new Error("'" + table + "' is not a ServiceNow table name (lowercase letters, digits and underscores).");
|
|
122
|
+
}
|
|
123
|
+
return name;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Parse a `sys_update_xml.name` into a pull target. Only per-record captures
|
|
127
|
+
* are named `<table>_<sys_id>`; schema captures (`sys_dictionary_<table>_<field>`,
|
|
128
|
+
* `sys_db_object_<table>`, choice sets, …) are not and return undefined.
|
|
129
|
+
*/
|
|
130
|
+
function parseUpdateXmlName(name) {
|
|
131
|
+
const m = UPDATE_XML_NAME_RE.exec(String(name || "").trim());
|
|
132
|
+
if (!m)
|
|
133
|
+
return undefined;
|
|
134
|
+
return { table: m[1], sysId: m[2] };
|
|
135
|
+
}
|
|
136
|
+
function dedupeTargets(targets) {
|
|
137
|
+
const seen = {};
|
|
138
|
+
const out = [];
|
|
139
|
+
for (const t of targets) {
|
|
140
|
+
const k = t.table + "/" + t.sysId;
|
|
141
|
+
if (seen[k])
|
|
142
|
+
continue;
|
|
143
|
+
seen[k] = true;
|
|
144
|
+
out.push({ table: t.table, sysId: t.sysId });
|
|
145
|
+
}
|
|
146
|
+
return out;
|
|
147
|
+
}
|
|
148
|
+
/** Strip content so an entry carries exactly what a refresh-written manifest does. */
|
|
149
|
+
function toManifestFiles(files) {
|
|
150
|
+
const out = [];
|
|
151
|
+
for (const f of files) {
|
|
152
|
+
if (f.name === "metaData" && f.type === "json")
|
|
153
|
+
continue;
|
|
154
|
+
out.push({ name: f.name, type: f.type });
|
|
155
|
+
}
|
|
156
|
+
return out;
|
|
157
|
+
}
|
|
158
|
+
function findRecordBySysId(tables, table, sysId) {
|
|
159
|
+
if (!tables)
|
|
160
|
+
return undefined;
|
|
161
|
+
const tableEntry = tables[table];
|
|
162
|
+
if (!tableEntry || !tableEntry.records)
|
|
163
|
+
return undefined;
|
|
164
|
+
const keys = Object.keys(tableEntry.records);
|
|
165
|
+
for (const key of keys) {
|
|
166
|
+
const rec = tableEntry.records[key];
|
|
167
|
+
if (rec && rec.sys_id === sysId)
|
|
168
|
+
return { key, record: rec };
|
|
169
|
+
}
|
|
170
|
+
return undefined;
|
|
171
|
+
}
|
|
172
|
+
function sameEntry(a, b) {
|
|
173
|
+
if (!a)
|
|
174
|
+
return false;
|
|
175
|
+
return JSON.stringify(entryShape(a)) === JSON.stringify(entryShape(b));
|
|
176
|
+
}
|
|
177
|
+
function entryShape(e) {
|
|
178
|
+
return { files: toManifestFiles(e.files || []), name: e.name, sys_id: e.sys_id };
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Splice ONE record entry into a scope manifest. Returns a NEW manifest; the
|
|
182
|
+
* input is never mutated. Every other table and record entry is carried over
|
|
183
|
+
* untouched and in its original order, so re-serializing changes exactly one
|
|
184
|
+
* key. A stale key holding the same sys_id (record renamed on the instance) is
|
|
185
|
+
* removed, since the manifest key must equal the folder name push looks up.
|
|
186
|
+
*/
|
|
187
|
+
function spliceManifestRecord(manifest, scope, table, entry, previousKey) {
|
|
188
|
+
const base = manifest
|
|
189
|
+
? { tables: manifest.tables || {}, scope: manifest.scope || scope }
|
|
190
|
+
: { tables: {}, scope };
|
|
191
|
+
const tables = {};
|
|
192
|
+
const tableNames = Object.keys(base.tables);
|
|
193
|
+
let placed = false;
|
|
194
|
+
for (const name of tableNames) {
|
|
195
|
+
if (name !== table) {
|
|
196
|
+
tables[name] = base.tables[name];
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
const records = {};
|
|
200
|
+
const existing = base.tables[name].records || {};
|
|
201
|
+
for (const key of Object.keys(existing)) {
|
|
202
|
+
if (previousKey !== undefined && key === previousKey && key !== entry.name)
|
|
203
|
+
continue;
|
|
204
|
+
if (key === entry.name) {
|
|
205
|
+
records[key] = entryShape(entry);
|
|
206
|
+
placed = true;
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
records[key] = existing[key];
|
|
210
|
+
}
|
|
211
|
+
if (!placed) {
|
|
212
|
+
records[entry.name] = entryShape(entry);
|
|
213
|
+
placed = true;
|
|
214
|
+
}
|
|
215
|
+
tables[name] = Object.assign({}, base.tables[name], { records });
|
|
216
|
+
}
|
|
217
|
+
if (!placed) {
|
|
218
|
+
tables[table] = { records: { [entry.name]: entryShape(entry) } };
|
|
219
|
+
}
|
|
220
|
+
// Preserve the original top-level key order (tables first, then scope) —
|
|
221
|
+
// that is what refresh writes, so the diff stays one entry.
|
|
222
|
+
const out = { tables, scope: base.scope };
|
|
223
|
+
return out;
|
|
224
|
+
}
|
|
225
|
+
function parseManifestText(text) {
|
|
226
|
+
if (text === undefined) {
|
|
227
|
+
return { manifest: undefined, formatPreserved: true, trailingNewline: false, existed: false };
|
|
228
|
+
}
|
|
229
|
+
let parsed;
|
|
230
|
+
try {
|
|
231
|
+
parsed = JSON.parse(text);
|
|
232
|
+
}
|
|
233
|
+
catch (e) {
|
|
234
|
+
throw new Error("Scope manifest is not valid JSON: " + (e instanceof Error ? e.message : String(e)));
|
|
235
|
+
}
|
|
236
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
237
|
+
throw new Error("Scope manifest is not a JSON object.");
|
|
238
|
+
}
|
|
239
|
+
const manifest = parsed;
|
|
240
|
+
const trailingNewline = text.endsWith("\n");
|
|
241
|
+
const reserialized = JSON.stringify(manifest, null, 2);
|
|
242
|
+
const body = trailingNewline ? text.slice(0, -1) : text;
|
|
243
|
+
return {
|
|
244
|
+
manifest,
|
|
245
|
+
formatPreserved: reserialized === body,
|
|
246
|
+
trailingNewline,
|
|
247
|
+
existed: true,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
function serializeManifest(manifest, trailingNewline) {
|
|
251
|
+
return JSON.stringify(manifest, null, 2) + (trailingNewline ? "\n" : "");
|
|
252
|
+
}
|
|
253
|
+
function scopeNameOf(rec) {
|
|
254
|
+
const dotWalked = rec["sys_scope.scope"];
|
|
255
|
+
if (typeof dotWalked === "string" && dotWalked.trim() !== "")
|
|
256
|
+
return dotWalked.trim();
|
|
257
|
+
return "";
|
|
258
|
+
}
|
|
259
|
+
function scopeSysIdOf(rec) {
|
|
260
|
+
const ref = rec.sys_scope;
|
|
261
|
+
if (typeof ref === "string")
|
|
262
|
+
return ref;
|
|
263
|
+
if (ref && typeof ref.value === "string")
|
|
264
|
+
return ref.value;
|
|
265
|
+
return "";
|
|
266
|
+
}
|
|
267
|
+
function httpStatusOf(e) {
|
|
268
|
+
if (typeof e !== "object" || e === null)
|
|
269
|
+
return undefined;
|
|
270
|
+
const resp = e.response;
|
|
271
|
+
if (resp && typeof resp.status === "number")
|
|
272
|
+
return resp.status;
|
|
273
|
+
return undefined;
|
|
274
|
+
}
|
|
275
|
+
// ---------------------------------------------------------------------------
|
|
276
|
+
// Instance reads
|
|
277
|
+
// ---------------------------------------------------------------------------
|
|
278
|
+
/**
|
|
279
|
+
* Expand `--from-update-set <sys_id>` into per-record targets: every
|
|
280
|
+
* INSERT_OR_UPDATE capture named `<table>_<sys_id>`. Captures that are not
|
|
281
|
+
* per-record (schema, choice sets, deletes) are reported in `skipped`.
|
|
282
|
+
*/
|
|
283
|
+
async function targetsFromUpdateSet(updateSetSysId, client = (0, snClient_1.defaultClient)()) {
|
|
284
|
+
const id = String(updateSetSysId || "").trim().toLowerCase();
|
|
285
|
+
if (!exports.SYS_ID_RE.test(id)) {
|
|
286
|
+
throw new Error("--from-update-set: '" + updateSetSysId + "' is not a sys_id (expected 32 hex characters).");
|
|
287
|
+
}
|
|
288
|
+
const targets = [];
|
|
289
|
+
const skipped = [];
|
|
290
|
+
let offset = 0;
|
|
291
|
+
let total = 0;
|
|
292
|
+
for (;;) {
|
|
293
|
+
const page = await (0, snClient_1.unwrapSNResponse)(client.getUpdateSetMembers(id, offset, UPDATE_SET_PAGE_SIZE));
|
|
294
|
+
if (!Array.isArray(page) || page.length === 0)
|
|
295
|
+
break;
|
|
296
|
+
total += page.length;
|
|
297
|
+
for (const row of page) {
|
|
298
|
+
const action = String(row.action || "").toUpperCase();
|
|
299
|
+
if (action === "DELETE") {
|
|
300
|
+
skipped.push(row.name + " (DELETE)");
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
const target = parseUpdateXmlName(row.name);
|
|
304
|
+
if (!target) {
|
|
305
|
+
skipped.push(row.name + " (not a per-record capture)");
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
targets.push(target);
|
|
309
|
+
}
|
|
310
|
+
if (page.length < UPDATE_SET_PAGE_SIZE)
|
|
311
|
+
break;
|
|
312
|
+
offset += UPDATE_SET_PAGE_SIZE;
|
|
313
|
+
}
|
|
314
|
+
FileLogger_1.fileLogger.debug("pull: update set " + id + " holds " + total + " captures → " + targets.length +
|
|
315
|
+
" per-record targets, " + skipped.length + " skipped");
|
|
316
|
+
if (total === 0) {
|
|
317
|
+
throw new Error("--from-update-set: no customer updates found for update set " + id +
|
|
318
|
+
" (wrong sys_id, empty set, or no read access to sys_update_xml).");
|
|
319
|
+
}
|
|
320
|
+
return { targets: dedupeTargets(targets), skipped };
|
|
321
|
+
}
|
|
322
|
+
async function resolveTargetScope(target, options, client) {
|
|
323
|
+
const label = target.table + " " + target.sysId;
|
|
324
|
+
let rec;
|
|
325
|
+
try {
|
|
326
|
+
rec = await (0, snClient_1.unwrapSNResponse)(client.getRecordScope(target.table, target.sysId));
|
|
327
|
+
}
|
|
328
|
+
catch (e) {
|
|
329
|
+
const status = httpStatusOf(e);
|
|
330
|
+
if (status === 404) {
|
|
331
|
+
throw new Error(label + ": not found on the instance (no such record, or no read access).");
|
|
332
|
+
}
|
|
333
|
+
if (status === 400) {
|
|
334
|
+
throw new Error(label + ": the instance rejected the read (status 400) — does table '" +
|
|
335
|
+
target.table + "' exist?");
|
|
336
|
+
}
|
|
337
|
+
throw e;
|
|
338
|
+
}
|
|
339
|
+
if (!rec || typeof rec !== "object" || rec.sys_id !== target.sysId) {
|
|
340
|
+
throw new Error(label + ": the instance returned an unexpected record (sys_id mismatch).");
|
|
341
|
+
}
|
|
342
|
+
const actualClass = typeof rec.sys_class_name === "string" ? rec.sys_class_name.trim() : "";
|
|
343
|
+
if (actualClass !== "" && actualClass !== target.table) {
|
|
344
|
+
throw new Error(label + ": the record's class is '" + actualClass + "', not '" + target.table +
|
|
345
|
+
"'. Dovetail mirrors a record under its own class — re-run with '" + actualClass + "'.");
|
|
346
|
+
}
|
|
347
|
+
let scope = scopeNameOf(rec);
|
|
348
|
+
if (scope === "") {
|
|
349
|
+
if (!options.scope) {
|
|
350
|
+
throw new Error(label + ": the record carries no sys_scope, so its scope cannot be resolved. " +
|
|
351
|
+
"Pass --scope <scope> to place it explicitly.");
|
|
352
|
+
}
|
|
353
|
+
scope = options.scope;
|
|
354
|
+
}
|
|
355
|
+
if (options.scope && scope !== options.scope) {
|
|
356
|
+
throw new Error(label + ": belongs to scope '" + scope + "', not the requested '" + options.scope +
|
|
357
|
+
"'. Nothing was written.");
|
|
358
|
+
}
|
|
359
|
+
return { table: target.table, sysId: target.sysId, scope, scopeSysId: scopeSysIdOf(rec) };
|
|
360
|
+
}
|
|
361
|
+
/** Files a scope's config says this table syncs, when the server manifest has nothing for it. */
|
|
362
|
+
function filesFromFieldOverrides(scope, table) {
|
|
363
|
+
let overrides = {};
|
|
364
|
+
try {
|
|
365
|
+
overrides = ConfigManager.resolveConfigForScope(scope).fieldOverrides || {};
|
|
366
|
+
}
|
|
367
|
+
catch (e) {
|
|
368
|
+
return [];
|
|
369
|
+
}
|
|
370
|
+
const tableOverride = overrides[table];
|
|
371
|
+
if (!tableOverride || typeof tableOverride !== "object" || Array.isArray(tableOverride))
|
|
372
|
+
return [];
|
|
373
|
+
const out = [];
|
|
374
|
+
const fieldMap = tableOverride;
|
|
375
|
+
for (const fieldName of Object.keys(fieldMap)) {
|
|
376
|
+
const cfg = fieldMap[fieldName];
|
|
377
|
+
const type = cfg && cfg.type ? cfg.type : "txt";
|
|
378
|
+
out.push({ name: fieldName, type });
|
|
379
|
+
}
|
|
380
|
+
return out;
|
|
381
|
+
}
|
|
382
|
+
function tableIsWhitelisted(scope, table) {
|
|
383
|
+
try {
|
|
384
|
+
const tables = ConfigManager.resolveConfigForScope(scope).tables || [];
|
|
385
|
+
return tables.indexOf(table) !== -1;
|
|
386
|
+
}
|
|
387
|
+
catch (e) {
|
|
388
|
+
return false;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
async function readIfExists(p) {
|
|
392
|
+
try {
|
|
393
|
+
return await fs_1.promises.readFile(p, "utf8");
|
|
394
|
+
}
|
|
395
|
+
catch (e) {
|
|
396
|
+
return undefined;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
// ---------------------------------------------------------------------------
|
|
400
|
+
// Plan (read-only)
|
|
401
|
+
// ---------------------------------------------------------------------------
|
|
402
|
+
/**
|
|
403
|
+
* Build the pull plan: resolve scopes (refusing any mismatch before a single
|
|
404
|
+
* byte is fetched for content), look each record up in its scope's server-side
|
|
405
|
+
* manifest, download its content, and diff against disk + the local manifest.
|
|
406
|
+
* Touches nothing on disk.
|
|
407
|
+
*/
|
|
408
|
+
async function planPull(rawTargets, options = {}, client = (0, snClient_1.defaultClient)()) {
|
|
409
|
+
const warnings = [];
|
|
410
|
+
const targets = dedupeTargets(rawTargets.map((t) => {
|
|
411
|
+
const ids = normalizeSysIds(t.sysId);
|
|
412
|
+
if (ids.length !== 1)
|
|
413
|
+
throw new Error("'" + t.table + "': a sys_id is required.");
|
|
414
|
+
return { table: assertTableName(t.table), sysId: ids[0] };
|
|
415
|
+
}));
|
|
416
|
+
if (targets.length === 0)
|
|
417
|
+
throw new Error("Nothing to pull: no table + sys_id given.");
|
|
418
|
+
const config = ConfigManager.getConfig();
|
|
419
|
+
const declaredScopes = (config.scopes && Object.keys(config.scopes)) || [];
|
|
420
|
+
if (options.scope && declaredScopes.length > 0 && declaredScopes.indexOf(options.scope) === -1) {
|
|
421
|
+
throw new Error("--scope '" + options.scope + "' is not declared in dove.config.js `scopes` (" +
|
|
422
|
+
declaredScopes.join(", ") + ").");
|
|
423
|
+
}
|
|
424
|
+
// 1. Resolve every record's scope FIRST. Any mismatch aborts the whole pull
|
|
425
|
+
// before content is fetched, so a bad batch writes nothing.
|
|
426
|
+
const resolved = [];
|
|
427
|
+
const failures = [];
|
|
428
|
+
for (const t of targets) {
|
|
429
|
+
try {
|
|
430
|
+
resolved.push(await resolveTargetScope(t, options, client));
|
|
431
|
+
}
|
|
432
|
+
catch (e) {
|
|
433
|
+
failures.push(e instanceof Error ? e.message : String(e));
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
for (const r of resolved) {
|
|
437
|
+
if (declaredScopes.length > 0 && declaredScopes.indexOf(r.scope) === -1) {
|
|
438
|
+
failures.push(r.table + " " + r.sysId + ": belongs to scope '" + r.scope +
|
|
439
|
+
"', which is not declared in dove.config.js `scopes`. Nothing was written.");
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
if (failures.length > 0) {
|
|
443
|
+
throw new Error((failures.length === 1 ? "Refusing to pull:\n " : "Refusing to pull (" + failures.length + " problems):\n ") +
|
|
444
|
+
failures.join("\n "));
|
|
445
|
+
}
|
|
446
|
+
// 2. One server-manifest read per scope (structure only, no content). This is
|
|
447
|
+
// what gives the record the SAME folder name and file list a full refresh
|
|
448
|
+
// would — including the duplicate-name suffix only the full set can know.
|
|
449
|
+
const scopes = [];
|
|
450
|
+
for (const r of resolved)
|
|
451
|
+
if (scopes.indexOf(r.scope) === -1)
|
|
452
|
+
scopes.push(r.scope);
|
|
453
|
+
const serverManifests = {};
|
|
454
|
+
for (const scope of scopes) {
|
|
455
|
+
FileLogger_1.fileLogger.debug("pull: reading server manifest for scope " + scope);
|
|
456
|
+
serverManifests[scope] = (0, appUtils_1.normalizeManifestKeys)(await (0, snClient_1.unwrapSNResponse)(client.getManifest(scope, config)));
|
|
457
|
+
}
|
|
458
|
+
const pending = resolved.map((r) => {
|
|
459
|
+
const hit = findRecordBySysId(serverManifests[r.scope].tables, r.table, r.sysId);
|
|
460
|
+
const requestFiles = hit ? toManifestFiles(hit.record.files) : filesFromFieldOverrides(r.scope, r.table);
|
|
461
|
+
return Object.assign({}, r, { serverKey: hit ? hit.key : undefined, requestFiles });
|
|
462
|
+
});
|
|
463
|
+
const missing = {};
|
|
464
|
+
for (const p of pending) {
|
|
465
|
+
if (!missing[p.table])
|
|
466
|
+
missing[p.table] = {};
|
|
467
|
+
missing[p.table][p.sysId] = p.requestFiles.map((f) => ({ name: f.name, type: f.type }));
|
|
468
|
+
}
|
|
469
|
+
const downloaded = await (0, snClient_1.unwrapSNResponse)(client.getMissingFiles(missing, config.tableOptions || {}));
|
|
470
|
+
// 4. Diff against disk + local manifests.
|
|
471
|
+
const manifestTexts = {};
|
|
472
|
+
const manifestsByScope = {};
|
|
473
|
+
const records = [];
|
|
474
|
+
const forceWrite = !!options.force;
|
|
475
|
+
for (const p of pending) {
|
|
476
|
+
const label = p.table + " " + p.sysId;
|
|
477
|
+
const dl = findRecordBySysId(downloaded, p.table, p.sysId);
|
|
478
|
+
if (!dl) {
|
|
479
|
+
throw new Error(label + ": the instance returned no content for this record (bulkDownload).");
|
|
480
|
+
}
|
|
481
|
+
const recWarnings = [];
|
|
482
|
+
const key = p.serverKey !== undefined
|
|
483
|
+
? p.serverKey
|
|
484
|
+
: (0, appUtils_1.toSafeFolderName)({ name: dl.record.name, sys_id: p.sysId, files: [] });
|
|
485
|
+
if (!manifestTexts[p.scope]) {
|
|
486
|
+
const manifestPath = ConfigManager.getScopeManifestPath(p.scope);
|
|
487
|
+
manifestTexts[p.scope] = parseManifestText(await readIfExists(manifestPath));
|
|
488
|
+
manifestsByScope[p.scope] = {
|
|
489
|
+
scope: p.scope,
|
|
490
|
+
manifestPath,
|
|
491
|
+
formatPreserved: manifestTexts[p.scope].formatPreserved,
|
|
492
|
+
existed: manifestTexts[p.scope].existed,
|
|
493
|
+
records: [],
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
const local = manifestTexts[p.scope].manifest;
|
|
497
|
+
const existing = findRecordBySysId(local ? local.tables : undefined, p.table, p.sysId);
|
|
498
|
+
const manifestEntry = { files: p.requestFiles.map((f) => ({ name: f.name, type: f.type })), name: key, sys_id: p.sysId };
|
|
499
|
+
let manifestAction = "add";
|
|
500
|
+
let previousKey;
|
|
501
|
+
if (existing) {
|
|
502
|
+
if (existing.key !== key)
|
|
503
|
+
previousKey = existing.key;
|
|
504
|
+
manifestAction = existing.key === key && sameEntry(existing.record, manifestEntry) ? "unchanged" : "update";
|
|
505
|
+
}
|
|
506
|
+
const recordDir = path_1.default.join(ConfigManager.getSourcePathForScope(p.scope), p.table, key);
|
|
507
|
+
if (previousKey !== undefined) {
|
|
508
|
+
recWarnings.push("renamed on the instance: manifest key '" + previousKey + "' → '" + key +
|
|
509
|
+
"'. The old folder " + path_1.default.join(ConfigManager.getSourcePathForScope(p.scope), p.table, previousKey) +
|
|
510
|
+
" is left in place — remove it by hand if it is now stale.");
|
|
511
|
+
}
|
|
512
|
+
// Field files first, then metaData (stamped exactly as refresh stamps it).
|
|
513
|
+
const files = [];
|
|
514
|
+
let serverMeta;
|
|
515
|
+
for (const f of dl.record.files || []) {
|
|
516
|
+
if (f.name === "metaData" && f.type === "json") {
|
|
517
|
+
serverMeta = f;
|
|
518
|
+
continue;
|
|
519
|
+
}
|
|
520
|
+
files.push({ path: path_1.default.join(recordDir, f.name + "." + f.type), file: f, action: "write" });
|
|
521
|
+
}
|
|
522
|
+
const metaFile = serverMeta && serverMeta.content ? (0, appUtils_1.stampMetadataContent)(serverMeta) : (0, appUtils_1.emptyMetadataFile)();
|
|
523
|
+
files.push({ path: path_1.default.join(recordDir, "metaData.json"), file: metaFile, action: "write" });
|
|
524
|
+
for (const pf of files) {
|
|
525
|
+
const current = await readIfExists(pf.path);
|
|
526
|
+
const wanted = pf.file.content || "";
|
|
527
|
+
pf.action = !forceWrite && current === wanted ? "unchanged" : "write";
|
|
528
|
+
}
|
|
529
|
+
const inWhitelist = tableIsWhitelisted(p.scope, p.table);
|
|
530
|
+
if (!inWhitelist) {
|
|
531
|
+
recWarnings.push("table '" + p.table + "' is not in includes._tables for scope '" + p.scope +
|
|
532
|
+
"'. The record is mirrored, but the next scope-wide 'dove refresh' will drop it from " +
|
|
533
|
+
"the manifest and 'dove push' will skip it — add the table to dove.config.js to keep it.");
|
|
534
|
+
}
|
|
535
|
+
if (p.serverKey === undefined) {
|
|
536
|
+
recWarnings.push((p.requestFiles.length === 0
|
|
537
|
+
? "no script/html column and no field override for '" + p.table + "' — metaData.json only. "
|
|
538
|
+
: "") +
|
|
539
|
+
"The server-side scope manifest does not list this record, so a scope-wide 'dove refresh' " +
|
|
540
|
+
"will drop its manifest entry.");
|
|
541
|
+
}
|
|
542
|
+
const planned = {
|
|
543
|
+
table: p.table,
|
|
544
|
+
sysId: p.sysId,
|
|
545
|
+
scope: p.scope,
|
|
546
|
+
key,
|
|
547
|
+
previousKey,
|
|
548
|
+
recordDir,
|
|
549
|
+
files,
|
|
550
|
+
manifestEntry,
|
|
551
|
+
manifestAction,
|
|
552
|
+
inWhitelist,
|
|
553
|
+
inServerManifest: p.serverKey !== undefined,
|
|
554
|
+
warnings: recWarnings,
|
|
555
|
+
};
|
|
556
|
+
records.push(planned);
|
|
557
|
+
manifestsByScope[p.scope].records.push(planned);
|
|
558
|
+
}
|
|
559
|
+
const manifests = scopes.filter((s) => !!manifestsByScope[s]).map((s) => manifestsByScope[s]);
|
|
560
|
+
for (const m of manifests) {
|
|
561
|
+
if (m.existed && !m.formatPreserved) {
|
|
562
|
+
warnings.push(m.manifestPath + " was not written by Dovetail's manifest writer (formatting differs), so " +
|
|
563
|
+
"re-serializing it will touch more than this record's entry. Run 'dove refresh' once to " +
|
|
564
|
+
"settle it, or review the diff carefully.");
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
return { records, manifests, warnings };
|
|
568
|
+
}
|
|
569
|
+
/**
|
|
570
|
+
* Perform the writes a plan describes: each record's folder + the files flagged
|
|
571
|
+
* `write`, then ONE splice per scope manifest. Everything else on disk is left
|
|
572
|
+
* exactly as it was.
|
|
573
|
+
*/
|
|
574
|
+
async function applyPull(plan) {
|
|
575
|
+
let filesWritten = 0;
|
|
576
|
+
let filesUnchanged = 0;
|
|
577
|
+
const manifestsWritten = [];
|
|
578
|
+
for (const rec of plan.records) {
|
|
579
|
+
await fUtils.createDirRecursively(rec.recordDir);
|
|
580
|
+
for (const pf of rec.files) {
|
|
581
|
+
if (pf.action !== "write") {
|
|
582
|
+
filesUnchanged++;
|
|
583
|
+
continue;
|
|
584
|
+
}
|
|
585
|
+
await fUtils.writeFileForce(pf.path, pf.file.content || "");
|
|
586
|
+
FileLogger_1.fileLogger.debug("pull: wrote " + pf.path);
|
|
587
|
+
filesWritten++;
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
for (const m of plan.manifests) {
|
|
591
|
+
const touched = m.records.filter((r) => r.manifestAction !== "unchanged");
|
|
592
|
+
if (touched.length === 0)
|
|
593
|
+
continue;
|
|
594
|
+
// Re-read at apply time: the plan's parse is for the diff, the splice must
|
|
595
|
+
// start from whatever is on disk NOW.
|
|
596
|
+
const text = parseManifestText(await readIfExists(m.manifestPath));
|
|
597
|
+
let manifest = text.manifest;
|
|
598
|
+
for (const r of touched) {
|
|
599
|
+
manifest = spliceManifestRecord(manifest, m.scope, r.table, r.manifestEntry, r.previousKey);
|
|
600
|
+
}
|
|
601
|
+
if (!manifest)
|
|
602
|
+
continue;
|
|
603
|
+
await fUtils.writeFileForce(m.manifestPath, serializeManifest(manifest, text.trailingNewline));
|
|
604
|
+
FileLogger_1.fileLogger.debug("pull: spliced " + touched.length + " entr" + (touched.length === 1 ? "y" : "ies") + " into " + m.manifestPath);
|
|
605
|
+
manifestsWritten.push(m.manifestPath);
|
|
606
|
+
}
|
|
607
|
+
return { filesWritten, filesUnchanged, manifestsWritten };
|
|
608
|
+
}
|