@tenonhq/sincronia-core 0.0.81 → 0.0.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/FileUtils.js +40 -1
- package/dist/MultiScopeWatcher.js +6 -3
- package/dist/allScopesCommands.js +4 -4
- package/dist/appUtils.js +238 -16
- package/dist/benchmark.js +104 -0
- package/dist/commander.js +25 -1
- package/dist/commands.js +5 -2
- package/dist/snClient.js +56 -2
- package/dist/tests/benchmarkRefresh.test.js +265 -0
- package/dist/tests/globalDebounce.test.js +3 -2
- package/dist/tests/multi-scope-watcher.test.js +10 -20
- package/dist/tests/scopeCaching.test.js +14 -23
- package/dist/tests/syncManifestRefresh.test.js +287 -0
- package/dist/updateSetCommands.js +3 -3
- package/package.json +2 -2
package/dist/FileUtils.js
CHANGED
|
@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.writeFileForce = exports.writeSNFileForce = exports.writeSNFileIfNotExists = exports.writeBuildFile = exports.summarizeFile = exports.encodedPathsToFilePaths = exports.isValidPath = exports.splitEncodedPaths = exports.getPathsInPath = exports.isDirectory = exports.toAbsolutePath = exports.getFileContextFromPath = exports.getFileContextWithSkipReason = exports.getBuildExt = exports.isUnderPath = exports.appendToPath = exports.pathExists = exports.createDirRecursively = exports.writeSNFileCurry = exports.writeScopeManifest = exports.writeManifestFile = exports.SNFileExists = void 0;
|
|
39
|
+
exports.writeFileForce = exports.writeSNFileForce = exports.writeSNFileIfNotExists = exports.writeBuildFile = exports.summarizeFile = exports.encodedPathsToFilePaths = exports.isValidPath = exports.splitEncodedPaths = exports.getPathsInPath = exports.isDirectory = exports.toAbsolutePath = exports.getFileContextFromPath = exports.getFileContextWithSkipReason = exports.getBuildExt = exports.isUnderPath = exports.appendToPath = exports.pathExists = exports.createDirRecursively = exports.writeSNFileIfDifferent = exports.writeSNFileCurry = exports.writeScopeManifest = exports.writeManifestFile = exports.SNFileExists = void 0;
|
|
40
40
|
exports.writeEnvVar = writeEnvVar;
|
|
41
41
|
exports.writeEnvVars = writeEnvVars;
|
|
42
42
|
const constants_1 = require("./constants");
|
|
@@ -103,6 +103,45 @@ const writeSNFileCurry = (checkExists) => async (file, parentPath) => {
|
|
|
103
103
|
}
|
|
104
104
|
};
|
|
105
105
|
exports.writeSNFileCurry = writeSNFileCurry;
|
|
106
|
+
/**
|
|
107
|
+
* Writes the SN file to disk only if the local content differs from `file.content`.
|
|
108
|
+
* Returns `true` if the file was written (missing or differed), `false` if skipped.
|
|
109
|
+
*
|
|
110
|
+
* Used by `sinc refresh` to pull ServiceNow-side edits without churning unchanged
|
|
111
|
+
* files (preserves mtime, avoids noisy watchers, keeps diff output clean).
|
|
112
|
+
*/
|
|
113
|
+
const writeSNFileIfDifferent = async (file, parentPath) => {
|
|
114
|
+
let { name, type, content = "" } = file;
|
|
115
|
+
if (!content)
|
|
116
|
+
content = "";
|
|
117
|
+
const fullPath = path_1.default.join(parentPath, `${name}.${type}`);
|
|
118
|
+
let localContent = null;
|
|
119
|
+
try {
|
|
120
|
+
localContent = await fs_1.promises.readFile(fullPath, "utf8");
|
|
121
|
+
}
|
|
122
|
+
catch (e) {
|
|
123
|
+
localContent = null;
|
|
124
|
+
}
|
|
125
|
+
if (localContent === content) {
|
|
126
|
+
FileLogger_1.fileLogger.debug("Unchanged: " + fullPath);
|
|
127
|
+
return false;
|
|
128
|
+
}
|
|
129
|
+
if (localContent !== null) {
|
|
130
|
+
FileLogger_1.fileLogger.debug("Overwriting local content (differs from instance): " + fullPath);
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
FileLogger_1.fileLogger.debug("Writing (missing locally): " + fullPath);
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
await fs_1.promises.writeFile(fullPath, content);
|
|
137
|
+
return true;
|
|
138
|
+
}
|
|
139
|
+
catch (error) {
|
|
140
|
+
FileLogger_1.fileLogger.error("Failed to write " + fullPath + ":", error);
|
|
141
|
+
throw error;
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
exports.writeSNFileIfDifferent = writeSNFileIfDifferent;
|
|
106
145
|
const createDirRecursively = async (path) => {
|
|
107
146
|
await fs_1.promises.mkdir(path, { recursive: true });
|
|
108
147
|
};
|
|
@@ -220,7 +220,8 @@ class MultiScopeWatcherManager {
|
|
|
220
220
|
if (curSysId && curName) {
|
|
221
221
|
var isDefault = curName === "Default" || curName.toLowerCase().indexOf("default") !== -1;
|
|
222
222
|
if (isDefault) {
|
|
223
|
-
Logger_1.logger.warn(`[${scopeName}] No update set configured
|
|
223
|
+
Logger_1.logger.warn(`[${scopeName}] No update set configured for scope ${scopeName}. ` +
|
|
224
|
+
`Changes will go to Default. ` +
|
|
224
225
|
`Use sinc createUpdateSet or activate a task in the dashboard.`);
|
|
225
226
|
}
|
|
226
227
|
else {
|
|
@@ -233,12 +234,14 @@ class MultiScopeWatcherManager {
|
|
|
233
234
|
}
|
|
234
235
|
else {
|
|
235
236
|
Logger_1.logger.warn(`[${scopeName}] No update set configured for scope ${scopeName}. ` +
|
|
237
|
+
`Changes will go to Default. ` +
|
|
236
238
|
`Use sinc createUpdateSet or activate a task in the dashboard.`);
|
|
237
239
|
}
|
|
238
240
|
}
|
|
239
241
|
catch (queryErr) {
|
|
240
|
-
Logger_1.logger.warn(`[${scopeName}] No update set configured
|
|
241
|
-
`Changes will
|
|
242
|
+
Logger_1.logger.warn(`[${scopeName}] No update set configured for scope ${scopeName}. ` +
|
|
243
|
+
`Changes will go to Default (could not query current update set). ` +
|
|
244
|
+
`Use sinc createUpdateSet or activate a task in the dashboard.`);
|
|
242
245
|
}
|
|
243
246
|
return;
|
|
244
247
|
}
|
|
@@ -90,7 +90,7 @@ async function processManifestForScope(manifest, sourceDirectory, forceWrite = f
|
|
|
90
90
|
const recordDirName = record.name || recordName;
|
|
91
91
|
const recordPath = path.join(tablePath, recordDirName);
|
|
92
92
|
// Check if metadata file exists in the files from server
|
|
93
|
-
const hasMetadataFromServer = record.files
|
|
93
|
+
const hasMetadataFromServer = record.files && record.files.some((f) => f.name === 'metaData' && f.type === 'json');
|
|
94
94
|
// Ensure the record directory exists
|
|
95
95
|
await fsp.mkdir(recordPath, { recursive: true });
|
|
96
96
|
// Process each file in the record
|
|
@@ -243,7 +243,7 @@ async function processScope(scopeName, scopeConfig, apiDelay = 0) {
|
|
|
243
243
|
}
|
|
244
244
|
}
|
|
245
245
|
}
|
|
246
|
-
const tableCount = Object.keys(manifest
|
|
246
|
+
const tableCount = Object.keys((manifest && manifest.tables) || {}).length;
|
|
247
247
|
Logger_1.logger.info("Writing " + tableCount + " tables for " + scopeName + "...");
|
|
248
248
|
await processManifestForScope(manifest, sourceDirectory, true);
|
|
249
249
|
// Create the scope-specific manifest structure
|
|
@@ -325,8 +325,8 @@ async function initScopesCommand(args) {
|
|
|
325
325
|
}
|
|
326
326
|
else {
|
|
327
327
|
failCount++;
|
|
328
|
-
const error = result.status === "rejected" ? result.reason : result.value
|
|
329
|
-
Logger_1.logger.error(`Failed to process ${scopeName}: ${error
|
|
328
|
+
const error = result.status === "rejected" ? result.reason : (result.value && result.value.error);
|
|
329
|
+
Logger_1.logger.error(`Failed to process ${scopeName}: ${(error && error.message) || "Unknown error"}`);
|
|
330
330
|
}
|
|
331
331
|
});
|
|
332
332
|
// Write per-scope manifest files instead of a single combined one
|
package/dist/appUtils.js
CHANGED
|
@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.createAndAssignUpdateSet = exports.swapScope = exports.buildFiles = exports.summarizeRecord = exports.pushFiles = exports.getAppFileList = exports.groupAppFiles = exports.processMissingFiles = exports.findMissingFiles = exports.syncManifest = exports.processManifest = exports.normalizeManifestKeys = void 0;
|
|
39
|
+
exports.createAndAssignUpdateSet = exports.swapScope = exports.buildFiles = exports.summarizeRecord = exports.pushFiles = exports.getAppFileList = exports.groupAppFiles = exports.processMissingFiles = exports.refreshAllFiles = exports.findMissingFiles = exports.syncManifest = exports.processManifest = exports.normalizeManifestKeys = void 0;
|
|
40
40
|
const path_1 = __importDefault(require("path"));
|
|
41
41
|
const fs_1 = __importDefault(require("fs"));
|
|
42
42
|
const progress_1 = __importDefault(require("progress"));
|
|
@@ -46,6 +46,7 @@ const constants_1 = require("./constants");
|
|
|
46
46
|
const PluginManager_1 = __importDefault(require("./PluginManager"));
|
|
47
47
|
const FileLogger_1 = require("./FileLogger");
|
|
48
48
|
const snClient_1 = require("./snClient");
|
|
49
|
+
const benchmark_1 = require("./benchmark");
|
|
49
50
|
const Logger_1 = require("./Logger");
|
|
50
51
|
const genericUtils_1 = require("./genericUtils");
|
|
51
52
|
const getUpdateSetConfig = () => {
|
|
@@ -60,19 +61,46 @@ const getUpdateSetConfig = () => {
|
|
|
60
61
|
}
|
|
61
62
|
return {};
|
|
62
63
|
};
|
|
64
|
+
// Merge _lastUpdatedOn into the server-provided metadata content. Preserves all
|
|
65
|
+
// record fields (sys_id, sys_scope, field value/display_value pairs, etc.) so
|
|
66
|
+
// the local metaData.json is a full snapshot of the record, not just a stub.
|
|
67
|
+
const stampMetadataContent = (file) => {
|
|
68
|
+
if (file.name !== "metaData" || file.type !== "json")
|
|
69
|
+
return file;
|
|
70
|
+
const stamp = new Date().toISOString();
|
|
71
|
+
if (!file.content) {
|
|
72
|
+
return { ...file, content: JSON.stringify({ _lastUpdatedOn: stamp }, null, 2) };
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
const metadata = JSON.parse(file.content);
|
|
76
|
+
if (metadata.sys_updated_on && metadata.sys_updated_on.value) {
|
|
77
|
+
metadata._lastUpdatedOn = metadata.sys_updated_on.value;
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
metadata._lastUpdatedOn = stamp;
|
|
81
|
+
}
|
|
82
|
+
return { ...file, content: JSON.stringify(metadata, null, 2) };
|
|
83
|
+
}
|
|
84
|
+
catch (e) {
|
|
85
|
+
// Content isn't JSON — leave as-is, it will be written verbatim.
|
|
86
|
+
return file;
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
const hasServerMetadata = (files) => files.some((f) => f.name === "metaData" && f.type === "json" && !!f.content);
|
|
63
90
|
const processFilesInManRec = async (recPath, rec, forceWrite) => {
|
|
64
91
|
FileLogger_1.fileLogger.debug("Processing record: " + rec.name + " (" + rec.files.length + " files)");
|
|
65
92
|
const fileWrite = fUtils.writeSNFileCurry(forceWrite);
|
|
66
|
-
//
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
93
|
+
// If the server did not provide a metadata file, fall back to a timestamp-only
|
|
94
|
+
// stub so the record directory always has a metaData.json.
|
|
95
|
+
if (!hasServerMetadata(rec.files)) {
|
|
96
|
+
const stubMetadata = {
|
|
97
|
+
name: "metaData",
|
|
98
|
+
type: "json",
|
|
99
|
+
content: JSON.stringify({ _lastUpdatedOn: new Date().toISOString() }, null, 2),
|
|
100
|
+
};
|
|
101
|
+
await fileWrite(stubMetadata, recPath);
|
|
102
|
+
}
|
|
103
|
+
const writeResults = await (0, genericUtils_1.allSettledBatched)(rec.files, constants_1.CONCURRENCY_FILES, function (file) { return fileWrite(stampMetadataContent(file), recPath); });
|
|
76
104
|
const writeFailures = writeResults.filter((r) => r.status === "rejected");
|
|
77
105
|
if (writeFailures.length > 0) {
|
|
78
106
|
writeFailures.forEach((f) => {
|
|
@@ -186,7 +214,16 @@ const processManifest = async (manifest, forceWrite = false, sourcePath) => {
|
|
|
186
214
|
}
|
|
187
215
|
};
|
|
188
216
|
exports.processManifest = processManifest;
|
|
189
|
-
const syncManifest = async (scope) => {
|
|
217
|
+
const syncManifest = async (scope, options = {}) => {
|
|
218
|
+
// Top-level entry owns the collector lifecycle. Recursive calls (all-scopes
|
|
219
|
+
// → per-scope) inherit the collector via options._benchmarkCollector.
|
|
220
|
+
var isBenchmarkOwner = false;
|
|
221
|
+
var collector = options._benchmarkCollector;
|
|
222
|
+
if (options.benchmark && !collector) {
|
|
223
|
+
collector = new benchmark_1.BenchmarkCollector();
|
|
224
|
+
(0, snClient_1.setBenchmarkSink)(collector);
|
|
225
|
+
isBenchmarkOwner = true;
|
|
226
|
+
}
|
|
190
227
|
try {
|
|
191
228
|
const curManifest = await ConfigManager.getManifest();
|
|
192
229
|
if (!curManifest)
|
|
@@ -239,7 +276,12 @@ const syncManifest = async (scope) => {
|
|
|
239
276
|
const refreshTableCount = Object.keys(newManifest.tables).length;
|
|
240
277
|
FileLogger_1.fileLogger.debug("Refreshed manifest for " + scope + ": " + refreshTableCount + " tables");
|
|
241
278
|
await fUtils.writeScopeManifest(scope, newManifest);
|
|
242
|
-
|
|
279
|
+
if (collector)
|
|
280
|
+
collector.startScope(scope);
|
|
281
|
+
await (0, exports.refreshAllFiles)(newManifest, scopeSourcePath, {
|
|
282
|
+
force: options.force,
|
|
283
|
+
benchmarkCollector: collector,
|
|
284
|
+
});
|
|
243
285
|
// Update the in-memory manifest for this scope
|
|
244
286
|
if (ConfigManager.isMultiScopeManifest(curManifest)) {
|
|
245
287
|
curManifest[scope] = newManifest;
|
|
@@ -250,20 +292,24 @@ const syncManifest = async (scope) => {
|
|
|
250
292
|
// Sync all scopes. Prefer the declared-scopes list (config.scopes) over
|
|
251
293
|
// the persisted manifest keys — the manifest may contain stale undeclared
|
|
252
294
|
// scopes that leaked in before the whitelist gate existed.
|
|
295
|
+
var childOptions = {
|
|
296
|
+
force: options.force,
|
|
297
|
+
_benchmarkCollector: collector,
|
|
298
|
+
};
|
|
253
299
|
if (declaredScopes.length > 0) {
|
|
254
300
|
for (var d = 0; d < declaredScopes.length; d++) {
|
|
255
|
-
await (0, exports.syncManifest)(declaredScopes[d]);
|
|
301
|
+
await (0, exports.syncManifest)(declaredScopes[d], childOptions);
|
|
256
302
|
}
|
|
257
303
|
}
|
|
258
304
|
else if (ConfigManager.isMultiScopeManifest(curManifest)) {
|
|
259
305
|
// No declared scopes — fall back to the persisted manifest's scopes.
|
|
260
306
|
for (const scopeName of Object.keys(curManifest)) {
|
|
261
|
-
await (0, exports.syncManifest)(scopeName);
|
|
307
|
+
await (0, exports.syncManifest)(scopeName, childOptions);
|
|
262
308
|
}
|
|
263
309
|
}
|
|
264
310
|
else if (curManifest.scope) {
|
|
265
311
|
// Single scope manifest
|
|
266
|
-
await (0, exports.syncManifest)(curManifest.scope);
|
|
312
|
+
await (0, exports.syncManifest)(curManifest.scope, childOptions);
|
|
267
313
|
}
|
|
268
314
|
}
|
|
269
315
|
}
|
|
@@ -275,6 +321,12 @@ const syncManifest = async (scope) => {
|
|
|
275
321
|
message = String(e);
|
|
276
322
|
Logger_1.logger.error("Refresh failed: " + message);
|
|
277
323
|
}
|
|
324
|
+
finally {
|
|
325
|
+
if (isBenchmarkOwner && collector) {
|
|
326
|
+
(0, snClient_1.setBenchmarkSink)(null);
|
|
327
|
+
Logger_1.logger.info(collector.formatSummary());
|
|
328
|
+
}
|
|
329
|
+
}
|
|
278
330
|
};
|
|
279
331
|
exports.syncManifest = syncManifest;
|
|
280
332
|
const markFileMissing = (missingObj) => (table) => (recordId) => (file) => {
|
|
@@ -351,6 +403,176 @@ exports.findMissingFiles = findMissingFiles;
|
|
|
351
403
|
// Must mirror the chunk size used by allScopesCommands.ts (watch path) so behaviour
|
|
352
404
|
// is consistent across `refresh` and `watch`.
|
|
353
405
|
const BULK_DOWNLOAD_TABLE_CHUNK_SIZE = 5;
|
|
406
|
+
/**
|
|
407
|
+
* Builds a MissingFileTableMap containing EVERY file in the manifest — ignores
|
|
408
|
+
* local disk state. Used by `sinc refresh` to pull instance-side edits down.
|
|
409
|
+
*/
|
|
410
|
+
const buildAllFilesMap = (manifest) => {
|
|
411
|
+
const result = {};
|
|
412
|
+
const { tables } = manifest;
|
|
413
|
+
const tableNames = Object.keys(tables);
|
|
414
|
+
for (var t = 0; t < tableNames.length; t++) {
|
|
415
|
+
var tableName = tableNames[t];
|
|
416
|
+
var records = tables[tableName].records;
|
|
417
|
+
var recNames = Object.keys(records);
|
|
418
|
+
if (recNames.length === 0)
|
|
419
|
+
continue;
|
|
420
|
+
var recMap = {};
|
|
421
|
+
for (var r = 0; r < recNames.length; r++) {
|
|
422
|
+
var rec = records[recNames[r]];
|
|
423
|
+
if (!rec.files || rec.files.length === 0)
|
|
424
|
+
continue;
|
|
425
|
+
// Strip any content that may be lingering on manifest file entries; the
|
|
426
|
+
// bulkDownload endpoint only needs name + type to resolve each field.
|
|
427
|
+
recMap[rec.sys_id] = rec.files.map(function (f) {
|
|
428
|
+
return { name: f.name, type: f.type };
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
if (Object.keys(recMap).length > 0)
|
|
432
|
+
result[tableName] = recMap;
|
|
433
|
+
}
|
|
434
|
+
return result;
|
|
435
|
+
};
|
|
436
|
+
/**
|
|
437
|
+
* Refreshes local files against the ServiceNow instance for every file in the
|
|
438
|
+
* given manifest. Unlike `processMissingFiles` (which only writes files absent
|
|
439
|
+
* from disk), this walks ALL manifest files, fetches their current content from
|
|
440
|
+
* the instance, and writes when content differs.
|
|
441
|
+
*
|
|
442
|
+
* @param options.force — when true, always overwrite local files even if their
|
|
443
|
+
* content matches the instance. Use for deliberate "reset local to instance".
|
|
444
|
+
*/
|
|
445
|
+
const refreshAllFiles = async (newManifest, sourcePath, options = {}) => {
|
|
446
|
+
try {
|
|
447
|
+
const allFiles = buildAllFilesMap(newManifest);
|
|
448
|
+
const tableNames = Object.keys(allFiles);
|
|
449
|
+
if (tableNames.length === 0)
|
|
450
|
+
return;
|
|
451
|
+
FileLogger_1.fileLogger.debug("Refreshing file content for " + tableNames.length + " tables (force=" + !!options.force + ")");
|
|
452
|
+
const { tableOptions = {} } = ConfigManager.getConfig();
|
|
453
|
+
const client = (0, snClient_1.defaultClient)();
|
|
454
|
+
const totalChunks = Math.ceil(tableNames.length / BULK_DOWNLOAD_TABLE_CHUNK_SIZE);
|
|
455
|
+
const filesToProcess = {};
|
|
456
|
+
for (var i = 0; i < tableNames.length; i += BULK_DOWNLOAD_TABLE_CHUNK_SIZE) {
|
|
457
|
+
const chunkTableNames = tableNames.slice(i, i + BULK_DOWNLOAD_TABLE_CHUNK_SIZE);
|
|
458
|
+
const chunkMissing = {};
|
|
459
|
+
for (var j = 0; j < chunkTableNames.length; j++) {
|
|
460
|
+
chunkMissing[chunkTableNames[j]] = allFiles[chunkTableNames[j]];
|
|
461
|
+
}
|
|
462
|
+
const batchNum = Math.floor(i / BULK_DOWNLOAD_TABLE_CHUNK_SIZE) + 1;
|
|
463
|
+
FileLogger_1.fileLogger.debug("Refresh download batch " + batchNum + "/" + totalChunks +
|
|
464
|
+
" (" + chunkTableNames.length + " tables): " + chunkTableNames.join(", "));
|
|
465
|
+
const chunkResult = await (0, snClient_1.unwrapSNResponse)(client.getMissingFiles(chunkMissing, tableOptions));
|
|
466
|
+
for (var tableName in chunkResult) {
|
|
467
|
+
filesToProcess[tableName] = chunkResult[tableName];
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
var basePath = sourcePath || ConfigManager.getSourcePath();
|
|
471
|
+
var recordCount = countRecordsInTables(filesToProcess);
|
|
472
|
+
var progress = createScopeProgress(Logger_1.logger.getLogLevel(), {
|
|
473
|
+
scope: newManifest.scope || "default",
|
|
474
|
+
total: recordCount,
|
|
475
|
+
});
|
|
476
|
+
var writtenCount = 0;
|
|
477
|
+
var unchangedCount = 0;
|
|
478
|
+
const forceWrite = !!options.force;
|
|
479
|
+
const forceWriter = fUtils.writeSNFileCurry(false);
|
|
480
|
+
const processedTableNames = Object.keys(filesToProcess);
|
|
481
|
+
await (0, genericUtils_1.processBatched)(processedTableNames, constants_1.CONCURRENCY_TABLES, async function (tableName) {
|
|
482
|
+
var tablePath = path_1.default.join(basePath, tableName);
|
|
483
|
+
var recs = filesToProcess[tableName].records;
|
|
484
|
+
var recKeys = Object.keys(recs);
|
|
485
|
+
await Promise.all(recKeys.map(function (k) {
|
|
486
|
+
return fUtils.createDirRecursively(path_1.default.join(tablePath, recs[k].name));
|
|
487
|
+
}));
|
|
488
|
+
await (0, genericUtils_1.processBatched)(recKeys, constants_1.CONCURRENCY_RECORDS, async function (recKey) {
|
|
489
|
+
var rec = recs[recKey];
|
|
490
|
+
var recPath = path_1.default.join(tablePath, rec.name);
|
|
491
|
+
// Split server-provided metadata off from the regular files so we can
|
|
492
|
+
// track whether any regular file actually changed — metaData shouldn't
|
|
493
|
+
// be the trigger for "this record changed" since we stamp it on every
|
|
494
|
+
// touch.
|
|
495
|
+
var metadataFiles = [];
|
|
496
|
+
var regularFiles = [];
|
|
497
|
+
for (var mi = 0; mi < rec.files.length; mi++) {
|
|
498
|
+
var rf = rec.files[mi];
|
|
499
|
+
if (rf.name === "metaData" && rf.type === "json") {
|
|
500
|
+
metadataFiles.push(rf);
|
|
501
|
+
}
|
|
502
|
+
else {
|
|
503
|
+
regularFiles.push(rf);
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
var results = await (0, genericUtils_1.allSettledBatched)(regularFiles, constants_1.CONCURRENCY_FILES, async function (file) {
|
|
507
|
+
if (forceWrite) {
|
|
508
|
+
await forceWriter(file, recPath);
|
|
509
|
+
return true;
|
|
510
|
+
}
|
|
511
|
+
return fUtils.writeSNFileIfDifferent(file, recPath);
|
|
512
|
+
});
|
|
513
|
+
var anyChanged = false;
|
|
514
|
+
for (var f = 0; f < results.length; f++) {
|
|
515
|
+
var res = results[f];
|
|
516
|
+
if (res.status === "rejected") {
|
|
517
|
+
FileLogger_1.fileLogger.error("File write failed: " + res.reason);
|
|
518
|
+
continue;
|
|
519
|
+
}
|
|
520
|
+
if (res.value) {
|
|
521
|
+
anyChanged = true;
|
|
522
|
+
writtenCount++;
|
|
523
|
+
}
|
|
524
|
+
else {
|
|
525
|
+
unchangedCount++;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
// Only touch metaData when at least one regular file in the record
|
|
529
|
+
// actually changed. Avoids rewriting _lastUpdatedOn for records that
|
|
530
|
+
// were already in sync with the instance. Prefer the server-provided
|
|
531
|
+
// metadata (full field snapshot) over a stub; fall back to a stub only
|
|
532
|
+
// when the server didn't send metadata at all.
|
|
533
|
+
if (anyChanged || forceWrite) {
|
|
534
|
+
let metadataFile;
|
|
535
|
+
if (metadataFiles.length > 0 && metadataFiles[0].content) {
|
|
536
|
+
metadataFile = stampMetadataContent(metadataFiles[0]);
|
|
537
|
+
}
|
|
538
|
+
else {
|
|
539
|
+
metadataFile = {
|
|
540
|
+
name: "metaData",
|
|
541
|
+
type: "json",
|
|
542
|
+
content: JSON.stringify({ _lastUpdatedOn: new Date().toISOString() }, null, 2),
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
await forceWriter(metadataFile, recPath);
|
|
546
|
+
}
|
|
547
|
+
// Strip content from manifest entries to keep memory bounded.
|
|
548
|
+
rec.files = rec.files.map(function (file) {
|
|
549
|
+
var copy = Object.assign({}, file);
|
|
550
|
+
delete copy.content;
|
|
551
|
+
return copy;
|
|
552
|
+
});
|
|
553
|
+
progress.tick();
|
|
554
|
+
});
|
|
555
|
+
});
|
|
556
|
+
FileLogger_1.fileLogger.debug("Refresh complete: " + writtenCount + " written, " + unchangedCount + " unchanged");
|
|
557
|
+
if (writtenCount > 0) {
|
|
558
|
+
Logger_1.logger.info("Refreshed " + writtenCount + " file(s) from instance" +
|
|
559
|
+
(unchangedCount > 0 ? " (" + unchangedCount + " already in sync)" : ""));
|
|
560
|
+
}
|
|
561
|
+
else {
|
|
562
|
+
Logger_1.logger.debug("No file changes detected from instance (" + unchangedCount + " checked)");
|
|
563
|
+
}
|
|
564
|
+
if (options.benchmarkCollector) {
|
|
565
|
+
options.benchmarkCollector.endScope(writtenCount, unchangedCount);
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
catch (e) {
|
|
569
|
+
if (options.benchmarkCollector) {
|
|
570
|
+
options.benchmarkCollector.endScope(0, 0);
|
|
571
|
+
}
|
|
572
|
+
throw e;
|
|
573
|
+
}
|
|
574
|
+
};
|
|
575
|
+
exports.refreshAllFiles = refreshAllFiles;
|
|
354
576
|
const processMissingFiles = async (newManifest, sourcePath) => {
|
|
355
577
|
try {
|
|
356
578
|
const missing = await (0, exports.findMissingFiles)(newManifest, sourcePath);
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Opt-in benchmark collector for `sinc refresh`. Wired into snClient via
|
|
4
|
+
* setBenchmarkSink — when null (the default), there is zero overhead.
|
|
5
|
+
*
|
|
6
|
+
* PR #36 changed refresh from "download files absent from disk" to "bulk-download
|
|
7
|
+
* every manifest file and compare before writing". That shifted the request
|
|
8
|
+
* profile silently. This collector surfaces the numbers so regressions are
|
|
9
|
+
* visible.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.BenchmarkCollector = void 0;
|
|
13
|
+
class BenchmarkCollector {
|
|
14
|
+
httpSamples = [];
|
|
15
|
+
scopeSamples = [];
|
|
16
|
+
currentScope = null;
|
|
17
|
+
recordHttp(sample) {
|
|
18
|
+
this.httpSamples.push(sample);
|
|
19
|
+
}
|
|
20
|
+
startScope(scopeName) {
|
|
21
|
+
this.currentScope = {
|
|
22
|
+
name: scopeName,
|
|
23
|
+
startedAt: Date.now(),
|
|
24
|
+
httpStart: this.httpSamples.length,
|
|
25
|
+
bytesStart: this.totalBytes(),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
endScope(filesWritten, filesUnchanged) {
|
|
29
|
+
if (!this.currentScope)
|
|
30
|
+
return;
|
|
31
|
+
var scope = this.currentScope;
|
|
32
|
+
this.scopeSamples.push({
|
|
33
|
+
scopeName: scope.name,
|
|
34
|
+
wallTimeMs: Date.now() - scope.startedAt,
|
|
35
|
+
httpRequests: this.httpSamples.length - scope.httpStart,
|
|
36
|
+
filesWritten: filesWritten,
|
|
37
|
+
filesUnchanged: filesUnchanged,
|
|
38
|
+
totalResponseBytes: this.totalBytes() - scope.bytesStart,
|
|
39
|
+
});
|
|
40
|
+
this.currentScope = null;
|
|
41
|
+
}
|
|
42
|
+
getHttpSamples() {
|
|
43
|
+
return this.httpSamples.slice();
|
|
44
|
+
}
|
|
45
|
+
getScopeSamples() {
|
|
46
|
+
return this.scopeSamples.slice();
|
|
47
|
+
}
|
|
48
|
+
totalBytes() {
|
|
49
|
+
var total = 0;
|
|
50
|
+
for (var i = 0; i < this.httpSamples.length; i++) {
|
|
51
|
+
total += this.httpSamples[i].responseBytes;
|
|
52
|
+
}
|
|
53
|
+
return total;
|
|
54
|
+
}
|
|
55
|
+
formatSummary() {
|
|
56
|
+
var lines = [];
|
|
57
|
+
lines.push("");
|
|
58
|
+
lines.push("=".repeat(72));
|
|
59
|
+
lines.push("Refresh Benchmark");
|
|
60
|
+
lines.push("=".repeat(72));
|
|
61
|
+
if (this.httpSamples.length === 0) {
|
|
62
|
+
lines.push("(no samples recorded)");
|
|
63
|
+
return lines.join("\n");
|
|
64
|
+
}
|
|
65
|
+
var latencies = this.httpSamples
|
|
66
|
+
.map(function (s) { return s.durationMs; })
|
|
67
|
+
.sort(function (a, b) { return a - b; });
|
|
68
|
+
var p50 = percentile(latencies, 0.5);
|
|
69
|
+
var p95 = percentile(latencies, 0.95);
|
|
70
|
+
var max = latencies[latencies.length - 1];
|
|
71
|
+
var totalBytes = this.totalBytes();
|
|
72
|
+
lines.push("Overall: " + this.httpSamples.length + " HTTP requests, " +
|
|
73
|
+
formatBytes(totalBytes) + " received");
|
|
74
|
+
lines.push("Latency: p50 " + p50 + "ms | p95 " + p95 + "ms | max " + max + "ms");
|
|
75
|
+
if (this.scopeSamples.length > 0) {
|
|
76
|
+
lines.push("");
|
|
77
|
+
lines.push("Per-scope:");
|
|
78
|
+
for (var i = 0; i < this.scopeSamples.length; i++) {
|
|
79
|
+
var s = this.scopeSamples[i];
|
|
80
|
+
lines.push(" " + s.scopeName + ": " +
|
|
81
|
+
s.wallTimeMs + "ms wall, " +
|
|
82
|
+
s.httpRequests + " req, " +
|
|
83
|
+
formatBytes(s.totalResponseBytes) + ", " +
|
|
84
|
+
s.filesWritten + " written / " + s.filesUnchanged + " unchanged");
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
lines.push("=".repeat(72));
|
|
88
|
+
return lines.join("\n");
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
exports.BenchmarkCollector = BenchmarkCollector;
|
|
92
|
+
function percentile(sortedAsc, p) {
|
|
93
|
+
if (sortedAsc.length === 0)
|
|
94
|
+
return 0;
|
|
95
|
+
var idx = Math.min(sortedAsc.length - 1, Math.floor(sortedAsc.length * p));
|
|
96
|
+
return sortedAsc[idx];
|
|
97
|
+
}
|
|
98
|
+
function formatBytes(n) {
|
|
99
|
+
if (n < 1024)
|
|
100
|
+
return n + "B";
|
|
101
|
+
if (n < 1024 * 1024)
|
|
102
|
+
return (n / 1024).toFixed(1) + "KB";
|
|
103
|
+
return (n / (1024 * 1024)).toFixed(2) + "MB";
|
|
104
|
+
}
|
package/dist/commander.js
CHANGED
|
@@ -50,7 +50,31 @@ async function initCommands() {
|
|
|
50
50
|
}, async (args) => {
|
|
51
51
|
await (0, allScopesCommands_1.watchAllScopesCommand)(args);
|
|
52
52
|
})
|
|
53
|
-
.command(["refresh", "r"], "
|
|
53
|
+
.command(["refresh", "r"], "Pull latest manifest and file contents from the ServiceNow instance", (cmdArgs) => {
|
|
54
|
+
cmdArgs.options({
|
|
55
|
+
...sharedOptions,
|
|
56
|
+
force: {
|
|
57
|
+
alias: "f",
|
|
58
|
+
type: "boolean",
|
|
59
|
+
default: false,
|
|
60
|
+
describe: "Overwrite local files even when content matches the instance",
|
|
61
|
+
},
|
|
62
|
+
scope: {
|
|
63
|
+
alias: "s",
|
|
64
|
+
type: "string",
|
|
65
|
+
describe: "Refresh a single scope (default: all declared scopes)",
|
|
66
|
+
},
|
|
67
|
+
benchmark: {
|
|
68
|
+
alias: "b",
|
|
69
|
+
type: "boolean",
|
|
70
|
+
default: false,
|
|
71
|
+
describe: "Log per-scope and aggregate HTTP latency, bytes, and file counts",
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
return cmdArgs;
|
|
75
|
+
}, async (args) => {
|
|
76
|
+
await (0, commands_1.refreshCommand)(args);
|
|
77
|
+
})
|
|
54
78
|
.command(["push [target]"], "[DESTRUCTIVE] Push all files from current local files to ServiceNow instance.", (cmdArgs) => {
|
|
55
79
|
cmdArgs.options({
|
|
56
80
|
...sharedOptions,
|
package/dist/commands.js
CHANGED
|
@@ -65,8 +65,11 @@ async function refreshCommand(args, log = true) {
|
|
|
65
65
|
try {
|
|
66
66
|
if (!log)
|
|
67
67
|
setLogLevel({ logLevel: "warn" });
|
|
68
|
-
FileLogger_1.fileLogger.debug("Syncing manifest from instance");
|
|
69
|
-
await AppUtils.syncManifest(
|
|
68
|
+
FileLogger_1.fileLogger.debug("Syncing manifest from instance (force=" + !!args.force + ", benchmark=" + !!args.benchmark + ")");
|
|
69
|
+
await AppUtils.syncManifest(args.scope, {
|
|
70
|
+
force: !!args.force,
|
|
71
|
+
benchmark: !!args.benchmark,
|
|
72
|
+
});
|
|
70
73
|
Logger_1.logger.success("Refresh complete!");
|
|
71
74
|
setLogLevel(args);
|
|
72
75
|
}
|