@tenonhq/dovetail-core 0.0.114 → 0.0.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/allScopesCommands.js +15 -8
- package/dist/appUtils.js +119 -47
- package/dist/commander.js +7 -0
- package/dist/commands.js +9 -0
- package/dist/reconcile/baseline.js +3 -3
- package/dist/reconcile/fields.js +2 -2
- package/dist/reconcile/recordSource.js +6 -3
- package/dist/tests/allScopesMetadata.test.js +193 -0
- package/dist/tests/stampMetadataContent.test.js +63 -5
- package/dist/tests/syncManifestRefresh.test.js +159 -0
- package/package.json +1 -1
- package/release-manifest.json +231 -222
|
@@ -106,8 +106,17 @@ async function processManifestForScope(manifest, sourceDirectory, forceWrite = f
|
|
|
106
106
|
await fsp.mkdir(recordPath, { recursive: true });
|
|
107
107
|
// Process each file in the record
|
|
108
108
|
for (const file of record.files || []) {
|
|
109
|
-
|
|
110
|
-
|
|
109
|
+
// Normalize metaData.json through the shared stamper before it hits
|
|
110
|
+
// disk. Without this the all-scopes path persisted the server's raw
|
|
111
|
+
// metadata — an absolute https://<instance>.service-now.com
|
|
112
|
+
// _record_link plus every field's live display_value — so a record
|
|
113
|
+
// written here churned against the same record written by `refresh`,
|
|
114
|
+
// and re-pulling from a different instance rewrote every file.
|
|
115
|
+
// stampMetadataContent is a no-op on non-metaData files and idempotent
|
|
116
|
+
// on already-normalized content.
|
|
117
|
+
const stamped = AppUtils.stampMetadataContent(file);
|
|
118
|
+
const filePath = path.join(recordPath, `${stamped.name}.${stamped.type}`);
|
|
119
|
+
const fileContent = stamped.content || "";
|
|
111
120
|
const fileDir = path.dirname(filePath);
|
|
112
121
|
await fsp.mkdir(fileDir, { recursive: true });
|
|
113
122
|
try {
|
|
@@ -118,15 +127,13 @@ async function processManifestForScope(manifest, sourceDirectory, forceWrite = f
|
|
|
118
127
|
throw writeError;
|
|
119
128
|
}
|
|
120
129
|
}
|
|
121
|
-
// If no metadata from server,
|
|
130
|
+
// If no metadata from server, write the deterministic placeholder. The
|
|
131
|
+
// previous stub stamped `_generatedAt: new Date().toISOString()`, which
|
|
132
|
+
// rewrote the file on every single run and read as a change downstream.
|
|
122
133
|
if (!hasMetadataFromServer) {
|
|
123
134
|
const metadataFilePath = path.join(recordPath, "metaData.json");
|
|
124
|
-
const metadataContent = {
|
|
125
|
-
_generatedAt: new Date().toISOString(),
|
|
126
|
-
_note: "Generated locally - metadata not provided by server"
|
|
127
|
-
};
|
|
128
135
|
try {
|
|
129
|
-
await fsp.writeFile(metadataFilePath,
|
|
136
|
+
await fsp.writeFile(metadataFilePath, AppUtils.EMPTY_METADATA_CONTENT, "utf8");
|
|
130
137
|
}
|
|
131
138
|
catch (metaError) {
|
|
132
139
|
Logger_1.logger.error("Failed to write metadata: " + metadataFilePath);
|
package/dist/appUtils.js
CHANGED
|
@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.createAndAssignUpdateSet = exports.swapScope = exports.buildFiles = exports.summarizeRecord = exports.pushFiles = exports.getAppFileList = exports.groupAppFiles = exports.processMissingFiles = exports.refreshAllFiles = exports.findMissingFiles = exports.syncManifest = exports.narrowManifestToRecord = exports.narrowManifestToTables = exports.processManifest = exports.normalizeManifestKeys = exports.toSafeFolderName = exports.stampMetadataContent = void 0;
|
|
39
|
+
exports.createAndAssignUpdateSet = exports.swapScope = exports.buildFiles = exports.summarizeRecord = exports.pushFiles = exports.getAppFileList = exports.groupAppFiles = exports.processMissingFiles = exports.refreshAllFiles = exports.findMissingFiles = exports.syncManifest = exports.narrowManifestToRecord = exports.narrowManifestToTables = exports.processManifest = exports.normalizeManifestKeys = exports.toSafeFolderName = exports.stampMetadataContent = exports.emptyMetadataFile = exports.EMPTY_METADATA_CONTENT = void 0;
|
|
40
40
|
const path_1 = __importDefault(require("path"));
|
|
41
41
|
const progress_1 = __importDefault(require("progress"));
|
|
42
42
|
const fUtils = __importStar(require("./FileUtils"));
|
|
@@ -81,24 +81,49 @@ const stripFieldDisplayValues = (metadata) => {
|
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
83
|
};
|
|
84
|
-
//
|
|
85
|
-
//
|
|
86
|
-
//
|
|
84
|
+
// The deterministic placeholder written when the server returns a record with no
|
|
85
|
+
// metadata at all. Every record directory still gets a metaData.json, but the
|
|
86
|
+
// file carries no generation-time value — the previous stub stamped
|
|
87
|
+
// `new Date().toISOString()`, so every run rewrote it and the record read as
|
|
88
|
+
// changed on a pull that changed nothing.
|
|
89
|
+
exports.EMPTY_METADATA_CONTENT = JSON.stringify({
|
|
90
|
+
_localOnly: true,
|
|
91
|
+
_description: "No metadata provided by server",
|
|
92
|
+
}, null, 2);
|
|
93
|
+
const emptyMetadataFile = () => ({
|
|
94
|
+
name: "metaData",
|
|
95
|
+
type: "json",
|
|
96
|
+
content: exports.EMPTY_METADATA_CONTENT,
|
|
97
|
+
});
|
|
98
|
+
exports.emptyMetadataFile = emptyMetadataFile;
|
|
99
|
+
// Finalize a record's metaData.json so its on-disk bytes are a pure function of
|
|
100
|
+
// the record — same record, same file, on any instance, from any machine, on
|
|
101
|
+
// every write path. Three things are normalized:
|
|
102
|
+
// 1. `_record_link` loses its instance host (stripRecordLinkHost).
|
|
103
|
+
// 2. Every field loses its live `display_value` (stripFieldDisplayValues).
|
|
104
|
+
// 3. `_lastUpdatedOn` is DELETED, not rewritten. It duplicated
|
|
105
|
+
// `sys_updated_on.value` exactly, and its no-sys_updated_on fallback was a
|
|
106
|
+
// wall-clock stamp that churned on every pull. Its only reader,
|
|
107
|
+
// extractUpdatedOn (reconcile/recordSource.ts), already prefers
|
|
108
|
+
// `sys_updated_on` and tolerates its absence.
|
|
109
|
+
// All record fields (sys_id, sys_scope, the value pairs) are preserved, so the
|
|
110
|
+
// file stays a full snapshot of the record rather than a stub.
|
|
87
111
|
const stampMetadataContent = (file) => {
|
|
88
112
|
if (file.name !== "metaData" || file.type !== "json")
|
|
89
113
|
return file;
|
|
90
|
-
const stamp = new Date().toISOString();
|
|
91
114
|
if (!file.content) {
|
|
92
|
-
return { ...file, content:
|
|
115
|
+
return { ...file, content: exports.EMPTY_METADATA_CONTENT };
|
|
93
116
|
}
|
|
94
117
|
try {
|
|
95
118
|
const metadata = JSON.parse(file.content);
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
metadata._lastUpdatedOn = stamp;
|
|
119
|
+
// Guard against non-object JSON (a bare string/number/array parses fine but
|
|
120
|
+
// has no keys to normalize) — writing it back verbatim is the safe default.
|
|
121
|
+
if (typeof metadata !== "object" || metadata === null || Array.isArray(metadata)) {
|
|
122
|
+
return file;
|
|
101
123
|
}
|
|
124
|
+
// Drop the key outright, including from legacy content that already carries
|
|
125
|
+
// it, so a re-stamp settles the file instead of preserving the stale stamp.
|
|
126
|
+
delete metadata._lastUpdatedOn;
|
|
102
127
|
if (typeof metadata._record_link === "string") {
|
|
103
128
|
metadata._record_link = stripRecordLinkHost(metadata._record_link);
|
|
104
129
|
}
|
|
@@ -115,15 +140,10 @@ const hasServerMetadata = (files) => files.some((f) => f.name === "metaData" &&
|
|
|
115
140
|
const processFilesInManRec = async (recPath, rec, forceWrite) => {
|
|
116
141
|
FileLogger_1.fileLogger.debug("Processing record: " + rec.name + " (" + rec.files.length + " files)");
|
|
117
142
|
const fileWrite = fUtils.writeSNFileCurry(forceWrite);
|
|
118
|
-
// If the server did not provide a metadata file, fall back to a
|
|
143
|
+
// If the server did not provide a metadata file, fall back to a deterministic
|
|
119
144
|
// stub so the record directory always has a metaData.json.
|
|
120
145
|
if (!hasServerMetadata(rec.files)) {
|
|
121
|
-
|
|
122
|
-
name: "metaData",
|
|
123
|
-
type: "json",
|
|
124
|
-
content: JSON.stringify({ _lastUpdatedOn: new Date().toISOString() }, null, 2),
|
|
125
|
-
};
|
|
126
|
-
await fileWrite(stubMetadata, recPath);
|
|
146
|
+
await fileWrite((0, exports.emptyMetadataFile)(), recPath);
|
|
127
147
|
}
|
|
128
148
|
const writeResults = await (0, genericUtils_1.allSettledBatched)(rec.files, constants_1.CONCURRENCY_FILES, function (file) { return fileWrite((0, exports.stampMetadataContent)(file), recPath); });
|
|
129
149
|
const writeFailures = writeResults.filter((r) => r.status === "rejected");
|
|
@@ -440,10 +460,14 @@ const syncManifest = async (scope, options = {}) => {
|
|
|
440
460
|
collector.startScope(scope);
|
|
441
461
|
await (0, exports.refreshAllFiles)(refreshManifest, scopeSourcePath, {
|
|
442
462
|
force: options.force,
|
|
463
|
+
metadataOnly: options.metadataOnly,
|
|
443
464
|
benchmarkCollector: collector,
|
|
444
465
|
});
|
|
445
|
-
// Update the in-memory manifest for this scope
|
|
446
|
-
|
|
466
|
+
// Update the in-memory manifest for this scope. Skipped under
|
|
467
|
+
// metadataOnly: persisting the manifest would put record adds, drops and
|
|
468
|
+
// renames into a diff that is supposed to contain metaData.json only. A
|
|
469
|
+
// normal `dove refresh` is what reconciles the manifest.
|
|
470
|
+
if (!options.metadataOnly && ConfigManager.isMultiScopeManifest(curManifest)) {
|
|
447
471
|
curManifest[scope] = newManifest;
|
|
448
472
|
ConfigManager.updateManifest(curManifest);
|
|
449
473
|
}
|
|
@@ -454,6 +478,7 @@ const syncManifest = async (scope, options = {}) => {
|
|
|
454
478
|
// scopes that leaked in before the whitelist gate existed.
|
|
455
479
|
var childOptions = {
|
|
456
480
|
force: options.force,
|
|
481
|
+
metadataOnly: options.metadataOnly,
|
|
457
482
|
tables: options.tables,
|
|
458
483
|
_benchmarkCollector: collector,
|
|
459
484
|
};
|
|
@@ -602,6 +627,13 @@ const buildAllFilesMap = (manifest) => {
|
|
|
602
627
|
*
|
|
603
628
|
* @param options.force — when true, always overwrite local files even if their
|
|
604
629
|
* content matches the instance. Use for deliberate "reset local to instance".
|
|
630
|
+
*
|
|
631
|
+
* @param options.metadataOnly — when true, rewrite every existing record's
|
|
632
|
+
* metaData.json and touch nothing else: no field file is written, and a record
|
|
633
|
+
* with no directory on disk is skipped entirely (writing one would leave an
|
|
634
|
+
* orphan directory holding a lone metaData.json and no fields). Use to settle a
|
|
635
|
+
* mirror after a change to the metadata writer, without the source churn
|
|
636
|
+
* `--force` produces.
|
|
605
637
|
*/
|
|
606
638
|
const refreshAllFiles = async (newManifest, sourcePath, options = {}) => {
|
|
607
639
|
try {
|
|
@@ -609,7 +641,8 @@ const refreshAllFiles = async (newManifest, sourcePath, options = {}) => {
|
|
|
609
641
|
const tableNames = Object.keys(allFiles);
|
|
610
642
|
if (tableNames.length === 0)
|
|
611
643
|
return;
|
|
612
|
-
FileLogger_1.fileLogger.debug("Refreshing file content for " + tableNames.length + " tables (force=" + !!options.force +
|
|
644
|
+
FileLogger_1.fileLogger.debug("Refreshing file content for " + tableNames.length + " tables (force=" + !!options.force +
|
|
645
|
+
", metadataOnly=" + !!options.metadataOnly + ")");
|
|
613
646
|
const { tableOptions = {} } = ConfigManager.getConfig();
|
|
614
647
|
const client = (0, snClient_1.defaultClient)();
|
|
615
648
|
const totalChunks = Math.ceil(tableNames.length / BULK_DOWNLOAD_TABLE_CHUNK_SIZE);
|
|
@@ -636,27 +669,55 @@ const refreshAllFiles = async (newManifest, sourcePath, options = {}) => {
|
|
|
636
669
|
});
|
|
637
670
|
var writtenCount = 0;
|
|
638
671
|
var unchangedCount = 0;
|
|
672
|
+
var skippedAbsentCount = 0;
|
|
673
|
+
var metadataWrittenCount = 0;
|
|
639
674
|
const forceWrite = !!options.force;
|
|
675
|
+
const metadataOnly = !!options.metadataOnly;
|
|
640
676
|
const forceWriter = fUtils.writeSNFileCurry(false);
|
|
641
677
|
const processedTableNames = Object.keys(filesToProcess);
|
|
642
678
|
await (0, genericUtils_1.processBatched)(processedTableNames, constants_1.CONCURRENCY_TABLES, async function (tableName) {
|
|
643
679
|
var tablePath = path_1.default.join(basePath, tableName);
|
|
644
680
|
var recs = filesToProcess[tableName].records;
|
|
645
681
|
var recKeys = Object.keys(recs);
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
682
|
+
// metadataOnly never materializes a record directory — it refreshes the
|
|
683
|
+
// metadata of records already on disk. Creating dirs here would leave an
|
|
684
|
+
// orphan folder holding a lone metaData.json with no field files.
|
|
685
|
+
if (!metadataOnly) {
|
|
686
|
+
await Promise.all(recKeys.map(function (k) {
|
|
687
|
+
// toSafeFolderName, not raw recs[k].name: the bulkDownload response
|
|
688
|
+
// carries the unmodified ServiceNow display name, which may be
|
|
689
|
+
// filesystem-unsafe (e.g. a `<table>.*` ACL). Without this, refresh
|
|
690
|
+
// recreates Windows-illegal folders that normalizeManifestKeys
|
|
691
|
+
// already removed from the manifest.
|
|
692
|
+
return fUtils.createDirRecursively(path_1.default.join(tablePath, (0, exports.toSafeFolderName)(recs[k])));
|
|
693
|
+
}));
|
|
694
|
+
}
|
|
654
695
|
await (0, genericUtils_1.processBatched)(recKeys, constants_1.CONCURRENCY_RECORDS, async function (recKey) {
|
|
655
696
|
var rec = recs[recKey];
|
|
656
697
|
var recPath = path_1.default.join(tablePath, (0, exports.toSafeFolderName)(rec));
|
|
698
|
+
// metadataOnly refreshes what is already checked out. A record the
|
|
699
|
+
// instance has but this branch does not is left alone — pulling it in
|
|
700
|
+
// would be a content change, which is exactly what this mode excludes.
|
|
701
|
+
// pathExists, NOT isDirectory: isDirectory stats unguarded and REJECTS
|
|
702
|
+
// on ENOENT, and processBatched propagates, so a single absent record
|
|
703
|
+
// would abort the whole scope — the common case, since "the instance
|
|
704
|
+
// has records this branch doesn't" is precisely what we're skipping.
|
|
705
|
+
if (metadataOnly && !(await fUtils.pathExists(recPath))) {
|
|
706
|
+
skippedAbsentCount++;
|
|
707
|
+
// Drop the downloaded content for skipped records too — filesToProcess
|
|
708
|
+
// accumulates across every chunk, so holding it would grow unbounded
|
|
709
|
+
// across a large scope.
|
|
710
|
+
rec.files = rec.files.map(function (file) {
|
|
711
|
+
var copy = Object.assign({}, file);
|
|
712
|
+
delete copy.content;
|
|
713
|
+
return copy;
|
|
714
|
+
});
|
|
715
|
+
progress.tick();
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
657
718
|
// Split server-provided metadata off from the regular files so we can
|
|
658
719
|
// track whether any regular file actually changed — metaData shouldn't
|
|
659
|
-
// be the trigger for "this record changed" since we
|
|
720
|
+
// be the trigger for "this record changed" since we rewrite it on every
|
|
660
721
|
// touch.
|
|
661
722
|
var metadataFiles = [];
|
|
662
723
|
var regularFiles = [];
|
|
@@ -669,13 +730,17 @@ const refreshAllFiles = async (newManifest, sourcePath, options = {}) => {
|
|
|
669
730
|
regularFiles.push(rf);
|
|
670
731
|
}
|
|
671
732
|
}
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
733
|
+
// metadataOnly writes no field file at all, so there is nothing to
|
|
734
|
+
// diff and the download result is discarded for regular files.
|
|
735
|
+
var results = metadataOnly
|
|
736
|
+
? []
|
|
737
|
+
: await (0, genericUtils_1.allSettledBatched)(regularFiles, constants_1.CONCURRENCY_FILES, async function (file) {
|
|
738
|
+
if (forceWrite) {
|
|
739
|
+
await forceWriter(file, recPath);
|
|
740
|
+
return true;
|
|
741
|
+
}
|
|
742
|
+
return fUtils.writeSNFileIfDifferent(file, recPath);
|
|
743
|
+
});
|
|
679
744
|
var anyChanged = false;
|
|
680
745
|
for (var f = 0; f < results.length; f++) {
|
|
681
746
|
var res = results[f];
|
|
@@ -692,23 +757,21 @@ const refreshAllFiles = async (newManifest, sourcePath, options = {}) => {
|
|
|
692
757
|
}
|
|
693
758
|
}
|
|
694
759
|
// Only touch metaData when at least one regular file in the record
|
|
695
|
-
// actually changed
|
|
696
|
-
//
|
|
760
|
+
// actually changed — otherwise a record already in sync with the
|
|
761
|
+
// instance would be rewritten for nothing. `force` and `metadataOnly`
|
|
762
|
+
// both mean "rewrite it regardless". Prefer the server-provided
|
|
697
763
|
// metadata (full field snapshot) over a stub; fall back to a stub only
|
|
698
764
|
// when the server didn't send metadata at all.
|
|
699
|
-
if (anyChanged || forceWrite) {
|
|
765
|
+
if (anyChanged || forceWrite || metadataOnly) {
|
|
700
766
|
let metadataFile;
|
|
701
767
|
if (metadataFiles.length > 0 && metadataFiles[0].content) {
|
|
702
768
|
metadataFile = (0, exports.stampMetadataContent)(metadataFiles[0]);
|
|
703
769
|
}
|
|
704
770
|
else {
|
|
705
|
-
metadataFile =
|
|
706
|
-
name: "metaData",
|
|
707
|
-
type: "json",
|
|
708
|
-
content: JSON.stringify({ _lastUpdatedOn: new Date().toISOString() }, null, 2),
|
|
709
|
-
};
|
|
771
|
+
metadataFile = (0, exports.emptyMetadataFile)();
|
|
710
772
|
}
|
|
711
773
|
await forceWriter(metadataFile, recPath);
|
|
774
|
+
metadataWrittenCount++;
|
|
712
775
|
}
|
|
713
776
|
// Strip content from manifest entries to keep memory bounded.
|
|
714
777
|
rec.files = rec.files.map(function (file) {
|
|
@@ -719,8 +782,17 @@ const refreshAllFiles = async (newManifest, sourcePath, options = {}) => {
|
|
|
719
782
|
progress.tick();
|
|
720
783
|
});
|
|
721
784
|
});
|
|
722
|
-
FileLogger_1.fileLogger.debug("Refresh complete: " + writtenCount + " written, " + unchangedCount + " unchanged"
|
|
723
|
-
|
|
785
|
+
FileLogger_1.fileLogger.debug("Refresh complete: " + writtenCount + " written, " + unchangedCount + " unchanged, " +
|
|
786
|
+
metadataWrittenCount + " metaData written, " + skippedAbsentCount + " records absent locally");
|
|
787
|
+
if (metadataOnly) {
|
|
788
|
+
// writtenCount is always 0 here — no field file is written — so report the
|
|
789
|
+
// metadata counts instead of the misleading "no changes detected".
|
|
790
|
+
Logger_1.logger.info("Rewrote metaData.json for " + metadataWrittenCount + " record(s)" +
|
|
791
|
+
(skippedAbsentCount > 0
|
|
792
|
+
? " (" + skippedAbsentCount + " skipped — not checked out locally)"
|
|
793
|
+
: ""));
|
|
794
|
+
}
|
|
795
|
+
else if (writtenCount > 0) {
|
|
724
796
|
Logger_1.logger.info("Refreshed " + writtenCount + " file(s) from instance" +
|
|
725
797
|
(unchangedCount > 0 ? " (" + unchangedCount + " already in sync)" : ""));
|
|
726
798
|
}
|
package/dist/commander.js
CHANGED
|
@@ -89,6 +89,13 @@ function configureCli(cli) {
|
|
|
89
89
|
default: false,
|
|
90
90
|
describe: "Overwrite local files even when content matches the instance",
|
|
91
91
|
},
|
|
92
|
+
"metadata-only": {
|
|
93
|
+
type: "boolean",
|
|
94
|
+
default: false,
|
|
95
|
+
describe: "Rewrite only each record's metaData.json — no field file is written, no " +
|
|
96
|
+
"record directory is created, and the manifest is left untouched. Records " +
|
|
97
|
+
"not already checked out locally are skipped. Cannot be combined with --force.",
|
|
98
|
+
},
|
|
92
99
|
scope: {
|
|
93
100
|
alias: "s",
|
|
94
101
|
type: "string",
|
package/dist/commands.js
CHANGED
|
@@ -123,11 +123,20 @@ async function refreshCommand(args, log = true) {
|
|
|
123
123
|
"Add " + pronoun + " to includes._tables (or includes._scopes.<scope>._tables) in dove.config.js first.");
|
|
124
124
|
}
|
|
125
125
|
}
|
|
126
|
+
// --force overwrites field files from the instance; --metadata-only writes
|
|
127
|
+
// no field file at all. Asking for both is contradictory, so refuse rather
|
|
128
|
+
// than silently picking one and clobbering the caller's source.
|
|
129
|
+
if (args.metadataOnly && args.force) {
|
|
130
|
+
throw new Error("--metadata-only cannot be combined with --force: --force overwrites every " +
|
|
131
|
+
"field file from the instance, while --metadata-only writes none. Pick one.");
|
|
132
|
+
}
|
|
126
133
|
FileLogger_1.fileLogger.debug("Syncing manifest from instance (force=" + !!args.force +
|
|
134
|
+
", metadataOnly=" + !!args.metadataOnly +
|
|
127
135
|
", benchmark=" + !!args.benchmark +
|
|
128
136
|
", tables=" + (tables.length > 0 ? tables.join(",") : "all") + ")");
|
|
129
137
|
await AppUtils.syncManifest(args.scope, {
|
|
130
138
|
force: !!args.force,
|
|
139
|
+
metadataOnly: !!args.metadataOnly,
|
|
131
140
|
benchmark: !!args.benchmark,
|
|
132
141
|
tables: tables.length > 0 ? tables : undefined,
|
|
133
142
|
});
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
// records the {sys_id -> sys_updated_on} the instance was known to hold at the
|
|
4
4
|
// last reconcile (or refresh), namespaced by instance host. It MUST live
|
|
5
5
|
// outside the git tree: after a `git checkout`, the on-disk
|
|
6
|
-
// `metaData.
|
|
7
|
-
// developer's instance state, so the branch files cannot answer "did the
|
|
8
|
-
// change the instance since we last synced?". The baseline can.
|
|
6
|
+
// `metaData.sys_updated_on` is whatever the *committer's* instance held, not
|
|
7
|
+
// this developer's instance state, so the branch files cannot answer "did the
|
|
8
|
+
// dev change the instance since we last synced?". The baseline can.
|
|
9
9
|
//
|
|
10
10
|
// Phase 1 only READS the baseline (to disambiguate deletes and surface drift).
|
|
11
11
|
// `writeBaseline` / `baselineFromLive` are the apply-phase write path and the
|
package/dist/reconcile/fields.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// Field-comparison rules for reconcile. A record's on-disk representation is a
|
|
3
3
|
// directory of per-field files plus a `metaData.json` snapshot. metaData is
|
|
4
|
-
// Dovetail bookkeeping — it carries
|
|
5
|
-
//
|
|
4
|
+
// Dovetail bookkeeping — it carries a host-stripped `_record_link` and a full
|
|
5
|
+
// field dump including volatile audit columns (sys_updated_on, sys_mod_count) —
|
|
6
6
|
// so including it in a content comparison would make every record read as
|
|
7
7
|
// modified. Excluding it leaves the genuine, content-bearing field files
|
|
8
8
|
// (script.js, etc.), which is exactly what `dove refresh` writes verbatim from
|
|
@@ -57,9 +57,12 @@ const fields_1 = require("./fields");
|
|
|
57
57
|
// Mirror the refresh chunk size so a large scope stays under ServiceNow's ~10 MB
|
|
58
58
|
// REST payload cap.
|
|
59
59
|
const BULK_DOWNLOAD_TABLE_CHUNK_SIZE = 5;
|
|
60
|
-
// Pull `sys_updated_on` out of a metaData.json blob.
|
|
61
|
-
// `sys_updated_on: { value }
|
|
62
|
-
//
|
|
60
|
+
// Pull `sys_updated_on` out of a metaData.json blob. Both sides carry
|
|
61
|
+
// `sys_updated_on: { value }` — that is the primary source. `_lastUpdatedOn` is
|
|
62
|
+
// a legacy fallback: Dovetail no longer writes it (it duplicated
|
|
63
|
+
// `sys_updated_on.value` and its no-sys_updated_on branch stamped a wall-clock
|
|
64
|
+
// that churned every pull), but a branch checked out before that change still
|
|
65
|
+
// has it on disk, so keep reading it. Neither present -> "".
|
|
63
66
|
function extractUpdatedOn(metaContent) {
|
|
64
67
|
if (!metaContent) {
|
|
65
68
|
return "";
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Regression: the all-scopes writer must normalize metaData.json, exactly like
|
|
4
|
+
* the `refresh` path does.
|
|
5
|
+
*
|
|
6
|
+
* `processManifestForScope` (allScopesCommands.ts) is what `dove initScopes` and
|
|
7
|
+
* `dove watchAllScopes` write through. It used to `fsp.writeFile` every server
|
|
8
|
+
* file verbatim, so on that path metaData.json kept the server's raw payload:
|
|
9
|
+
*
|
|
10
|
+
* - `_record_link` with the full `https://<instance>.service-now.com` host,
|
|
11
|
+
* making the on-disk artifact instance-bound — re-pull from another instance
|
|
12
|
+
* and every file rewrites;
|
|
13
|
+
* - every field's live `display_value`, which is resolved server-side at pull
|
|
14
|
+
* time, so renaming one referenced record re-churns every file pointing at
|
|
15
|
+
* it (and timestamp display_values render in the *puller's* timezone, so two
|
|
16
|
+
* developers churn each other);
|
|
17
|
+
*
|
|
18
|
+
* and when the server sent no metadata at all it wrote a stub stamped
|
|
19
|
+
* `_generatedAt: new Date().toISOString()` — a value that changed on every run,
|
|
20
|
+
* by construction.
|
|
21
|
+
*
|
|
22
|
+
* The result: records written here diffed against the same records written by
|
|
23
|
+
* `refresh`, and a pull that changed nothing still dirtied the tree. The writer
|
|
24
|
+
* now routes every file through the shared `stampMetadataContent` and writes a
|
|
25
|
+
* deterministic placeholder instead of the timestamp stub.
|
|
26
|
+
*/
|
|
27
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
28
|
+
if (k2 === undefined) k2 = k;
|
|
29
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
30
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
31
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
32
|
+
}
|
|
33
|
+
Object.defineProperty(o, k2, desc);
|
|
34
|
+
}) : (function(o, m, k, k2) {
|
|
35
|
+
if (k2 === undefined) k2 = k;
|
|
36
|
+
o[k2] = m[k];
|
|
37
|
+
}));
|
|
38
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
39
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
40
|
+
}) : function(o, v) {
|
|
41
|
+
o["default"] = v;
|
|
42
|
+
});
|
|
43
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
44
|
+
var ownKeys = function(o) {
|
|
45
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
46
|
+
var ar = [];
|
|
47
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
48
|
+
return ar;
|
|
49
|
+
};
|
|
50
|
+
return ownKeys(o);
|
|
51
|
+
};
|
|
52
|
+
return function (mod) {
|
|
53
|
+
if (mod && mod.__esModule) return mod;
|
|
54
|
+
var result = {};
|
|
55
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
56
|
+
__setModuleDefault(result, mod);
|
|
57
|
+
return result;
|
|
58
|
+
};
|
|
59
|
+
})();
|
|
60
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
61
|
+
const fs = __importStar(require("fs"));
|
|
62
|
+
const os = __importStar(require("os"));
|
|
63
|
+
const path = __importStar(require("path"));
|
|
64
|
+
// ---------- mocks (must come before importing the module under test) ----------
|
|
65
|
+
var mockLogger = {
|
|
66
|
+
info: jest.fn(),
|
|
67
|
+
debug: jest.fn(),
|
|
68
|
+
warn: jest.fn(),
|
|
69
|
+
error: jest.fn(),
|
|
70
|
+
success: jest.fn(),
|
|
71
|
+
getLogLevel: function () { return "warn"; },
|
|
72
|
+
};
|
|
73
|
+
var mockFileLogger = {
|
|
74
|
+
debug: jest.fn(),
|
|
75
|
+
info: jest.fn(),
|
|
76
|
+
warn: jest.fn(),
|
|
77
|
+
error: jest.fn(),
|
|
78
|
+
};
|
|
79
|
+
jest.mock("../Logger", function () { return { logger: mockLogger }; });
|
|
80
|
+
jest.mock("../FileLogger", function () { return { fileLogger: mockFileLogger }; });
|
|
81
|
+
jest.mock("../config", function () { return {}; });
|
|
82
|
+
jest.mock("../snClient", function () {
|
|
83
|
+
return {
|
|
84
|
+
defaultClient: function () { return {}; },
|
|
85
|
+
unwrapSNResponse: function (p) { return Promise.resolve(p); },
|
|
86
|
+
};
|
|
87
|
+
});
|
|
88
|
+
jest.mock("../wizard", function () {
|
|
89
|
+
return { setupDotEnv: jest.fn(), getLoginInfo: jest.fn() };
|
|
90
|
+
});
|
|
91
|
+
jest.mock("../commands", function () { return { setLogLevel: jest.fn() }; });
|
|
92
|
+
jest.mock("progress", function () {
|
|
93
|
+
return jest.fn().mockImplementation(function () {
|
|
94
|
+
return { tick: jest.fn() };
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
const AllScopes = __importStar(require("../allScopesCommands"));
|
|
98
|
+
const appUtils_1 = require("../appUtils");
|
|
99
|
+
// ---------- helpers ----------
|
|
100
|
+
var tmpRoot;
|
|
101
|
+
var SYS_ID = "24a3a804c3eb7250d4ddf1db05013165";
|
|
102
|
+
var ISO_STAMP = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/;
|
|
103
|
+
/** The metaData.json payload a real instance returns: absolute link, display_values. */
|
|
104
|
+
function serverMetadata() {
|
|
105
|
+
return JSON.stringify({
|
|
106
|
+
name: { value: "UIFilterApiMS", display_value: "UIFilterApiMS" },
|
|
107
|
+
sys_updated_on: {
|
|
108
|
+
value: "2026-04-29 14:38:51",
|
|
109
|
+
// Rendered in the puller's local timezone — the churn source.
|
|
110
|
+
display_value: "2026-04-29 07:38:51",
|
|
111
|
+
},
|
|
112
|
+
sys_scope: {
|
|
113
|
+
value: "4e4449a5475c255085d19fd8036d43a0",
|
|
114
|
+
display_value: "Tenon Marketing Work Management",
|
|
115
|
+
},
|
|
116
|
+
_table: "sys_script_include",
|
|
117
|
+
_sys_id: SYS_ID,
|
|
118
|
+
_record_link: "https://tenonworkstudio.service-now.com/sys_script_include.do?sys_id=" + SYS_ID,
|
|
119
|
+
_localOnly: true,
|
|
120
|
+
_lastUpdatedOn: "2026-04-29 14:38:51",
|
|
121
|
+
_description: "Complete field metadata for record - DO NOT SYNC TO SERVICENOW",
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
function manifestWith(files) {
|
|
125
|
+
return {
|
|
126
|
+
tables: {
|
|
127
|
+
sys_script_include: {
|
|
128
|
+
records: {
|
|
129
|
+
UIFilterApiMS: { name: "UIFilterApiMS", sys_id: SYS_ID, files: files },
|
|
130
|
+
},
|
|
131
|
+
},
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
function readRecordFile(name) {
|
|
136
|
+
return fs.readFileSync(path.join(tmpRoot, "sys_script_include", "UIFilterApiMS", name), "utf8");
|
|
137
|
+
}
|
|
138
|
+
describe("all-scopes writer — metaData.json is normalized, not written raw", function () {
|
|
139
|
+
beforeEach(function () {
|
|
140
|
+
jest.clearAllMocks();
|
|
141
|
+
tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dove-allscopes-meta-"));
|
|
142
|
+
});
|
|
143
|
+
afterEach(function () {
|
|
144
|
+
try {
|
|
145
|
+
fs.rmSync(tmpRoot, { recursive: true, force: true });
|
|
146
|
+
}
|
|
147
|
+
catch (e) { }
|
|
148
|
+
});
|
|
149
|
+
test("strips the instance host from _record_link", async function () {
|
|
150
|
+
await AllScopes.processManifestForScope(manifestWith([{ name: "metaData", type: "json", content: serverMetadata() }]), tmpRoot, true);
|
|
151
|
+
var written = JSON.parse(readRecordFile("metaData.json"));
|
|
152
|
+
expect(written._record_link).toBe("/sys_script_include.do?sys_id=" + SYS_ID);
|
|
153
|
+
expect(readRecordFile("metaData.json")).not.toContain("service-now.com");
|
|
154
|
+
});
|
|
155
|
+
test("strips every display_value, keeping value", async function () {
|
|
156
|
+
await AllScopes.processManifestForScope(manifestWith([{ name: "metaData", type: "json", content: serverMetadata() }]), tmpRoot, true);
|
|
157
|
+
var raw = readRecordFile("metaData.json");
|
|
158
|
+
var written = JSON.parse(raw);
|
|
159
|
+
expect(raw).not.toContain("display_value");
|
|
160
|
+
expect(written.name).toEqual({ value: "UIFilterApiMS" });
|
|
161
|
+
expect(written.sys_updated_on).toEqual({ value: "2026-04-29 14:38:51" });
|
|
162
|
+
expect(written.sys_scope).toEqual({ value: "4e4449a5475c255085d19fd8036d43a0" });
|
|
163
|
+
});
|
|
164
|
+
test("drops _lastUpdatedOn and writes no wall-clock value", async function () {
|
|
165
|
+
await AllScopes.processManifestForScope(manifestWith([{ name: "metaData", type: "json", content: serverMetadata() }]), tmpRoot, true);
|
|
166
|
+
var raw = readRecordFile("metaData.json");
|
|
167
|
+
expect(JSON.parse(raw)._lastUpdatedOn).toBeUndefined();
|
|
168
|
+
expect(raw).not.toMatch(ISO_STAMP);
|
|
169
|
+
});
|
|
170
|
+
test("two runs over the same server payload produce identical bytes", async function () {
|
|
171
|
+
// The contract in one assertion: a pull that changed nothing changes nothing.
|
|
172
|
+
await AllScopes.processManifestForScope(manifestWith([{ name: "metaData", type: "json", content: serverMetadata() }]), tmpRoot, true);
|
|
173
|
+
var first = readRecordFile("metaData.json");
|
|
174
|
+
await AllScopes.processManifestForScope(manifestWith([{ name: "metaData", type: "json", content: serverMetadata() }]), tmpRoot, true);
|
|
175
|
+
expect(readRecordFile("metaData.json")).toBe(first);
|
|
176
|
+
});
|
|
177
|
+
test("writes the deterministic placeholder when the server sends no metadata", async function () {
|
|
178
|
+
await AllScopes.processManifestForScope(manifestWith([{ name: "script", type: "js", content: "// no metadata came back" }]), tmpRoot, true);
|
|
179
|
+
var raw = readRecordFile("metaData.json");
|
|
180
|
+
expect(raw).toBe(appUtils_1.EMPTY_METADATA_CONTENT);
|
|
181
|
+
// The old stub's two churn keys are gone.
|
|
182
|
+
expect(raw).not.toContain("_generatedAt");
|
|
183
|
+
expect(raw).not.toMatch(ISO_STAMP);
|
|
184
|
+
});
|
|
185
|
+
test("leaves regular field files untouched", async function () {
|
|
186
|
+
// stampMetadataContent must be a no-op on anything that isn't metaData.json.
|
|
187
|
+
await AllScopes.processManifestForScope(manifestWith([
|
|
188
|
+
{ name: "script", type: "js", content: "var x = 1;\n" },
|
|
189
|
+
{ name: "metaData", type: "json", content: serverMetadata() },
|
|
190
|
+
]), tmpRoot, true);
|
|
191
|
+
expect(readRecordFile("script.js")).toBe("var x = 1;\n");
|
|
192
|
+
});
|
|
193
|
+
});
|