fhirsmith 0.16.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +26 -18
- package/config-template.json +25 -4
- package/extension-tracker/extension-tracker.js +111 -3
- package/extension-tracker/readme.md +3 -0
- package/library/html-server.js +16 -16
- package/library/html.js +9 -0
- package/library/narrative-check.js +203 -0
- package/package.json +2 -1
- package/packages/package-crawler.js +189 -82
- package/packages/packages.js +4 -62
- package/packages/utilities/dedupe-versions.js +22 -8
- package/registry/crawler.js +8 -1
- package/registry/registry.js +18 -11
- package/release.sh +80 -0
- package/testing/readme.md +1 -1
- package/tx/cm/cm-database.js +62 -0
- package/tx/cm/cm-package.js +20 -3
- package/tx/cs/cs-cs.js +86 -18
- package/tx/cs/cs-loinc.js +32 -14
- package/tx/library.js +9 -2
- package/tx/ocl/README.md +62 -6
- package/tx/ocl/cm-ocl.cjs +166 -44
- package/tx/ocl/cs-ocl.cjs +163 -13
- package/tx/ocl/resolve/public-access.js +143 -0
- package/tx/ocl/resolve/reference-resolver.js +458 -0
- package/tx/ocl/vs-ocl.cjs +167 -4
- package/tx/problems.js +7 -5
- package/tx/provider.js +5 -1
- package/tx/tx-html.js +22 -7
- package/tx/tx.js +4 -3
- package/tx/vs/vs-database.js +26 -0
- package/tx/vs/vs-package.js +42 -10
- package/tx/workers/expand.js +1 -1
- package/tx/workers/metadata.js +6 -1
- package/utilities/submit-test-report.js +1 -1
- package/xig/xig.js +88 -12
- package/gr.sh +0 -3
|
@@ -16,6 +16,33 @@ const http = require('http');
|
|
|
16
16
|
const https = require('https');
|
|
17
17
|
const ipaddr = require('ipaddr.js');
|
|
18
18
|
const {debugLog} = require("../tx/operation-context");
|
|
19
|
+
const {Parser: TarParser} = require('tar');
|
|
20
|
+
|
|
21
|
+
const MB = 1024 * 1024;
|
|
22
|
+
|
|
23
|
+
// Limits on what a package can make the crawler download, unpack and hold in memory, so
|
|
24
|
+
// that a hostile or broken package (a decompression bomb, say) costs a bounded amount.
|
|
25
|
+
// All can be set in the packages "crawler" config section:
|
|
26
|
+
// maxDownloadMB the package file as downloaded (or read from a local feed)
|
|
27
|
+
// maxUnpackedMB the total of the package's files once unzipped. The files are streamed,
|
|
28
|
+
// not held, so this limits the work, not the memory
|
|
29
|
+
// maxFileMB a file the crawler reads: package.json, .index.json, ig.ini, and each
|
|
30
|
+
// .json file (for its url). Memory is bounded by this, not by the package
|
|
31
|
+
// maxFiles the number of files in the package
|
|
32
|
+
const LIMIT_DEFAULTS = { maxDownloadMB: 250, maxUnpackedMB: 4096, maxFileMB: 50, maxFiles: 250000 };
|
|
33
|
+
|
|
34
|
+
function crawlerLimits(config) {
|
|
35
|
+
const c = { ...LIMIT_DEFAULTS, ...(config?.crawler || {}) };
|
|
36
|
+
return {
|
|
37
|
+
maxDownloadBytes: c.maxDownloadMB * MB,
|
|
38
|
+
maxUnpackedBytes: c.maxUnpackedMB * MB,
|
|
39
|
+
maxFileBytes: c.maxFileMB * MB,
|
|
40
|
+
maxFiles: c.maxFiles
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// The files in a package that the crawler reads for its metadata
|
|
45
|
+
const META_FILES = new Set(['package.json', '.index.json', 'ig.ini']);
|
|
19
46
|
|
|
20
47
|
// True if an IP literal is anything other than a normal public (unicast) address:
|
|
21
48
|
// loopback, private, link-local (incl. 169.254.169.254 cloud metadata), unique-local,
|
|
@@ -350,8 +377,14 @@ class PackageCrawler {
|
|
|
350
377
|
|
|
351
378
|
async fetchUrl(url) {
|
|
352
379
|
try {
|
|
380
|
+
const limits = crawlerLimits(this.config);
|
|
353
381
|
if (url.startsWith("/")) {
|
|
354
|
-
const
|
|
382
|
+
const localPath = this.resolveLocalReadPath(url);
|
|
383
|
+
const size = (await fs.promises.stat(localPath)).size;
|
|
384
|
+
if (size > limits.maxDownloadBytes) {
|
|
385
|
+
throw new Error(`the package is ${Math.round(size / MB)} MB, more than the limit of ${limits.maxDownloadBytes / MB} MB (maxDownloadMB)`);
|
|
386
|
+
}
|
|
387
|
+
const buffer = await fs.promises.readFile(localPath);
|
|
355
388
|
this.totalBytes += buffer.byteLength;
|
|
356
389
|
return buffer;
|
|
357
390
|
} else {
|
|
@@ -360,6 +393,7 @@ class PackageCrawler {
|
|
|
360
393
|
responseType: 'arraybuffer',
|
|
361
394
|
// We want the package file's own bytes: decodePackageBody undoes the Content-Encoding
|
|
362
395
|
decompress: false,
|
|
396
|
+
maxContentLength: limits.maxDownloadBytes,
|
|
363
397
|
signal: this.abortController?.signal,
|
|
364
398
|
headers: {
|
|
365
399
|
'User-Agent': 'FHIR Package Crawler/1.0'
|
|
@@ -373,6 +407,9 @@ class PackageCrawler {
|
|
|
373
407
|
}
|
|
374
408
|
} catch (error) {
|
|
375
409
|
debugLog(`Failed to fetch ${url}: ${error.message}`);
|
|
410
|
+
if (/maxContentLength/.test(error.message)) {
|
|
411
|
+
throw new Error(`Failed to fetch ${url}: the package is more than the limit of ${crawlerLimits(this.config).maxDownloadBytes / MB} MB (maxDownloadMB)`);
|
|
412
|
+
}
|
|
376
413
|
if (error.response && error.response.status === 429) {
|
|
377
414
|
throw new Error(`RATE_LIMITED: Server returned 429 Too Many Requests for ${url}`);
|
|
378
415
|
}
|
|
@@ -387,21 +424,41 @@ class PackageCrawler {
|
|
|
387
424
|
// package unless unzipping it gives another gzip file - a server that really did compress
|
|
388
425
|
// a .tgz for transfer.
|
|
389
426
|
decodePackageBody(body, contentEncoding) {
|
|
427
|
+
// undoing the transfer encoding gives the package file, so the download limit applies
|
|
428
|
+
const max = { maxOutputLength: crawlerLimits(this.config).maxDownloadBytes };
|
|
429
|
+
const tooBig = (e) => e.code === 'ERR_BUFFER_TOO_LARGE'
|
|
430
|
+
? new Error(`the package is more than the limit of ${max.maxOutputLength / MB} MB (maxDownloadMB) once its Content-Encoding is undone`) : e;
|
|
390
431
|
const encodings = String(contentEncoding || '').toLowerCase().split(',').map(e => e.trim()).filter(e => e && e !== 'identity');
|
|
391
432
|
for (const enc of encodings.reverse()) {
|
|
392
433
|
if (enc === 'gzip' || enc === 'x-gzip') {
|
|
434
|
+
let start;
|
|
393
435
|
try {
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
}
|
|
436
|
+
// only the start is needed to see whether it is gzip inside: a tar unzipped
|
|
437
|
+
// from a .tgz can be far larger than the download limit
|
|
438
|
+
start = zlib.gunzipSync(body.subarray(0, 4096), { finishFlush: zlib.constants.Z_SYNC_FLUSH });
|
|
398
439
|
} catch (e) {
|
|
399
440
|
// not gzip after all: leave it for extractNpmPackage to report
|
|
441
|
+
continue;
|
|
442
|
+
}
|
|
443
|
+
if (start[0] === 0x1f && start[1] === 0x8b) {
|
|
444
|
+
try {
|
|
445
|
+
body = zlib.gunzipSync(body, max);
|
|
446
|
+
} catch (e) {
|
|
447
|
+
throw tooBig(e);
|
|
448
|
+
}
|
|
400
449
|
}
|
|
401
450
|
} else if (enc === 'br') {
|
|
402
|
-
|
|
451
|
+
try {
|
|
452
|
+
body = zlib.brotliDecompressSync(body, max);
|
|
453
|
+
} catch (e) {
|
|
454
|
+
throw tooBig(e);
|
|
455
|
+
}
|
|
403
456
|
} else if (enc === 'deflate') {
|
|
404
|
-
|
|
457
|
+
try {
|
|
458
|
+
body = zlib.inflateSync(body, max);
|
|
459
|
+
} catch (e) {
|
|
460
|
+
throw tooBig(e);
|
|
461
|
+
}
|
|
405
462
|
} else {
|
|
406
463
|
throw new Error(`Unsupported Content-Encoding ${enc}`);
|
|
407
464
|
}
|
|
@@ -854,6 +911,10 @@ class PackageCrawler {
|
|
|
854
911
|
let fid = this.fixPrefix(id);
|
|
855
912
|
const filename = `${fid}-${version}.tgz`;
|
|
856
913
|
const filepath = path.join(this.config.mirrorPath, filename);
|
|
914
|
+
// id and version are validated, but the file must land in the mirror whatever they say
|
|
915
|
+
if (path.dirname(path.resolve(filepath)) !== path.resolve(this.config.mirrorPath)) {
|
|
916
|
+
throw new Error(`Package ${id}#${version} rejected: its mirror file name is not a plain file name`);
|
|
917
|
+
}
|
|
857
918
|
fs.writeFileSync(filepath, packageBuffer);
|
|
858
919
|
}
|
|
859
920
|
|
|
@@ -866,60 +927,123 @@ class PackageCrawler {
|
|
|
866
927
|
}
|
|
867
928
|
}
|
|
868
929
|
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
let
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
if (
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
// Check if this is the end (null header)
|
|
890
|
-
if (header[0] === 0) break;
|
|
891
|
-
|
|
892
|
-
// Extract filename (first 100 bytes, null-terminated)
|
|
893
|
-
let filename = '';
|
|
894
|
-
for (let i = 0; i < 100; i++) {
|
|
895
|
-
if (header[i] === 0) break;
|
|
896
|
-
filename += String.fromCharCode(header[i]);
|
|
930
|
+
// Reads what the crawler needs from a package as the package is unzipped and untarred:
|
|
931
|
+
// package.json, .index.json and ig.ini, the file names, and the url of each resource. Only
|
|
932
|
+
// those files are held, one at a time, so memory is bounded by maxFileMB however large
|
|
933
|
+
// the package unpacks to (see crawlerLimits)
|
|
934
|
+
async unpackPackage(packageBuffer) {
|
|
935
|
+
const limits = crawlerLimits(this.config);
|
|
936
|
+
const meta = {};
|
|
937
|
+
const fileNames = [];
|
|
938
|
+
const resourceUrls = [];
|
|
939
|
+
const skipped = [];
|
|
940
|
+
|
|
941
|
+
await new Promise((resolve, reject) => {
|
|
942
|
+
let unpacked = 0;
|
|
943
|
+
let failed = false;
|
|
944
|
+
const gunzip = zlib.createGunzip();
|
|
945
|
+
const fail = (e) => {
|
|
946
|
+
if (!failed) {
|
|
947
|
+
failed = true;
|
|
948
|
+
gunzip.destroy();
|
|
949
|
+
reject(e);
|
|
897
950
|
}
|
|
951
|
+
};
|
|
898
952
|
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
953
|
+
const parser = new TarParser({
|
|
954
|
+
onReadEntry: (entry) => {
|
|
955
|
+
if (failed || !['File', 'OldFile', 'ContiguousFile'].includes(entry.type)) {
|
|
956
|
+
entry.resume();
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
const name = entry.path.replace(/^package\//, '');
|
|
960
|
+
fileNames.push(name);
|
|
961
|
+
if (fileNames.length > limits.maxFiles) {
|
|
962
|
+
fail(new Error(`the package has more than ${limits.maxFiles} files (maxFiles)`));
|
|
963
|
+
entry.resume();
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
const isMeta = META_FILES.has(name);
|
|
967
|
+
if (!isMeta && !name.endsWith('.json')) {
|
|
968
|
+
entry.resume();
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
if (entry.size > limits.maxFileBytes) {
|
|
972
|
+
if (name === 'package.json') {
|
|
973
|
+
fail(new Error(`package.json is ${Math.round(entry.size / MB)} MB, more than the limit of ${limits.maxFileBytes / MB} MB (maxFileMB)`));
|
|
974
|
+
} else {
|
|
975
|
+
skipped.push(name);
|
|
976
|
+
}
|
|
977
|
+
entry.resume();
|
|
978
|
+
return;
|
|
979
|
+
}
|
|
980
|
+
const chunks = [];
|
|
981
|
+
entry.on('data', (chunk) => chunks.push(chunk));
|
|
982
|
+
entry.on('end', () => {
|
|
983
|
+
if (failed) {
|
|
984
|
+
return;
|
|
985
|
+
}
|
|
986
|
+
const text = Buffer.concat(chunks).toString('utf8');
|
|
987
|
+
if (isMeta) {
|
|
988
|
+
meta[name] = text;
|
|
989
|
+
}
|
|
990
|
+
if (name.endsWith('.json') && name !== 'package.json' && name !== '.index.json') {
|
|
991
|
+
try {
|
|
992
|
+
const json = JSON.parse(text);
|
|
993
|
+
if (json.url && json.resourceType) {
|
|
994
|
+
resourceUrls.push(json.url);
|
|
995
|
+
}
|
|
996
|
+
} catch (e) {
|
|
997
|
+
// not a resource
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
});
|
|
904
1001
|
}
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
1002
|
+
});
|
|
1003
|
+
parser.on('warn', (code, message) => {
|
|
1004
|
+
if (code === 'TAR_BAD_ARCHIVE') {
|
|
1005
|
+
fail(new Error(`not a tar file once unzipped (${message})`));
|
|
1006
|
+
}
|
|
1007
|
+
});
|
|
1008
|
+
parser.on('error', fail);
|
|
1009
|
+
parser.on('end', () => {
|
|
1010
|
+
if (!failed) {
|
|
1011
|
+
resolve();
|
|
1012
|
+
}
|
|
1013
|
+
});
|
|
913
1014
|
|
|
914
|
-
|
|
915
|
-
|
|
1015
|
+
gunzip.on('data', (chunk) => {
|
|
1016
|
+
if (failed) {
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
unpacked += chunk.length;
|
|
1020
|
+
if (unpacked > limits.maxUnpackedBytes) {
|
|
1021
|
+
fail(new Error(`the package unzips to more than the limit of ${limits.maxUnpackedBytes / MB} MB (maxUnpackedMB)`));
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1024
|
+
parser.write(chunk);
|
|
1025
|
+
});
|
|
1026
|
+
gunzip.on('end', () => {
|
|
1027
|
+
if (!failed) {
|
|
1028
|
+
parser.end();
|
|
916
1029
|
}
|
|
1030
|
+
});
|
|
1031
|
+
gunzip.on('error', (e) => fail(new Error(`could not unzip the package: ${e.message}`)));
|
|
1032
|
+
gunzip.end(packageBuffer);
|
|
1033
|
+
});
|
|
1034
|
+
|
|
1035
|
+
return { meta, fileNames, resourceUrls, skipped };
|
|
1036
|
+
}
|
|
917
1037
|
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
1038
|
+
async extractNpmPackage(packageBuffer, source) {
|
|
1039
|
+
try {
|
|
1040
|
+
if (packageBuffer.length < 2 || packageBuffer[0] !== 0x1f || packageBuffer[1] !== 0x8b) {
|
|
1041
|
+
const start = packageBuffer.subarray(0, 8).toString('hex');
|
|
1042
|
+
throw new Error(`not a gzip file (it starts with ${start || 'nothing'})`);
|
|
921
1043
|
}
|
|
922
1044
|
|
|
1045
|
+
const { meta: files, fileNames, resourceUrls, skipped } = await this.unpackPackage(packageBuffer);
|
|
1046
|
+
|
|
923
1047
|
// Parse package.json (required)
|
|
924
1048
|
if (!files['package.json']) {
|
|
925
1049
|
throw new Error('package.json not found in extracted package');
|
|
@@ -933,7 +1057,7 @@ class PackageCrawler {
|
|
|
933
1057
|
packageJson.scripts.postinstall
|
|
934
1058
|
)
|
|
935
1059
|
);
|
|
936
|
-
const hasJavaScript =
|
|
1060
|
+
const hasJavaScript = fileNames.some(f => f.endsWith('.js') || f.endsWith('.mjs') || f.endsWith('.cjs'));
|
|
937
1061
|
|
|
938
1062
|
// Extract basic NPM fields
|
|
939
1063
|
const id = packageJson.name || '';
|
|
@@ -1025,6 +1149,10 @@ class PackageCrawler {
|
|
|
1025
1149
|
}
|
|
1026
1150
|
}
|
|
1027
1151
|
|
|
1152
|
+
if (skipped.length > 0) {
|
|
1153
|
+
this.log.warn(`Package ${id}#${version}: ${skipped.length} file(s) are more than the limit of ${crawlerLimits(this.config).maxFileBytes / MB} MB (maxFileMB), so were not read for their url: ${skipped.slice(0, 5).join(', ')}${skipped.length > 5 ? ', ...' : ''}`);
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1028
1156
|
// Default fhirVersion if not found
|
|
1029
1157
|
if (!fhirVersion) {
|
|
1030
1158
|
fhirVersion = '4.0.1'; // Default to R4
|
|
@@ -1046,7 +1174,8 @@ class PackageCrawler {
|
|
|
1046
1174
|
hasInstallScripts,
|
|
1047
1175
|
hasJavaScript,
|
|
1048
1176
|
notForPublication,
|
|
1049
|
-
|
|
1177
|
+
fileNames,
|
|
1178
|
+
resourceUrls
|
|
1050
1179
|
};
|
|
1051
1180
|
|
|
1052
1181
|
} catch (error) {
|
|
@@ -1103,8 +1232,9 @@ class PackageCrawler {
|
|
|
1103
1232
|
}
|
|
1104
1233
|
|
|
1105
1234
|
isValidSemVersion(version) {
|
|
1106
|
-
//
|
|
1107
|
-
|
|
1235
|
+
// The whole version must be semver: it goes into the database, URLs and the mirror
|
|
1236
|
+
// file name, so a version like 1.0.0/../../x must not get through
|
|
1237
|
+
return /^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$/.test(version);
|
|
1108
1238
|
}
|
|
1109
1239
|
|
|
1110
1240
|
isAbsoluteUrl(url) {
|
|
@@ -1117,31 +1247,8 @@ class PackageCrawler {
|
|
|
1117
1247
|
}
|
|
1118
1248
|
|
|
1119
1249
|
processPackageUrls(npmPackage) {
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
try {
|
|
1123
|
-
|
|
1124
|
-
for (const filename of Object.keys(npmPackage.files)) {
|
|
1125
|
-
try {
|
|
1126
|
-
const bytes = npmPackage.files[filename];
|
|
1127
|
-
if (filename.endsWith('.json')) {
|
|
1128
|
-
try {
|
|
1129
|
-
const jsonContent = JSON.parse(bytes);
|
|
1130
|
-
|
|
1131
|
-
if (jsonContent.url && jsonContent.resourceType) {
|
|
1132
|
-
urls.push(jsonContent.url);
|
|
1133
|
-
}
|
|
1134
|
-
} catch (fileError) {
|
|
1135
|
-
// this.log.warn(`Error processing package file ${npmPackage.name}#${npmPackage.version}/package/${filename}: ${fileError.message}`);
|
|
1136
|
-
}
|
|
1137
|
-
}
|
|
1138
|
-
} catch (fileError) {
|
|
1139
|
-
this.log.warn(`Error processing package file ${npmPackage.name}#${npmPackage.version}/package/${filename}: ${fileError.message}`);
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1142
|
-
} catch (error) {
|
|
1143
|
-
this.log.warn(`Error processing package URLs for ${npmPackage.name}#${npmPackage.version}:`, error.message);
|
|
1144
|
-
}
|
|
1250
|
+
// the resources' urls are read as the package is unpacked (unpackPackage)
|
|
1251
|
+
const urls = [...(npmPackage.resourceUrls || [])];
|
|
1145
1252
|
|
|
1146
1253
|
// Include main package URL
|
|
1147
1254
|
if (npmPackage.url) {
|
package/packages/packages.js
CHANGED
|
@@ -1026,15 +1026,16 @@ class PackagesModule {
|
|
|
1026
1026
|
if (this.config.crawler.enabled) {
|
|
1027
1027
|
pckLog.info('Starting initial package crawler...');
|
|
1028
1028
|
|
|
1029
|
-
// Run crawler in background (non-blocking)
|
|
1030
|
-
|
|
1029
|
+
// Run crawler in background (non-blocking); errors are logged inside
|
|
1030
|
+
const crawl = async () => {
|
|
1031
1031
|
try {
|
|
1032
1032
|
await this.runCrawler();
|
|
1033
1033
|
pckLog.info('Initial package crawler completed successfully');
|
|
1034
1034
|
} catch (error) {
|
|
1035
1035
|
pckLog.error('Initial package crawler failed:', error.message);
|
|
1036
1036
|
}
|
|
1037
|
-
}
|
|
1037
|
+
};
|
|
1038
|
+
setImmediate(() => void crawl());
|
|
1038
1039
|
}
|
|
1039
1040
|
}
|
|
1040
1041
|
|
|
@@ -2804,65 +2805,6 @@ class PackagesModule {
|
|
|
2804
2805
|
return va.pre.localeCompare(vb.pre, undefined, { numeric: true });
|
|
2805
2806
|
}
|
|
2806
2807
|
|
|
2807
|
-
generateSearchHtml(req, results, params) {
|
|
2808
|
-
// Simplified HTML generation - you'd want to use a proper template engine
|
|
2809
|
-
const {name, dependson, canonicalPkg, canonicalUrl, fhirVersion} = params;
|
|
2810
|
-
return `
|
|
2811
|
-
<!DOCTYPE html>
|
|
2812
|
-
<html>
|
|
2813
|
-
<head>
|
|
2814
|
-
<title>FHIR Package Search</title>
|
|
2815
|
-
<style>
|
|
2816
|
-
body { font-family: Arial, sans-serif; margin: 20px; }
|
|
2817
|
-
.search-form { background: #f5f5f5; padding: 20px; margin-bottom: 20px; }
|
|
2818
|
-
.search-form input, .search-form select { margin: 5px; padding: 5px; }
|
|
2819
|
-
.results { margin-top: 20px; }
|
|
2820
|
-
.package { border: 1px solid #ddd; margin: 10px 0; padding: 15px; }
|
|
2821
|
-
.package-name { font-weight: bold; font-size: 1.2em; }
|
|
2822
|
-
.package-details { color: #666; margin-top: 5px; }
|
|
2823
|
-
</style>
|
|
2824
|
-
</head>
|
|
2825
|
-
<body>
|
|
2826
|
-
<h1>FHIR Package Search</h1>
|
|
2827
|
-
|
|
2828
|
-
<form class="search-form" method="GET">
|
|
2829
|
-
<input type="text" name="name" placeholder="Package name" value="${escape(name)}">
|
|
2830
|
-
<input type="text" name="dependson" placeholder="Depends on" value="${escape(dependson)}">
|
|
2831
|
-
<input type="text" name="canonicalPkg" placeholder="Canonical package" value="${escape(canonicalPkg)}">
|
|
2832
|
-
<input type="text" name="canonicalUrl" placeholder="Canonical URL" value="${escape(canonicalUrl)}">
|
|
2833
|
-
<select name="fhirVersion">
|
|
2834
|
-
<option value="">Any FHIR version</option>
|
|
2835
|
-
<option value="R2" ${fhirVersion === 'R2' ? 'selected' : ''}>R2</option>
|
|
2836
|
-
<option value="R3" ${fhirVersion === 'R3' ? 'selected' : ''}>R3</option>
|
|
2837
|
-
<option value="R4" ${fhirVersion === 'R4' ? 'selected' : ''}>R4</option>
|
|
2838
|
-
<option value="R5" ${fhirVersion === 'R5' ? 'selected' : ''}>R5</option>
|
|
2839
|
-
</select>
|
|
2840
|
-
<button type="submit">Search</button>
|
|
2841
|
-
</form>
|
|
2842
|
-
|
|
2843
|
-
<div class="results">
|
|
2844
|
-
<h2>Results (${results.length} packages found)</h2>
|
|
2845
|
-
${results.map(pkg => `
|
|
2846
|
-
<div class="package">
|
|
2847
|
-
<div class="package-name">
|
|
2848
|
-
<a href="${pkg.url}">${escape(pkg.name)}</a> v${escape(pkg.version)}
|
|
2849
|
-
</div>
|
|
2850
|
-
<div class="package-details">
|
|
2851
|
-
<strong>FHIR Version:</strong> ${escape(pkg.fhirVersion)}<br>
|
|
2852
|
-
<strong>Type:</strong> ${escape(pkg.kind)}<br>
|
|
2853
|
-
<strong>Canonical:</strong> ${escape(pkg.canonical)}<br>
|
|
2854
|
-
${pkg.description ? `<strong>Description:</strong> ${escape(pkg.description)}<br>` : ''}
|
|
2855
|
-
${pkg.date ? `<strong>Published:</strong> ${new Date(pkg.date).toLocaleDateString()}<br>` : ''}
|
|
2856
|
-
${pkg.count ? `<strong>Downloads:</strong> ${pkg.count}<br>` : ''}
|
|
2857
|
-
</div>
|
|
2858
|
-
</div>
|
|
2859
|
-
`).join('')}
|
|
2860
|
-
</div>
|
|
2861
|
-
</body>
|
|
2862
|
-
</html>
|
|
2863
|
-
`;
|
|
2864
|
-
}
|
|
2865
|
-
|
|
2866
2808
|
async shutdown() {
|
|
2867
2809
|
pckLog.info('Shutting down Packages module...');
|
|
2868
2810
|
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* One-off repair of a package server's database and mirror, for a database built before
|
|
10
10
|
* the crawler stored each id#version only once.
|
|
11
11
|
*
|
|
12
|
-
* node packages/utilities/dedupe-versions.js -db <packages.db> -mirror <mirror folder> [-apply] [-verbose]
|
|
12
|
+
* node packages/utilities/dedupe-versions.js -db <packages.db> -mirror <mirror folder> [-apply] [-verbose] [-keep <GUID>]...
|
|
13
13
|
*
|
|
14
14
|
* Without -apply it reports what it would do and changes nothing. Back up the database
|
|
15
15
|
* first. It is safe to run while the server is up, but best between crawler runs. It can be
|
|
@@ -19,7 +19,8 @@
|
|
|
19
19
|
* crawler's rule: a direct copy beats a Simplifier copy (a secondary echo); otherwise the
|
|
20
20
|
* first stored wins. Two direct copies with different content are a conflict - only their
|
|
21
21
|
* Simplifier rows are removed, and they are listed for manual cleanup. Deleted rows'
|
|
22
|
-
* download counts are added to the kept row.
|
|
22
|
+
* download counts are added to the kept row. To settle a conflict, run it again with
|
|
23
|
+
* -keep and the GUID (the URL it lists) of the copy to keep, once per conflict.
|
|
23
24
|
* 2. The unique index on (Id, Version), once there are no duplicates left.
|
|
24
25
|
* 3. The mirror. Every row's mirror file must be the bytes that row's Hash describes; a file
|
|
25
26
|
* that is missing or different is rewritten from the row's Content. Conflicts are skipped.
|
|
@@ -59,8 +60,13 @@ function keepOwnership(file, like) {
|
|
|
59
60
|
}
|
|
60
61
|
const sha1 = buf => crypto.createHash('sha1').update(buf).digest('hex');
|
|
61
62
|
|
|
62
|
-
// Which rows of one id#version to keep and delete. rows are in PackageVersionKey order
|
|
63
|
-
|
|
63
|
+
// Which rows of one id#version to keep and delete. rows are in PackageVersionKey order;
|
|
64
|
+
// keepGuids are the copies chosen (-keep) to settle conflicts.
|
|
65
|
+
function plan(rows, keepGuids = new Set()) {
|
|
66
|
+
const chosen = rows.filter(r => keepGuids.has(r.GUID));
|
|
67
|
+
if (chosen.length === 1) {
|
|
68
|
+
return { keep: chosen, conflict: false };
|
|
69
|
+
}
|
|
64
70
|
const direct = rows.filter(r => !isSimplifier(r.GUID));
|
|
65
71
|
if (direct.length === 0) {
|
|
66
72
|
return { keep: [rows[0]], conflict: false };
|
|
@@ -71,7 +77,8 @@ function plan(rows) {
|
|
|
71
77
|
return { keep: direct, conflict: true };
|
|
72
78
|
}
|
|
73
79
|
|
|
74
|
-
function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, log = console.log }) {
|
|
80
|
+
function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, keep = [], log = console.log }) {
|
|
81
|
+
const keepGuids = new Set(keep);
|
|
75
82
|
if (!fs.existsSync(dbPath)) {
|
|
76
83
|
throw new Error(`No database at ${dbPath}`); // DatabaseSync would create an empty one
|
|
77
84
|
}
|
|
@@ -87,7 +94,7 @@ function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, log = cons
|
|
|
87
94
|
const changes = [];
|
|
88
95
|
for (const g of groups) {
|
|
89
96
|
const rows = rowsOf.all(g.Id, g.Version);
|
|
90
|
-
const { keep, conflict } = plan(rows);
|
|
97
|
+
const { keep, conflict } = plan(rows, keepGuids);
|
|
91
98
|
const del = rows.filter(r => !keep.includes(r));
|
|
92
99
|
if (conflict) {
|
|
93
100
|
report.conflicts.push({ id: g.Id, version: g.Version, kept: keep.map(r => r.GUID) });
|
|
@@ -100,6 +107,12 @@ function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, log = cons
|
|
|
100
107
|
}
|
|
101
108
|
}
|
|
102
109
|
report.groups = groups.length;
|
|
110
|
+
const used = new Set(changes.flatMap(c => [c.keep.GUID]));
|
|
111
|
+
for (const g of keepGuids) {
|
|
112
|
+
if (!used.has(g)) {
|
|
113
|
+
log(`-keep ${g}: not a copy of any duplicated id#version, so ignored`);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
103
116
|
const toDelete = new Set(changes.flatMap(c => c.del.map(r => r.PackageVersionKey)));
|
|
104
117
|
const conflicted = new Set(report.conflicts.map(c => `${c.id}#${c.version}`));
|
|
105
118
|
|
|
@@ -219,15 +232,16 @@ function main(argv) {
|
|
|
219
232
|
const dbPath = arg('-db');
|
|
220
233
|
const mirrorPath = arg('-mirror');
|
|
221
234
|
if (!dbPath) {
|
|
222
|
-
console.error('usage: node packages/utilities/dedupe-versions.js -db <packages.db> [-mirror <mirror folder>] [-apply] [-verbose]');
|
|
235
|
+
console.error('usage: node packages/utilities/dedupe-versions.js -db <packages.db> [-mirror <mirror folder>] [-apply] [-verbose] [-keep <GUID>]...');
|
|
223
236
|
return 1;
|
|
224
237
|
}
|
|
225
238
|
const apply = argv.includes('-apply');
|
|
239
|
+
const keep = argv.flatMap((a, i) => a === '-keep' && argv[i + 1] ? [argv[i + 1]] : []);
|
|
226
240
|
if (!apply) {
|
|
227
241
|
console.log('Dry run: nothing will be changed (use -apply)');
|
|
228
242
|
}
|
|
229
243
|
try {
|
|
230
|
-
dedupe({ dbPath, mirrorPath, apply, verbose: argv.includes('-verbose') });
|
|
244
|
+
dedupe({ dbPath, mirrorPath, apply, keep, verbose: argv.includes('-verbose') });
|
|
231
245
|
} catch (e) {
|
|
232
246
|
console.error(e.message);
|
|
233
247
|
return 1;
|
package/registry/crawler.js
CHANGED
|
@@ -42,7 +42,9 @@ class RegistryCrawler {
|
|
|
42
42
|
userAgent: config.userAgent || 'HealthIntersections/FhirServer',
|
|
43
43
|
crawlInterval: config.crawlInterval || 5 * 60 * 1000, // 5 minutes default
|
|
44
44
|
apiKeys: config.apiKeys || {}, // Map of server URL or code to API key
|
|
45
|
-
allowPrivateAddresses: config.allowPrivateAddresses || false // SSRF opt-out for local test servers
|
|
45
|
+
allowPrivateAddresses: config.allowPrivateAddresses || false, // SSRF opt-out for local test servers
|
|
46
|
+
// the most a server's response can be, once unzipped: the servers are other people's
|
|
47
|
+
maxResponseBytes: (config.maxResponseMB || 100) * 1024 * 1024
|
|
46
48
|
};
|
|
47
49
|
this.stats = stats;
|
|
48
50
|
|
|
@@ -588,6 +590,8 @@ class RegistryCrawler {
|
|
|
588
590
|
headers: headers,
|
|
589
591
|
signal: this.abortController?.signal,
|
|
590
592
|
validateStatus: (status) => status < 500, // Don't throw on 4xx
|
|
593
|
+
// counted as it is unzipped, so a small response that unzips to a lot is stopped too
|
|
594
|
+
maxContentLength: this.config.maxResponseBytes,
|
|
591
595
|
httpAgent: this.guardedAgents().httpAgent,
|
|
592
596
|
httpsAgent: this.guardedAgents().httpsAgent
|
|
593
597
|
});
|
|
@@ -608,6 +612,9 @@ class RegistryCrawler {
|
|
|
608
612
|
|
|
609
613
|
} catch (error) {
|
|
610
614
|
debugLog(error);
|
|
615
|
+
if (/maxContentLength/.test(error.message)) {
|
|
616
|
+
throw new Error(`The response from ${url} is more than ${this.config.maxResponseBytes / (1024 * 1024)} MB`);
|
|
617
|
+
}
|
|
611
618
|
if (error.response) {
|
|
612
619
|
throw new Error(`HTTP ${error.response.status}: ${error.response.statusText}`);
|
|
613
620
|
} else if (error.request) {
|