fhirsmith 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -2
- package/config-template.json +18 -1
- package/package.json +1 -1
- package/publisher/publisher.js +153 -5
- package/publisher/validation.js +198 -0
- package/server.js +12 -12
- package/stats-db.js +127 -0
- package/stats.js +172 -6
- package/translations/Messages.properties +4 -1
- package/tx/cs/cs-api.md +1 -1
- package/tx/cs/cs-snomed.js +103 -54
- package/tx/library/conceptmap.js +39 -0
- package/tx/library/parameter-check.js +139 -0
- package/tx/operation-context.js +220 -10
- package/tx/params.js +226 -175
- package/tx/sct/expressions.js +485 -0
- package/tx/tx.fhir.org.yml +1 -1
- package/tx/tx.js +57 -50
- package/tx/workers/batch-validate.js +5 -2
- package/tx/workers/cache-control.js +82 -6
- package/tx/workers/expand.js +80 -23
- package/tx/workers/lookup.js +14 -1
- package/tx/workers/translate.js +461 -30
- package/tx/workers/validate.js +7 -1
- package/tx/workers/worker.js +22 -10
- package/tx/xml/xml-base.js +21 -1
- package/tx/xversion/xv-conceptmap.js +81 -0
- package/tx/xversion/xv-parameters.js +8 -0
- package/utilities/merge-audit.py +178 -0
- package/utilities/why-merge-audit.md +39 -0
- package/tx/data/snomed-subset.txt +0 -84
package/CHANGELOG.md
CHANGED
|
@@ -5,7 +5,77 @@ All notable changes to the Health Intersections Node Server will be documented i
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
-
## 0.
|
|
8
|
+
## [0.12.0] - 2026-08-27
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `$cache-control?mode=check`: ask the server whether a cache-id is still alive. The check is
|
|
13
|
+
also a keepalive - it resets the cache's idle clock - because a client whose own local cache is
|
|
14
|
+
absorbing the work is invisible to the server, and its cache would otherwise time out mid-job.
|
|
15
|
+
An unknown id answers 200 with `valid` = false (not 404), so a client can tell "the server says
|
|
16
|
+
my cache is gone" from "I could not reach the server"; a live cache also reports `sealed`,
|
|
17
|
+
`resource-count`, `idle` and the server's idle timeout, so a client can work out how often to
|
|
18
|
+
check instead of guessing
|
|
19
|
+
- Cache tombstones: the server now records why a cache-id it issued stopped existing - closed by
|
|
20
|
+
the client, expired after N minutes idle, or cleared with everything else - and says which in
|
|
21
|
+
the error, with the timings. New `CACHE_ID_CLOSED`, `CACHE_ID_EXPIRED` and `CACHE_ID_CLEARED`
|
|
22
|
+
messages; `CACHE_ID_UNKNOWN` now means "never issued here", and says that a cache-id is only
|
|
23
|
+
valid on the server instance and endpoint that issued it (#279)
|
|
24
|
+
- SNOMED CT: MRCM validation of postcoordinated expressions - attribute domain (including the
|
|
25
|
+
lateralizable body structure rule), range (evaluated as ECL, or as a concrete-value range such
|
|
26
|
+
as `dec(>#0..)`), cardinality and grouping. A constraint that cannot be resolved leaves the
|
|
27
|
+
value unchecked rather than rejecting it. The MRCM and lateralizable reference sets are now
|
|
28
|
+
included in the test SNOMED distribution
|
|
29
|
+
- `$translate`: R4 parameter names are accepted as aliases for the R5 ones (`source`/`target` for
|
|
30
|
+
the scopes, `targetsystem`, `targetcode`), and R4's `reverse` parameter now does literally what
|
|
31
|
+
it says - swaps the source and target sides once the parameters are read - while remaining an
|
|
32
|
+
error in R5+, which names the target concept directly instead
|
|
33
|
+
- `$translate`: reverse translation by naming the target concept (`targetCode`, `targetCoding`,
|
|
34
|
+
`targetCodeableConcept`), with the source system saying which system the answers come from
|
|
35
|
+
- `$translate`: `ConceptMap.group.element.comment` is read from R6 resources and from the
|
|
36
|
+
cross-version extension on R5 ones, so a preadopted R5 map and a native R6 map behave the same
|
|
37
|
+
- `$translate`: `originMap` names where a chain of maps started, with every other map consulted
|
|
38
|
+
along the way reported as a `used-conceptmap`; `group.unmapped` is applied per group, so a
|
|
39
|
+
mapping in one group no longer suppresses another group's fallback
|
|
40
|
+
- Persistent usage statistics: request counts per module, endpoint and operation are written to a
|
|
41
|
+
SQLite database every `intervalMinutes`, with both the interval count and the all-time total, so
|
|
42
|
+
they survive restarts and upgrades. New optional `stats` config block (#255)
|
|
43
|
+
- Support for `ValueSet.compose.property` (R6): a value set can name the properties to return in
|
|
44
|
+
its own expansion, rather than leaving it to the request
|
|
45
|
+
- Publisher: input validation for GitHub owner and repository, git branch, package id and version -
|
|
46
|
+
enforced on the server, with matching patterns on the form so the browser objects first
|
|
47
|
+
- Publisher: `large-file-archive` config setting - files the IG Publisher leaves in the web output
|
|
48
|
+
that are too big for GitHub (>100MB) are moved aside for GitHub-hosted websites, instead of
|
|
49
|
+
leaving the push to fail
|
|
50
|
+
|
|
51
|
+
### Changed
|
|
52
|
+
|
|
53
|
+
- `tx/params.js`: parameter names are now interpreted in exactly one place (`seeParameter`), used
|
|
54
|
+
by both the request `Parameters` resource and the `valueset-expansion-parameter` extension. A
|
|
55
|
+
parameter from the request always wins over one embedded in a ValueSet's expansion parameters;
|
|
56
|
+
accumulating parameters (version rules, designations, properties, supplements) still add
|
|
57
|
+
- An inactive display is now governed by `lenient-display-validation` like every other display
|
|
58
|
+
check - a warning (and `result` = true) when lenient, an error when not - rather than always
|
|
59
|
+
being a warning. The display is a designation of the concept, just not a current one
|
|
60
|
+
- tx.fhir.org now loads `fhir.tx.support` rather than `fhir.tx.support.r4`
|
|
61
|
+
|
|
62
|
+
### Fixed
|
|
63
|
+
|
|
64
|
+
- `$expand`: `status` was lost from imported property declarations, and a concept carrying the
|
|
65
|
+
same property more than once had the repeats collapsed to a single value
|
|
66
|
+
- Expansion properties are de-duplicated when they arrive from more than one place (the request,
|
|
67
|
+
an expansion parameter extension, `compose.property`) - a repeat emitted the property twice and
|
|
68
|
+
changed the cache key
|
|
69
|
+
- `no-cache=true` never busted the cache: the parameter wrote `uid`, which nothing read, instead of
|
|
70
|
+
the field the cache key hashes
|
|
71
|
+
- Boolean parameters passed as strings (as they always are on a GET) are now accepted, which
|
|
72
|
+
revives five parameters that were dead on GET requests
|
|
73
|
+
|
|
74
|
+
### Tx Conformance Statement
|
|
75
|
+
|
|
76
|
+
FHIRsmith passed all 2822 HL7 terminology service tests (modes tx.fhir.org+omop+general+snomed, tests v1.9.3, runner v6.10.3)
|
|
77
|
+
|
|
78
|
+
## [0.11.2] - 2026-08-12
|
|
9
79
|
|
|
10
80
|
### Fixed
|
|
11
81
|
|
|
@@ -14,7 +84,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
14
84
|
### Fixed
|
|
15
85
|
|
|
16
86
|
- Fix snomed designation handling for languages
|
|
17
|
-
- Fix for
|
|
87
|
+
- Fix for SSRF protection in the validator-
|
|
18
88
|
|
|
19
89
|
### Tx Conformance Statement
|
|
20
90
|
|
package/config-template.json
CHANGED
|
@@ -6,6 +6,18 @@
|
|
|
6
6
|
"credentials": true
|
|
7
7
|
}
|
|
8
8
|
},
|
|
9
|
+
// Usage statistics. Request counts are kept per module, per endpoint and per
|
|
10
|
+
// operation, and written to a SQLite database every intervalMinutes so they
|
|
11
|
+
// survive restarts and upgrades. Each row carries both the count for that
|
|
12
|
+
// interval and the all-time total. The whole block is optional.
|
|
13
|
+
"stats": {
|
|
14
|
+
// set false to keep statistics in memory only (nothing is written to disk)
|
|
15
|
+
"enabled": true,
|
|
16
|
+
// relative paths resolve under the data folder's databases directory
|
|
17
|
+
"database": "stats.db",
|
|
18
|
+
// how often the counters are written out
|
|
19
|
+
"intervalMinutes": 10
|
|
20
|
+
},
|
|
9
21
|
"modules": {
|
|
10
22
|
"shl": {
|
|
11
23
|
"enabled": false,
|
|
@@ -64,7 +76,12 @@
|
|
|
64
76
|
"database": "/absolute/path/to/publisher.db",
|
|
65
77
|
"sessionSecret": "change-this-to-a-secure-random-string",
|
|
66
78
|
"workspaceRoot": "/absolute/path/to/workspaces",
|
|
67
|
-
"igPublisherTimeoutMinutes": 60
|
|
79
|
+
"igPublisherTimeoutMinutes": 60,
|
|
80
|
+
// where to put files the IG Publisher leaves in the web folder that are too big for GitHub
|
|
81
|
+
// (>100MB). Only applies to websites whose publish-setup.json has $.website.github = true.
|
|
82
|
+
// Omit to leave them in place - the push will then fail. Relative paths resolve under the
|
|
83
|
+
// publisher data folder
|
|
84
|
+
"large-file-archive": "/absolute/path/to/large-file-archive"
|
|
68
85
|
},
|
|
69
86
|
"npmprojector": {
|
|
70
87
|
"enabled": true,
|
package/package.json
CHANGED
package/publisher/publisher.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
const express = require('express');
|
|
2
2
|
const path = require('path');
|
|
3
3
|
const fs = require('fs');
|
|
4
|
+
const validation = require('./validation');
|
|
4
5
|
const Database = require('sqlite3').Database;
|
|
5
6
|
const bcrypt = require('bcrypt');
|
|
6
7
|
const session = require('express-session');
|
|
@@ -8,6 +9,10 @@ const folders = require('../library/folder-setup');
|
|
|
8
9
|
const escape = require('escape-html');
|
|
9
10
|
const {Utilities} = require("../library/utilities");
|
|
10
11
|
|
|
12
|
+
// GitHub refuses any file larger than this at the pre-receive hook, and the rejection takes down
|
|
13
|
+
// the whole push rather than just the offending file.
|
|
14
|
+
const GITHUB_MAX_FILE_SIZE = 100 * 1024 * 1024;
|
|
15
|
+
|
|
11
16
|
|
|
12
17
|
class PublisherModule {
|
|
13
18
|
constructor(stats) {
|
|
@@ -928,6 +933,137 @@ class PublisherModule {
|
|
|
928
933
|
}
|
|
929
934
|
}
|
|
930
935
|
|
|
936
|
+
// The IG Publisher can leave a file in the web folder that GitHub will not accept - typically a
|
|
937
|
+
// technical correction archive, which is a zip of an entire published version folder and so
|
|
938
|
+
// carries that folder's own full-ig.zip inside it. GitHub rejects the entire push over one such
|
|
939
|
+
// file, at the very last step of a publication run. The IG Publisher will keep these out of the
|
|
940
|
+
// web folder itself from the next release; until then, move them aside here.
|
|
941
|
+
//
|
|
942
|
+
// Does nothing unless publisher.large-file-archive says where to put them, and the website is
|
|
943
|
+
// one that's hosted on GitHub in the first place.
|
|
944
|
+
async archiveLargeFiles(task, website) {
|
|
945
|
+
const configured = this.config['large-file-archive'];
|
|
946
|
+
if (!configured || !this.isGitHubHosted(website)) {
|
|
947
|
+
return;
|
|
948
|
+
}
|
|
949
|
+
const root = website.git_root || website.local_folder;
|
|
950
|
+
const oversized = this.findLargeFiles(root, GITHUB_MAX_FILE_SIZE);
|
|
951
|
+
if (oversized.length === 0) {
|
|
952
|
+
return;
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
const archiveDir = path.isAbsolute(configured)
|
|
956
|
+
? configured
|
|
957
|
+
: folders.filePath('publisher', configured);
|
|
958
|
+
if (!fs.existsSync(archiveDir)) {
|
|
959
|
+
fs.mkdirSync(archiveDir, { recursive: true });
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
for (const file of oversized) {
|
|
963
|
+
const sizeMB = Math.round(fs.statSync(file).size / (1024 * 1024));
|
|
964
|
+
const dest = this.nonClashingPath(archiveDir, path.basename(file));
|
|
965
|
+
this.moveFile(file, dest);
|
|
966
|
+
await this.logTaskMessage(task.id, 'warn', 'Large file ' + path.relative(root, file) + ' (' + sizeMB +
|
|
967
|
+
' MB) is over the 100 MB GitHub limit - moved to ' + dest + ' instead of being published');
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
// Only a website kept in a GitHub repository has a 100MB problem - others can carry files of any
|
|
972
|
+
// size. $.website.github in the web root's publish-setup.json says which is which; if that can't
|
|
973
|
+
// be read, or doesn't say, assume not and leave the files where they are.
|
|
974
|
+
isGitHubHosted(website) {
|
|
975
|
+
if (!website.local_folder) {
|
|
976
|
+
return false;
|
|
977
|
+
}
|
|
978
|
+
const setupPath = path.join(website.local_folder, 'publish-setup.json');
|
|
979
|
+
try {
|
|
980
|
+
const setup = JSON.parse(fs.readFileSync(setupPath, 'utf8'));
|
|
981
|
+
return !!(setup.website && setup.website.github === true);
|
|
982
|
+
} catch (err) {
|
|
983
|
+
this.logger.warn('Could not read ' + setupPath + ' to check whether the website is on GitHub: ' + err.message);
|
|
984
|
+
return false;
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
// Files in the web folder that are too big to push. Never descends into .git: a pack file over the
|
|
989
|
+
// limit is perfectly normal there, and moving it would destroy the repository. Files git is already
|
|
990
|
+
// ignoring are left alone, since they were never going to be pushed anyway.
|
|
991
|
+
findLargeFiles(root, limit) {
|
|
992
|
+
const found = [];
|
|
993
|
+
const walk = (dir) => {
|
|
994
|
+
let entries;
|
|
995
|
+
try {
|
|
996
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
997
|
+
} catch (err) {
|
|
998
|
+
this.logger.warn('Large file scan could not read ' + dir + ': ' + err.message);
|
|
999
|
+
return;
|
|
1000
|
+
}
|
|
1001
|
+
for (const entry of entries) {
|
|
1002
|
+
const full = path.join(dir, entry.name);
|
|
1003
|
+
if (entry.isDirectory()) {
|
|
1004
|
+
if (entry.name !== '.git') {
|
|
1005
|
+
walk(full);
|
|
1006
|
+
}
|
|
1007
|
+
} else if (entry.isFile()) {
|
|
1008
|
+
try {
|
|
1009
|
+
if (fs.statSync(full).size > limit) {
|
|
1010
|
+
found.push(full);
|
|
1011
|
+
}
|
|
1012
|
+
} catch (err) {
|
|
1013
|
+
// the file went away between readdir and stat - nothing left to move
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
};
|
|
1018
|
+
walk(root);
|
|
1019
|
+
return found.length === 0 ? found : this.removeGitIgnored(root, found);
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
removeGitIgnored(root, files) {
|
|
1023
|
+
try {
|
|
1024
|
+
const result = require('child_process').spawnSync('git', ['check-ignore', '--stdin'],
|
|
1025
|
+
{ cwd: root, input: files.join('\n'), encoding: 'utf8' });
|
|
1026
|
+
// exit 0: at least one path is ignored. 1: none are. anything else: git could not tell us,
|
|
1027
|
+
// in which case treat everything as pushable rather than leaving a file that breaks the push.
|
|
1028
|
+
if (result.status !== 0 || !result.stdout) {
|
|
1029
|
+
return files;
|
|
1030
|
+
}
|
|
1031
|
+
const ignored = new Set(result.stdout.split('\n').map((s) => s.trim()).filter((s) => s));
|
|
1032
|
+
return files.filter((f) => !ignored.has(f));
|
|
1033
|
+
} catch (err) {
|
|
1034
|
+
this.logger.warn('Could not check the gitignore status of large files: ' + err.message);
|
|
1035
|
+
return files;
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
nonClashingPath(dir, name) {
|
|
1040
|
+
let candidate = path.join(dir, name);
|
|
1041
|
+
if (!fs.existsSync(candidate)) {
|
|
1042
|
+
return candidate;
|
|
1043
|
+
}
|
|
1044
|
+
const ext = path.extname(name);
|
|
1045
|
+
const stem = path.basename(name, ext);
|
|
1046
|
+
let i = 2;
|
|
1047
|
+
do {
|
|
1048
|
+
candidate = path.join(dir, stem + '-' + i + ext);
|
|
1049
|
+
i++;
|
|
1050
|
+
} while (fs.existsSync(candidate));
|
|
1051
|
+
return candidate;
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
// rename() fails across filesystems, and the archive folder will not always be on the same one
|
|
1055
|
+
moveFile(src, dest) {
|
|
1056
|
+
try {
|
|
1057
|
+
fs.renameSync(src, dest);
|
|
1058
|
+
} catch (err) {
|
|
1059
|
+
if (err.code !== 'EXDEV') {
|
|
1060
|
+
throw err;
|
|
1061
|
+
}
|
|
1062
|
+
fs.copyFileSync(src, dest);
|
|
1063
|
+
fs.unlinkSync(src);
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
|
|
931
1067
|
async runPublication(task) {
|
|
932
1068
|
const website = await this.getWebsite(task.website_id);
|
|
933
1069
|
if (!website) {
|
|
@@ -1016,6 +1152,9 @@ class PublisherModule {
|
|
|
1016
1152
|
// url - can survive into the web tree. Catch that here, before anything is committed.
|
|
1017
1153
|
await this.verifyPublishedPackage(task, website, draftDir);
|
|
1018
1154
|
|
|
1155
|
+
// Step 5c: Move anything too big for GitHub out of the web folder, before it gets staged
|
|
1156
|
+
await this.archiveLargeFiles(task, website);
|
|
1157
|
+
|
|
1019
1158
|
// Step 6: Commit and push the web folder
|
|
1020
1159
|
await this.logTaskMessage(task.id, 'info', 'Committing changes to web folder...');
|
|
1021
1160
|
const gitUrl = 'https://github.com/' + task.github_org + '/' + task.github_repo + '.git';
|
|
@@ -1363,23 +1502,23 @@ class PublisherModule {
|
|
|
1363
1502
|
content += '</div>';
|
|
1364
1503
|
content += '<div class="col-md-3">';
|
|
1365
1504
|
content += '<label for="github_org" class="form-label">GitHub Org</label>';
|
|
1366
|
-
content += '<input type="text" class="form-control" id="github_org" name="github_org" required placeholder="hl7">';
|
|
1505
|
+
content += '<input type="text" class="form-control" id="github_org" name="github_org" required maxlength="39" pattern="' + validation.HTML_PATTERNS.github_org + '" title="Letters, digits and single hyphens" placeholder="hl7">';
|
|
1367
1506
|
content += '</div>';
|
|
1368
1507
|
content += '<div class="col-md-3">';
|
|
1369
1508
|
content += '<label for="github_repo" class="form-label">GitHub Repo</label>';
|
|
1370
|
-
content += '<input type="text" class="form-control" id="github_repo" name="github_repo" required placeholder="fhir-us-core">';
|
|
1509
|
+
content += '<input type="text" class="form-control" id="github_repo" name="github_repo" required maxlength="100" pattern="' + validation.HTML_PATTERNS.github_repo + '" title="Letters, digits, dots, hyphens and underscores" placeholder="fhir-us-core">';
|
|
1371
1510
|
content += '</div>';
|
|
1372
1511
|
content += '<div class="col-md-3">';
|
|
1373
1512
|
content += '<label for="git_branch" class="form-label">Branch</label>';
|
|
1374
|
-
content += '<input type="text" class="form-control" id="git_branch" name="git_branch" required placeholder="main">';
|
|
1513
|
+
content += '<input type="text" class="form-control" id="git_branch" name="git_branch" required maxlength="255" pattern="' + validation.HTML_PATTERNS.git_branch + '" title="A git branch name - slashes are fine, but not spaces, backslashes or any of ~ ^ : ? * [" placeholder="main">';
|
|
1375
1514
|
content += '</div>';
|
|
1376
1515
|
content += '<div class="col-md-4">';
|
|
1377
1516
|
content += '<label for="npm_package_id" class="form-label">NPM Package ID</label>';
|
|
1378
|
-
content += '<input type="text" class="form-control" id="npm_package_id" name="npm_package_id" required placeholder="hl7.fhir.us.core">';
|
|
1517
|
+
content += '<input type="text" class="form-control" id="npm_package_id" name="npm_package_id" required maxlength="128" pattern="' + validation.HTML_PATTERNS.npm_package_id + '" title="Letters, digits, dots, hyphens and underscores" placeholder="hl7.fhir.us.core">';
|
|
1379
1518
|
content += '</div>';
|
|
1380
1519
|
content += '<div class="col-md-4">';
|
|
1381
1520
|
content += '<label for="version" class="form-label">Version</label>';
|
|
1382
|
-
content += '<input type="text" class="form-control" id="version" name="version" required placeholder="6.0.0">';
|
|
1521
|
+
content += '<input type="text" class="form-control" id="version" name="version" required maxlength="64" pattern="' + validation.HTML_PATTERNS.version + '" title="Letters, digits, dots, hyphens and plus signs" placeholder="6.0.0">';
|
|
1383
1522
|
content += '</div>';
|
|
1384
1523
|
content += '<div class="col-md-4 d-flex align-items-end">';
|
|
1385
1524
|
content += '<button type="submit" class="btn btn-primary">Create Task</button>';
|
|
@@ -1495,6 +1634,15 @@ class PublisherModule {
|
|
|
1495
1634
|
try {
|
|
1496
1635
|
const {website_id, github_org, github_repo, git_branch, npm_package_id, version} = req.body;
|
|
1497
1636
|
|
|
1637
|
+
// Check the input before it goes anywhere near a git command line or a
|
|
1638
|
+
// file name. The browser checks these too (see HTML_PATTERNS), but the
|
|
1639
|
+
// form isn't the only way to reach this route.
|
|
1640
|
+
const problems = validation.validateTaskInput(req.body);
|
|
1641
|
+
if (problems.length > 0) {
|
|
1642
|
+
this.logger.warn('Rejected task creation from user ' + req.session.userId + ': ' + problems.join('; '));
|
|
1643
|
+
return res.status(400).send('Invalid task details: ' + problems.join('; '));
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1498
1646
|
// Verify user has permission for this website
|
|
1499
1647
|
const canQueue = await this.userCanQueue(req.session.userId, website_id);
|
|
1500
1648
|
if (!canQueue) {
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Validation for the fields a user types into the publication task form.
|
|
3
|
+
*
|
|
4
|
+
* None of these values can cause command injection - they're passed to spawn()
|
|
5
|
+
* as argv, and nothing goes near a shell - but they do end up in git command
|
|
6
|
+
* lines, in URLs, and in file names, so they're checked at the boundary rather
|
|
7
|
+
* than trusted to be harmless everywhere they're later used.
|
|
8
|
+
*
|
|
9
|
+
* The rules are the ones GitHub and git themselves enforce (see
|
|
10
|
+
* git-check-ref-format(1) for the branch rules): anything rejected here would
|
|
11
|
+
* have failed at clone time anyway, with a much worse error.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
// GitHub owner names: letters, digits and single hyphens, no hyphen at either
|
|
15
|
+
// end, 39 characters at most.
|
|
16
|
+
const MAX_OWNER = 39;
|
|
17
|
+
// Repository names: letters, digits, dot, hyphen, underscore, 100 at most.
|
|
18
|
+
const MAX_REPO = 100;
|
|
19
|
+
// git imposes no length limit on a ref, but a branch name past this is a
|
|
20
|
+
// filesystem problem waiting to happen.
|
|
21
|
+
const MAX_BRANCH = 255;
|
|
22
|
+
const MAX_PACKAGE_ID = 128;
|
|
23
|
+
const MAX_VERSION = 64;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* A GitHub organisation or user name.
|
|
27
|
+
*/
|
|
28
|
+
function validateGithubOwner(value) {
|
|
29
|
+
if (!value) {
|
|
30
|
+
return 'GitHub org is required';
|
|
31
|
+
}
|
|
32
|
+
if (value.length > MAX_OWNER) {
|
|
33
|
+
return `GitHub org must be ${MAX_OWNER} characters or less`;
|
|
34
|
+
}
|
|
35
|
+
if (!/^[A-Za-z0-9](?:-?[A-Za-z0-9])*$/.test(value)) {
|
|
36
|
+
return 'GitHub org may only contain letters, digits and single hyphens, and may not start or end with a hyphen';
|
|
37
|
+
}
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* A GitHub repository name.
|
|
43
|
+
*/
|
|
44
|
+
function validateGithubRepo(value) {
|
|
45
|
+
if (!value) {
|
|
46
|
+
return 'GitHub repo is required';
|
|
47
|
+
}
|
|
48
|
+
if (value.length > MAX_REPO) {
|
|
49
|
+
return `GitHub repo must be ${MAX_REPO} characters or less`;
|
|
50
|
+
}
|
|
51
|
+
if (!/^[A-Za-z0-9._-]+$/.test(value)) {
|
|
52
|
+
return 'GitHub repo may only contain letters, digits, dots, hyphens and underscores';
|
|
53
|
+
}
|
|
54
|
+
if (value === '.' || value === '..') {
|
|
55
|
+
return 'GitHub repo is not a valid name';
|
|
56
|
+
}
|
|
57
|
+
// the code appends '.git' when it builds the clone URL, and GitHub won't
|
|
58
|
+
// create a repository with that suffix anyway
|
|
59
|
+
if (value.toLowerCase().endsWith('.git')) {
|
|
60
|
+
return 'GitHub repo should not include the .git suffix';
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* A branch name, by the rules in git-check-ref-format(1). Slashes are fine -
|
|
67
|
+
* that's how branches are grouped - but the characters git reserves for
|
|
68
|
+
* revision syntax are not. Note that git itself rejects a backslash in a ref
|
|
69
|
+
* name, so this does too.
|
|
70
|
+
*/
|
|
71
|
+
function validateGitBranch(value) {
|
|
72
|
+
if (!value) {
|
|
73
|
+
return 'Branch is required';
|
|
74
|
+
}
|
|
75
|
+
if (value.length > MAX_BRANCH) {
|
|
76
|
+
return `Branch must be ${MAX_BRANCH} characters or less`;
|
|
77
|
+
}
|
|
78
|
+
// space, the ASCII control characters, and DEL
|
|
79
|
+
// eslint-disable-next-line no-control-regex
|
|
80
|
+
if (/[\u0000-\u001F\u007F ]/.test(value)) {
|
|
81
|
+
return 'Branch may not contain spaces or control characters';
|
|
82
|
+
}
|
|
83
|
+
// the characters git reserves for revision syntax
|
|
84
|
+
if (/[~^:?*[\\]/.test(value)) {
|
|
85
|
+
return 'Branch may not contain any of ~ ^ : ? * [ \\';
|
|
86
|
+
}
|
|
87
|
+
if (value.includes('..')) {
|
|
88
|
+
return 'Branch may not contain ..';
|
|
89
|
+
}
|
|
90
|
+
if (value.includes('@{')) {
|
|
91
|
+
return 'Branch may not contain @{';
|
|
92
|
+
}
|
|
93
|
+
if (value === '@') {
|
|
94
|
+
return 'Branch may not be @';
|
|
95
|
+
}
|
|
96
|
+
if (value.startsWith('-')) {
|
|
97
|
+
return 'Branch may not start with a hyphen';
|
|
98
|
+
}
|
|
99
|
+
if (value.startsWith('/') || value.endsWith('/') || value.includes('//')) {
|
|
100
|
+
return 'Branch may not start or end with /, or contain //';
|
|
101
|
+
}
|
|
102
|
+
if (value.endsWith('.')) {
|
|
103
|
+
return 'Branch may not end with .';
|
|
104
|
+
}
|
|
105
|
+
for (const part of value.split('/')) {
|
|
106
|
+
if (part.startsWith('.')) {
|
|
107
|
+
return 'No part of a branch name may start with .';
|
|
108
|
+
}
|
|
109
|
+
if (part.endsWith('.lock')) {
|
|
110
|
+
return 'No part of a branch name may end with .lock';
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* An NPM package id. This one is not about git at all: the package id and the
|
|
118
|
+
* version are concatenated into file names under the zips directory
|
|
119
|
+
* (<id>#<version>.log, <id>#<version>-announcement.txt), so a value containing
|
|
120
|
+
* a path separator would look outside that directory.
|
|
121
|
+
*/
|
|
122
|
+
function validatePackageId(value) {
|
|
123
|
+
if (!value) {
|
|
124
|
+
return 'NPM package id is required';
|
|
125
|
+
}
|
|
126
|
+
if (value.length > MAX_PACKAGE_ID) {
|
|
127
|
+
return `NPM package id must be ${MAX_PACKAGE_ID} characters or less`;
|
|
128
|
+
}
|
|
129
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value)) {
|
|
130
|
+
return 'NPM package id may only contain letters, digits, dots, hyphens and underscores, and must start with a letter or digit';
|
|
131
|
+
}
|
|
132
|
+
if (value.includes('..')) {
|
|
133
|
+
return 'NPM package id may not contain ..';
|
|
134
|
+
}
|
|
135
|
+
return null;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* A package version. Loose enough for semver with pre-release and build
|
|
140
|
+
* metadata, tight enough that it can't become a path.
|
|
141
|
+
*/
|
|
142
|
+
function validateVersion(value) {
|
|
143
|
+
if (!value) {
|
|
144
|
+
return 'Version is required';
|
|
145
|
+
}
|
|
146
|
+
if (value.length > MAX_VERSION) {
|
|
147
|
+
return `Version must be ${MAX_VERSION} characters or less`;
|
|
148
|
+
}
|
|
149
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9.+-]*$/.test(value)) {
|
|
150
|
+
return 'Version may only contain letters, digits, dots, hyphens and plus signs, and must start with a letter or digit';
|
|
151
|
+
}
|
|
152
|
+
if (value.includes('..')) {
|
|
153
|
+
return 'Version may not contain ..';
|
|
154
|
+
}
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Validate everything the task form collects.
|
|
160
|
+
* @returns {string[]} the problems found, empty if the input is acceptable
|
|
161
|
+
*/
|
|
162
|
+
function validateTaskInput(input) {
|
|
163
|
+
const errors = [];
|
|
164
|
+
const checks = [
|
|
165
|
+
validateGithubOwner(input.github_org),
|
|
166
|
+
validateGithubRepo(input.github_repo),
|
|
167
|
+
validateGitBranch(input.git_branch),
|
|
168
|
+
validatePackageId(input.npm_package_id),
|
|
169
|
+
validateVersion(input.version)
|
|
170
|
+
];
|
|
171
|
+
for (const error of checks) {
|
|
172
|
+
if (error) {
|
|
173
|
+
errors.push(error);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return errors;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Patterns for the HTML form, so the browser objects before the round trip.
|
|
180
|
+
// Deliberately a subset of the checks above - the server is what decides -
|
|
181
|
+
// but they catch the obvious mistakes as the user types.
|
|
182
|
+
const HTML_PATTERNS = {
|
|
183
|
+
github_org: '[A-Za-z0-9](-?[A-Za-z0-9])*',
|
|
184
|
+
github_repo: '[A-Za-z0-9._-]+',
|
|
185
|
+
git_branch: '[^\\\\ ~^:?*\\[]+',
|
|
186
|
+
npm_package_id: '[A-Za-z0-9][A-Za-z0-9._-]*',
|
|
187
|
+
version: '[A-Za-z0-9][A-Za-z0-9.+-]*'
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
module.exports = {
|
|
191
|
+
validateGithubOwner,
|
|
192
|
+
validateGithubRepo,
|
|
193
|
+
validateGitBranch,
|
|
194
|
+
validatePackageId,
|
|
195
|
+
validateVersion,
|
|
196
|
+
validateTaskInput,
|
|
197
|
+
HTML_PATTERNS
|
|
198
|
+
};
|
package/server.js
CHANGED
|
@@ -99,13 +99,13 @@ let stats = null;
|
|
|
99
99
|
|
|
100
100
|
// Initialize modules based on configuration
|
|
101
101
|
async function initializeModules() {
|
|
102
|
-
stats = new ServerStats();
|
|
102
|
+
stats = new ServerStats(config.stats, serverLog);
|
|
103
103
|
|
|
104
104
|
// Initialize SHL module
|
|
105
105
|
if (config.modules?.shl?.enabled) {
|
|
106
106
|
try {
|
|
107
107
|
serverLog.info('Initializing module: shl...');
|
|
108
|
-
modules.shl = new SHLModule(stats);
|
|
108
|
+
modules.shl = new SHLModule(stats.forModule('shl'));
|
|
109
109
|
await modules.shl.initialize(config.modules.shl);
|
|
110
110
|
app.use('/shl', modules.shl.router);
|
|
111
111
|
} catch (error) {
|
|
@@ -118,7 +118,7 @@ async function initializeModules() {
|
|
|
118
118
|
if (config.modules?.vcl?.enabled) {
|
|
119
119
|
try {
|
|
120
120
|
serverLog.info('Initializing module: vcl...');
|
|
121
|
-
modules.vcl = new VCLModule(stats);
|
|
121
|
+
modules.vcl = new VCLModule(stats.forModule('vcl'));
|
|
122
122
|
await modules.vcl.initialize(config.modules.vcl);
|
|
123
123
|
app.use('/VCL', modules.vcl.router);
|
|
124
124
|
} catch (error) {
|
|
@@ -131,7 +131,7 @@ async function initializeModules() {
|
|
|
131
131
|
if (config.modules?.xig?.enabled) {
|
|
132
132
|
try {
|
|
133
133
|
serverLog.info('Initializing module: xig...');
|
|
134
|
-
await xigModule.initializeXigModule(stats, config.modules.xig);
|
|
134
|
+
await xigModule.initializeXigModule(stats.forModule('xig'), config.modules.xig);
|
|
135
135
|
app.use('/xig', xigModule.router);
|
|
136
136
|
modules.xig = xigModule;
|
|
137
137
|
} catch (error) {
|
|
@@ -144,7 +144,7 @@ async function initializeModules() {
|
|
|
144
144
|
if (config.modules?.packages?.enabled) {
|
|
145
145
|
try {
|
|
146
146
|
serverLog.info('Initializing module: packages...');
|
|
147
|
-
modules.packages = new PackagesModule(stats);
|
|
147
|
+
modules.packages = new PackagesModule(stats.forModule('packages'));
|
|
148
148
|
await modules.packages.initialize(config.modules.packages);
|
|
149
149
|
app.use('/packages', modules.packages.router);
|
|
150
150
|
} catch (error) {
|
|
@@ -158,7 +158,7 @@ async function initializeModules() {
|
|
|
158
158
|
if (config.modules?.registry?.enabled) {
|
|
159
159
|
try {
|
|
160
160
|
serverLog.info('Initializing module: registry...');
|
|
161
|
-
modules.registry = new RegistryModule(stats);
|
|
161
|
+
modules.registry = new RegistryModule(stats.forModule('registry'));
|
|
162
162
|
await modules.registry.initialize(config.modules.registry);
|
|
163
163
|
app.use('/tx-reg', modules.registry.router);
|
|
164
164
|
} catch (error) {
|
|
@@ -171,7 +171,7 @@ async function initializeModules() {
|
|
|
171
171
|
if (config.modules?.publisher?.enabled) {
|
|
172
172
|
try {
|
|
173
173
|
serverLog.info('Initializing module: publisher...');
|
|
174
|
-
modules.publisher = new PublisherModule(stats);
|
|
174
|
+
modules.publisher = new PublisherModule(stats.forModule('publisher'));
|
|
175
175
|
await modules.publisher.initialize(config.modules.publisher);
|
|
176
176
|
app.use('/publisher', modules.publisher.router);
|
|
177
177
|
} catch (error) {
|
|
@@ -184,7 +184,7 @@ async function initializeModules() {
|
|
|
184
184
|
if (config.modules?.token?.enabled) {
|
|
185
185
|
try {
|
|
186
186
|
serverLog.info('Initializing module: token...');
|
|
187
|
-
modules.token = new TokenModule(stats);
|
|
187
|
+
modules.token = new TokenModule(stats.forModule('token'));
|
|
188
188
|
await modules.token.initialize(config.modules.token);
|
|
189
189
|
app.use('/token', modules.token.router);
|
|
190
190
|
} catch (error) {
|
|
@@ -197,7 +197,7 @@ async function initializeModules() {
|
|
|
197
197
|
if (config.modules?.npmprojector?.enabled) {
|
|
198
198
|
try {
|
|
199
199
|
serverLog.info('Initializing module: npmprojector...');
|
|
200
|
-
modules.npmprojector = new NpmProjectorModule(stats);
|
|
200
|
+
modules.npmprojector = new NpmProjectorModule(stats.forModule('npmprojector'));
|
|
201
201
|
await modules.npmprojector.initialize(config.modules.npmprojector);
|
|
202
202
|
const basePath = NpmProjectorModule.getBasePath(config.modules.npmprojector);
|
|
203
203
|
app.use(basePath, modules.npmprojector.router);
|
|
@@ -211,7 +211,7 @@ async function initializeModules() {
|
|
|
211
211
|
if (config.modules?.['ext-tracker']?.enabled) {
|
|
212
212
|
try {
|
|
213
213
|
serverLog.info('Initializing module: ext-tracker...');
|
|
214
|
-
modules.extTracker = new ExtensionTrackerModule(stats);
|
|
214
|
+
modules.extTracker = new ExtensionTrackerModule(stats.forModule('ext-tracker'));
|
|
215
215
|
await modules.extTracker.initialize(config.modules['ext-tracker'], app);
|
|
216
216
|
} catch (error) {
|
|
217
217
|
serverLog.error('Failed to initialize extension tracker module:', error);
|
|
@@ -224,7 +224,7 @@ async function initializeModules() {
|
|
|
224
224
|
if (config.modules?.tx?.enabled) {
|
|
225
225
|
try {
|
|
226
226
|
serverLog.info('Initializing module: tx...');
|
|
227
|
-
modules.tx = new TXModule(stats);
|
|
227
|
+
modules.tx = new TXModule(stats.forModule('tx'));
|
|
228
228
|
await modules.tx.initialize(config.modules.tx, app);
|
|
229
229
|
} catch (error) {
|
|
230
230
|
serverLog.error('Failed to initialize TX module:', error);
|
|
@@ -235,7 +235,7 @@ async function initializeModules() {
|
|
|
235
235
|
if (config.modules?.folder?.enabled) {
|
|
236
236
|
try {
|
|
237
237
|
serverLog.info('Initializing module: folder...');
|
|
238
|
-
modules.folder = new FolderModule(stats);
|
|
238
|
+
modules.folder = new FolderModule(stats.forModule('folder'));
|
|
239
239
|
await modules.folder.initialize(config.modules.folder, app);
|
|
240
240
|
// mount the router
|
|
241
241
|
} catch (error) {
|