fhirsmith 0.11.2 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +377 -2
- package/README.md +1 -1
- package/config-template.json +31 -1
- package/library/languages.js +191 -34
- package/library/path-safety.js +51 -0
- package/library/request-token.js +56 -0
- package/package.json +2 -2
- package/packages/packages.js +17 -118
- package/publisher/publisher.js +208 -30
- package/publisher/validation.js +210 -0
- package/server.js +24 -15
- package/stats-db.js +127 -0
- package/stats.js +172 -6
- package/translations/Messages.properties +5 -1
- package/tx/README.md +21 -0
- package/tx/cs/cs-api.js +30 -1
- package/tx/cs/cs-api.md +1 -1
- package/tx/cs/cs-cs.js +1 -1
- package/tx/cs/cs-icd11.js +805 -0
- package/tx/cs/cs-lang.js +331 -85
- package/tx/cs/cs-loinc.js +45 -4
- package/tx/cs/cs-mimetypes.js +492 -21
- package/tx/cs/cs-snomed.js +141 -58
- package/tx/cs/cs-ucum.js +33 -3
- package/tx/cs/icd11-expressions.js +243 -0
- package/tx/importers/export-icd11-codesystem.js +823 -0
- package/tx/importers/icd11-schema.md +285 -0
- package/tx/importers/import-icd11.js +861 -0
- package/tx/importers/import-loinc.module.js +14 -7
- package/tx/importers/import-sct.module.js +10 -1
- package/tx/library/codesystem.js +21 -27
- package/tx/library/conceptmap.js +39 -0
- package/tx/library/nesting-limit.js +79 -0
- package/tx/library/operation-outcome.js +97 -3
- package/tx/library/parameter-check.js +139 -0
- package/tx/library/valueset.js +16 -20
- package/tx/library.js +23 -1
- package/tx/ocl/cache/cache-utils.cjs +5 -3
- package/tx/ocl/cs-ocl.cjs +73 -17
- package/tx/ocl/jobs/background-queue.cjs +6 -3
- package/tx/ocl/mappers/concept-mapper.cjs +9 -1
- package/tx/ocl/shared/patches.cjs +48 -1
- package/tx/ocl/vs-ocl.cjs +15 -13
- package/tx/operation-context.js +220 -10
- package/tx/params.js +226 -175
- package/tx/sct/expressions.js +677 -15
- package/tx/tx.fhir.org.yml +3 -2
- package/tx/tx.js +72 -92
- package/tx/workers/batch-validate.js +11 -18
- package/tx/workers/batch.js +16 -28
- package/tx/workers/cache-control.js +87 -17
- package/tx/workers/closure.js +3 -8
- package/tx/workers/compare.js +6 -34
- package/tx/workers/expand.js +108 -60
- package/tx/workers/lookup.js +35 -40
- package/tx/workers/read.js +6 -40
- package/tx/workers/search.js +2 -8
- package/tx/workers/subsumes.js +58 -41
- package/tx/workers/translate.js +472 -47
- package/tx/workers/validate.js +83 -78
- package/tx/workers/worker.js +92 -11
- package/tx/xml/xml-base.js +264 -77
- package/tx/xversion/xv-conceptmap.js +81 -0
- package/tx/xversion/xv-parameters.js +8 -0
- package/tx/xversion/xv-valueset.js +39 -1
- package/utilities/merge-audit.py +178 -0
- package/utilities/why-merge-audit.md +39 -0
- package/tx/data/snomed-subset.txt +0 -84
package/publisher/publisher.js
CHANGED
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
const express = require('express');
|
|
2
2
|
const path = require('path');
|
|
3
3
|
const fs = require('fs');
|
|
4
|
+
const validation = require('./validation');
|
|
4
5
|
const Database = require('sqlite3').Database;
|
|
5
6
|
const bcrypt = require('bcrypt');
|
|
6
7
|
const session = require('express-session');
|
|
8
|
+
const rateLimit = require('express-rate-limit');
|
|
9
|
+
const crypto = require('crypto');
|
|
7
10
|
const folders = require('../library/folder-setup');
|
|
8
11
|
const escape = require('escape-html');
|
|
9
12
|
const {Utilities} = require("../library/utilities");
|
|
10
13
|
|
|
14
|
+
// GitHub refuses any file larger than this at the pre-receive hook, and the rejection takes down
|
|
15
|
+
// the whole push rather than just the offending file.
|
|
16
|
+
const GITHUB_MAX_FILE_SIZE = 100 * 1024 * 1024;
|
|
17
|
+
|
|
11
18
|
|
|
12
19
|
class PublisherModule {
|
|
13
20
|
constructor(stats) {
|
|
@@ -30,11 +37,28 @@ class PublisherModule {
|
|
|
30
37
|
await this.initializeDatabase();
|
|
31
38
|
|
|
32
39
|
// Set up session middleware - use in-memory sessions or separate session db
|
|
40
|
+
if (!this.config.sessionSecret) {
|
|
41
|
+
// Never fall back to a constant. This project is open source, so a fixed default
|
|
42
|
+
// secret is a published signing key: anyone could mint themselves an admin session
|
|
43
|
+
// on any deployment that forgot the setting, and nothing about the server would
|
|
44
|
+
// look wrong. A random secret fails safe instead - sessions simply do not survive
|
|
45
|
+
// a restart, which is visible and harmless. token.js does the same.
|
|
46
|
+
this.logger.warn('publisher: no sessionSecret configured - using a random one, so ' +
|
|
47
|
+
'logins will not survive a restart. Set modules.publisher.sessionSecret.');
|
|
48
|
+
}
|
|
33
49
|
this.router.use(session({
|
|
34
|
-
secret: this.config.sessionSecret || '
|
|
50
|
+
secret: this.config.sessionSecret || crypto.randomBytes(64).toString('hex'),
|
|
35
51
|
resave: false,
|
|
36
52
|
saveUninitialized: false,
|
|
37
|
-
cookie: {
|
|
53
|
+
cookie: {
|
|
54
|
+
// Secure by default: the cookie is then only ever sent over HTTPS. Express is
|
|
55
|
+
// configured to trust the proxy, so this works behind an NGINX TLS terminator
|
|
56
|
+
// reporting X-Forwarded-Proto. Set cookieSecure false to run the publisher over
|
|
57
|
+
// plain HTTP - without it the browser will not send the cookie back and login
|
|
58
|
+
// appears to succeed and then silently do nothing.
|
|
59
|
+
secure: this.config.cookieSecure ?? true,
|
|
60
|
+
maxAge: 24 * 60 * 60 * 1000 // 24 hours
|
|
61
|
+
}
|
|
38
62
|
// Not using SQLiteStore to avoid the database conflict
|
|
39
63
|
}));
|
|
40
64
|
|
|
@@ -272,9 +296,20 @@ class PublisherModule {
|
|
|
272
296
|
// Main dashboard
|
|
273
297
|
this.router.get('/', this.renderDashboard.bind(this));
|
|
274
298
|
|
|
275
|
-
// Authentication
|
|
299
|
+
// Authentication. The login post is rate limited: besides slowing down guessing, it
|
|
300
|
+
// is the one unauthenticated route that runs bcrypt, and bcrypt.compare holds a libuv
|
|
301
|
+
// threadpool thread (4 by default) for ~100ms - so without a limit a handful of
|
|
302
|
+
// concurrent attempts stall every file operation the server wants to do.
|
|
303
|
+
const loginLimiter = rateLimit({
|
|
304
|
+
windowMs: 15 * 60 * 1000,
|
|
305
|
+
max: this.config.loginAttemptsPerWindow ?? 20,
|
|
306
|
+
message: 'Too many login attempts, please try again later.',
|
|
307
|
+
standardHeaders: true,
|
|
308
|
+
legacyHeaders: false
|
|
309
|
+
});
|
|
310
|
+
|
|
276
311
|
this.router.get('/login', this.renderLogin.bind(this));
|
|
277
|
-
this.router.post('/login', this.handleLogin.bind(this));
|
|
312
|
+
this.router.post('/login', loginLimiter, this.handleLogin.bind(this));
|
|
278
313
|
this.router.post('/logout', this.handleLogout.bind(this));
|
|
279
314
|
|
|
280
315
|
// Tasks
|
|
@@ -928,6 +963,137 @@ class PublisherModule {
|
|
|
928
963
|
}
|
|
929
964
|
}
|
|
930
965
|
|
|
966
|
+
// The IG Publisher can leave a file in the web folder that GitHub will not accept - typically a
|
|
967
|
+
// technical correction archive, which is a zip of an entire published version folder and so
|
|
968
|
+
// carries that folder's own full-ig.zip inside it. GitHub rejects the entire push over one such
|
|
969
|
+
// file, at the very last step of a publication run. The IG Publisher will keep these out of the
|
|
970
|
+
// web folder itself from the next release; until then, move them aside here.
|
|
971
|
+
//
|
|
972
|
+
// Does nothing unless publisher.large-file-archive says where to put them, and the website is
|
|
973
|
+
// one that's hosted on GitHub in the first place.
|
|
974
|
+
async archiveLargeFiles(task, website) {
|
|
975
|
+
const configured = this.config['large-file-archive'];
|
|
976
|
+
if (!configured || !this.isGitHubHosted(website)) {
|
|
977
|
+
return;
|
|
978
|
+
}
|
|
979
|
+
const root = website.git_root || website.local_folder;
|
|
980
|
+
const oversized = this.findLargeFiles(root, GITHUB_MAX_FILE_SIZE);
|
|
981
|
+
if (oversized.length === 0) {
|
|
982
|
+
return;
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
const archiveDir = path.isAbsolute(configured)
|
|
986
|
+
? configured
|
|
987
|
+
: folders.filePath('publisher', configured);
|
|
988
|
+
if (!fs.existsSync(archiveDir)) {
|
|
989
|
+
fs.mkdirSync(archiveDir, { recursive: true });
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
for (const file of oversized) {
|
|
993
|
+
const sizeMB = Math.round(fs.statSync(file).size / (1024 * 1024));
|
|
994
|
+
const dest = this.nonClashingPath(archiveDir, path.basename(file));
|
|
995
|
+
this.moveFile(file, dest);
|
|
996
|
+
await this.logTaskMessage(task.id, 'warn', 'Large file ' + path.relative(root, file) + ' (' + sizeMB +
|
|
997
|
+
' MB) is over the 100 MB GitHub limit - moved to ' + dest + ' instead of being published');
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
// Only a website kept in a GitHub repository has a 100MB problem - others can carry files of any
|
|
1002
|
+
// size. $.website.github in the web root's publish-setup.json says which is which; if that can't
|
|
1003
|
+
// be read, or doesn't say, assume not and leave the files where they are.
|
|
1004
|
+
isGitHubHosted(website) {
|
|
1005
|
+
if (!website.local_folder) {
|
|
1006
|
+
return false;
|
|
1007
|
+
}
|
|
1008
|
+
const setupPath = path.join(website.local_folder, 'publish-setup.json');
|
|
1009
|
+
try {
|
|
1010
|
+
const setup = JSON.parse(fs.readFileSync(setupPath, 'utf8'));
|
|
1011
|
+
return !!(setup.website && setup.website.github === true);
|
|
1012
|
+
} catch (err) {
|
|
1013
|
+
this.logger.warn('Could not read ' + setupPath + ' to check whether the website is on GitHub: ' + err.message);
|
|
1014
|
+
return false;
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
// Files in the web folder that are too big to push. Never descends into .git: a pack file over the
|
|
1019
|
+
// limit is perfectly normal there, and moving it would destroy the repository. Files git is already
|
|
1020
|
+
// ignoring are left alone, since they were never going to be pushed anyway.
|
|
1021
|
+
findLargeFiles(root, limit) {
|
|
1022
|
+
const found = [];
|
|
1023
|
+
const walk = (dir) => {
|
|
1024
|
+
let entries;
|
|
1025
|
+
try {
|
|
1026
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
1027
|
+
} catch (err) {
|
|
1028
|
+
this.logger.warn('Large file scan could not read ' + dir + ': ' + err.message);
|
|
1029
|
+
return;
|
|
1030
|
+
}
|
|
1031
|
+
for (const entry of entries) {
|
|
1032
|
+
const full = path.join(dir, entry.name);
|
|
1033
|
+
if (entry.isDirectory()) {
|
|
1034
|
+
if (entry.name !== '.git') {
|
|
1035
|
+
walk(full);
|
|
1036
|
+
}
|
|
1037
|
+
} else if (entry.isFile()) {
|
|
1038
|
+
try {
|
|
1039
|
+
if (fs.statSync(full).size > limit) {
|
|
1040
|
+
found.push(full);
|
|
1041
|
+
}
|
|
1042
|
+
} catch (err) {
|
|
1043
|
+
// the file went away between readdir and stat - nothing left to move
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
};
|
|
1048
|
+
walk(root);
|
|
1049
|
+
return found.length === 0 ? found : this.removeGitIgnored(root, found);
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
removeGitIgnored(root, files) {
|
|
1053
|
+
try {
|
|
1054
|
+
const result = require('child_process').spawnSync('git', ['check-ignore', '--stdin'],
|
|
1055
|
+
{ cwd: root, input: files.join('\n'), encoding: 'utf8' });
|
|
1056
|
+
// exit 0: at least one path is ignored. 1: none are. anything else: git could not tell us,
|
|
1057
|
+
// in which case treat everything as pushable rather than leaving a file that breaks the push.
|
|
1058
|
+
if (result.status !== 0 || !result.stdout) {
|
|
1059
|
+
return files;
|
|
1060
|
+
}
|
|
1061
|
+
const ignored = new Set(result.stdout.split('\n').map((s) => s.trim()).filter((s) => s));
|
|
1062
|
+
return files.filter((f) => !ignored.has(f));
|
|
1063
|
+
} catch (err) {
|
|
1064
|
+
this.logger.warn('Could not check the gitignore status of large files: ' + err.message);
|
|
1065
|
+
return files;
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
nonClashingPath(dir, name) {
|
|
1070
|
+
let candidate = path.join(dir, name);
|
|
1071
|
+
if (!fs.existsSync(candidate)) {
|
|
1072
|
+
return candidate;
|
|
1073
|
+
}
|
|
1074
|
+
const ext = path.extname(name);
|
|
1075
|
+
const stem = path.basename(name, ext);
|
|
1076
|
+
let i = 2;
|
|
1077
|
+
do {
|
|
1078
|
+
candidate = path.join(dir, stem + '-' + i + ext);
|
|
1079
|
+
i++;
|
|
1080
|
+
} while (fs.existsSync(candidate));
|
|
1081
|
+
return candidate;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// rename() fails across filesystems, and the archive folder will not always be on the same one
|
|
1085
|
+
moveFile(src, dest) {
|
|
1086
|
+
try {
|
|
1087
|
+
fs.renameSync(src, dest);
|
|
1088
|
+
} catch (err) {
|
|
1089
|
+
if (err.code !== 'EXDEV') {
|
|
1090
|
+
throw err;
|
|
1091
|
+
}
|
|
1092
|
+
fs.copyFileSync(src, dest);
|
|
1093
|
+
fs.unlinkSync(src);
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
|
|
931
1097
|
async runPublication(task) {
|
|
932
1098
|
const website = await this.getWebsite(task.website_id);
|
|
933
1099
|
if (!website) {
|
|
@@ -1016,6 +1182,9 @@ class PublisherModule {
|
|
|
1016
1182
|
// url - can survive into the web tree. Catch that here, before anything is committed.
|
|
1017
1183
|
await this.verifyPublishedPackage(task, website, draftDir);
|
|
1018
1184
|
|
|
1185
|
+
// Step 5c: Move anything too big for GitHub out of the web folder, before it gets staged
|
|
1186
|
+
await this.archiveLargeFiles(task, website);
|
|
1187
|
+
|
|
1019
1188
|
// Step 6: Commit and push the web folder
|
|
1020
1189
|
await this.logTaskMessage(task.id, 'info', 'Committing changes to web folder...');
|
|
1021
1190
|
const gitUrl = 'https://github.com/' + task.github_org + '/' + task.github_repo + '.git';
|
|
@@ -1181,7 +1350,7 @@ class PublisherModule {
|
|
|
1181
1350
|
content += '<div class="col-12">';
|
|
1182
1351
|
|
|
1183
1352
|
if (req.session.userId) {
|
|
1184
|
-
content += '<p>Welcome, ' + req.session.userName + '!</p>';
|
|
1353
|
+
content += '<p>Welcome, ' + escape(req.session.userName) + '!</p>';
|
|
1185
1354
|
content += '<div class="mb-3">';
|
|
1186
1355
|
content += '<a href="/publisher/tasks" class="btn btn-primary me-2">View All Tasks</a>';
|
|
1187
1356
|
if (req.session.isAdmin) {
|
|
@@ -1209,11 +1378,11 @@ class PublisherModule {
|
|
|
1209
1378
|
tasks.forEach(task => {
|
|
1210
1379
|
content += '<tr>';
|
|
1211
1380
|
content += '<td><strong>#' + task.id + '</strong></td>';
|
|
1212
|
-
content += '<td>' + task.npm_package_id + '</td>';
|
|
1213
|
-
content += '<td>' + task.version + '</td>';
|
|
1214
|
-
content += '<td><span class="badge bg-' + this.getStatusColor(task.status) + '">' + task.status + '</span></td>';
|
|
1381
|
+
content += '<td>' + escape(task.npm_package_id) + '</td>';
|
|
1382
|
+
content += '<td>' + escape(task.version) + '</td>';
|
|
1383
|
+
content += '<td><span class="badge bg-' + this.getStatusColor(task.status) + '">' + escape(task.status) + '</span></td>';
|
|
1215
1384
|
content += '<td>' + new Date(task.queued_at).toLocaleString() + '</td>';
|
|
1216
|
-
content += '<td>' + task.user_name + '</td>';
|
|
1385
|
+
content += '<td>' + escape(task.user_name) + '</td>';
|
|
1217
1386
|
content += '</tr>';
|
|
1218
1387
|
});
|
|
1219
1388
|
|
|
@@ -1363,23 +1532,23 @@ class PublisherModule {
|
|
|
1363
1532
|
content += '</div>';
|
|
1364
1533
|
content += '<div class="col-md-3">';
|
|
1365
1534
|
content += '<label for="github_org" class="form-label">GitHub Org</label>';
|
|
1366
|
-
content += '<input type="text" class="form-control" id="github_org" name="github_org" required placeholder="hl7">';
|
|
1535
|
+
content += '<input type="text" class="form-control" id="github_org" name="github_org" required maxlength="39" pattern="' + escape(validation.HTML_PATTERNS.github_org) + '" title="Letters, digits and single hyphens" placeholder="hl7">';
|
|
1367
1536
|
content += '</div>';
|
|
1368
1537
|
content += '<div class="col-md-3">';
|
|
1369
1538
|
content += '<label for="github_repo" class="form-label">GitHub Repo</label>';
|
|
1370
|
-
content += '<input type="text" class="form-control" id="github_repo" name="github_repo" required placeholder="fhir-us-core">';
|
|
1539
|
+
content += '<input type="text" class="form-control" id="github_repo" name="github_repo" required maxlength="100" pattern="' + escape(validation.HTML_PATTERNS.github_repo) + '" title="Letters, digits, dots, hyphens and underscores" placeholder="fhir-us-core">';
|
|
1371
1540
|
content += '</div>';
|
|
1372
1541
|
content += '<div class="col-md-3">';
|
|
1373
1542
|
content += '<label for="git_branch" class="form-label">Branch</label>';
|
|
1374
|
-
content += '<input type="text" class="form-control" id="git_branch" name="git_branch" required placeholder="main">';
|
|
1543
|
+
content += '<input type="text" class="form-control" id="git_branch" name="git_branch" required maxlength="255" pattern="' + escape(validation.HTML_PATTERNS.git_branch) + '" title="A git branch name - slashes are fine, but not spaces, backslashes, markup characters, or any of ~ ^ : ? * [" placeholder="main">';
|
|
1375
1544
|
content += '</div>';
|
|
1376
1545
|
content += '<div class="col-md-4">';
|
|
1377
1546
|
content += '<label for="npm_package_id" class="form-label">NPM Package ID</label>';
|
|
1378
|
-
content += '<input type="text" class="form-control" id="npm_package_id" name="npm_package_id" required placeholder="hl7.fhir.us.core">';
|
|
1547
|
+
content += '<input type="text" class="form-control" id="npm_package_id" name="npm_package_id" required maxlength="128" pattern="' + escape(validation.HTML_PATTERNS.npm_package_id) + '" title="Letters, digits, dots, hyphens and underscores" placeholder="hl7.fhir.us.core">';
|
|
1379
1548
|
content += '</div>';
|
|
1380
1549
|
content += '<div class="col-md-4">';
|
|
1381
1550
|
content += '<label for="version" class="form-label">Version</label>';
|
|
1382
|
-
content += '<input type="text" class="form-control" id="version" name="version" required placeholder="6.0.0">';
|
|
1551
|
+
content += '<input type="text" class="form-control" id="version" name="version" required maxlength="64" pattern="' + escape(validation.HTML_PATTERNS.version) + '" title="Letters, digits, dots, hyphens and plus signs" placeholder="6.0.0">';
|
|
1383
1552
|
content += '</div>';
|
|
1384
1553
|
content += '<div class="col-md-4 d-flex align-items-end">';
|
|
1385
1554
|
content += '<button type="submit" class="btn btn-primary">Create Task</button>';
|
|
@@ -1415,13 +1584,13 @@ class PublisherModule {
|
|
|
1415
1584
|
|
|
1416
1585
|
content += '<tr>';
|
|
1417
1586
|
content += '<td><strong>#' + task.id + '</strong></td>';
|
|
1418
|
-
content += '<td><code>' + task.npm_package_id + '</code></td>';
|
|
1419
|
-
content += '<td>' + task.version + '</td>';
|
|
1420
|
-
content += '<td>' + task.website_name + '</td>';
|
|
1421
|
-
content += '<td><span class="badge bg-' + this.getStatusColor(task.status) + '">' + task.status + '</span></td>';
|
|
1587
|
+
content += '<td><code>' + escape(task.npm_package_id) + '</code></td>';
|
|
1588
|
+
content += '<td>' + escape(task.version) + '</td>';
|
|
1589
|
+
content += '<td>' + escape(task.website_name) + '</td>';
|
|
1590
|
+
content += '<td><span class="badge bg-' + this.getStatusColor(task.status) + '">' + escape(task.status) + '</span></td>';
|
|
1422
1591
|
content += '<td>' + (task.publisher_version ? '<code>' + escape(task.publisher_version) + '</code>' : '<span class="text-muted">—</span>') + '</td>';
|
|
1423
1592
|
content += '<td>' + new Date(task.queued_at).toLocaleString() + '</td>';
|
|
1424
|
-
content += '<td>' + task.user_name + '</td>';
|
|
1593
|
+
content += '<td>' + escape(task.user_name) + '</td>';
|
|
1425
1594
|
content += '<td class="task-actions">';
|
|
1426
1595
|
content += '<a href="/publisher/tasks/' + task.id + '/history" class="btn btn-sm btn-outline-secondary me-1">History</a>';
|
|
1427
1596
|
|
|
@@ -1495,6 +1664,15 @@ class PublisherModule {
|
|
|
1495
1664
|
try {
|
|
1496
1665
|
const {website_id, github_org, github_repo, git_branch, npm_package_id, version} = req.body;
|
|
1497
1666
|
|
|
1667
|
+
// Check the input before it goes anywhere near a git command line or a
|
|
1668
|
+
// file name. The browser checks these too (see HTML_PATTERNS), but the
|
|
1669
|
+
// form isn't the only way to reach this route.
|
|
1670
|
+
const problems = validation.validateTaskInput(req.body);
|
|
1671
|
+
if (problems.length > 0) {
|
|
1672
|
+
this.logger.warn('Rejected task creation from user ' + req.session.userId + ': ' + problems.join('; '));
|
|
1673
|
+
return res.status(400).send('Invalid task details: ' + problems.join('; '));
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1498
1676
|
// Verify user has permission for this website
|
|
1499
1677
|
const canQueue = await this.userCanQueue(req.session.userId, website_id);
|
|
1500
1678
|
if (!canQueue) {
|
|
@@ -1720,20 +1898,20 @@ class PublisherModule {
|
|
|
1720
1898
|
|
|
1721
1899
|
if (req.headers.accept && req.headers.accept.includes('text/html')) {
|
|
1722
1900
|
const htmlServer = require('../library/html-server');
|
|
1723
|
-
let content = '<h3>Task Output: #' + task.id + ' - ' + task.npm_package_id + '#' + task.version + '</h3>';
|
|
1724
|
-
content += '<p><strong>Status:</strong> <span class="badge bg-' + this.getStatusColor(task.status) + '">' + task.status + '</span></p>';
|
|
1725
|
-
content += '<p><strong>GitHub:</strong> ' + task.github_org + '/' + task.github_repo + ' (' + task.git_branch + ')</p>';
|
|
1901
|
+
let content = '<h3>Task Output: #' + task.id + ' - ' + escape(task.npm_package_id) + '#' + escape(task.version) + '</h3>';
|
|
1902
|
+
content += '<p><strong>Status:</strong> <span class="badge bg-' + this.getStatusColor(task.status) + '">' + escape(task.status) + '</span></p>';
|
|
1903
|
+
content += '<p><strong>GitHub:</strong> ' + escape(task.github_org) + '/' + escape(task.github_repo) + ' (' + escape(task.git_branch) + ')</p>';
|
|
1726
1904
|
|
|
1727
1905
|
if (task.publisher_version) {
|
|
1728
1906
|
content += '<p><strong>IG Publisher:</strong> <code>' + escape(task.publisher_version) + '</code></p>';
|
|
1729
1907
|
}
|
|
1730
1908
|
|
|
1731
1909
|
if (task.local_folder) {
|
|
1732
|
-
content += '<p><strong>Local Folder:</strong> <code>' + task.local_folder + '</code></p>';
|
|
1910
|
+
content += '<p><strong>Local Folder:</strong> <code>' + escape(task.local_folder) + '</code></p>';
|
|
1733
1911
|
}
|
|
1734
1912
|
|
|
1735
1913
|
if (task.failure_reason) {
|
|
1736
|
-
content += '<div class="alert alert-danger"><strong>Failure Reason:</strong> ' + task.failure_reason + '</div>';
|
|
1914
|
+
content += '<div class="alert alert-danger"><strong>Failure Reason:</strong> ' + escape(task.failure_reason) + '</div>';
|
|
1737
1915
|
}
|
|
1738
1916
|
|
|
1739
1917
|
// Task logs section
|
|
@@ -1745,7 +1923,7 @@ class PublisherModule {
|
|
|
1745
1923
|
logs.forEach(log => {
|
|
1746
1924
|
const timestamp = new Date(log.timestamp).toLocaleString();
|
|
1747
1925
|
const levelClass = log.level === 'error' ? 'text-danger' : (log.level === 'warn' ? 'text-warning' : '');
|
|
1748
|
-
content += '<div class="' + levelClass + '">[' + timestamp + '] [' + log.level.toUpperCase() + '] ' + log.message + '</div>';
|
|
1926
|
+
content += '<div class="' + levelClass + '">[' + timestamp + '] [' + escape(log.level.toUpperCase()) + '] ' + escape(log.message) + '</div>';
|
|
1749
1927
|
});
|
|
1750
1928
|
content += '</div>';
|
|
1751
1929
|
}
|
|
@@ -1861,7 +2039,7 @@ class PublisherModule {
|
|
|
1861
2039
|
content += '<div class="card mb-4"><div class="card-body">';
|
|
1862
2040
|
content += '<div class="row">';
|
|
1863
2041
|
content += '<div class="col-md-6">';
|
|
1864
|
-
content += '<p><strong>Status:</strong> <span class="badge bg-' + this.getStatusColor(task.status) + '">' + task.status + '</span></p>';
|
|
2042
|
+
content += '<p><strong>Status:</strong> <span class="badge bg-' + this.getStatusColor(task.status) + '">' + escape(task.status) + '</span></p>';
|
|
1865
2043
|
content += '<p><strong>Package:</strong> <code>' + escape(task.npm_package_id) + '</code></p>';
|
|
1866
2044
|
content += '<p><strong>Version:</strong> ' + escape(task.version) + '</p>';
|
|
1867
2045
|
content += '<p><strong>Website:</strong> ' + escape(task.website_name) + '</p>';
|
|
@@ -1937,7 +2115,7 @@ class PublisherModule {
|
|
|
1937
2115
|
if (action.action === 'create_task') actionLabel = 'Created task';
|
|
1938
2116
|
else if (action.action === 'approve_task') actionLabel = 'Approved task';
|
|
1939
2117
|
else if (action.action === 'delete_task') actionLabel = 'Deleted task';
|
|
1940
|
-
else actionLabel = action.action.replace(/_/g, ' ');
|
|
2118
|
+
else actionLabel = escape(action.action.replace(/_/g, ' '));
|
|
1941
2119
|
|
|
1942
2120
|
events.push({
|
|
1943
2121
|
timestamp: action.timestamp,
|
|
@@ -2158,7 +2336,7 @@ class PublisherModule {
|
|
|
2158
2336
|
|
|
2159
2337
|
websites.forEach(website => {
|
|
2160
2338
|
content += '<tr>';
|
|
2161
|
-
content += '<td>' + website.name + '</td>';
|
|
2339
|
+
content += '<td>' + escape(website.name) + '</td>';
|
|
2162
2340
|
content += '<td><code>' + escape(website.local_folder) + '</code></td>';
|
|
2163
2341
|
content += '<td><code>' + escape(website.git_root || '') + '</code></td>';
|
|
2164
2342
|
content += '<td><code>' + escape(website.history_templates) + '</code></td>';
|
|
@@ -2279,7 +2457,7 @@ class PublisherModule {
|
|
|
2279
2457
|
|
|
2280
2458
|
content += '<div class="card mb-3">';
|
|
2281
2459
|
content += '<div class="card-header">';
|
|
2282
|
-
content += '<h5>' + user.name + ' (' + user.login + ') ' + (user.is_admin ? '<span class="badge bg-warning">Admin</span>' : '') + '</h5>';
|
|
2460
|
+
content += '<h5>' + escape(user.name) + ' (' + escape(user.login) + ') ' + (user.is_admin ? '<span class="badge bg-warning">Admin</span>' : '') + '</h5>';
|
|
2283
2461
|
content += '</div>';
|
|
2284
2462
|
content += '<div class="card-body">';
|
|
2285
2463
|
|
|
@@ -2295,7 +2473,7 @@ class PublisherModule {
|
|
|
2295
2473
|
|
|
2296
2474
|
websites.forEach(website => {
|
|
2297
2475
|
const perm = permissions.find(p => p.website_id === website.id) || {};
|
|
2298
|
-
content += '<div>' + website.name + '</div>';
|
|
2476
|
+
content += '<div>' + escape(website.name) + '</div>';
|
|
2299
2477
|
content += '<div>';
|
|
2300
2478
|
content += '<input type="checkbox" name="queue_' + website.id + '"' + (perm.can_queue ? ' checked' : '') + '>';
|
|
2301
2479
|
content += '</div>';
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Validation for the fields a user types into the publication task form.
|
|
3
|
+
*
|
|
4
|
+
* None of these values can cause command injection - they're passed to spawn()
|
|
5
|
+
* as argv, and nothing goes near a shell - but they do end up in git command
|
|
6
|
+
* lines, in URLs, and in file names, so they're checked at the boundary rather
|
|
7
|
+
* than trusted to be harmless everywhere they're later used.
|
|
8
|
+
*
|
|
9
|
+
* The rules are mostly the ones GitHub and git themselves enforce (see
|
|
10
|
+
* git-check-ref-format(1) for the branch rules): anything rejected here would
|
|
11
|
+
* have failed at clone time anyway, with a much worse error. The exception is
|
|
12
|
+
* the markup characters in a branch name, which git permits and we do not - see
|
|
13
|
+
* validateGitBranch.
|
|
14
|
+
*
|
|
15
|
+
* These checks are NOT the defence against markup in the pages that display
|
|
16
|
+
* these values. Output escaping is, and it has to stay there whatever this file
|
|
17
|
+
* allows; the branch rule below is a second line, not the first.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
// GitHub owner names: letters, digits and single hyphens, no hyphen at either
|
|
21
|
+
// end, 39 characters at most.
|
|
22
|
+
const MAX_OWNER = 39;
|
|
23
|
+
// Repository names: letters, digits, dot, hyphen, underscore, 100 at most.
|
|
24
|
+
const MAX_REPO = 100;
|
|
25
|
+
// git imposes no length limit on a ref, but a branch name past this is a
|
|
26
|
+
// filesystem problem waiting to happen.
|
|
27
|
+
const MAX_BRANCH = 255;
|
|
28
|
+
const MAX_PACKAGE_ID = 128;
|
|
29
|
+
const MAX_VERSION = 64;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A GitHub organisation or user name.
|
|
33
|
+
*/
|
|
34
|
+
function validateGithubOwner(value) {
|
|
35
|
+
if (!value) {
|
|
36
|
+
return 'GitHub org is required';
|
|
37
|
+
}
|
|
38
|
+
if (value.length > MAX_OWNER) {
|
|
39
|
+
return `GitHub org must be ${MAX_OWNER} characters or less`;
|
|
40
|
+
}
|
|
41
|
+
if (!/^[A-Za-z0-9](?:-?[A-Za-z0-9])*$/.test(value)) {
|
|
42
|
+
return 'GitHub org may only contain letters, digits and single hyphens, and may not start or end with a hyphen';
|
|
43
|
+
}
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* A GitHub repository name.
|
|
49
|
+
*/
|
|
50
|
+
function validateGithubRepo(value) {
|
|
51
|
+
if (!value) {
|
|
52
|
+
return 'GitHub repo is required';
|
|
53
|
+
}
|
|
54
|
+
if (value.length > MAX_REPO) {
|
|
55
|
+
return `GitHub repo must be ${MAX_REPO} characters or less`;
|
|
56
|
+
}
|
|
57
|
+
if (!/^[A-Za-z0-9._-]+$/.test(value)) {
|
|
58
|
+
return 'GitHub repo may only contain letters, digits, dots, hyphens and underscores';
|
|
59
|
+
}
|
|
60
|
+
if (value === '.' || value === '..') {
|
|
61
|
+
return 'GitHub repo is not a valid name';
|
|
62
|
+
}
|
|
63
|
+
// the code appends '.git' when it builds the clone URL, and GitHub won't
|
|
64
|
+
// create a repository with that suffix anyway
|
|
65
|
+
if (value.toLowerCase().endsWith('.git')) {
|
|
66
|
+
return 'GitHub repo should not include the .git suffix';
|
|
67
|
+
}
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* A branch name, by the rules in git-check-ref-format(1). Slashes are fine -
|
|
73
|
+
* that's how branches are grouped - but the characters git reserves for
|
|
74
|
+
* revision syntax are not. Note that git itself rejects a backslash in a ref
|
|
75
|
+
* name, so this does too.
|
|
76
|
+
*/
|
|
77
|
+
function validateGitBranch(value) {
|
|
78
|
+
if (!value) {
|
|
79
|
+
return 'Branch is required';
|
|
80
|
+
}
|
|
81
|
+
if (value.length > MAX_BRANCH) {
|
|
82
|
+
return `Branch must be ${MAX_BRANCH} characters or less`;
|
|
83
|
+
}
|
|
84
|
+
// space, the ASCII control characters, and DEL
|
|
85
|
+
// eslint-disable-next-line no-control-regex
|
|
86
|
+
if (/[\u0000-\u001F\u007F ]/.test(value)) {
|
|
87
|
+
return 'Branch may not contain spaces or control characters';
|
|
88
|
+
}
|
|
89
|
+
// the characters git reserves for revision syntax
|
|
90
|
+
if (/[~^:?*[\\]/.test(value)) {
|
|
91
|
+
return 'Branch may not contain any of ~ ^ : ? * [ \\';
|
|
92
|
+
}
|
|
93
|
+
// git itself allows these, but a branch name is displayed on the task pages, and a value that
|
|
94
|
+
// has to be escaped everywhere it is shown is one escape away from being a stored XSS. Nobody
|
|
95
|
+
// needs a branch called <script>, so refuse it at the boundary as well as escaping on output
|
|
96
|
+
if (/[<>"'&]/.test(value)) {
|
|
97
|
+
return 'Branch may not contain any of < > " \' &';
|
|
98
|
+
}
|
|
99
|
+
if (value.includes('..')) {
|
|
100
|
+
return 'Branch may not contain ..';
|
|
101
|
+
}
|
|
102
|
+
if (value.includes('@{')) {
|
|
103
|
+
return 'Branch may not contain @{';
|
|
104
|
+
}
|
|
105
|
+
if (value === '@') {
|
|
106
|
+
return 'Branch may not be @';
|
|
107
|
+
}
|
|
108
|
+
if (value.startsWith('-')) {
|
|
109
|
+
return 'Branch may not start with a hyphen';
|
|
110
|
+
}
|
|
111
|
+
if (value.startsWith('/') || value.endsWith('/') || value.includes('//')) {
|
|
112
|
+
return 'Branch may not start or end with /, or contain //';
|
|
113
|
+
}
|
|
114
|
+
if (value.endsWith('.')) {
|
|
115
|
+
return 'Branch may not end with .';
|
|
116
|
+
}
|
|
117
|
+
for (const part of value.split('/')) {
|
|
118
|
+
if (part.startsWith('.')) {
|
|
119
|
+
return 'No part of a branch name may start with .';
|
|
120
|
+
}
|
|
121
|
+
if (part.endsWith('.lock')) {
|
|
122
|
+
return 'No part of a branch name may end with .lock';
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* An NPM package id. This one is not about git at all: the package id and the
|
|
130
|
+
* version are concatenated into file names under the zips directory
|
|
131
|
+
* (<id>#<version>.log, <id>#<version>-announcement.txt), so a value containing
|
|
132
|
+
* a path separator would look outside that directory.
|
|
133
|
+
*/
|
|
134
|
+
function validatePackageId(value) {
|
|
135
|
+
if (!value) {
|
|
136
|
+
return 'NPM package id is required';
|
|
137
|
+
}
|
|
138
|
+
if (value.length > MAX_PACKAGE_ID) {
|
|
139
|
+
return `NPM package id must be ${MAX_PACKAGE_ID} characters or less`;
|
|
140
|
+
}
|
|
141
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value)) {
|
|
142
|
+
return 'NPM package id may only contain letters, digits, dots, hyphens and underscores, and must start with a letter or digit';
|
|
143
|
+
}
|
|
144
|
+
if (value.includes('..')) {
|
|
145
|
+
return 'NPM package id may not contain ..';
|
|
146
|
+
}
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* A package version. Loose enough for semver with pre-release and build
|
|
152
|
+
* metadata, tight enough that it can't become a path.
|
|
153
|
+
*/
|
|
154
|
+
function validateVersion(value) {
|
|
155
|
+
if (!value) {
|
|
156
|
+
return 'Version is required';
|
|
157
|
+
}
|
|
158
|
+
if (value.length > MAX_VERSION) {
|
|
159
|
+
return `Version must be ${MAX_VERSION} characters or less`;
|
|
160
|
+
}
|
|
161
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9.+-]*$/.test(value)) {
|
|
162
|
+
return 'Version may only contain letters, digits, dots, hyphens and plus signs, and must start with a letter or digit';
|
|
163
|
+
}
|
|
164
|
+
if (value.includes('..')) {
|
|
165
|
+
return 'Version may not contain ..';
|
|
166
|
+
}
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Validate everything the task form collects.
|
|
172
|
+
* @returns {string[]} the problems found, empty if the input is acceptable
|
|
173
|
+
*/
|
|
174
|
+
function validateTaskInput(input) {
|
|
175
|
+
const errors = [];
|
|
176
|
+
const checks = [
|
|
177
|
+
validateGithubOwner(input.github_org),
|
|
178
|
+
validateGithubRepo(input.github_repo),
|
|
179
|
+
validateGitBranch(input.git_branch),
|
|
180
|
+
validatePackageId(input.npm_package_id),
|
|
181
|
+
validateVersion(input.version)
|
|
182
|
+
];
|
|
183
|
+
for (const error of checks) {
|
|
184
|
+
if (error) {
|
|
185
|
+
errors.push(error);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return errors;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Patterns for the HTML form, so the browser objects before the round trip.
|
|
192
|
+
// Deliberately a subset of the checks above - the server is what decides -
|
|
193
|
+
// but they catch the obvious mistakes as the user types.
|
|
194
|
+
const HTML_PATTERNS = {
|
|
195
|
+
github_org: '[A-Za-z0-9](-?[A-Za-z0-9])*',
|
|
196
|
+
github_repo: '[A-Za-z0-9._-]+',
|
|
197
|
+
git_branch: '[^\\\\ ~^:?*\\[<>"\'&]+',
|
|
198
|
+
npm_package_id: '[A-Za-z0-9][A-Za-z0-9._-]*',
|
|
199
|
+
version: '[A-Za-z0-9][A-Za-z0-9.+-]*'
|
|
200
|
+
};
|
|
201
|
+
|
|
202
|
+
module.exports = {
|
|
203
|
+
validateGithubOwner,
|
|
204
|
+
validateGithubRepo,
|
|
205
|
+
validateGitBranch,
|
|
206
|
+
validatePackageId,
|
|
207
|
+
validateVersion,
|
|
208
|
+
validateTaskInput,
|
|
209
|
+
HTML_PATTERNS
|
|
210
|
+
};
|