fhirsmith 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/config-template.json +18 -1
- package/library/folder-content-loader.js +111 -21
- package/package.json +3 -3
- package/publisher/publisher.js +153 -5
- package/publisher/validation.js +198 -0
- package/server.js +12 -12
- package/stats-db.js +127 -0
- package/stats.js +172 -6
- package/translations/Messages.properties +4 -1
- package/tx/cs/cs-api.js +9 -3
- package/tx/cs/cs-api.md +1 -1
- package/tx/cs/cs-snomed.js +131 -65
- package/tx/library/conceptmap.js +39 -0
- package/tx/library/designations.js +9 -2
- package/tx/library/parameter-check.js +139 -0
- package/tx/library.js +62 -9
- package/tx/operation-context.js +220 -10
- package/tx/params.js +226 -175
- package/tx/sct/expressions.js +485 -0
- package/tx/tx.fhir.org.yml +1 -1
- package/tx/tx.js +57 -50
- package/tx/workers/batch-validate.js +5 -2
- package/tx/workers/cache-control.js +82 -6
- package/tx/workers/expand.js +80 -23
- package/tx/workers/lookup.js +14 -1
- package/tx/workers/translate.js +461 -30
- package/tx/workers/validate.js +7 -1
- package/tx/workers/worker.js +22 -10
- package/tx/xml/xml-base.js +21 -1
- package/tx/xversion/xv-conceptmap.js +81 -0
- package/tx/xversion/xv-parameters.js +8 -0
- package/utilities/merge-audit.py +178 -0
- package/utilities/why-merge-audit.md +39 -0
- package/tx/data/snomed-subset.txt +0 -84
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Validation for the fields a user types into the publication task form.
|
|
3
|
+
*
|
|
4
|
+
* None of these values can cause command injection - they're passed to spawn()
|
|
5
|
+
* as argv, and nothing goes near a shell - but they do end up in git command
|
|
6
|
+
* lines, in URLs, and in file names, so they're checked at the boundary rather
|
|
7
|
+
* than trusted to be harmless everywhere they're later used.
|
|
8
|
+
*
|
|
9
|
+
* The rules are the ones GitHub and git themselves enforce (see
|
|
10
|
+
* git-check-ref-format(1) for the branch rules): anything rejected here would
|
|
11
|
+
* have failed at clone time anyway, with a much worse error.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
// GitHub owner names: letters, digits and single hyphens, no hyphen at either
|
|
15
|
+
// end, 39 characters at most.
|
|
16
|
+
const MAX_OWNER = 39;
|
|
17
|
+
// Repository names: letters, digits, dot, hyphen, underscore, 100 at most.
|
|
18
|
+
const MAX_REPO = 100;
|
|
19
|
+
// git imposes no length limit on a ref, but a branch name past this is a
|
|
20
|
+
// filesystem problem waiting to happen.
|
|
21
|
+
const MAX_BRANCH = 255;
|
|
22
|
+
const MAX_PACKAGE_ID = 128;
|
|
23
|
+
const MAX_VERSION = 64;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* A GitHub organisation or user name.
|
|
27
|
+
*/
|
|
28
|
+
function validateGithubOwner(value) {
|
|
29
|
+
if (!value) {
|
|
30
|
+
return 'GitHub org is required';
|
|
31
|
+
}
|
|
32
|
+
if (value.length > MAX_OWNER) {
|
|
33
|
+
return `GitHub org must be ${MAX_OWNER} characters or less`;
|
|
34
|
+
}
|
|
35
|
+
if (!/^[A-Za-z0-9](?:-?[A-Za-z0-9])*$/.test(value)) {
|
|
36
|
+
return 'GitHub org may only contain letters, digits and single hyphens, and may not start or end with a hyphen';
|
|
37
|
+
}
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* A GitHub repository name.
|
|
43
|
+
*/
|
|
44
|
+
function validateGithubRepo(value) {
|
|
45
|
+
if (!value) {
|
|
46
|
+
return 'GitHub repo is required';
|
|
47
|
+
}
|
|
48
|
+
if (value.length > MAX_REPO) {
|
|
49
|
+
return `GitHub repo must be ${MAX_REPO} characters or less`;
|
|
50
|
+
}
|
|
51
|
+
if (!/^[A-Za-z0-9._-]+$/.test(value)) {
|
|
52
|
+
return 'GitHub repo may only contain letters, digits, dots, hyphens and underscores';
|
|
53
|
+
}
|
|
54
|
+
if (value === '.' || value === '..') {
|
|
55
|
+
return 'GitHub repo is not a valid name';
|
|
56
|
+
}
|
|
57
|
+
// the code appends '.git' when it builds the clone URL, and GitHub won't
|
|
58
|
+
// create a repository with that suffix anyway
|
|
59
|
+
if (value.toLowerCase().endsWith('.git')) {
|
|
60
|
+
return 'GitHub repo should not include the .git suffix';
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* A branch name, by the rules in git-check-ref-format(1). Slashes are fine -
|
|
67
|
+
* that's how branches are grouped - but the characters git reserves for
|
|
68
|
+
* revision syntax are not. Note that git itself rejects a backslash in a ref
|
|
69
|
+
* name, so this does too.
|
|
70
|
+
*/
|
|
71
|
+
function validateGitBranch(value) {
|
|
72
|
+
if (!value) {
|
|
73
|
+
return 'Branch is required';
|
|
74
|
+
}
|
|
75
|
+
if (value.length > MAX_BRANCH) {
|
|
76
|
+
return `Branch must be ${MAX_BRANCH} characters or less`;
|
|
77
|
+
}
|
|
78
|
+
// space, the ASCII control characters, and DEL
|
|
79
|
+
// eslint-disable-next-line no-control-regex
|
|
80
|
+
if (/[\u0000-\u001F\u007F ]/.test(value)) {
|
|
81
|
+
return 'Branch may not contain spaces or control characters';
|
|
82
|
+
}
|
|
83
|
+
// the characters git reserves for revision syntax
|
|
84
|
+
if (/[~^:?*[\\]/.test(value)) {
|
|
85
|
+
return 'Branch may not contain any of ~ ^ : ? * [ \\';
|
|
86
|
+
}
|
|
87
|
+
if (value.includes('..')) {
|
|
88
|
+
return 'Branch may not contain ..';
|
|
89
|
+
}
|
|
90
|
+
if (value.includes('@{')) {
|
|
91
|
+
return 'Branch may not contain @{';
|
|
92
|
+
}
|
|
93
|
+
if (value === '@') {
|
|
94
|
+
return 'Branch may not be @';
|
|
95
|
+
}
|
|
96
|
+
if (value.startsWith('-')) {
|
|
97
|
+
return 'Branch may not start with a hyphen';
|
|
98
|
+
}
|
|
99
|
+
if (value.startsWith('/') || value.endsWith('/') || value.includes('//')) {
|
|
100
|
+
return 'Branch may not start or end with /, or contain //';
|
|
101
|
+
}
|
|
102
|
+
if (value.endsWith('.')) {
|
|
103
|
+
return 'Branch may not end with .';
|
|
104
|
+
}
|
|
105
|
+
for (const part of value.split('/')) {
|
|
106
|
+
if (part.startsWith('.')) {
|
|
107
|
+
return 'No part of a branch name may start with .';
|
|
108
|
+
}
|
|
109
|
+
if (part.endsWith('.lock')) {
|
|
110
|
+
return 'No part of a branch name may end with .lock';
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* An NPM package id. This one is not about git at all: the package id and the
|
|
118
|
+
* version are concatenated into file names under the zips directory
|
|
119
|
+
* (<id>#<version>.log, <id>#<version>-announcement.txt), so a value containing
|
|
120
|
+
* a path separator would look outside that directory.
|
|
121
|
+
*/
|
|
122
|
+
function validatePackageId(value) {
|
|
123
|
+
if (!value) {
|
|
124
|
+
return 'NPM package id is required';
|
|
125
|
+
}
|
|
126
|
+
if (value.length > MAX_PACKAGE_ID) {
|
|
127
|
+
return `NPM package id must be ${MAX_PACKAGE_ID} characters or less`;
|
|
128
|
+
}
|
|
129
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value)) {
|
|
130
|
+
return 'NPM package id may only contain letters, digits, dots, hyphens and underscores, and must start with a letter or digit';
|
|
131
|
+
}
|
|
132
|
+
if (value.includes('..')) {
|
|
133
|
+
return 'NPM package id may not contain ..';
|
|
134
|
+
}
|
|
135
|
+
return null;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* A package version. Loose enough for semver with pre-release and build
|
|
140
|
+
* metadata, tight enough that it can't become a path.
|
|
141
|
+
*/
|
|
142
|
+
function validateVersion(value) {
|
|
143
|
+
if (!value) {
|
|
144
|
+
return 'Version is required';
|
|
145
|
+
}
|
|
146
|
+
if (value.length > MAX_VERSION) {
|
|
147
|
+
return `Version must be ${MAX_VERSION} characters or less`;
|
|
148
|
+
}
|
|
149
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9.+-]*$/.test(value)) {
|
|
150
|
+
return 'Version may only contain letters, digits, dots, hyphens and plus signs, and must start with a letter or digit';
|
|
151
|
+
}
|
|
152
|
+
if (value.includes('..')) {
|
|
153
|
+
return 'Version may not contain ..';
|
|
154
|
+
}
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Validate everything the task form collects.
|
|
160
|
+
* @returns {string[]} the problems found, empty if the input is acceptable
|
|
161
|
+
*/
|
|
162
|
+
function validateTaskInput(input) {
|
|
163
|
+
const errors = [];
|
|
164
|
+
const checks = [
|
|
165
|
+
validateGithubOwner(input.github_org),
|
|
166
|
+
validateGithubRepo(input.github_repo),
|
|
167
|
+
validateGitBranch(input.git_branch),
|
|
168
|
+
validatePackageId(input.npm_package_id),
|
|
169
|
+
validateVersion(input.version)
|
|
170
|
+
];
|
|
171
|
+
for (const error of checks) {
|
|
172
|
+
if (error) {
|
|
173
|
+
errors.push(error);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return errors;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Patterns for the HTML form, so the browser objects before the round trip.
|
|
180
|
+
// Deliberately a subset of the checks above - the server is what decides -
|
|
181
|
+
// but they catch the obvious mistakes as the user types.
|
|
182
|
+
const HTML_PATTERNS = {
|
|
183
|
+
github_org: '[A-Za-z0-9](-?[A-Za-z0-9])*',
|
|
184
|
+
github_repo: '[A-Za-z0-9._-]+',
|
|
185
|
+
git_branch: '[^\\\\ ~^:?*\\[]+',
|
|
186
|
+
npm_package_id: '[A-Za-z0-9][A-Za-z0-9._-]*',
|
|
187
|
+
version: '[A-Za-z0-9][A-Za-z0-9.+-]*'
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
module.exports = {
|
|
191
|
+
validateGithubOwner,
|
|
192
|
+
validateGithubRepo,
|
|
193
|
+
validateGitBranch,
|
|
194
|
+
validatePackageId,
|
|
195
|
+
validateVersion,
|
|
196
|
+
validateTaskInput,
|
|
197
|
+
HTML_PATTERNS
|
|
198
|
+
};
|
package/server.js
CHANGED
|
@@ -99,13 +99,13 @@ let stats = null;
|
|
|
99
99
|
|
|
100
100
|
// Initialize modules based on configuration
|
|
101
101
|
async function initializeModules() {
|
|
102
|
-
stats = new ServerStats();
|
|
102
|
+
stats = new ServerStats(config.stats, serverLog);
|
|
103
103
|
|
|
104
104
|
// Initialize SHL module
|
|
105
105
|
if (config.modules?.shl?.enabled) {
|
|
106
106
|
try {
|
|
107
107
|
serverLog.info('Initializing module: shl...');
|
|
108
|
-
modules.shl = new SHLModule(stats);
|
|
108
|
+
modules.shl = new SHLModule(stats.forModule('shl'));
|
|
109
109
|
await modules.shl.initialize(config.modules.shl);
|
|
110
110
|
app.use('/shl', modules.shl.router);
|
|
111
111
|
} catch (error) {
|
|
@@ -118,7 +118,7 @@ async function initializeModules() {
|
|
|
118
118
|
if (config.modules?.vcl?.enabled) {
|
|
119
119
|
try {
|
|
120
120
|
serverLog.info('Initializing module: vcl...');
|
|
121
|
-
modules.vcl = new VCLModule(stats);
|
|
121
|
+
modules.vcl = new VCLModule(stats.forModule('vcl'));
|
|
122
122
|
await modules.vcl.initialize(config.modules.vcl);
|
|
123
123
|
app.use('/VCL', modules.vcl.router);
|
|
124
124
|
} catch (error) {
|
|
@@ -131,7 +131,7 @@ async function initializeModules() {
|
|
|
131
131
|
if (config.modules?.xig?.enabled) {
|
|
132
132
|
try {
|
|
133
133
|
serverLog.info('Initializing module: xig...');
|
|
134
|
-
await xigModule.initializeXigModule(stats, config.modules.xig);
|
|
134
|
+
await xigModule.initializeXigModule(stats.forModule('xig'), config.modules.xig);
|
|
135
135
|
app.use('/xig', xigModule.router);
|
|
136
136
|
modules.xig = xigModule;
|
|
137
137
|
} catch (error) {
|
|
@@ -144,7 +144,7 @@ async function initializeModules() {
|
|
|
144
144
|
if (config.modules?.packages?.enabled) {
|
|
145
145
|
try {
|
|
146
146
|
serverLog.info('Initializing module: packages...');
|
|
147
|
-
modules.packages = new PackagesModule(stats);
|
|
147
|
+
modules.packages = new PackagesModule(stats.forModule('packages'));
|
|
148
148
|
await modules.packages.initialize(config.modules.packages);
|
|
149
149
|
app.use('/packages', modules.packages.router);
|
|
150
150
|
} catch (error) {
|
|
@@ -158,7 +158,7 @@ async function initializeModules() {
|
|
|
158
158
|
if (config.modules?.registry?.enabled) {
|
|
159
159
|
try {
|
|
160
160
|
serverLog.info('Initializing module: registry...');
|
|
161
|
-
modules.registry = new RegistryModule(stats);
|
|
161
|
+
modules.registry = new RegistryModule(stats.forModule('registry'));
|
|
162
162
|
await modules.registry.initialize(config.modules.registry);
|
|
163
163
|
app.use('/tx-reg', modules.registry.router);
|
|
164
164
|
} catch (error) {
|
|
@@ -171,7 +171,7 @@ async function initializeModules() {
|
|
|
171
171
|
if (config.modules?.publisher?.enabled) {
|
|
172
172
|
try {
|
|
173
173
|
serverLog.info('Initializing module: publisher...');
|
|
174
|
-
modules.publisher = new PublisherModule(stats);
|
|
174
|
+
modules.publisher = new PublisherModule(stats.forModule('publisher'));
|
|
175
175
|
await modules.publisher.initialize(config.modules.publisher);
|
|
176
176
|
app.use('/publisher', modules.publisher.router);
|
|
177
177
|
} catch (error) {
|
|
@@ -184,7 +184,7 @@ async function initializeModules() {
|
|
|
184
184
|
if (config.modules?.token?.enabled) {
|
|
185
185
|
try {
|
|
186
186
|
serverLog.info('Initializing module: token...');
|
|
187
|
-
modules.token = new TokenModule(stats);
|
|
187
|
+
modules.token = new TokenModule(stats.forModule('token'));
|
|
188
188
|
await modules.token.initialize(config.modules.token);
|
|
189
189
|
app.use('/token', modules.token.router);
|
|
190
190
|
} catch (error) {
|
|
@@ -197,7 +197,7 @@ async function initializeModules() {
|
|
|
197
197
|
if (config.modules?.npmprojector?.enabled) {
|
|
198
198
|
try {
|
|
199
199
|
serverLog.info('Initializing module: npmprojector...');
|
|
200
|
-
modules.npmprojector = new NpmProjectorModule(stats);
|
|
200
|
+
modules.npmprojector = new NpmProjectorModule(stats.forModule('npmprojector'));
|
|
201
201
|
await modules.npmprojector.initialize(config.modules.npmprojector);
|
|
202
202
|
const basePath = NpmProjectorModule.getBasePath(config.modules.npmprojector);
|
|
203
203
|
app.use(basePath, modules.npmprojector.router);
|
|
@@ -211,7 +211,7 @@ async function initializeModules() {
|
|
|
211
211
|
if (config.modules?.['ext-tracker']?.enabled) {
|
|
212
212
|
try {
|
|
213
213
|
serverLog.info('Initializing module: ext-tracker...');
|
|
214
|
-
modules.extTracker = new ExtensionTrackerModule(stats);
|
|
214
|
+
modules.extTracker = new ExtensionTrackerModule(stats.forModule('ext-tracker'));
|
|
215
215
|
await modules.extTracker.initialize(config.modules['ext-tracker'], app);
|
|
216
216
|
} catch (error) {
|
|
217
217
|
serverLog.error('Failed to initialize extension tracker module:', error);
|
|
@@ -224,7 +224,7 @@ async function initializeModules() {
|
|
|
224
224
|
if (config.modules?.tx?.enabled) {
|
|
225
225
|
try {
|
|
226
226
|
serverLog.info('Initializing module: tx...');
|
|
227
|
-
modules.tx = new TXModule(stats);
|
|
227
|
+
modules.tx = new TXModule(stats.forModule('tx'));
|
|
228
228
|
await modules.tx.initialize(config.modules.tx, app);
|
|
229
229
|
} catch (error) {
|
|
230
230
|
serverLog.error('Failed to initialize TX module:', error);
|
|
@@ -235,7 +235,7 @@ async function initializeModules() {
|
|
|
235
235
|
if (config.modules?.folder?.enabled) {
|
|
236
236
|
try {
|
|
237
237
|
serverLog.info('Initializing module: folder...');
|
|
238
|
-
modules.folder = new FolderModule(stats);
|
|
238
|
+
modules.folder = new FolderModule(stats.forModule('folder'));
|
|
239
239
|
await modules.folder.initialize(config.modules.folder, app);
|
|
240
240
|
// mount the router
|
|
241
241
|
} catch (error) {
|
package/stats-db.js
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
const path = require('path');
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const Database = require('better-sqlite3');
|
|
4
|
+
const folders = require('./library/folder-setup');
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Persistent storage for the server's request statistics (issue #255).
|
|
8
|
+
*
|
|
9
|
+
* The in-memory counters in ServerStats reset every time the server restarts,
|
|
10
|
+
* which makes them useless for anything but "what has happened since the last
|
|
11
|
+
* deploy". This class writes them to a SQLite database that outlives restarts
|
|
12
|
+
* and upgrades.
|
|
13
|
+
*
|
|
14
|
+
* One row per (module, endpoint, operation) per collection interval, holding
|
|
15
|
+
* both numbers:
|
|
16
|
+
*
|
|
17
|
+
* - the *delta*: what happened in that interval. This is what you graph -
|
|
18
|
+
* plot count_delta against time and you have requests per interval without
|
|
19
|
+
* any differencing at the query end.
|
|
20
|
+
* - the *total*: the all-time cumulative count, carried across restarts by
|
|
21
|
+
* reloading it at startup. This is what you quote as "how much has this
|
|
22
|
+
* endpoint ever been asked to do", and it means a restart shows up as a
|
|
23
|
+
* flat spot in the graph rather than a cliff back to zero.
|
|
24
|
+
*
|
|
25
|
+
* Rows are only written for keys that saw traffic in the interval, so idle
|
|
26
|
+
* endpoints cost nothing. Nothing is pruned: at 10 minute intervals a
|
|
27
|
+
* continuously busy (endpoint, operation) pair produces ~52,000 rows a year,
|
|
28
|
+
* which SQLite does not notice.
|
|
29
|
+
*
|
|
30
|
+
* better-sqlite3 (rather than sqlite3) on purpose: the writes are synchronous,
|
|
31
|
+
* so the final flush in ServerStats.finishStats() lands on disk before
|
|
32
|
+
* server.js calls process.exit(). A handful of inserts every ten minutes is
|
|
33
|
+
* well under a millisecond, so this does not put the event loop at risk.
|
|
34
|
+
*/
|
|
35
|
+
class StatsDatabase {
|
|
36
|
+
|
|
37
|
+
constructor(config, log) {
|
|
38
|
+
this.log = log || console;
|
|
39
|
+
this.config = config || {};
|
|
40
|
+
this.db = null;
|
|
41
|
+
this.path = StatsDatabase.resolvePath(this.config);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Where the database lives: config.database if given (relative paths resolve
|
|
46
|
+
* under the data folder's databases directory), otherwise stats.db there.
|
|
47
|
+
*/
|
|
48
|
+
static resolvePath(config) {
|
|
49
|
+
const configured = config && config.database;
|
|
50
|
+
if (!configured) {
|
|
51
|
+
return path.join(folders.databasesDir(), 'stats.db');
|
|
52
|
+
}
|
|
53
|
+
return path.isAbsolute(configured) ? configured : path.join(folders.databasesDir(), configured);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Open the database, creating the file and the schema if they aren't there.
|
|
58
|
+
*/
|
|
59
|
+
open() {
|
|
60
|
+
fs.mkdirSync(path.dirname(this.path), {recursive: true});
|
|
61
|
+
this.db = new Database(this.path);
|
|
62
|
+
this.db.pragma('journal_mode = WAL');
|
|
63
|
+
this.db.exec(`
|
|
64
|
+
CREATE TABLE IF NOT EXISTS request_counts (
|
|
65
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
66
|
+
time INTEGER NOT NULL,
|
|
67
|
+
interval_ms INTEGER NOT NULL,
|
|
68
|
+
module TEXT NOT NULL,
|
|
69
|
+
endpoint TEXT NOT NULL,
|
|
70
|
+
operation TEXT NOT NULL,
|
|
71
|
+
count_delta INTEGER NOT NULL,
|
|
72
|
+
count_total INTEGER NOT NULL,
|
|
73
|
+
time_delta INTEGER NOT NULL,
|
|
74
|
+
time_total INTEGER NOT NULL
|
|
75
|
+
)
|
|
76
|
+
`);
|
|
77
|
+
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_request_counts_time ON request_counts(time)`);
|
|
78
|
+
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_request_counts_key ON request_counts(module, endpoint, operation, id)`);
|
|
79
|
+
|
|
80
|
+
this.insertStatement = this.db.prepare(`
|
|
81
|
+
INSERT INTO request_counts
|
|
82
|
+
(time, interval_ms, module, endpoint, operation, count_delta, count_total, time_delta, time_total)
|
|
83
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
84
|
+
`);
|
|
85
|
+
this.insertAll = this.db.transaction((time, intervalMs, rows) => {
|
|
86
|
+
for (const row of rows) {
|
|
87
|
+
this.insertStatement.run(time, intervalMs, row.module, row.endpoint, row.operation,
|
|
88
|
+
row.countDelta, row.countTotal, row.timeDelta, row.timeTotal);
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
return this;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The most recent cumulative total for each key, so counting resumes where
|
|
96
|
+
* the last run left off instead of starting again from zero.
|
|
97
|
+
*/
|
|
98
|
+
loadTotals() {
|
|
99
|
+
if (!this.db) {
|
|
100
|
+
return [];
|
|
101
|
+
}
|
|
102
|
+
return this.db.prepare(`
|
|
103
|
+
SELECT module, endpoint, operation, count_total AS countTotal, time_total AS timeTotal
|
|
104
|
+
FROM request_counts
|
|
105
|
+
WHERE id IN (SELECT MAX(id) FROM request_counts GROUP BY module, endpoint, operation)
|
|
106
|
+
`).all();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Write one interval's worth of rows. All or nothing - if the transaction
|
|
111
|
+
* fails the caller keeps its deltas and tries again next interval.
|
|
112
|
+
*/
|
|
113
|
+
write(time, intervalMs, rows) {
|
|
114
|
+
if (this.db && rows && rows.length > 0) {
|
|
115
|
+
this.insertAll(time, intervalMs, rows);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
close() {
|
|
120
|
+
if (this.db) {
|
|
121
|
+
this.db.close();
|
|
122
|
+
this.db = null;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
module.exports = StatsDatabase;
|
package/stats.js
CHANGED
|
@@ -1,6 +1,49 @@
|
|
|
1
1
|
const { monitorEventLoopDelay } = require('perf_hooks');
|
|
2
2
|
const {Utilities} = require("./library/utilities");
|
|
3
3
|
const escape = require('escape-html');
|
|
4
|
+
const StatsDatabase = require('./stats-db');
|
|
5
|
+
|
|
6
|
+
// Requests that aren't counted by a module (the root page, the dashboard) are
|
|
7
|
+
// attributed here, and requests that have no endpoint dimension (everything
|
|
8
|
+
// outside the tx module) carry NO_ENDPOINT.
|
|
9
|
+
const SERVER_MODULE = '(server)';
|
|
10
|
+
const NO_ENDPOINT = '';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* A per-module view of the server statistics.
|
|
14
|
+
*
|
|
15
|
+
* Modules are handed one of these instead of the ServerStats itself, so that
|
|
16
|
+
* everything they count is attributed to them without each of the ~76 call
|
|
17
|
+
* sites having to name its own module. The API is deliberately the same shape
|
|
18
|
+
* as ServerStats, so a module can't tell the difference - except that
|
|
19
|
+
* countRequest takes an optional endpoint, which is how the tx module
|
|
20
|
+
* distinguishes /tx/r4 from /tx/r5.
|
|
21
|
+
*/
|
|
22
|
+
class ModuleStats {
|
|
23
|
+
|
|
24
|
+
constructor(parent, module) {
|
|
25
|
+
this.parent = parent;
|
|
26
|
+
this.module = module;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
countRequest(name, tat, endpoint = NO_ENDPOINT) {
|
|
30
|
+
this.parent.countRequest(name, tat, this.module, endpoint);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
addTask(name, frequency) { this.parent.addTask(name, frequency); }
|
|
34
|
+
task(name, state) { this.parent.task(name, state); }
|
|
35
|
+
taskDone(name, state) { this.parent.taskDone(name, state); }
|
|
36
|
+
taskError(name, state) { this.parent.taskError(name, state); }
|
|
37
|
+
|
|
38
|
+
// Pass-throughs for the handful of properties modules read off the stats
|
|
39
|
+
// object. These have to be getters, not copies, or a module would capture
|
|
40
|
+
// the value at construction time and never see it change.
|
|
41
|
+
get cachingModules() { return this.parent.cachingModules; }
|
|
42
|
+
get requestCount() { return this.parent.requestCount; }
|
|
43
|
+
get staticRequestCount() { return this.parent.staticRequestCount; }
|
|
44
|
+
get history() { return this.parent.history; }
|
|
45
|
+
get startTime() { return this.parent.startTime; }
|
|
46
|
+
}
|
|
4
47
|
|
|
5
48
|
class ServerStats {
|
|
6
49
|
started = false;
|
|
@@ -16,13 +59,53 @@ class ServerStats {
|
|
|
16
59
|
timer;
|
|
17
60
|
cachingModules = [];
|
|
18
61
|
taskMap = new Map();
|
|
62
|
+
// key (module/endpoint/operation) -> counter. See countRequest().
|
|
63
|
+
counters = new Map();
|
|
64
|
+
db = null;
|
|
19
65
|
|
|
20
|
-
constructor() {
|
|
66
|
+
constructor(config, log) {
|
|
67
|
+
this.config = config || {};
|
|
68
|
+
this.log = log || console;
|
|
69
|
+
if (this.config.intervalMinutes) {
|
|
70
|
+
this.intervalMs = this.config.intervalMinutes * 60 * 1000;
|
|
71
|
+
}
|
|
72
|
+
this.openDatabase();
|
|
21
73
|
this.timer = setInterval(() => {
|
|
22
74
|
this.recordMetrics();
|
|
23
75
|
}, this.intervalMs);
|
|
24
76
|
}
|
|
25
77
|
|
|
78
|
+
/**
|
|
79
|
+
* Open the persistent counter store and pick up where the last run of the
|
|
80
|
+
* server left off. Statistics must never be the reason the server won't
|
|
81
|
+
* start, so a failure here is logged and then ignored: the counters carry
|
|
82
|
+
* on in memory, they just don't survive the next restart.
|
|
83
|
+
*/
|
|
84
|
+
openDatabase() {
|
|
85
|
+
if (this.config.enabled === false) {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
this.db = new StatsDatabase(this.config, this.log).open();
|
|
90
|
+
for (const row of this.db.loadTotals()) {
|
|
91
|
+
const counter = this.counterFor(row.module, row.endpoint, row.operation);
|
|
92
|
+
counter.countTotal = row.countTotal;
|
|
93
|
+
counter.timeTotal = row.timeTotal;
|
|
94
|
+
}
|
|
95
|
+
} catch (e) {
|
|
96
|
+
this.log.error(`Unable to open the statistics database: ${e.message}. Statistics will not be persisted`);
|
|
97
|
+
this.db = null;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* A view of the statistics that attributes everything counted through it to
|
|
103
|
+
* the named module (see ModuleStats).
|
|
104
|
+
*/
|
|
105
|
+
forModule(module) {
|
|
106
|
+
return new ModuleStats(this, module);
|
|
107
|
+
}
|
|
108
|
+
|
|
26
109
|
recordMetrics() {
|
|
27
110
|
if (this.started) {
|
|
28
111
|
const now = Date.now();
|
|
@@ -61,9 +144,12 @@ class ServerStats {
|
|
|
61
144
|
this.requestTime = 0;
|
|
62
145
|
this.lastTime = now;
|
|
63
146
|
|
|
64
|
-
// Prune old data (keep 24 hours)
|
|
147
|
+
// Prune old data (keep 24 hours). Only the in-memory history is pruned -
|
|
148
|
+
// the per-endpoint counters are persisted and kept indefinitely.
|
|
65
149
|
const cutoff = now - (24 * 60 * 60 * 1000); // 24 hours ago
|
|
66
150
|
this.history = this.history.filter(m => m.time > cutoff);
|
|
151
|
+
|
|
152
|
+
this.flushCounters(now, this.intervalMs);
|
|
67
153
|
}
|
|
68
154
|
}
|
|
69
155
|
|
|
@@ -77,11 +163,78 @@ class ServerStats {
|
|
|
77
163
|
this.recordMetrics();
|
|
78
164
|
}
|
|
79
165
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
166
|
+
/**
|
|
167
|
+
* The counter for one (module, endpoint, operation), created on first use.
|
|
168
|
+
*
|
|
169
|
+
* countDelta/timeDelta accumulate within the current interval and are reset
|
|
170
|
+
* each time they're written out; countTotal/timeTotal are cumulative for the
|
|
171
|
+
* life of the database, not the life of the process.
|
|
172
|
+
*/
|
|
173
|
+
counterFor(module, endpoint, operation) {
|
|
174
|
+
const key = `${module}\u001f${endpoint}\u001f${operation}`;
|
|
175
|
+
let counter = this.counters.get(key);
|
|
176
|
+
if (!counter) {
|
|
177
|
+
counter = {
|
|
178
|
+
module: module, endpoint: endpoint, operation: operation,
|
|
179
|
+
countDelta: 0, timeDelta: 0, countTotal: 0, timeTotal: 0
|
|
180
|
+
};
|
|
181
|
+
this.counters.set(key, counter);
|
|
182
|
+
}
|
|
183
|
+
return counter;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Count one request. Modules go through their own ModuleStats, which fills
|
|
188
|
+
* in module (and, for tx, the endpoint) for them.
|
|
189
|
+
*
|
|
190
|
+
* @param {string} name - the operation, e.g. '$expand' or 'login'
|
|
191
|
+
* @param {number} tat - turnaround time in ms
|
|
192
|
+
* @param {string} module - the module that served it
|
|
193
|
+
* @param {string} endpoint - the endpoint within the module, where the
|
|
194
|
+
* module has more than one (the tx module does; nothing else does yet)
|
|
195
|
+
*/
|
|
196
|
+
countRequest(name, tat, module = SERVER_MODULE, endpoint = NO_ENDPOINT) {
|
|
83
197
|
this.requestCount++;
|
|
84
198
|
this.requestTime = this.requestTime + tat;
|
|
199
|
+
|
|
200
|
+
const counter = this.counterFor(module || SERVER_MODULE, endpoint || NO_ENDPOINT, name || '(unnamed)');
|
|
201
|
+
counter.countDelta++;
|
|
202
|
+
counter.countTotal++;
|
|
203
|
+
counter.timeDelta = counter.timeDelta + tat;
|
|
204
|
+
counter.timeTotal = counter.timeTotal + tat;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Write out the counters that saw traffic this interval, and start the next
|
|
209
|
+
* one. The deltas are only cleared if the write succeeded, so a transient
|
|
210
|
+
* database problem delays the numbers rather than losing them (the interval
|
|
211
|
+
* they're eventually written against is longer, which is what interval_ms on
|
|
212
|
+
* the row is for).
|
|
213
|
+
*/
|
|
214
|
+
flushCounters(now, intervalMs) {
|
|
215
|
+
if (!this.db) {
|
|
216
|
+
return 0;
|
|
217
|
+
}
|
|
218
|
+
const rows = [];
|
|
219
|
+
for (const counter of this.counters.values()) {
|
|
220
|
+
if (counter.countDelta > 0) {
|
|
221
|
+
rows.push({...counter});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if (rows.length === 0) {
|
|
225
|
+
return 0;
|
|
226
|
+
}
|
|
227
|
+
try {
|
|
228
|
+
this.db.write(now, intervalMs, rows);
|
|
229
|
+
} catch (e) {
|
|
230
|
+
this.log.error(`Unable to write statistics: ${e.message}`);
|
|
231
|
+
return 0;
|
|
232
|
+
}
|
|
233
|
+
for (const counter of this.counters.values()) {
|
|
234
|
+
counter.countDelta = 0;
|
|
235
|
+
counter.timeDelta = 0;
|
|
236
|
+
}
|
|
237
|
+
return rows.length;
|
|
85
238
|
}
|
|
86
239
|
|
|
87
240
|
addTask(name, frequency) {
|
|
@@ -144,6 +297,17 @@ class ServerStats {
|
|
|
144
297
|
|
|
145
298
|
finishStats() {
|
|
146
299
|
clearInterval(this.timer);
|
|
300
|
+
// Synchronous, so the last partial interval is on disk before server.js
|
|
301
|
+
// calls process.exit().
|
|
302
|
+
this.flushCounters(Date.now(), Date.now() - (this.lastTime || this.startTime));
|
|
303
|
+
if (this.db) {
|
|
304
|
+
try {
|
|
305
|
+
this.db.close();
|
|
306
|
+
} catch (e) {
|
|
307
|
+
this.log.error(`Unable to close the statistics database: ${e.message}`);
|
|
308
|
+
}
|
|
309
|
+
this.db = null;
|
|
310
|
+
}
|
|
147
311
|
}
|
|
148
312
|
|
|
149
313
|
// Live cache aggregates across all registered caching modules (defensive: a
|
|
@@ -174,4 +338,6 @@ class ServerStats {
|
|
|
174
338
|
}
|
|
175
339
|
}
|
|
176
340
|
}
|
|
177
|
-
module.exports = ServerStats;
|
|
341
|
+
module.exports = ServerStats;
|
|
342
|
+
module.exports.ModuleStats = ModuleStats;
|
|
343
|
+
module.exports.SERVER_MODULE = SERVER_MODULE;
|
|
@@ -1101,7 +1101,10 @@ Unable_to_resolve_system__value_set_has_multiple_matches = The System URI could
|
|
|
1101
1101
|
Unable_to_resolve_system__value_set_has_no_includes_or_expansion = The System URI could not be determined for the code ''{0}'' in the ValueSet ''{1}'': value set has no includes or expansion
|
|
1102
1102
|
UNABLE_TO_RESOLVE_SYSTEM_SYSTEM_IS_INDETERMINATE = The code system {1} referred to from value set {0} has a grammar, and the code might be valid in it
|
|
1103
1103
|
Unable_to_resolve_value_Set_ = A definition for the value Set ''{0}'' could not be found
|
|
1104
|
-
CACHE_ID_UNKNOWN = The cache ''{0}''
|
|
1104
|
+
CACHE_ID_UNKNOWN = The cache ''{0}'' was never issued by this server, and this server has no record of it ever existing. Caches are created with $cache-control?mode=start, and a cache-id is only valid on the server instance and endpoint that issued it - either it was issued somewhere else, or this server has restarted since it was issued
|
|
1105
|
+
CACHE_ID_CLOSED = The cache ''{0}'' was closed {1} ago by the client, using $cache-control?mode=end (it had been open for {2}). It did not expire and it was not idle: something in the client released this cache while this request was still using it
|
|
1106
|
+
CACHE_ID_EXPIRED = The cache ''{0}'' expired: it went {1} without being used, and this server closes caches that have been idle for more than {2} (this one was dropped {3} ago). Every request that uses a cache resets its idle clock, so a client that keeps using a cache will not see this
|
|
1107
|
+
CACHE_ID_CLEARED = The cache ''{0}'' was dropped {1} ago, when this server cleared all of its caches
|
|
1105
1108
|
Unable_to_validate_code_without_using_server = Unable to validate code without using server because: {0}
|
|
1106
1109
|
Undefined_attribute__on__for_type__properties__ = Undefined attribute ''@{0}'' on {1} for type {2}
|
|
1107
1110
|
Undefined_element_ = Undefined element ''{0}'' at {1}
|
package/tx/cs/cs-api.js
CHANGED
|
@@ -404,9 +404,15 @@ class CodeSystemProvider {
|
|
|
404
404
|
* @param {boolean} [significantOnly=false] - when true, emit only the
|
|
405
405
|
* designations that are significant for a value set expansion (the display,
|
|
406
406
|
* and - for code systems that distinguish them - the fully specified name
|
|
407
|
-
* and the preferred term), rather than every designation
|
|
408
|
-
*
|
|
409
|
-
*
|
|
407
|
+
* and the preferred term), rather than every designation. This narrows which
|
|
408
|
+
* designations are emitted, NOT which languages: the display for a request is
|
|
409
|
+
* chosen downstream by preferredDesignation(displayLanguage), so a provider
|
|
410
|
+
* that emits significant designations for one language only makes localized
|
|
411
|
+
* expansions fall back to that language. Emit the significant designations
|
|
412
|
+
* for every language the concept has.
|
|
413
|
+
* The $expand worker passes true when the client did not ask for
|
|
414
|
+
* designations; $lookup and includeDesignations=true use the default (false)
|
|
415
|
+
* to return the complete set. Providers that do not distinguish significant
|
|
410
416
|
* designations may ignore this parameter and return all designations either
|
|
411
417
|
* way.
|
|
412
418
|
* @returns {Designation[]} whatever designations exist (in all languages)
|