@jwilleke/ngdpbase-types 5.25.0 → 5.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/packages/ngdpbase-types/index.d.ts +9 -2
- package/lib/packages/ngdpbase-types/testing.d.ts +7 -3
- package/lib/packages/ngdpbase-types/testing.js +7 -3
- package/lib/src/converters/DocxConverter.d.ts +30 -0
- package/lib/src/converters/MarkdownConverter.d.ts +31 -0
- package/lib/src/export/ChromeServicePdfProvider.d.ts +52 -0
- package/lib/src/export/ChromeServicePdfProvider.js +153 -0
- package/lib/src/export/LocalChromePdfProvider.d.ts +25 -0
- package/lib/src/export/LocalChromePdfProvider.js +33 -0
- package/lib/src/export/PdfProvider.d.ts +51 -0
- package/lib/src/export/PdfProvider.js +37 -0
- package/lib/src/export/pdfEngine.d.ts +113 -0
- package/lib/src/export/pdfEngine.js +195 -0
- package/lib/src/http/egressPolicy.js +61 -0
- package/lib/src/http/guardedElasticsearch.d.ts +27 -0
- package/lib/src/http/guardedElasticsearch.js +99 -0
- package/lib/src/ice/converterShim.d.ts +5 -0
- package/lib/src/ice/fileHandling.d.ts +30 -0
- package/lib/src/ice/ledger.d.ts +37 -0
- package/lib/src/ice/runImport.d.ts +51 -0
- package/lib/src/ice/types.d.ts +262 -0
- package/lib/src/mail/MailProvider.d.ts +15 -0
- package/lib/src/mail/NodemailerMailProvider.d.ts +1 -1
- package/lib/src/managers/AddonsManager.d.ts +55 -3
- package/lib/src/managers/AttachmentManager.d.ts +4 -5
- package/lib/src/managers/AuthManager.d.ts +150 -32
- package/lib/src/managers/BackgroundJobManager.d.ts +22 -12
- package/lib/src/managers/BackupManager.d.ts +779 -0
- package/lib/src/managers/BackupManager.js +2042 -0
- package/lib/src/managers/BaseManager.d.ts +156 -6
- package/lib/src/managers/BaseManager.js +64 -2
- package/lib/src/managers/CommentManager.d.ts +6 -6
- package/lib/src/managers/ConsentManager.d.ts +130 -0
- package/lib/src/managers/DatabaseManager.d.ts +6 -0
- package/lib/src/managers/DatabaseManager.js +25 -15
- package/lib/src/managers/EmailManager.d.ts +32 -5
- package/lib/src/managers/ExportManager.d.ts +62 -0
- package/lib/src/managers/ExportManager.js +147 -2
- package/lib/src/managers/FootnoteManager.d.ts +6 -6
- package/lib/src/managers/ImportManager.d.ts +590 -0
- package/lib/src/managers/OrganizationManager.d.ts +8 -6
- package/lib/src/managers/PageManager.d.ts +61 -1
- package/lib/src/managers/PersonManager.d.ts +8 -6
- package/lib/src/managers/RoleManager.d.ts +6 -8
- package/lib/src/providers/BaseAttachmentProvider.d.ts +4 -4
- package/lib/src/providers/BaseBackupProvider.d.ts +77 -0
- package/lib/src/providers/BaseBackupProvider.js +52 -0
- package/lib/src/providers/BasicAttachmentProvider.d.ts +4 -4
- package/lib/src/providers/FileBackupProvider.d.ts +34 -0
- package/lib/src/providers/FileBackupProvider.js +116 -0
- package/lib/src/providers/FileConsentProvider.d.ts +18 -0
- package/lib/src/providers/SqliteDatabaseProvider.d.ts +2 -2
- package/lib/src/providers/SqliteDatabaseProvider.js +12 -11
- package/lib/src/providers/attachmentFilesBackup.d.ts +5 -4
- package/lib/src/providers/databaseMigrations.js +45 -0
- package/lib/src/security/PolicyDecisionPoint.d.ts +14 -0
- package/lib/src/security/PolicyInformationPoint.d.ts +25 -2
- package/lib/src/types/Config.d.ts +2 -0
- package/lib/src/types/Consent.d.ts +90 -0
- package/lib/src/types/express.d.ts +6 -0
- package/lib/src/utils/addonHost.d.ts +45 -2
- package/lib/src/utils/addonHost.js +39 -3
- package/lib/src/utils/auditEventNames.generated.d.ts +20 -0
- package/lib/src/utils/auditEventNames.generated.js +20 -0
- package/lib/src/utils/backupCrypto.js +275 -0
- package/lib/src/utils/backupDocument.d.ts +36 -0
- package/lib/src/utils/backupDocument.js +99 -0
- package/lib/src/utils/browserDescription.d.ts +11 -0
- package/lib/src/utils/directoryBackup.d.ts +48 -48
- package/lib/src/utils/directoryBackup.js +291 -0
- package/lib/src/utils/frontmatterAccess.d.ts +54 -11
- package/lib/src/utils/importRunSummary.d.ts +64 -0
- package/lib/src/utils/instanceEnvBackup.d.ts +46 -0
- package/lib/src/utils/instanceEnvBackup.js +120 -0
- package/lib/src/utils/instanceEnvSecret.js +79 -0
- package/lib/src/utils/ipLocation.d.ts +11 -0
- package/lib/src/utils/ncmNotify.d.ts +30 -0
- package/lib/src/utils/privateStoreImport.d.ts +58 -0
- package/lib/src/utils/schedule.js +171 -0
- package/lib/src/utils/secretFileMode.js +78 -0
- package/lib/src/utils/siteRestore.d.ts +13 -16
- package/lib/src/utils/siteRestore.js +33 -6
- package/lib/src/utils/treeArchive.d.ts +2 -0
- package/lib/src/utils/treeArchive.js +242 -0
- package/lib/src/utils/version.js +408 -0
- package/lib/src/utils/zipArchive.d.ts +71 -0
- package/package.json +6 -1
- package/lib/src/utils/administrators.d.ts +0 -21
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The HTML → PDF engine (#1974): the options and the error every provider
|
|
3
|
+
* shares, and the child process the `local-chrome` provider renders in
|
|
4
|
+
* (headless Chromium, driven by `playwright-core`, both optional at run time).
|
|
5
|
+
*
|
|
6
|
+
* - `resolveChromium` finds a Chromium to run: the configured path, then the
|
|
7
|
+
* one Playwright installed (`npx playwright install chromium`), then the
|
|
8
|
+
* usual system locations. Nothing is downloaded at `npm install`.
|
|
9
|
+
* - `renderPdfInChild` renders one document in a child process
|
|
10
|
+
* (`pdfRenderWorker.ts`), killed at the timeout, so the site never waits on
|
|
11
|
+
* Chromium or on PDF parsing.
|
|
12
|
+
* - `normalizePdfInChild` gives a PDF a PDF service made (the
|
|
13
|
+
* `chrome-service` provider) the same metadata pass, in the same kind of
|
|
14
|
+
* child, so parsing it never runs on the thread that answers requests.
|
|
15
|
+
*
|
|
16
|
+
* `ExportManager.htmlToPdf` is the door, through its providers
|
|
17
|
+
* (`PdfProvider.ts`); nothing else calls these.
|
|
18
|
+
*/
|
|
19
|
+
import { fork } from 'node:child_process';
|
|
20
|
+
import { existsSync } from 'node:fs';
|
|
21
|
+
import { createRequire } from 'node:module';
|
|
22
|
+
/** Page sizes Chromium knows by name. */
|
|
23
|
+
export const PDF_PAGE_SIZES = ['Letter', 'Legal', 'Tabloid', 'Ledger', 'A0', 'A1', 'A2', 'A3', 'A4', 'A5', 'A6'];
|
|
24
|
+
/** A CSS length Chromium's PDF margins accept: a number and px, in, cm or mm. */
|
|
25
|
+
const MARGIN = /^\d+(\.\d+)?(px|in|cm|mm)$/;
|
|
26
|
+
/**
|
|
27
|
+
* No PDF engine: `playwright-core` is not installed, no Chromium was found, or
|
|
28
|
+
* it could not start. An add-on catches it by `code` (it cannot import core
|
|
29
|
+
* values) and falls back to HTML.
|
|
30
|
+
*/
|
|
31
|
+
export class PdfUnavailableError extends Error {
|
|
32
|
+
code = 'PDF_UNAVAILABLE';
|
|
33
|
+
constructor(message, options) {
|
|
34
|
+
super(message, options);
|
|
35
|
+
this.name = 'PdfUnavailableError';
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Validate a caller's options and fill the defaults. Throws on anything
|
|
40
|
+
* Chromium would misread, so a typo is an error rather than a different page.
|
|
41
|
+
*
|
|
42
|
+
* @param options - The caller's options
|
|
43
|
+
* @param defaultFormat - `ngdpbase.export.pdf.page-size`
|
|
44
|
+
*/
|
|
45
|
+
export function resolvePdfOptions(options, defaultFormat) {
|
|
46
|
+
const opts = options ?? {};
|
|
47
|
+
if (typeof opts !== 'object')
|
|
48
|
+
throw new TypeError('htmlToPdf: options must be an object');
|
|
49
|
+
const format = opts.format ?? defaultFormat;
|
|
50
|
+
if (!PDF_PAGE_SIZES.includes(format)) {
|
|
51
|
+
throw new TypeError(`htmlToPdf: format must be one of ${PDF_PAGE_SIZES.join(', ')} (got ${JSON.stringify(format)})`);
|
|
52
|
+
}
|
|
53
|
+
const margin = {};
|
|
54
|
+
for (const side of ['top', 'right', 'bottom', 'left']) {
|
|
55
|
+
const value = opts.margin?.[side];
|
|
56
|
+
if (value === undefined)
|
|
57
|
+
continue;
|
|
58
|
+
if (typeof value !== 'string' || !MARGIN.test(value)) {
|
|
59
|
+
throw new TypeError(`htmlToPdf: margin.${side} must be a length such as '20mm' or '0.5in' (got ${JSON.stringify(value)})`);
|
|
60
|
+
}
|
|
61
|
+
margin[side] = value;
|
|
62
|
+
}
|
|
63
|
+
const printBackground = opts.printBackground ?? true;
|
|
64
|
+
if (typeof printBackground !== 'boolean')
|
|
65
|
+
throw new TypeError('htmlToPdf: printBackground must be a boolean');
|
|
66
|
+
return { format: format, margin, printBackground };
|
|
67
|
+
}
|
|
68
|
+
/** Where Chromium or Chrome usually is, per platform. Checked after the configured path and Playwright's own. */
|
|
69
|
+
export const SYSTEM_CHROMIUM_PATHS = {
|
|
70
|
+
linux: ['/usr/bin/chromium', '/usr/bin/chromium-browser', '/usr/lib/chromium/chromium', '/usr/bin/google-chrome', '/usr/bin/google-chrome-stable', '/snap/bin/chromium'],
|
|
71
|
+
darwin: ['/Applications/Chromium.app/Contents/MacOS/Chromium', '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'],
|
|
72
|
+
win32: ['C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe']
|
|
73
|
+
};
|
|
74
|
+
const requireHere = createRequire(import.meta.url);
|
|
75
|
+
/** Whether `playwright-core` is installed (it is an optional dependency). */
|
|
76
|
+
export function playwrightInstalled() {
|
|
77
|
+
try {
|
|
78
|
+
requireHere.resolve('playwright-core');
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
async function playwrightChromiumPath() {
|
|
86
|
+
try {
|
|
87
|
+
const { chromium } = await import('playwright-core');
|
|
88
|
+
return chromium.executablePath() || null;
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Find a Chromium to render with. Read on every call, so installing Chromium
|
|
96
|
+
* or setting the path takes effect without a restart.
|
|
97
|
+
*
|
|
98
|
+
* @param configuredPath - `ngdpbase.export.pdf.provider.local-chrome.chromium-path` ('' = find one)
|
|
99
|
+
*/
|
|
100
|
+
export async function resolveChromium(configuredPath, deps = {}) {
|
|
101
|
+
const exists = deps.exists ?? existsSync;
|
|
102
|
+
if (!(deps.playwrightInstalled ?? playwrightInstalled)()) {
|
|
103
|
+
return { ok: false, reason: 'playwright-core is not installed (it is an optional dependency; npm install without --omit=optional)' };
|
|
104
|
+
}
|
|
105
|
+
if (configuredPath) {
|
|
106
|
+
return exists(configuredPath)
|
|
107
|
+
? { ok: true, executablePath: configuredPath, source: 'configured' }
|
|
108
|
+
: { ok: false, reason: `ngdpbase.export.pdf.provider.local-chrome.chromium-path is set, but nothing is there: ${configuredPath}` };
|
|
109
|
+
}
|
|
110
|
+
const bundled = await (deps.playwrightChromium ?? playwrightChromiumPath)();
|
|
111
|
+
if (bundled && exists(bundled))
|
|
112
|
+
return { ok: true, executablePath: bundled, source: 'playwright' };
|
|
113
|
+
for (const candidate of SYSTEM_CHROMIUM_PATHS[deps.platform ?? process.platform] ?? []) {
|
|
114
|
+
if (exists(candidate))
|
|
115
|
+
return { ok: true, executablePath: candidate, source: 'system' };
|
|
116
|
+
}
|
|
117
|
+
return { ok: false, reason: 'no Chromium found: install Chromium, or set ngdpbase.export.pdf.provider.local-chrome.chromium-path' };
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* The environment the child gets: enough to find a home, a temp folder and
|
|
121
|
+
* fonts, and none of the site's secrets (the session secret, keys and
|
|
122
|
+
* passwords the site holds in its environment).
|
|
123
|
+
*/
|
|
124
|
+
const CHILD_ENV_KEYS = [
|
|
125
|
+
'PATH', 'HOME', 'TMPDIR', 'TMP', 'TEMP', 'LANG', 'LC_ALL', 'PLAYWRIGHT_BROWSERS_PATH',
|
|
126
|
+
'FONTCONFIG_PATH', 'FONTCONFIG_FILE', 'XDG_CONFIG_HOME', 'XDG_CACHE_HOME', 'XDG_RUNTIME_DIR',
|
|
127
|
+
'SystemRoot', 'LOCALAPPDATA', 'USERPROFILE', 'APPDATA', 'ProgramFiles', 'ProgramFiles(x86)'
|
|
128
|
+
];
|
|
129
|
+
export function childEnv(env = process.env) {
|
|
130
|
+
const out = {};
|
|
131
|
+
for (const key of CHILD_ENV_KEYS)
|
|
132
|
+
if (env[key] !== undefined)
|
|
133
|
+
out[key] = env[key];
|
|
134
|
+
return out;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Render one job in a child process. Resolves with the child's answer; rejects
|
|
138
|
+
* only when the child timed out, crashed or never answered.
|
|
139
|
+
*
|
|
140
|
+
* @param job - The render
|
|
141
|
+
*/
|
|
142
|
+
export function renderPdfInChild(job) {
|
|
143
|
+
return runPdfChild(job);
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Normalise a PDF in a child process (`normalizePdf`, the same pass a local
|
|
147
|
+
* render gets). Rejects when the bytes are not a PDF pdf-lib can read, or the
|
|
148
|
+
* child times out.
|
|
149
|
+
*
|
|
150
|
+
* @param pdf - A PDF a PDF service returned
|
|
151
|
+
* @param timeoutMs - The child is killed after this
|
|
152
|
+
*/
|
|
153
|
+
export async function normalizePdfInChild(pdf, timeoutMs) {
|
|
154
|
+
const result = await runPdfChild({ kind: 'normalize', pdf, timeoutMs });
|
|
155
|
+
if (!result.ok)
|
|
156
|
+
throw new Error(`the PDF could not be read: ${result.error}`);
|
|
157
|
+
return result.pdf;
|
|
158
|
+
}
|
|
159
|
+
function runPdfChild(job) {
|
|
160
|
+
// Compiled, the worker is the .js beside this file. From source (vitest,
|
|
161
|
+
// tsx) it is the .ts, which needs tsx's loader.
|
|
162
|
+
const fromSource = import.meta.url.endsWith('.ts');
|
|
163
|
+
const entry = new URL(fromSource ? './pdfRenderWorker.ts' : './pdfRenderWorker.js', import.meta.url);
|
|
164
|
+
return new Promise((resolve, reject) => {
|
|
165
|
+
const child = fork(entry, [], {
|
|
166
|
+
execArgv: fromSource ? ['--import', 'tsx'] : [],
|
|
167
|
+
env: childEnv(),
|
|
168
|
+
serialization: 'advanced',
|
|
169
|
+
stdio: ['ignore', 'ignore', 'pipe', 'ipc']
|
|
170
|
+
});
|
|
171
|
+
let settled = false;
|
|
172
|
+
let stderr = '';
|
|
173
|
+
child.stderr?.on('data', (chunk) => { if (stderr.length < 2000)
|
|
174
|
+
stderr += chunk.toString(); });
|
|
175
|
+
const settle = (fn) => {
|
|
176
|
+
if (settled)
|
|
177
|
+
return;
|
|
178
|
+
settled = true;
|
|
179
|
+
clearTimeout(timer);
|
|
180
|
+
fn();
|
|
181
|
+
};
|
|
182
|
+
const timer = setTimeout(() => {
|
|
183
|
+
settle(() => reject(new Error(`PDF render timed out after ${job.timeoutMs} ms`)));
|
|
184
|
+
child.kill('SIGKILL');
|
|
185
|
+
}, job.timeoutMs);
|
|
186
|
+
child.once('message', (result) => {
|
|
187
|
+
settle(() => resolve(result.ok ? { ok: true, pdf: Buffer.from(result.pdf) } : result));
|
|
188
|
+
});
|
|
189
|
+
child.once('error', (err) => settle(() => reject(err)));
|
|
190
|
+
child.once('exit', (code, signal) => {
|
|
191
|
+
settle(() => reject(new Error(`PDF render process exited (${signal ?? code}) without an answer${stderr ? `: ${stderr.trim().split('\n')[0]}` : ''}`)));
|
|
192
|
+
});
|
|
193
|
+
child.send(job);
|
|
194
|
+
});
|
|
195
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration to policy (#1133).
|
|
3
|
+
*
|
|
4
|
+
* The operator sets two CIDR lists; this turns them into the `EgressPolicy`
|
|
5
|
+
* the guard enforces, and reconciles them at BOOT so a contradiction is a
|
|
6
|
+
* startup message rather than one dropped image months later — which would
|
|
7
|
+
* surface as a `fetch` warning indistinguishable from the remote host being
|
|
8
|
+
* down.
|
|
9
|
+
*
|
|
10
|
+
* Contradictions resolve by FIREWALL CONVENTION and none of them stops the
|
|
11
|
+
* boot (#1144, D8). #1133 chose longest-prefix match — the routing rule,
|
|
12
|
+
* because the values are routes — so a general overlap is not a conflict at
|
|
13
|
+
* all. The cases longest prefix cannot decide are settled here: an allow entry
|
|
14
|
+
* reaching the mechanism is unsatisfiable and dropped, and a range in both
|
|
15
|
+
* lists is a prefix-length tie the DENY wins.
|
|
16
|
+
*
|
|
17
|
+
* `ngdpbase.security.profile` used to decide whether a contradiction stopped
|
|
18
|
+
* the boot. That was the profile looking for a job: `iptables` rejects a bad
|
|
19
|
+
* rule and keeps the chain, and the Kubernetes API server rejects an invalid
|
|
20
|
+
* NetworkPolicy while the others keep applying. Neither takes the workload
|
|
21
|
+
* down. The one case with no safe silent resolution — a malformed range, where
|
|
22
|
+
* dropping a DENY fails open — is reported separately for #1152.
|
|
23
|
+
*/
|
|
24
|
+
import { reconcilePolicy } from './ssrf.js';
|
|
25
|
+
export const DENIED_RANGES_KEY = 'ngdpbase.security.egress.denied-ranges';
|
|
26
|
+
export const ALLOWED_RANGES_KEY = 'ngdpbase.security.egress.allowed-ranges';
|
|
27
|
+
/** Only strings can be CIDRs; anything else in the list is discarded, not coerced. */
|
|
28
|
+
function stringList(value) {
|
|
29
|
+
return Array.isArray(value) ? value.filter((v) => typeof v === 'string') : [];
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Read the configured ranges, reconcile them, and report what could not be
|
|
33
|
+
* resolved.
|
|
34
|
+
*
|
|
35
|
+
* A contradiction never widens access. Where the two lists disagree the
|
|
36
|
+
* stricter reading wins, which is the only direction a security control should
|
|
37
|
+
* fail when nobody can be asked.
|
|
38
|
+
*/
|
|
39
|
+
export function resolveEgressPolicy(read) {
|
|
40
|
+
const deniedRanges = stringList(read(DENIED_RANGES_KEY, []));
|
|
41
|
+
const allowedRanges = stringList(read(ALLOWED_RANGES_KEY, []));
|
|
42
|
+
const { conflicts, malformed, duplicates } = reconcilePolicy(deniedRanges, allowedRanges);
|
|
43
|
+
const unusable = new Set(malformed);
|
|
44
|
+
// An allow entry is dropped when it cannot be parsed, when it reaches the
|
|
45
|
+
// mechanism, or when the same range is also denied. The last is D8's tie
|
|
46
|
+
// break: equal prefix lengths give longest-prefix nothing to work with, so
|
|
47
|
+
// the deny wins.
|
|
48
|
+
const safeAllowed = allowedRanges.filter((range) => !unusable.has(range)
|
|
49
|
+
&& !duplicates.includes(range)
|
|
50
|
+
&& !conflicts.some((c) => c.startsWith('allowed-ranges:') && c.includes(`'${range}'`)));
|
|
51
|
+
// A denied range is dropped ONLY when it cannot be parsed. Previously a
|
|
52
|
+
// range appearing in both lists was removed from here too, so the operator's
|
|
53
|
+
// stricter statement was discarded along with the looser one and the range
|
|
54
|
+
// fell back to the built-in defaults.
|
|
55
|
+
const safeDenied = deniedRanges.filter((range) => !unusable.has(range));
|
|
56
|
+
return {
|
|
57
|
+
policy: { deniedRanges: safeDenied, allowedRanges: safeAllowed },
|
|
58
|
+
conflicts,
|
|
59
|
+
malformed
|
|
60
|
+
};
|
|
61
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { Client, type ClientOptions } from '@elastic/elasticsearch';
|
|
2
|
+
import { type Resolver } from './guardedLookup.js';
|
|
3
|
+
import { type ConfigReader } from './egressPolicy.js';
|
|
4
|
+
import { type EgressPolicy } from './ssrf.js';
|
|
5
|
+
export interface GuardedElasticsearchOptions {
|
|
6
|
+
/** Per-request timeout in ms; the SDK's default when omitted. */
|
|
7
|
+
requestTimeout?: number;
|
|
8
|
+
/**
|
|
9
|
+
* The resolution step, for tests only — the same single seam `guardedLookup`
|
|
10
|
+
* exposes. Production leaves it unset and `node:dns` answers.
|
|
11
|
+
*/
|
|
12
|
+
resolver?: Resolver;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The options that make a client guarded. Exported so a test can assert the
|
|
16
|
+
* wiring — that the connection class is the `http` one and that its agents
|
|
17
|
+
* carry the guarded lookup and the guarded connect — without opening a socket.
|
|
18
|
+
*/
|
|
19
|
+
export declare function guardedClientOptions(node: string, policy: EgressPolicy, options?: GuardedElasticsearchOptions): ClientOptions;
|
|
20
|
+
/** An Elasticsearch client whose every connection is subject to the instance's egress policy. */
|
|
21
|
+
export declare function createGuardedElasticsearchClient(node: string, read: ConfigReader, options?: GuardedElasticsearchOptions): Client;
|
|
22
|
+
/**
|
|
23
|
+
* The boot-time line both call sites log when the configured node would be
|
|
24
|
+
* refused — one sentence, the same shape as sist2's (#1186), so the operator
|
|
25
|
+
* is sent to the egress configuration and not to the Elasticsearch host.
|
|
26
|
+
*/
|
|
27
|
+
export declare function refusedNodeMessage(who: string, node: string, reason: string): string;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Elasticsearch client, built inside the boundary (#1188).
|
|
3
|
+
*
|
|
4
|
+
* `@elastic/elasticsearch` opens sockets the same way `fetch` does, and until
|
|
5
|
+
* #1188 it did so from two places outside `src/http/` — the search provider
|
|
6
|
+
* and the elasticsearch addon — with `new Client({ node })` and no policy at
|
|
7
|
+
* all. The boundary check (#1139) could not see it because the SDK was not in
|
|
8
|
+
* its list of client modules, so the guard read green while the process
|
|
9
|
+
* connected to whatever `es-url` said, loopback and RFC1918 included.
|
|
10
|
+
*
|
|
11
|
+
* This is the only place the SDK is constructed. It hands the client Node's
|
|
12
|
+
* `http` connection class with agents whose `lookup` is the guarded one, so
|
|
13
|
+
* every socket the client opens — the first request, a retry, a sniffed node —
|
|
14
|
+
* is judged against the instance's egress policy at connect time, exactly as
|
|
15
|
+
* `guardedFetch` does. There is no ungated constructor to reach for: the
|
|
16
|
+
* scanner now names the SDK, so an `import { Client }` outside this directory
|
|
17
|
+
* fails `lint:http`.
|
|
18
|
+
*
|
|
19
|
+
* What this means for an operator: an Elasticsearch node on `localhost` is
|
|
20
|
+
* refused — loopback is mechanism, not policy (#1186), and no allow entry
|
|
21
|
+
* opens it. A node on the LAN needs its prefix in
|
|
22
|
+
* `ngdpbase.security.egress.allowed-ranges`, the same as sist2. Both call
|
|
23
|
+
* sites say so at boot when the configured address would be refused.
|
|
24
|
+
*/
|
|
25
|
+
import { Agent as HttpAgent } from 'node:http';
|
|
26
|
+
import { Agent as HttpsAgent } from 'node:https';
|
|
27
|
+
import { isIP } from 'node:net';
|
|
28
|
+
import { Client, HttpConnection } from '@elastic/elasticsearch';
|
|
29
|
+
import { ESSRF_BLOCKED, guardedLookup } from './guardedLookup.js';
|
|
30
|
+
import { resolveEgressPolicy } from './egressPolicy.js';
|
|
31
|
+
import { isAddressAllowed, REFUSAL } from './ssrf.js';
|
|
32
|
+
/**
|
|
33
|
+
* The connect step, guarding the case the lookup never sees.
|
|
34
|
+
*
|
|
35
|
+
* Node calls an agent's `lookup` only for a NAME. A URL whose host is an IP
|
|
36
|
+
* literal — `http://192.168.0.10:9200`, `http://127.0.0.1:9200` — goes
|
|
37
|
+
* straight to the socket, so a guard that lives only in `lookup` waves every
|
|
38
|
+
* literal address through. Found by pointing the first version of this at
|
|
39
|
+
* jimstest's LAN node under an empty policy and watching `ping` return true.
|
|
40
|
+
* `guardedFetch` covers the same hole with `validateUrl` before each hop;
|
|
41
|
+
* here the judgement sits in `createConnection`, which every socket passes
|
|
42
|
+
* through, literal or resolved.
|
|
43
|
+
*/
|
|
44
|
+
function guardedCreateConnection(policy, connect) {
|
|
45
|
+
return function guarded(options, callback) {
|
|
46
|
+
const host = String(options.host ?? options.hostname ?? 'localhost');
|
|
47
|
+
if (isIP(host) && !isAddressAllowed(host, policy)) {
|
|
48
|
+
const err = Object.assign(new Error(`${REFUSAL}: ${host}`), { code: ESSRF_BLOCKED });
|
|
49
|
+
if (callback) {
|
|
50
|
+
process.nextTick(callback, err);
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
53
|
+
throw err;
|
|
54
|
+
}
|
|
55
|
+
// Node's own connect for this agent — `net.createConnection` for http,
|
|
56
|
+
// the TLS one for https. The agent has merged its options in, so `lookup`
|
|
57
|
+
// (the guarded one) rides along for a host that is a name.
|
|
58
|
+
return connect.call(this, options, callback);
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The options that make a client guarded. Exported so a test can assert the
|
|
63
|
+
* wiring — that the connection class is the `http` one and that its agents
|
|
64
|
+
* carry the guarded lookup and the guarded connect — without opening a socket.
|
|
65
|
+
*/
|
|
66
|
+
export function guardedClientOptions(node, policy, options = {}) {
|
|
67
|
+
// keepAlive stays on, as the SDK's own default agent has it: a kept socket
|
|
68
|
+
// was judged when it was opened, and both guards run again for every new one.
|
|
69
|
+
const lookup = guardedLookup(policy, options.resolver);
|
|
70
|
+
const agents = { http: new HttpAgent({ lookup, keepAlive: true }), https: new HttpsAgent({ lookup, keepAlive: true }) };
|
|
71
|
+
// `createConnection` is a method of the agent, not an option it accepts:
|
|
72
|
+
// Node reads `this.createConnection` when it opens a socket. Replaced on the
|
|
73
|
+
// instance, wrapping the prototype's own so TLS setup stays Node's.
|
|
74
|
+
const method = (agent) => Object.getPrototypeOf(agent).createConnection;
|
|
75
|
+
agents.http.createConnection =
|
|
76
|
+
guardedCreateConnection(policy, method(agents.http));
|
|
77
|
+
agents.https.createConnection =
|
|
78
|
+
guardedCreateConnection(policy, method(agents.https));
|
|
79
|
+
return {
|
|
80
|
+
node,
|
|
81
|
+
...(options.requestTimeout !== undefined ? { requestTimeout: options.requestTimeout } : {}),
|
|
82
|
+
Connection: HttpConnection,
|
|
83
|
+
agent: (connection) => (connection.url.protocol === 'https:' ? agents.https : agents.http)
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/** An Elasticsearch client whose every connection is subject to the instance's egress policy. */
|
|
87
|
+
export function createGuardedElasticsearchClient(node, read, options = {}) {
|
|
88
|
+
return new Client(guardedClientOptions(node, resolveEgressPolicy(read).policy, options));
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* The boot-time line both call sites log when the configured node would be
|
|
92
|
+
* refused — one sentence, the same shape as sist2's (#1186), so the operator
|
|
93
|
+
* is sent to the egress configuration and not to the Elasticsearch host.
|
|
94
|
+
*/
|
|
95
|
+
export function refusedNodeMessage(who, node, reason) {
|
|
96
|
+
return (`[${who}] Elasticsearch node ${node} is refused by the egress policy (${reason}). ` +
|
|
97
|
+
'Loopback and link-local can never be opened; a LAN address needs its prefix in ' +
|
|
98
|
+
'ngdpbase.security.egress.allowed-ranges. Every request to it will fail. (#1188)');
|
|
99
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { IContentConverter } from '../converters/IContentConverter.js';
|
|
2
|
+
import type { IceShim } from './types.js';
|
|
3
|
+
/** The receiver today's converters hand to. */
|
|
4
|
+
export declare const PAGES_RECEIVER = "core.pages";
|
|
5
|
+
export declare function converterShim(converter: IContentConverter, tableClasses: () => string[]): IceShim;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { type ZipReadLimits } from '../utils/zipArchive.js';
|
|
2
|
+
import type { IceUpload } from './types.js';
|
|
3
|
+
/** A file's fingerprint: SHA-256 of its bytes, hex. */
|
|
4
|
+
export declare function fingerprint(bytes: Buffer): string;
|
|
5
|
+
/**
|
|
6
|
+
* Is this path noise — something an operating system or archiver adds that is
|
|
7
|
+
* nobody's data? Patterns (configuration, `ngdpbase.import.engine.noise`):
|
|
8
|
+
*
|
|
9
|
+
* - `name/` — any path with a directory segment of that name (`__MACOSX/`);
|
|
10
|
+
* - `prefix*` / `*suffix` — a file name starting or ending so (`._*`);
|
|
11
|
+
* - anything else — a file name exactly (`.DS_Store`), case-insensitive.
|
|
12
|
+
*/
|
|
13
|
+
export declare function isNoise(filePath: string, patterns: readonly string[]): boolean;
|
|
14
|
+
/** Is this upload a zip archive the engine unpacks? By its name, or its media type. */
|
|
15
|
+
export declare function isZip(upload: IceUpload): boolean;
|
|
16
|
+
/**
|
|
17
|
+
* Unpack the zips among the uploads into the files they hold, named
|
|
18
|
+
* `archive.zip/inner/path`. A nested zip is unpacked too. Everything else is
|
|
19
|
+
* passed through. Limits are per archive (`ZipReadLimits`), so a small
|
|
20
|
+
* archive that expands enormously stops at the budget. An archive that cannot
|
|
21
|
+
* be read is returned in `unreadable`, with why; the rest go on.
|
|
22
|
+
*/
|
|
23
|
+
export declare function expandUploads(uploads: readonly IceUpload[], limits: ZipReadLimits, maxDepth: number, depth?: number): {
|
|
24
|
+
files: IceUpload[];
|
|
25
|
+
unreadable: Array<{
|
|
26
|
+
name: string;
|
|
27
|
+
size: number;
|
|
28
|
+
message: string;
|
|
29
|
+
}>;
|
|
30
|
+
};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { IceLedger, LedgerFileEntry } from './types.js';
|
|
2
|
+
interface ScopeFile {
|
|
3
|
+
scope: string;
|
|
4
|
+
files: LedgerFileEntry[];
|
|
5
|
+
items: string[];
|
|
6
|
+
}
|
|
7
|
+
/** Keeps the newest entry per fingerprint, at most `maxEntries` files and item keys per scope. */
|
|
8
|
+
declare abstract class ScopedLedger implements IceLedger {
|
|
9
|
+
private readonly maxEntries;
|
|
10
|
+
constructor(maxEntries: number);
|
|
11
|
+
protected abstract load(scope: string): Promise<ScopeFile>;
|
|
12
|
+
protected abstract save(data: ScopeFile): Promise<void>;
|
|
13
|
+
/** One writer per scope at a time, so two runs in one scope cannot lose each other's entries. */
|
|
14
|
+
private readonly chains;
|
|
15
|
+
private serial;
|
|
16
|
+
lastFile(scope: string, sha256: string): Promise<LedgerFileEntry | null>;
|
|
17
|
+
recordFile(scope: string, entry: LedgerFileEntry): Promise<void>;
|
|
18
|
+
hasItem(scope: string, itemKey: string): Promise<boolean>;
|
|
19
|
+
recordItem(scope: string, itemKey: string): Promise<void>;
|
|
20
|
+
files(scope: string): Promise<LedgerFileEntry[]>;
|
|
21
|
+
}
|
|
22
|
+
/** The ledger in memory: for tests, and for a site that keeps no import history. */
|
|
23
|
+
export declare class MemoryIceLedger extends ScopedLedger {
|
|
24
|
+
private readonly scopes;
|
|
25
|
+
constructor(maxEntries?: number);
|
|
26
|
+
protected load(scope: string): Promise<ScopeFile>;
|
|
27
|
+
protected save(data: ScopeFile): Promise<void>;
|
|
28
|
+
}
|
|
29
|
+
/** One JSON file per scope, named by the scope's hash (a scope holds a username). */
|
|
30
|
+
export declare class FileIceLedger extends ScopedLedger {
|
|
31
|
+
private readonly dir;
|
|
32
|
+
constructor(dir: string, maxEntries: number);
|
|
33
|
+
private fileFor;
|
|
34
|
+
protected load(scope: string): Promise<ScopeFile>;
|
|
35
|
+
protected save(data: ScopeFile): Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
export {};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { ActorContext } from '../context/ActorContext.js';
|
|
2
|
+
import type { ZipReadLimits } from '../utils/zipArchive.js';
|
|
3
|
+
import type { IceItem, IceLedger, IceRunReport, IceRunRequest, OriginalChoice, RegisteredReceiver, RegisteredShim, ShimTools } from './types.js';
|
|
4
|
+
/** Configuration the engine reads (each a key under `ngdpbase.import.engine.`). */
|
|
5
|
+
export interface IceEngineSettings {
|
|
6
|
+
noise: string[];
|
|
7
|
+
zip: ZipReadLimits;
|
|
8
|
+
maxZipDepth: number;
|
|
9
|
+
maxFileBytes: number;
|
|
10
|
+
maxChainDepth: number;
|
|
11
|
+
defaultOriginal: OriginalChoice;
|
|
12
|
+
}
|
|
13
|
+
export interface IceRunDeps {
|
|
14
|
+
/** Enabled shims, in registration order. */
|
|
15
|
+
shims: RegisteredShim[];
|
|
16
|
+
receivers: ReadonlyMap<string, RegisteredReceiver>;
|
|
17
|
+
ledger: IceLedger;
|
|
18
|
+
settings: IceEngineSettings;
|
|
19
|
+
tools(shim: RegisteredShim): ShimTools;
|
|
20
|
+
now?: () => Date;
|
|
21
|
+
}
|
|
22
|
+
/** The id of the receiver that keeps an original when the chosen receiver does not keep its own (D11). */
|
|
23
|
+
export declare const FILES_RECEIVER = "core.files";
|
|
24
|
+
/** The scope a run's fingerprints are checked in, unless the caller names one. */
|
|
25
|
+
export declare function defaultScope(receiverId: string, ctx: ActorContext): string;
|
|
26
|
+
export interface ChainLeaf {
|
|
27
|
+
item: IceItem;
|
|
28
|
+
chain: string[];
|
|
29
|
+
}
|
|
30
|
+
/** What a chain needs: the shims it may continue through, and where it stops. */
|
|
31
|
+
export interface ChainSpec {
|
|
32
|
+
shims: RegisteredShim[];
|
|
33
|
+
tools(shim: RegisteredShim): ShimTools;
|
|
34
|
+
maxChainDepth: number;
|
|
35
|
+
/** Does this item, from this shim, stop here? */
|
|
36
|
+
done(shim: RegisteredShim, item: IceItem): boolean;
|
|
37
|
+
/** Named in the error when nothing takes an item further. */
|
|
38
|
+
target: string;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Run a shim, then the shims after it (D9), until each item is done. What
|
|
42
|
+
* came before travels with each item: earlier metadata under later, earlier
|
|
43
|
+
* warnings first. Partial notes are collected into `partial`.
|
|
44
|
+
*/
|
|
45
|
+
export declare function convertChain(shim: RegisteredShim, input: IceItem, chain: string[], spec: ChainSpec, partial: string[]): Promise<ChainLeaf[]>;
|
|
46
|
+
/**
|
|
47
|
+
* Run one import. Fills and returns `report`, which the caller may be
|
|
48
|
+
* showing while it runs. Never throws for one file's failure; that file's
|
|
49
|
+
* status says why.
|
|
50
|
+
*/
|
|
51
|
+
export declare function runImport(deps: IceRunDeps, request: IceRunRequest, ctx: ActorContext, report: IceRunReport): Promise<IceRunReport>;
|