@jwilleke/ngdpbase-types 5.25.0 → 5.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/README.md +1 -1
  2. package/lib/packages/ngdpbase-types/index.d.ts +9 -2
  3. package/lib/packages/ngdpbase-types/testing.d.ts +7 -3
  4. package/lib/packages/ngdpbase-types/testing.js +7 -3
  5. package/lib/src/converters/DocxConverter.d.ts +30 -0
  6. package/lib/src/converters/MarkdownConverter.d.ts +31 -0
  7. package/lib/src/export/ChromeServicePdfProvider.d.ts +52 -0
  8. package/lib/src/export/ChromeServicePdfProvider.js +153 -0
  9. package/lib/src/export/LocalChromePdfProvider.d.ts +25 -0
  10. package/lib/src/export/LocalChromePdfProvider.js +33 -0
  11. package/lib/src/export/PdfProvider.d.ts +51 -0
  12. package/lib/src/export/PdfProvider.js +37 -0
  13. package/lib/src/export/pdfEngine.d.ts +113 -0
  14. package/lib/src/export/pdfEngine.js +195 -0
  15. package/lib/src/http/egressPolicy.js +61 -0
  16. package/lib/src/http/guardedElasticsearch.d.ts +27 -0
  17. package/lib/src/http/guardedElasticsearch.js +99 -0
  18. package/lib/src/ice/converterShim.d.ts +5 -0
  19. package/lib/src/ice/fileHandling.d.ts +30 -0
  20. package/lib/src/ice/ledger.d.ts +37 -0
  21. package/lib/src/ice/runImport.d.ts +51 -0
  22. package/lib/src/ice/types.d.ts +262 -0
  23. package/lib/src/mail/MailProvider.d.ts +15 -0
  24. package/lib/src/mail/NodemailerMailProvider.d.ts +1 -1
  25. package/lib/src/managers/AddonsManager.d.ts +55 -3
  26. package/lib/src/managers/AttachmentManager.d.ts +4 -5
  27. package/lib/src/managers/AuthManager.d.ts +150 -32
  28. package/lib/src/managers/BackgroundJobManager.d.ts +22 -12
  29. package/lib/src/managers/BackupManager.d.ts +779 -0
  30. package/lib/src/managers/BackupManager.js +2042 -0
  31. package/lib/src/managers/BaseManager.d.ts +156 -6
  32. package/lib/src/managers/BaseManager.js +64 -2
  33. package/lib/src/managers/CommentManager.d.ts +6 -6
  34. package/lib/src/managers/ConsentManager.d.ts +130 -0
  35. package/lib/src/managers/DatabaseManager.d.ts +6 -0
  36. package/lib/src/managers/DatabaseManager.js +25 -15
  37. package/lib/src/managers/EmailManager.d.ts +32 -5
  38. package/lib/src/managers/ExportManager.d.ts +62 -0
  39. package/lib/src/managers/ExportManager.js +147 -2
  40. package/lib/src/managers/FootnoteManager.d.ts +6 -6
  41. package/lib/src/managers/ImportManager.d.ts +590 -0
  42. package/lib/src/managers/OrganizationManager.d.ts +8 -6
  43. package/lib/src/managers/PageManager.d.ts +61 -1
  44. package/lib/src/managers/PersonManager.d.ts +8 -6
  45. package/lib/src/managers/RoleManager.d.ts +6 -8
  46. package/lib/src/providers/BaseAttachmentProvider.d.ts +4 -4
  47. package/lib/src/providers/BaseBackupProvider.d.ts +77 -0
  48. package/lib/src/providers/BaseBackupProvider.js +52 -0
  49. package/lib/src/providers/BasicAttachmentProvider.d.ts +4 -4
  50. package/lib/src/providers/FileBackupProvider.d.ts +34 -0
  51. package/lib/src/providers/FileBackupProvider.js +116 -0
  52. package/lib/src/providers/FileConsentProvider.d.ts +18 -0
  53. package/lib/src/providers/SqliteDatabaseProvider.d.ts +2 -2
  54. package/lib/src/providers/SqliteDatabaseProvider.js +12 -11
  55. package/lib/src/providers/attachmentFilesBackup.d.ts +5 -4
  56. package/lib/src/providers/databaseMigrations.js +45 -0
  57. package/lib/src/security/PolicyDecisionPoint.d.ts +14 -0
  58. package/lib/src/security/PolicyInformationPoint.d.ts +25 -2
  59. package/lib/src/types/Config.d.ts +2 -0
  60. package/lib/src/types/Consent.d.ts +90 -0
  61. package/lib/src/types/express.d.ts +6 -0
  62. package/lib/src/utils/addonHost.d.ts +45 -2
  63. package/lib/src/utils/addonHost.js +39 -3
  64. package/lib/src/utils/auditEventNames.generated.d.ts +20 -0
  65. package/lib/src/utils/auditEventNames.generated.js +20 -0
  66. package/lib/src/utils/backupCrypto.js +275 -0
  67. package/lib/src/utils/backupDocument.d.ts +36 -0
  68. package/lib/src/utils/backupDocument.js +99 -0
  69. package/lib/src/utils/browserDescription.d.ts +11 -0
  70. package/lib/src/utils/directoryBackup.d.ts +48 -48
  71. package/lib/src/utils/directoryBackup.js +291 -0
  72. package/lib/src/utils/frontmatterAccess.d.ts +54 -11
  73. package/lib/src/utils/importRunSummary.d.ts +64 -0
  74. package/lib/src/utils/instanceEnvBackup.d.ts +46 -0
  75. package/lib/src/utils/instanceEnvBackup.js +120 -0
  76. package/lib/src/utils/instanceEnvSecret.js +79 -0
  77. package/lib/src/utils/ipLocation.d.ts +11 -0
  78. package/lib/src/utils/ncmNotify.d.ts +30 -0
  79. package/lib/src/utils/privateStoreImport.d.ts +58 -0
  80. package/lib/src/utils/schedule.js +171 -0
  81. package/lib/src/utils/secretFileMode.js +78 -0
  82. package/lib/src/utils/siteRestore.d.ts +13 -16
  83. package/lib/src/utils/siteRestore.js +33 -6
  84. package/lib/src/utils/treeArchive.d.ts +2 -0
  85. package/lib/src/utils/treeArchive.js +242 -0
  86. package/lib/src/utils/version.js +408 -0
  87. package/lib/src/utils/zipArchive.d.ts +71 -0
  88. package/package.json +6 -1
  89. package/lib/src/utils/administrators.d.ts +0 -21
@@ -1,6 +1,16 @@
1
1
  import type { ActorContext } from '../context/ActorContext.js';
2
2
  import BaseManager from './BaseManager.js';
3
3
  import type { WikiEngine } from '../types/WikiEngine.js';
4
+ import { PdfUnavailableError, type PdfJob, type PdfJobResult, type PdfOptions, type ChromiumResolution } from '../export/pdfEngine.js';
5
+ import type { Transport } from '../http/guardedFetch.js';
6
+ export { PdfUnavailableError };
7
+ export type { PdfOptions, PdfMargin, PdfPageSize } from '../export/pdfEngine.js';
8
+ /**
9
+ * The needs-attention reason while PDF export has no engine (#1974). Set only
10
+ * when something asked for a PDF and could not have one; cleared by the next
11
+ * render that works.
12
+ */
13
+ export declare const PDF_UNAVAILABLE_REASON = "PDF export unavailable: choose a PDF provider (local-chrome or chrome-service) in ngdpbase.export.pdf.provider, or fix the one chosen (the server log says why)";
4
14
  /**
5
15
  * Export file metadata interface
6
16
  */
@@ -41,6 +51,14 @@ export interface PageForExport {
41
51
  }
42
52
  declare class ExportManager extends BaseManager {
43
53
  private exportDirectory;
54
+ /** What the providers call out to; replaced in tests. A site never sets these. */
55
+ protected pdfRenderer: (job: PdfJob) => Promise<PdfJobResult>;
56
+ protected chromiumResolver: (configuredPath: string) => Promise<ChromiumResolution>;
57
+ protected pdfServiceTransport: Transport | undefined;
58
+ protected pdfNormalizer: (pdf: Uint8Array, timeoutMs: number) => Promise<Uint8Array>;
59
+ /** Renders running now, and renders waiting for a slot (`ngdpbase.export.pdf.max-concurrent`). */
60
+ private pdfActive;
61
+ private pdfWaiting;
44
62
  /**
45
63
  * Creates a new ExportManager instance
46
64
  *
@@ -70,6 +88,50 @@ declare class ExportManager extends BaseManager {
70
88
  * @returns The CSV document
71
89
  */
72
90
  toCsv(headers: readonly string[], rows: readonly (readonly unknown[])[]): string;
91
+ /**
92
+ * A rendered HTML document (print CSS and all) as PDF bytes (#1974).
93
+ *
94
+ * - __A provider renders it:__ `ngdpbase.export.pdf.provider` — `none`
95
+ * (default), `local-chrome` (a Chromium on this machine) or
96
+ * `chrome-service` (a Chromium service by URL, e.g. Gotenberg); see
97
+ * `src/export/PdfProvider.ts`. Read on every render, so a change applies
98
+ * without a restart.
99
+ * - __Sandboxed:__ `local-chrome` runs Chromium with JavaScript off and every
100
+ * request refused except `data:` URIs, so images, stylesheets and fonts
101
+ * must be inlined. A `chrome-service` is reached through the egress policy,
102
+ * and what its Chromium may load is the service's own setting.
103
+ * - __Deterministic:__ the same HTML on the same core version, provider and
104
+ * engine version gives the same bytes (no dates, no random ID), so an
105
+ * add-on can store the document's SHA-256.
106
+ * - __Off the request path:__ a local render, and the metadata pass on a
107
+ * service's PDF, each run in a child process; everything is bounded by
108
+ * `ngdpbase.export.pdf.timeout-ms`, and at most
109
+ * `ngdpbase.export.pdf.max-concurrent` run at once while the rest wait.
110
+ * - __Optional:__ with no engine (provider `none`, or the chosen one cannot
111
+ * be used) it throws `PdfUnavailableError` (`code: 'PDF_UNAVAILABLE'`),
112
+ * logs why, and this manager shows "needs attention" on the admin
113
+ * dashboard until a render works. A site that never asks for a PDF is
114
+ * never flagged.
115
+ *
116
+ * Records no audit event: it reads nothing and changes nothing. The caller
117
+ * that issues the document records that, with its own event.
118
+ *
119
+ * @param ctx - Who is asking; named in the log line
120
+ * @param html - A complete HTML document
121
+ * @param options - Page size, margins, printBackground; an `@page` rule in the HTML wins
122
+ * @returns The PDF
123
+ */
124
+ htmlToPdf(ctx: ActorContext, html: string, options?: PdfOptions): Promise<Buffer>;
125
+ /**
126
+ * The provider `ngdpbase.export.pdf.provider` names, built for this render
127
+ * from the current settings. A name that is not a provider is unavailable,
128
+ * with the reason, as `none` is.
129
+ */
130
+ private pdfProvider;
131
+ /** Log why, flag the manager (unless something worse is already shown), and refuse. */
132
+ private pdfUnavailable;
133
+ private acquirePdfSlot;
134
+ private releasePdfSlot;
73
135
  /**
74
136
  * Initialize the ExportManager
75
137
  *
@@ -3,11 +3,27 @@ import fs from 'fs/promises';
3
3
  import path from 'path';
4
4
  import LocaleUtils from '../utils/LocaleUtils.js';
5
5
  import logger from '../utils/logger.js';
6
+ import { actorOf } from '../context/ActorContext.js';
7
+ import { PdfUnavailableError, normalizePdfInChild, renderPdfInChild, resolveChromium, resolvePdfOptions } from '../export/pdfEngine.js';
8
+ import { NonePdfProvider, PDF_PROVIDERS } from '../export/PdfProvider.js';
9
+ import { LocalChromePdfProvider } from '../export/LocalChromePdfProvider.js';
10
+ import { ChromeServicePdfProvider } from '../export/ChromeServicePdfProvider.js';
11
+ import { resolveEgressPolicy } from '../http/egressPolicy.js';
12
+ export { PdfUnavailableError };
13
+ /**
14
+ * The needs-attention reason while PDF export has no engine (#1974). Set only
15
+ * when something asked for a PDF and could not have one; cleared by the next
16
+ * render that works.
17
+ */
18
+ export const PDF_UNAVAILABLE_REASON = 'PDF export unavailable: choose a PDF provider (local-chrome or chrome-service) in ngdpbase.export.pdf.provider, or fix the one chosen (the server log says why)';
19
+ /** Shipped defaults of the `ngdpbase.export.pdf.*` keys. */
20
+ const PDF_DEFAULTS = { provider: 'none', pageSize: 'Letter', timeoutMs: 30_000, maxConcurrent: 2, serviceMaxBytes: 52_428_800 };
6
21
  /**
7
22
  * ExportManager - Handles page exports to multiple formats
8
23
  *
9
24
  * Similar to JSPWiki's export functionality, provides page export capabilities
10
- * to HTML, PDF, markdown, and other formats.
25
+ * to HTML and markdown, the one CSV writer, and HTML → PDF (`htmlToPdf`, #1974)
26
+ * through an optional headless Chromium.
11
27
  *
12
28
  * @class ExportManager
13
29
  * @extends BaseManager
@@ -24,6 +40,14 @@ import logger from '../utils/logger.js';
24
40
  const CSV_FORMULA_START = /^[=+\-@\t\r]/;
25
41
  class ExportManager extends BaseManager {
26
42
  exportDirectory;
43
+ /** What the providers call out to; replaced in tests. A site never sets these. */
44
+ pdfRenderer = renderPdfInChild;
45
+ chromiumResolver = (p) => resolveChromium(p);
46
+ pdfServiceTransport = undefined;
47
+ pdfNormalizer = normalizePdfInChild;
48
+ /** Renders running now, and renders waiting for a slot (`ngdpbase.export.pdf.max-concurrent`). */
49
+ pdfActive = 0;
50
+ pdfWaiting = [];
27
51
  /**
28
52
  * Creates a new ExportManager instance
29
53
  *
@@ -67,6 +91,121 @@ class ExportManager extends BaseManager {
67
91
  toCsv(headers, rows) {
68
92
  return [headers, ...rows].map((row) => row.map((cell) => this.csvCell(cell)).join(',')).join('\r\n') + '\r\n';
69
93
  }
94
+ /**
95
+ * A rendered HTML document (print CSS and all) as PDF bytes (#1974).
96
+ *
97
+ * - __A provider renders it:__ `ngdpbase.export.pdf.provider` — `none`
98
+ * (default), `local-chrome` (a Chromium on this machine) or
99
+ * `chrome-service` (a Chromium service by URL, e.g. Gotenberg); see
100
+ * `src/export/PdfProvider.ts`. Read on every render, so a change applies
101
+ * without a restart.
102
+ * - __Sandboxed:__ `local-chrome` runs Chromium with JavaScript off and every
103
+ * request refused except `data:` URIs, so images, stylesheets and fonts
104
+ * must be inlined. A `chrome-service` is reached through the egress policy,
105
+ * and what its Chromium may load is the service's own setting.
106
+ * - __Deterministic:__ the same HTML on the same core version, provider and
107
+ * engine version gives the same bytes (no dates, no random ID), so an
108
+ * add-on can store the document's SHA-256.
109
+ * - __Off the request path:__ a local render, and the metadata pass on a
110
+ * service's PDF, each run in a child process; everything is bounded by
111
+ * `ngdpbase.export.pdf.timeout-ms`, and at most
112
+ * `ngdpbase.export.pdf.max-concurrent` run at once while the rest wait.
113
+ * - __Optional:__ with no engine (provider `none`, or the chosen one cannot
114
+ * be used) it throws `PdfUnavailableError` (`code: 'PDF_UNAVAILABLE'`),
115
+ * logs why, and this manager shows "needs attention" on the admin
116
+ * dashboard until a render works. A site that never asks for a PDF is
117
+ * never flagged.
118
+ *
119
+ * Records no audit event: it reads nothing and changes nothing. The caller
120
+ * that issues the document records that, with its own event.
121
+ *
122
+ * @param ctx - Who is asking; named in the log line
123
+ * @param html - A complete HTML document
124
+ * @param options - Page size, margins, printBackground; an `@page` rule in the HTML wins
125
+ * @returns The PDF
126
+ */
127
+ async htmlToPdf(ctx, html, options) {
128
+ if (!ctx)
129
+ throw new TypeError('htmlToPdf: a context is required (security-posture P1)');
130
+ if (typeof html !== 'string' || html.length === 0)
131
+ throw new TypeError('htmlToPdf: html must be a non-empty string');
132
+ const config = this.engine.getManager('ConfigurationManager');
133
+ const setting = (key, fallback) => config?.getProperty(key, fallback) ?? fallback;
134
+ const resolved = resolvePdfOptions(options, String(setting('ngdpbase.export.pdf.page-size', PDF_DEFAULTS.pageSize)));
135
+ const timeoutMs = positiveInt(setting('ngdpbase.export.pdf.timeout-ms', PDF_DEFAULTS.timeoutMs), PDF_DEFAULTS.timeoutMs);
136
+ const maxConcurrent = positiveInt(setting('ngdpbase.export.pdf.max-concurrent', PDF_DEFAULTS.maxConcurrent), PDF_DEFAULTS.maxConcurrent);
137
+ const provider = this.pdfProvider(setting);
138
+ const started = Date.now();
139
+ await this.acquirePdfSlot(maxConcurrent);
140
+ let rendered;
141
+ try {
142
+ rendered = await provider.render({ html, options: resolved, timeoutMs });
143
+ }
144
+ catch (err) {
145
+ if (err instanceof PdfUnavailableError)
146
+ this.pdfUnavailable(err.message);
147
+ throw err;
148
+ }
149
+ finally {
150
+ this.releasePdfSlot();
151
+ }
152
+ this.markReadyIfDegradedBy(PDF_UNAVAILABLE_REASON);
153
+ const pdf = Buffer.from(rendered.pdf);
154
+ logger.info(`[ExportManager] PDF rendered bytes=${pdf.length} ms=${Date.now() - started} engine=${rendered.engine} by=${actorOf(ctx).user}`);
155
+ return pdf;
156
+ }
157
+ /**
158
+ * The provider `ngdpbase.export.pdf.provider` names, built for this render
159
+ * from the current settings. A name that is not a provider is unavailable,
160
+ * with the reason, as `none` is.
161
+ */
162
+ pdfProvider(setting) {
163
+ const name = String(setting('ngdpbase.export.pdf.provider', PDF_DEFAULTS.provider));
164
+ switch (name) {
165
+ case 'none':
166
+ return new NonePdfProvider();
167
+ case 'local-chrome':
168
+ return new LocalChromePdfProvider(String(setting('ngdpbase.export.pdf.provider.local-chrome.chromium-path', '')), {
169
+ render: (job) => this.pdfRenderer(job),
170
+ resolve: (p) => this.chromiumResolver(p)
171
+ });
172
+ case 'chrome-service': {
173
+ const config = this.engine.getManager('ConfigurationManager');
174
+ const egress = resolveEgressPolicy((key, fallback) => config?.getProperty(key, fallback));
175
+ return new ChromeServicePdfProvider({
176
+ url: String(setting('ngdpbase.export.pdf.provider.chrome-service.url', '')),
177
+ policy: egress.policy,
178
+ maxBytes: positiveInt(setting('ngdpbase.export.pdf.provider.chrome-service.max-bytes', PDF_DEFAULTS.serviceMaxBytes), PDF_DEFAULTS.serviceMaxBytes)
179
+ }, { ...(this.pdfServiceTransport ? { transport: this.pdfServiceTransport } : {}), normalize: (pdf, ms) => this.pdfNormalizer(pdf, ms) });
180
+ }
181
+ default:
182
+ return new NonePdfProvider(`ngdpbase.export.pdf.provider '${name}' is not a PDF provider: use ${PDF_PROVIDERS.join(', ')}`);
183
+ }
184
+ }
185
+ /** Log why, flag the manager (unless something worse is already shown), and refuse. */
186
+ pdfUnavailable(reason) {
187
+ logger.warn(`[ExportManager] PDF export unavailable: ${reason}`);
188
+ const status = this.getManagerStatus();
189
+ if (status.state === 'ready' || status.reason === PDF_UNAVAILABLE_REASON) {
190
+ this.markNeedsAttention(PDF_UNAVAILABLE_REASON, 'ngdpbase.export.pdf.provider');
191
+ }
192
+ throw new PdfUnavailableError(`PDF export unavailable: ${reason}`);
193
+ }
194
+ async acquirePdfSlot(limit) {
195
+ if (this.pdfActive < limit) {
196
+ this.pdfActive++;
197
+ return;
198
+ }
199
+ // The slot is handed over by releasePdfSlot, so pdfActive stays as it is.
200
+ await new Promise((resolve) => this.pdfWaiting.push(resolve));
201
+ }
202
+ releasePdfSlot() {
203
+ const next = this.pdfWaiting.shift();
204
+ if (next)
205
+ next();
206
+ else
207
+ this.pdfActive--;
208
+ }
70
209
  /**
71
210
  * Initialize the ExportManager
72
211
  *
@@ -90,7 +229,8 @@ class ExportManager extends BaseManager {
90
229
  }
91
230
  // Create exports directory
92
231
  await fs.mkdir(this.exportDirectory, { recursive: true });
93
- logger.info('ExportManager initialized');
232
+ const pdfProviderName = this.engine.getManager('ConfigurationManager')?.getProperty('ngdpbase.export.pdf.provider', PDF_DEFAULTS.provider);
233
+ logger.info(`ExportManager initialized (PDF provider: ${pdfProviderName ?? PDF_DEFAULTS.provider})`);
94
234
  }
95
235
  /**
96
236
  * Export a single page to HTML
@@ -417,4 +557,9 @@ class ExportManager extends BaseManager {
417
557
  return date.toLocaleString();
418
558
  }
419
559
  }
560
+ /** A positive whole number from config, or the shipped default. */
561
+ function positiveInt(value, fallback) {
562
+ const n = Number(value);
563
+ return Number.isInteger(n) && n > 0 ? n : fallback;
564
+ }
420
565
  export default ExportManager;
@@ -1,4 +1,4 @@
1
- import BaseManager, { type BackupData, type ManagerBackupOptions, type ManagerRestoreOptions } from './BaseManager.js';
1
+ import BaseManager, { type BackupFolder } from './BaseManager.js';
2
2
  import type { WikiEngine } from '../types/WikiEngine.js';
3
3
  import { type ActorContext } from '../context/ActorContext.js';
4
4
  import { type VaultPageNotesOutside, type PageRef } from '../utils/pageNotes.js';
@@ -17,15 +17,15 @@ export default class FootnoteManager extends BaseManager {
17
17
  private readonly outsideCheck;
18
18
  constructor(engine: WikiEngine);
19
19
  initialize(): Promise<void>;
20
+ /** `ngdpbase.footnotes.storagedir`, resolved, or the default when there is no configuration (a test). */
21
+ private configuredDir;
20
22
  /**
21
23
  * The public folder is in the backup (#1876), as `<stem>.footnotes.tar` beside
22
24
  * the document: the public pages' footnotes. A vault page's are in its vault, which
23
- * PageManager backs up with the vault, sealed as stored.
25
+ * PageManager backs up with the vault, sealed as stored. BackupManager backs
26
+ * it up, stages it, and puts it in place at the next start (#1985).
24
27
  */
25
- backup(options?: ManagerBackupOptions): Promise<BackupData>;
26
- /** Staged for the next start with the rest of a site restore (#1876); the live folder is untouched. */
27
- restore(backupData: BackupData, options?: ManagerRestoreOptions): Promise<void>;
28
- restoreIsStaged(): boolean;
28
+ backupFolders(): BackupFolder[];
29
29
  isEnabled(): boolean;
30
30
  /** The public folder and the vaults, as configured now (#1839). */
31
31
  private roots;