@harperfast/harper-pro 5.0.0-alpha.9 → 5.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/core/.dockerignore +9 -0
  2. package/core/.git-blame-ignore-revs +2 -0
  3. package/core/.github/workflows/create-release.yaml +4 -4
  4. package/core/.github/workflows/integration-tests.yml +12 -10
  5. package/core/.github/workflows/notify-release-published.yaml +1 -1
  6. package/core/.github/workflows/publish-docker.yaml +2 -2
  7. package/core/.github/workflows/publish-npm.yaml +4 -4
  8. package/core/CONTRIBUTING.md +1 -1
  9. package/core/Dockerfile +62 -0
  10. package/core/build-tools/build-studio.sh +12 -0
  11. package/core/build-tools/build.sh +22 -0
  12. package/core/build-tools/download-prebuilds.js +13 -0
  13. package/core/components/Logger.ts +14 -0
  14. package/core/components/Scope.ts +35 -11
  15. package/core/components/componentLoader.ts +27 -10
  16. package/core/components/operations.js +10 -2
  17. package/core/config/configUtils.js +1 -1
  18. package/core/dataLayer/CreateTableObject.js +2 -2
  19. package/core/dataLayer/schema.js +7 -5
  20. package/core/dataLayer/schemaDescribe.js +1 -1
  21. package/core/index.d.ts +11 -6
  22. package/core/index.js +2 -0
  23. package/core/integrationTests/README.md +24 -0
  24. package/core/integrationTests/apiTests/tests/10_otherRoleTests.mjs +6 -6
  25. package/core/integrationTests/apiTests/tests/12_configuration.mjs +1 -1
  26. package/core/integrationTests/apiTests/tests/14_tokenAuth.mjs +2 -2
  27. package/core/integrationTests/apiTests/tests/16_terminologyUpdates.mjs +4 -4
  28. package/core/integrationTests/apiTests/tests/1_environmentSetup.mjs +1 -1
  29. package/core/integrationTests/apiTests/tests/2_dataLoad.mjs +4 -4
  30. package/core/integrationTests/apiTests/tests/3_sqlTests.mjs +3 -3
  31. package/core/integrationTests/apiTests/tests/4_noSqlTests.mjs +12 -12
  32. package/core/integrationTests/apiTests/tests/5_noSqlRoleTesting.mjs +8 -8
  33. package/core/integrationTests/apiTests/tests/7_jobsAndJobRoleTesting.mjs +10 -12
  34. package/core/integrationTests/apiTests/tests/8_deleteTests.mjs +8 -8
  35. package/core/integrationTests/apiTests/tests/9_transactions.mjs +2 -2
  36. package/core/integrationTests/apiTests/utils/search.mjs +1 -1
  37. package/core/integrationTests/apiTests/utils/table.mjs +1 -1
  38. package/core/integrationTests/server/operation-user-rbac.test.ts +1 -1
  39. package/core/integrationTests/server/operations-server.test.ts +1 -1
  40. package/core/integrationTests/server/storage-reclamation.test.ts +1 -1
  41. package/core/integrationTests/utils/README.md +1 -15
  42. package/core/integrationTests/utils/harperLifecycle.ts +33 -21
  43. package/core/package.json +23 -5
  44. package/core/resources/ResourceInterface.ts +1 -1
  45. package/core/resources/Table.ts +26 -11
  46. package/core/resources/analytics/read.ts +33 -26
  47. package/core/resources/analytics/write.ts +3 -7
  48. package/core/resources/databases.ts +29 -18
  49. package/core/resources/search.ts +10 -5
  50. package/core/security/auth.ts +1 -1
  51. package/core/security/jsLoader.ts +302 -83
  52. package/core/security/keys.js +11 -12
  53. package/core/security/user.ts +3 -3
  54. package/core/server/REST.ts +18 -2
  55. package/core/server/Server.ts +2 -1
  56. package/core/server/fastifyRoutes.ts +1 -0
  57. package/core/server/http.ts +13 -9
  58. package/core/server/loadRootComponents.js +1 -0
  59. package/core/server/operationsServer.ts +2 -1
  60. package/core/server/threads/manageThreads.js +49 -35
  61. package/core/static/defaultConfig.yaml +3 -0
  62. package/core/unitTests/apiTests/RESTProperties-test.mjs +2 -2
  63. package/core/unitTests/apiTests/basicREST-test.mjs +2 -2
  64. package/core/unitTests/components/Scope.test.js +54 -16
  65. package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/circular.js +4 -0
  66. package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/in-child-dir.js +4 -0
  67. package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/typestrip.ts +2 -0
  68. package/core/unitTests/components/fixtures/testJSWithDeps/resources.js +43 -0
  69. package/core/unitTests/components/fixtures/testJSWithDeps/test-child-process.js +18 -0
  70. package/core/unitTests/components/globalIsolation.test.js +87 -1
  71. package/core/unitTests/config/configUtils.test.js +1 -260
  72. package/core/unitTests/resources/query.test.js +16 -1
  73. package/core/unitTests/resources/vectorIndex.test.js +1 -1
  74. package/core/unitTests/server/fastifyRoutes/operations.test.js +1 -1
  75. package/core/unitTests/testUtils.js +0 -17
  76. package/core/utility/hdbTerms.ts +3 -0
  77. package/core/utility/installation.ts +2 -5
  78. package/core/utility/lmdb/commonUtility.js +21 -10
  79. package/dist/core/{resources/ResourceInterfaceV2.js → components/Logger.js} +1 -1
  80. package/dist/core/components/Logger.js.map +1 -0
  81. package/dist/core/components/Scope.js +18 -10
  82. package/dist/core/components/Scope.js.map +1 -1
  83. package/dist/core/components/componentLoader.js +17 -10
  84. package/dist/core/components/componentLoader.js.map +1 -1
  85. package/dist/core/components/operations.js +2 -2
  86. package/dist/core/components/operations.js.map +1 -1
  87. package/dist/core/config/configUtils.js +1 -1
  88. package/dist/core/config/configUtils.js.map +1 -1
  89. package/dist/core/dataLayer/CreateTableObject.js +2 -2
  90. package/dist/core/dataLayer/CreateTableObject.js.map +1 -1
  91. package/dist/core/dataLayer/schema.js +6 -5
  92. package/dist/core/dataLayer/schema.js.map +1 -1
  93. package/dist/core/dataLayer/schemaDescribe.js +1 -1
  94. package/dist/core/dataLayer/schemaDescribe.js.map +1 -1
  95. package/dist/core/index.js +2 -0
  96. package/dist/core/index.js.map +1 -1
  97. package/dist/core/resources/Table.js +12 -4
  98. package/dist/core/resources/Table.js.map +1 -1
  99. package/dist/core/resources/analytics/read.js +32 -22
  100. package/dist/core/resources/analytics/read.js.map +1 -1
  101. package/dist/core/resources/analytics/write.js +3 -6
  102. package/dist/core/resources/analytics/write.js.map +1 -1
  103. package/dist/core/resources/databases.js +22 -19
  104. package/dist/core/resources/databases.js.map +1 -1
  105. package/dist/core/resources/search.js +11 -5
  106. package/dist/core/resources/search.js.map +1 -1
  107. package/dist/core/security/auth.js +1 -1
  108. package/dist/core/security/auth.js.map +1 -1
  109. package/dist/core/security/jsLoader.js +265 -73
  110. package/dist/core/security/jsLoader.js.map +1 -1
  111. package/dist/core/security/keys.js +11 -12
  112. package/dist/core/security/keys.js.map +1 -1
  113. package/dist/core/security/user.js +3 -3
  114. package/dist/core/security/user.js.map +1 -1
  115. package/dist/core/server/REST.js +16 -2
  116. package/dist/core/server/REST.js.map +1 -1
  117. package/dist/core/server/Server.js.map +1 -1
  118. package/dist/core/server/fastifyRoutes.js +2 -0
  119. package/dist/core/server/fastifyRoutes.js.map +1 -1
  120. package/dist/core/server/http.js +12 -6
  121. package/dist/core/server/http.js.map +1 -1
  122. package/dist/core/server/loadRootComponents.js +1 -0
  123. package/dist/core/server/loadRootComponents.js.map +1 -1
  124. package/dist/core/server/operationsServer.js +3 -1
  125. package/dist/core/server/operationsServer.js.map +1 -1
  126. package/dist/core/server/threads/manageThreads.js +50 -35
  127. package/dist/core/server/threads/manageThreads.js.map +1 -1
  128. package/dist/core/utility/hdbTerms.js +3 -0
  129. package/dist/core/utility/hdbTerms.js.map +1 -1
  130. package/dist/core/utility/installation.js.map +1 -1
  131. package/dist/core/utility/lmdb/commonUtility.js +20 -13
  132. package/dist/core/utility/lmdb/commonUtility.js.map +1 -1
  133. package/dist/licensing/usageLicensing.js.map +1 -1
  134. package/dist/replication/knownNodes.js +5 -37
  135. package/dist/replication/knownNodes.js.map +1 -1
  136. package/dist/replication/nodeIdMapping.js +2 -35
  137. package/dist/replication/nodeIdMapping.js.map +1 -1
  138. package/dist/replication/replicationConnection.js +15 -6
  139. package/dist/replication/replicationConnection.js.map +1 -1
  140. package/dist/replication/replicator.js +3 -2
  141. package/dist/replication/replicator.js.map +1 -1
  142. package/dist/replication/setNode.js +1 -1
  143. package/dist/replication/setNode.js.map +1 -1
  144. package/dist/security/certificate.js.map +1 -1
  145. package/licensing/usageLicensing.ts +3 -2
  146. package/npm-shrinkwrap.json +303 -282
  147. package/package.json +4 -3
  148. package/replication/knownNodes.ts +3 -2
  149. package/replication/nodeIdMapping.ts +1 -1
  150. package/replication/replicationConnection.ts +33 -8
  151. package/replication/replicator.ts +7 -2
  152. package/replication/setNode.ts +1 -1
  153. package/security/certificate.ts +2 -1
  154. package/studio/web/assets/{index-v3wIpSYx.js → index-CWN9Wp5V.js} +2 -2
  155. package/studio/web/assets/{index-v3wIpSYx.js.map → index-CWN9Wp5V.js.map} +1 -1
  156. package/studio/web/assets/{index-ChCctErQ.js → index-CzghSAn2.js} +2 -2
  157. package/studio/web/assets/{index-ChCctErQ.js.map → index-CzghSAn2.js.map} +1 -1
  158. package/studio/web/assets/{index-Qu8D43wo.js → index-DMDhGP7N.js} +5 -5
  159. package/studio/web/assets/{index-Qu8D43wo.js.map → index-DMDhGP7N.js.map} +1 -1
  160. package/studio/web/assets/{index.lazy-tVSPM7bX.js → index.lazy-C-yDTGUy.js} +2 -2
  161. package/studio/web/assets/{index.lazy-tVSPM7bX.js.map → index.lazy-C-yDTGUy.js.map} +1 -1
  162. package/studio/web/assets/{profiler-C9as4sv-.js → profiler-0fZAOscv.js} +2 -2
  163. package/studio/web/assets/{profiler-C9as4sv-.js.map → profiler-0fZAOscv.js.map} +1 -1
  164. package/studio/web/assets/{react-redux-RRIhZnM6.js → react-redux-BIxqK8O6.js} +2 -2
  165. package/studio/web/assets/{react-redux-RRIhZnM6.js.map → react-redux-BIxqK8O6.js.map} +1 -1
  166. package/studio/web/assets/{startRecording-DYa4zCXV.js → startRecording-Ca3Gf2MY.js} +2 -2
  167. package/studio/web/assets/{startRecording-DYa4zCXV.js.map → startRecording-Ca3Gf2MY.js.map} +1 -1
  168. package/studio/web/index.html +1 -1
  169. package/core/resources/ResourceInterfaceV2.ts +0 -53
  170. package/core/resources/ResourceV2.ts +0 -67
  171. package/core/resources/analytics/profile.ts +0 -109
  172. package/core/unitTests/apiTests/analytics-test.mjs +0 -38
  173. package/core/v1.d.ts +0 -47
  174. package/core/v1.js +0 -38
  175. package/core/v2.d.ts +0 -47
  176. package/core/v2.js +0 -38
  177. package/dist/core/resources/ResourceInterfaceV2.js.map +0 -1
  178. package/dist/core/resources/ResourceV2.js +0 -27
  179. package/dist/core/resources/ResourceV2.js.map +0 -1
  180. package/dist/core/resources/analytics/profile.js +0 -144
  181. package/dist/core/resources/analytics/profile.js.map +0 -1
@@ -3,7 +3,6 @@ import { contextStorage, transaction } from '../resources/transaction.ts';
3
3
  import { RequestTarget } from '../resources/RequestTarget.ts';
4
4
  import { tables, databases } from '../resources/databases.ts';
5
5
  import { readFile } from 'node:fs/promises';
6
- import { readFileSync } from 'node:fs';
7
6
  import { dirname, isAbsolute } from 'node:path';
8
7
  import { pathToFileURL, fileURLToPath } from 'node:url';
9
8
  import { SourceTextModule, SyntheticModule, createContext, runInContext } from 'node:vm';
@@ -11,9 +10,13 @@ import { ApplicationScope } from '../components/ApplicationScope.ts';
11
10
  import logger from '../utility/logging/harper_logger.js';
12
11
  import { createRequire } from 'node:module';
13
12
  import * as env from '../utility/environment/environmentManager';
13
+ import * as child_process from 'node:child_process';
14
14
  import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
15
15
  import { contentTypes } from '../server/serverHelpers/contentTypes.ts';
16
16
  import type { CompartmentOptions } from 'ses';
17
+ import { mkdirSync, readFileSync, writeFileSync, unlinkSync, openSync, closeSync, statSync } from 'node:fs';
18
+ import { join } from 'node:path';
19
+ import { EventEmitter } from 'node:events';
17
20
 
18
21
  type Lockdown = 'none' | 'freeze' | 'ses';
19
22
  const APPLICATIONS_LOCKDOWN: Lockdown = env.get(CONFIG_PARAMS.APPLICATIONS_LOCKDOWN);
@@ -86,12 +89,13 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
86
89
  if (!scope.compartment) scope.compartment = getCompartment(scope, globals);
87
90
  const result = await (await scope.compartment).import(moduleUrl);
88
91
  return result.namespace;
89
- } // else use standard node:vm module to do containment
90
- return await loadModuleWithVM(moduleUrl, scope);
91
- } else {
92
- // important! we need to await the import, otherwise the error will not be caught
93
- return await import(moduleUrl);
92
+ } else if (SourceTextModule) {
93
+ // else use standard node:vm module to do containment (if it is available)
94
+ return await loadModuleWithVM(moduleUrl, scope);
95
+ }
94
96
  }
97
+ // important! we need to await the import, otherwise the error will not be caught
98
+ return await import(moduleUrl);
95
99
  } catch (err) {
96
100
  try {
97
101
  // the actual parse error (internally known as the "arrow message")
@@ -107,11 +111,23 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
107
111
  }
108
112
  }
109
113
 
114
+ let amaro: typeof import('amaro') | undefined;
115
+ /**
116
+ * Strip TypeScript types using the amaro library (what Node.js uses internally)
117
+ * Falls back to regex-based stripping if amaro is not available
118
+ */
119
+ async function stripTypeScriptTypes(source: string): Promise<string> {
120
+ // Use amaro - the library that Node.js uses internally for type stripping
121
+ amaro = await import('amaro');
122
+ return amaro.transformSync(source, { mode: 'strip-only' }).code;
123
+ }
124
+
110
125
  /**
111
126
  * Load a module using Node's vm.Module API with (not really secure) sandboxing
112
127
  */
113
128
  async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
114
- const moduleCache = new Map<string, Promise<SourceTextModule | SyntheticModule>>();
129
+ const moduleCache = new Map<string, SourceTextModule | SyntheticModule>();
130
+ const linkingPromises = new Map<string, Promise<void>>();
115
131
 
116
132
  // Create a secure context with limited globals
117
133
  const contextObject = getGlobalObject(scope);
@@ -128,11 +144,16 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
128
144
  if (specifier.startsWith('file://')) {
129
145
  return specifier;
130
146
  }
131
- const resolved = createRequire(referrer).resolve(specifier);
132
- if (isAbsolute(resolved)) {
133
- return pathToFileURL(resolved).toString();
147
+ // For relative paths, resolve to absolute file URL
148
+ if (specifier.startsWith('.')) {
149
+ const resolved = createRequire(referrer).resolve(specifier);
150
+ if (isAbsolute(resolved)) {
151
+ return pathToFileURL(resolved).toString();
152
+ }
153
+ return resolved;
134
154
  }
135
- return resolved;
155
+ // For package names and node: specifiers, keep as-is for proper require() handling
156
+ return specifier;
136
157
  }
137
158
 
138
159
  /**
@@ -208,33 +229,54 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
208
229
  async function linker(specifier: string, referencingModule: SourceTextModule | SyntheticModule) {
209
230
  const resolvedUrl = resolveModule(specifier, referencingModule.identifier);
210
231
 
211
- // Check cache first
212
- if (moduleCache.has(resolvedUrl)) {
213
- return moduleCache.get(resolvedUrl)!;
214
- }
215
-
216
232
  const useContainment = specifier.startsWith('.') || scope.dependencyContainment;
217
- // Load the module
218
- return await loadModuleWithCache(resolvedUrl, useContainment);
233
+ // Return the module immediately (even if not yet linked) to support circular dependencies
234
+ return await getOrCreateModule(resolvedUrl, useContainment);
219
235
  }
220
236
 
221
- function loadModuleWithCache(url: string, usePrivateGlobal: boolean): Promise<SourceTextModule | SyntheticModule> {
222
- // Check cache
237
+ async function getOrCreateModule(
238
+ url: string,
239
+ usePrivateGlobal: boolean
240
+ ): Promise<SourceTextModule | SyntheticModule> {
241
+ // Check cache first - return cached module immediately (even if not linked yet)
223
242
  if (moduleCache.has(url)) {
224
243
  return moduleCache.get(url)!;
225
244
  }
226
- const loadingModule = loadModule(url, usePrivateGlobal);
227
- moduleCache.set(url, loadingModule);
228
- return loadingModule;
245
+
246
+ // Create the module and cache it immediately (before linking)
247
+ const module = await createModule(url, usePrivateGlobal);
248
+ moduleCache.set(url, module);
249
+
250
+ return module;
251
+ }
252
+
253
+ async function loadModuleWithCache(
254
+ url: string,
255
+ usePrivateGlobal: boolean
256
+ ): Promise<SourceTextModule | SyntheticModule> {
257
+ const module = await getOrCreateModule(url, usePrivateGlobal);
258
+
259
+ // Only link/evaluate once per module
260
+ if (!linkingPromises.has(url)) {
261
+ linkingPromises.set(
262
+ url,
263
+ module.link(linker).then(() => module.evaluate())
264
+ );
265
+ }
266
+
267
+ // Wait for linking to complete
268
+ await linkingPromises.get(url);
269
+
270
+ return module;
229
271
  }
230
272
  /**
231
- * Load a module from URL and create appropriate vm.Module
273
+ * Create a module from URL without linking or evaluating
232
274
  */
233
- async function loadModule(url: string, usePrivateGlobal: boolean): Promise<SourceTextModule | SyntheticModule> {
275
+ async function createModule(url: string, usePrivateGlobal: boolean): Promise<SourceTextModule | SyntheticModule> {
234
276
  let module: SourceTextModule | SyntheticModule;
235
277
 
236
278
  // Handle special built-in modules
237
- if (url === 'harper') {
279
+ if (url === 'harper' || url === 'harperdb') {
238
280
  let harperExports = getHarperExports(scope);
239
281
  module = new SyntheticModule(
240
282
  Object.keys(harperExports),
@@ -245,10 +287,15 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
245
287
  },
246
288
  { identifier: url, context }
247
289
  );
248
- } else if (usePrivateGlobal && url.startsWith('file://')) {
290
+ } else if (url.startsWith('file://')) {
249
291
  checkAllowedModulePath(url, scope.verifyPath);
250
292
  // Load source text from file
251
- const source = await readFile(new URL(url), { encoding: 'utf-8' });
293
+ let source = await readFile(new URL(url), { encoding: 'utf-8' });
294
+
295
+ // Strip TypeScript types if this is a .ts file
296
+ if (url.endsWith('.ts') || url.endsWith('.tsx')) {
297
+ source = await stripTypeScriptTypes(source);
298
+ }
252
299
 
253
300
  // Try to parse as ESM first
254
301
  try {
@@ -264,17 +311,8 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
264
311
  return dynamicModule;
265
312
  },
266
313
  });
267
- // Cache the module
268
- moduleCache.set(url, module);
269
- // Link the module (resolve all imports)
270
- await module.link(linker);
271
-
272
- // Evaluate the module
273
- await module.evaluate();
274
- return module;
275
314
  } catch (err) {
276
315
  // If ESM parsing fails, try to load as CommonJS
277
- // but first try the cache again
278
316
  if (
279
317
  err.message?.includes('require is not defined') ||
280
318
  source.includes('module.exports') ||
@@ -286,28 +324,41 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
286
324
  }
287
325
  }
288
326
  } else {
289
- checkAllowedModulePath(url, scope.verifyPath);
290
- // For Node.js built-in modules (node:) and npm packages, use dynamic import
291
- const importedModule = await import(url);
292
- const exportNames = Object.keys(importedModule);
293
-
294
- module = new SyntheticModule(
295
- exportNames,
296
- function () {
297
- for (const key of exportNames) {
298
- this.setExport(key, importedModule[key]);
299
- }
300
- },
301
- { identifier: url, context }
302
- );
327
+ const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
328
+ // For Node.js built-in modules (node:) and npm packages
329
+ // Always try require first to properly handle CJS modules with named exports
330
+ try {
331
+ const cjsExports = replacedModule ?? require(url);
332
+ // It's a CJS module - expose all properties as named exports
333
+ const exportNames = Object.keys(cjsExports);
334
+ module = new SyntheticModule(
335
+ exportNames.length > 0 ? [...exportNames, 'default'] : ['default'],
336
+ function () {
337
+ if (exportNames.length > 0) {
338
+ for (const key of exportNames) {
339
+ this.setExport(key, cjsExports[key]);
340
+ }
341
+ }
342
+ this.setExport('default', cjsExports);
343
+ },
344
+ { identifier: url, context }
345
+ );
346
+ } catch {
347
+ // Fall back to dynamic import for ESM packages
348
+ const importedModule = await import(url);
349
+ const exportNames = Object.keys(importedModule);
350
+ module = new SyntheticModule(
351
+ exportNames,
352
+ function () {
353
+ for (const key of exportNames) {
354
+ this.setExport(key, importedModule[key]);
355
+ }
356
+ },
357
+ { identifier: url, context }
358
+ );
359
+ }
303
360
  }
304
361
 
305
- // Link the module (resolve all imports)
306
- await module.link(linker);
307
-
308
- // Evaluate the module
309
- await module.evaluate();
310
-
311
362
  return module;
312
363
  }
313
364
 
@@ -439,40 +490,208 @@ function getHarperExports(scope: ApplicationScope) {
439
490
  contentTypes,
440
491
  };
441
492
  }
442
- const ALLOWED_NODE_BUILTIN_MODULES = new Set([
443
- 'assert',
444
- 'http',
445
- 'https',
446
- 'path',
447
- 'url',
448
- 'util',
449
- 'stream',
450
- 'crypto',
451
- 'buffer',
452
- 'string_decoder',
453
- 'querystring',
454
- 'punycode',
455
- 'zlib',
456
- 'events',
457
- 'timers',
458
- 'process',
459
- 'async_hooks',
460
- 'console',
461
- 'perf_hooks',
462
- 'diagnostics_channel',
463
- 'fs',
464
- ]);
465
- function checkAllowedModulePath(moduleUrl: string, containingFolder: string): boolean {
493
+ const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES)
494
+ ? new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES))
495
+ : {
496
+ // if we don't have a list of allowed modules, allow everything
497
+ has() {
498
+ return true;
499
+ },
500
+ };
501
+ const ALLOWED_COMMANDS = new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDSPAWNCOMMANDS) ?? []);
502
+ const REPLACED_BUILTIN_MODULES = {
503
+ child_process: {
504
+ exec: createSpawn(child_process.exec),
505
+ execFile: createSpawn(child_process.execFile),
506
+ fork: createSpawn(child_process.fork, true), // this is launching node, so deemed safe
507
+ spawn: createSpawn(child_process.spawn),
508
+ },
509
+ };
510
+ /**
511
+ * Creates a ChildProcess-like object for an existing process
512
+ */
513
+ class ExistingProcessWrapper extends EventEmitter {
514
+ pid: number;
515
+ private checkInterval: NodeJS.Timeout;
516
+
517
+ constructor(pid: number) {
518
+ super();
519
+ this.pid = pid;
520
+
521
+ // Monitor process and emit exit event when it terminates
522
+ this.checkInterval = setInterval(() => {
523
+ try {
524
+ // Signal 0 checks if process exists without actually killing it
525
+ process.kill(pid, 0);
526
+ } catch {
527
+ // Process no longer exists
528
+ clearInterval(this.checkInterval);
529
+ this.emit('exit', null, null);
530
+ }
531
+ }, 1000);
532
+ }
533
+
534
+ // Kill the process
535
+ kill(signal?: NodeJS.Signals | number) {
536
+ try {
537
+ process.kill(this.pid, signal);
538
+ return true;
539
+ } catch {
540
+ return false;
541
+ }
542
+ }
543
+
544
+ // Clean up interval when wrapper is no longer needed
545
+ unref() {
546
+ clearInterval(this.checkInterval);
547
+ return this;
548
+ }
549
+ }
550
+
551
+ /**
552
+ * Checks if a process with the given PID is running
553
+ */
554
+ function isProcessRunning(pid: number): boolean {
555
+ try {
556
+ // Signal 0 checks existence without killing
557
+ process.kill(pid, 0);
558
+ return true;
559
+ } catch {
560
+ return false;
561
+ }
562
+ }
563
+
564
+ /**
565
+ * Acquires an exclusive lock using the PID file itself (synchronously with busy-wait)
566
+ * Returns 0 if lock was acquired (need to spawn new process), or the existing PID if process is running
567
+ */
568
+ function acquirePidFileLock(pidFilePath: string, maxRetries = 100, retryDelay = 5): number {
569
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
570
+ try {
571
+ // Try to open exclusively - 'wx' fails if file exists
572
+ const fd = openSync(pidFilePath, 'wx');
573
+ closeSync(fd);
574
+ return 0; // Successfully acquired lock (file created), caller should spawn process
575
+ } catch (err) {
576
+ if (err.code === 'EEXIST') {
577
+ // File exists - check if it contains a valid running process
578
+ try {
579
+ const pidContent = readFileSync(pidFilePath, 'utf-8');
580
+ const existingPid = parseInt(pidContent.trim(), 10);
581
+
582
+ if (!isNaN(existingPid) && isProcessRunning(existingPid)) {
583
+ // Valid process is running, return its PID immediately
584
+ return existingPid;
585
+ }
586
+
587
+ // Invalid/empty PID - check file age to determine if it's stale or being written
588
+ const stats = statSync(pidFilePath);
589
+ const fileAge = Date.now() - stats.mtimeMs;
590
+
591
+ // If file is very new (less than 100ms) and empty/invalid, another thread is likely still writing to it
592
+ if (fileAge < 100) {
593
+ // Just wait and retry, don't try to remove
594
+ } else {
595
+ // Stale PID file (old and invalid), try to remove it
596
+ try {
597
+ unlinkSync(pidFilePath);
598
+ } catch {
599
+ // Another thread may have removed it, retry
600
+ }
601
+ }
602
+ } catch {
603
+ // Couldn't read/stat file, another thread might be modifying it, retry
604
+ }
605
+
606
+ // Wait a bit before retrying
607
+ const start = Date.now();
608
+ while (Date.now() - start < retryDelay) {
609
+ // Busy wait
610
+ }
611
+ } else {
612
+ throw err;
613
+ }
614
+ }
615
+ }
616
+
617
+ throw new Error(`Failed to acquire PID file lock after ${maxRetries} attempts`);
618
+ }
619
+
620
+ function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess, alwaysAllow?: boolean) {
621
+ const basePath = env.getHdbBasePath();
622
+ return function (command: string, args?: any, options?: any, callback?: (...args: any[]) => void) {
623
+ if (!ALLOWED_COMMANDS.has(command.split(' ')[0]) && !alwaysAllow) {
624
+ throw new Error(`Command ${command} is not allowed`);
625
+ }
626
+ const processName = options?.name;
627
+ if (!processName)
628
+ throw new Error(
629
+ `Calling ${spawnFunction.name} in Harper must have a process "name" in the options to ensure that a single process is started and reused`
630
+ );
631
+
632
+ // Ensure PID directory exists
633
+ const pidDir = join(basePath, 'pids');
634
+ mkdirSync(pidDir, { recursive: true });
635
+
636
+ const pidFilePath = join(pidDir, `${processName}.pid`);
637
+
638
+ // Try to acquire lock - returns 0 if acquired, or existing PID
639
+ const existingPid = acquirePidFileLock(pidFilePath);
640
+
641
+ if (existingPid !== 0) {
642
+ // Existing process is running, return wrapper
643
+ return new ExistingProcessWrapper(existingPid);
644
+ }
645
+
646
+ // We acquired the lock (file was created), spawn new process
647
+ const childProcess = spawnFunction(command, args, options, callback);
648
+
649
+ // Write PID to the file we just created
650
+ try {
651
+ writeFileSync(pidFilePath, childProcess.pid.toString(), 'utf-8');
652
+ } catch (err) {
653
+ // Failed to write PID, clean up
654
+ try {
655
+ childProcess.kill();
656
+ unlinkSync(pidFilePath);
657
+ } catch {}
658
+ throw err;
659
+ }
660
+
661
+ // Clean up PID file when process exits
662
+ childProcess.on('exit', () => {
663
+ try {
664
+ unlinkSync(pidFilePath);
665
+ } catch {
666
+ // File may already be removed
667
+ }
668
+ });
669
+
670
+ return childProcess;
671
+ };
672
+ }
673
+
674
+ /**
675
+ * Validates whether a module can be loaded based on security restrictions and returns the module path or replacement.
676
+ * For file URLs, ensures the module is within the containing folder.
677
+ * For node built-in modules, checks against an allowlist and returns any replacements.
678
+ *
679
+ * @param {string} moduleUrl - The URL or identifier of the module to be loaded, which may be a file: URL, node: URL, or bare module specifier.
680
+ * @param {string} containingFolder - The absolute path of the folder that contains the application, used to validate file: URLs are within bounds.
681
+ * @return {any} Returns undefined for allowed file paths, or a replacement module identifier for allowed node built-in modules.
682
+ * @throws {Error} Throws an error if the module is outside the application folder or if the module is not in the allowed list.
683
+ */
684
+ function checkAllowedModulePath(moduleUrl: string, containingFolder?: string): boolean {
466
685
  if (moduleUrl.startsWith('file:')) {
467
686
  const path = moduleUrl.slice(7);
468
687
  if (!containingFolder || path.startsWith(containingFolder)) {
469
- return true;
688
+ return;
470
689
  }
471
690
  throw new Error(`Can not load module outside of application folder ${containingFolder}`);
472
691
  }
473
692
  let simpleName = moduleUrl.startsWith('node:') ? moduleUrl.slice(5) : moduleUrl;
474
693
  simpleName = simpleName.split('/')[0];
475
- if (ALLOWED_NODE_BUILTIN_MODULES.has(simpleName)) return true;
694
+ if (ALLOWED_NODE_BUILTIN_MODULES.has(simpleName)) return REPLACED_BUILTIN_MODULES[simpleName];
476
695
  throw new Error(`Module ${moduleUrl} is not allowed to be imported`);
477
696
  }
478
697
 
@@ -202,7 +202,7 @@ function loadCertificates() {
202
202
  const x509Cert = new X509Certificate(certificatePem);
203
203
  let certCn;
204
204
  try {
205
- certCn = getPrimaryHostName(x509Cert);
205
+ certCn = (!ca && config.name) || getPrimaryHostName(x509Cert);
206
206
  } catch (err) {
207
207
  logger.error?.('error extracting host name from certificate', err);
208
208
  return;
@@ -238,7 +238,7 @@ function loadCertificates() {
238
238
 
239
239
  promise = certificateTable.put({
240
240
  name: certCn,
241
- uses: ['https', ...(configKey.includes('operations') ? ['operations'] : [])],
241
+ uses: config.uses ?? ['https', ...(configKey.includes('operations') ? ['operations'] : [])],
242
242
  ciphers: config.ciphers,
243
243
  certificate: certificatePem,
244
244
  private_key_name,
@@ -459,7 +459,7 @@ async function getCertAuthority() {
459
459
  let match;
460
460
  for (let cert of allCerts) {
461
461
  if (!cert.is_authority) continue;
462
- const matchingPrivateKey = await getPrivateKeyByName(cert.private_key_name);
462
+ const matchingPrivateKey = getPrivateKeyByName(cert.private_key_name);
463
463
  if (cert.private_key_name && matchingPrivateKey) {
464
464
  const keyCheck = new X509Certificate(cert.certificate).checkPrivateKey(createPrivateKey(matchingPrivateKey));
465
465
  if (keyCheck) {
@@ -724,7 +724,7 @@ function createTLSSelector(type, mtlsOptions) {
724
724
  server.secureContextsListeners = [];
725
725
  }
726
726
  return (SNICallback.ready = new Promise((resolve, reject) => {
727
- async function updateTLS() {
727
+ function updateTLS() {
728
728
  try {
729
729
  secureContexts.clear();
730
730
  caCerts.clear();
@@ -733,7 +733,7 @@ function createTLSSelector(type, mtlsOptions) {
733
733
  resolve();
734
734
  return;
735
735
  }
736
- for await (const cert of databases.system.hdb_certificate.search([])) {
736
+ for (const cert of databases.system.hdb_certificate.search([])) {
737
737
  const certificate = cert.certificate;
738
738
  const certParsed = new X509Certificate(certificate);
739
739
  if (cert.is_authority) {
@@ -742,17 +742,16 @@ function createTLSSelector(type, mtlsOptions) {
742
742
  }
743
743
  }
744
744
 
745
- for await (const cert of databases.system.hdb_certificate.search([])) {
745
+ for (const cert of databases.system.hdb_certificate.search([])) {
746
746
  try {
747
747
  if (cert.is_authority) {
748
748
  continue;
749
749
  }
750
- let isOperations = type === 'operations-api';
751
750
  let quality = cert.is_self_signed ? 1 : 3;
752
751
  // prefer operations certificates for operations API
753
- if (isOperations && cert.uses?.includes?.('operations')) quality += 1;
752
+ if (cert.uses?.includes(type)) quality += 1;
754
753
 
755
- const private_key = await getPrivateKeyByName(cert.private_key_name);
754
+ const private_key = getPrivateKeyByName(cert.private_key_name);
756
755
 
757
756
  let certificate = cert.certificate;
758
757
  const certParsed = new X509Certificate(certificate);
@@ -776,7 +775,7 @@ function createTLSSelector(type, mtlsOptions) {
776
775
  let hostnames = cert.hostnames ?? hostnamesFromCert(certParsed);
777
776
  if (!Array.isArray(hostnames)) hostnames = [hostnames];
778
777
  for (let hostname of hostnames) {
779
- if (hostname === getHost()) quality += 1; // prefer a certificate that has our hostname in the SANs
778
+ if (hostname === getHost()) quality += 0.1; // prefer a certificate that has our hostname in the SANs
780
779
  }
781
780
  let secureContext = tls.createSecureContext(secureOptions);
782
781
  secureContext.name = cert.name;
@@ -876,10 +875,10 @@ function createTLSSelector(type, mtlsOptions) {
876
875
  }
877
876
  }
878
877
 
879
- async function getPrivateKeyByName(private_key_name) {
878
+ function getPrivateKeyByName(private_key_name) {
880
879
  const private_key = privateKeys.get(private_key_name);
881
880
  if (!private_key && private_key_name) {
882
- return await fs.readFile(
881
+ return fs.readFileSync(
883
882
  path.join(envManager.get(CONFIG_PARAMS.ROOTPATH), hdbTerms.LICENSE_KEY_DIR_NAME, private_key_name),
884
883
  'utf8'
885
884
  );
@@ -165,7 +165,7 @@ async function addUser(user: User | any): Promise<string> {
165
165
  throw new ClientError(HDB_ERROR_MSGS.USER_ALREADY_EXISTS(cleanUser.username), HTTP_STATUS_CODES.CONFLICT);
166
166
  }
167
167
 
168
- signalling.signalUserChange(new UserEventMsg(process.pid));
168
+ await signalling.signalUserChange(new UserEventMsg(process.pid));
169
169
  return `${cleanUser.username} successfully added`;
170
170
  }
171
171
 
@@ -228,7 +228,7 @@ async function alterUser(jsonMessage) {
228
228
  });
229
229
 
230
230
  await setUsersWithRolesCache();
231
- signalling.signalUserChange(new UserEventMsg(process.pid));
231
+ await signalling.signalUserChange(new UserEventMsg(process.pid));
232
232
 
233
233
  return updateResponse;
234
234
  }
@@ -248,7 +248,7 @@ async function dropUser(user: User | any): Promise<string> {
248
248
 
249
249
  logger.debug(deleteResponse);
250
250
  await setUsersWithRolesCache();
251
- signalling.signalUserChange(new UserEventMsg(process.pid));
251
+ await signalling.signalUserChange(new UserEventMsg(process.pid));
252
252
  return `${user.username} successfully deleted`;
253
253
  }
254
254
 
@@ -220,6 +220,7 @@ async function http(request: Context & Request, nextHandler) {
220
220
  }
221
221
  return responseObject;
222
222
  } catch (error) {
223
+ error ??= new Error('Unknown error occurred');
223
224
  let statusCode = error.statusCode ?? request.response.status;
224
225
  if (statusCode) {
225
226
  if (statusCode === 500) harperLogger.warn(error);
@@ -232,12 +233,27 @@ async function http(request: Context & Request, nextHandler) {
232
233
  }
233
234
  }
234
235
  } else harperLogger.error(error);
236
+
237
+ // RFC 9457 Problem Details
238
+ const status = statusCode || 500;
239
+ // we prefer to use error classes for error codes (constructor.name), but if there is a code, it is probably a node.js
240
+ // error that denotes error codes with a separate property
241
+ const code = error.code ?? error.constructor.name;
242
+ const problemDetail = {
243
+ type: `error:${code}`, // eventually we want this to be a resolvable URI to our docs
244
+ code,
245
+ title: error.message ?? error.toString(),
246
+ status,
247
+ detail: error.detail,
248
+ instance: request.url,
249
+ };
250
+
235
251
  const responseObject = {
236
- status: statusCode || 500, // use specified error status, or default to generic server error
252
+ status,
237
253
  headers,
238
254
  body: undefined,
239
255
  };
240
- responseObject.body = serialize(error instanceof Error ? errorToString(error) : error, request, responseObject);
256
+ responseObject.body = serialize(problemDetail, request, responseObject);
241
257
  return responseObject;
242
258
  }
243
259
  }
@@ -49,7 +49,8 @@ interface Node {
49
49
  export interface ServerOptions {
50
50
  port?: number;
51
51
  securePort?: number;
52
- isOperationsServer?: boolean;
52
+ mtls?: boolean;
53
+ usageType?: string;
53
54
  }
54
55
  interface WebSocketOptions extends ServerOptions {
55
56
  subProtocol: string;
@@ -179,6 +179,7 @@ async function buildServer(isHttps) {
179
179
 
180
180
  app.register(function (instance, options, done) {
181
181
  instance.setNotFoundHandler(function (request, reply) {
182
+ if (reply.sent || reply.raw.headersSent || reply.raw.writableEnded) return;
182
183
  app.server.emit('unhandled', request.raw, reply.raw);
183
184
  });
184
185
  done();