@harperfast/harper-pro 5.0.0-alpha.9 → 5.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/core/.dockerignore +9 -0
  2. package/core/.git-blame-ignore-revs +2 -0
  3. package/core/.github/workflows/create-release.yaml +4 -4
  4. package/core/.github/workflows/integration-tests.yml +12 -10
  5. package/core/.github/workflows/notify-release-published.yaml +1 -1
  6. package/core/.github/workflows/publish-docker.yaml +2 -2
  7. package/core/.github/workflows/publish-npm.yaml +4 -4
  8. package/core/CONTRIBUTING.md +1 -1
  9. package/core/Dockerfile +62 -0
  10. package/core/build-tools/build-studio.sh +12 -0
  11. package/core/build-tools/build.sh +22 -0
  12. package/core/build-tools/download-prebuilds.js +13 -0
  13. package/core/components/Logger.ts +14 -0
  14. package/core/components/Scope.ts +35 -11
  15. package/core/components/componentLoader.ts +27 -10
  16. package/core/components/operations.js +10 -2
  17. package/core/config/configUtils.js +1 -1
  18. package/core/dataLayer/CreateTableObject.js +2 -2
  19. package/core/dataLayer/schema.js +7 -5
  20. package/core/dataLayer/schemaDescribe.js +1 -1
  21. package/core/index.d.ts +11 -6
  22. package/core/index.js +2 -0
  23. package/core/integrationTests/README.md +24 -0
  24. package/core/integrationTests/apiTests/tests/10_otherRoleTests.mjs +6 -6
  25. package/core/integrationTests/apiTests/tests/12_configuration.mjs +1 -1
  26. package/core/integrationTests/apiTests/tests/14_tokenAuth.mjs +2 -2
  27. package/core/integrationTests/apiTests/tests/16_terminologyUpdates.mjs +4 -4
  28. package/core/integrationTests/apiTests/tests/1_environmentSetup.mjs +1 -1
  29. package/core/integrationTests/apiTests/tests/2_dataLoad.mjs +4 -4
  30. package/core/integrationTests/apiTests/tests/3_sqlTests.mjs +3 -3
  31. package/core/integrationTests/apiTests/tests/4_noSqlTests.mjs +12 -12
  32. package/core/integrationTests/apiTests/tests/5_noSqlRoleTesting.mjs +8 -8
  33. package/core/integrationTests/apiTests/tests/7_jobsAndJobRoleTesting.mjs +10 -12
  34. package/core/integrationTests/apiTests/tests/8_deleteTests.mjs +8 -8
  35. package/core/integrationTests/apiTests/tests/9_transactions.mjs +2 -2
  36. package/core/integrationTests/apiTests/utils/search.mjs +1 -1
  37. package/core/integrationTests/apiTests/utils/table.mjs +1 -1
  38. package/core/integrationTests/server/operation-user-rbac.test.ts +1 -1
  39. package/core/integrationTests/server/operations-server.test.ts +1 -1
  40. package/core/integrationTests/server/storage-reclamation.test.ts +1 -1
  41. package/core/integrationTests/utils/README.md +1 -15
  42. package/core/integrationTests/utils/harperLifecycle.ts +33 -21
  43. package/core/package.json +23 -5
  44. package/core/resources/ResourceInterface.ts +1 -1
  45. package/core/resources/Table.ts +26 -11
  46. package/core/resources/analytics/read.ts +33 -26
  47. package/core/resources/analytics/write.ts +3 -7
  48. package/core/resources/databases.ts +29 -18
  49. package/core/resources/search.ts +10 -5
  50. package/core/security/auth.ts +1 -1
  51. package/core/security/jsLoader.ts +302 -83
  52. package/core/security/keys.js +11 -12
  53. package/core/security/user.ts +3 -3
  54. package/core/server/REST.ts +18 -2
  55. package/core/server/Server.ts +2 -1
  56. package/core/server/fastifyRoutes.ts +1 -0
  57. package/core/server/http.ts +13 -9
  58. package/core/server/loadRootComponents.js +1 -0
  59. package/core/server/operationsServer.ts +2 -1
  60. package/core/server/threads/manageThreads.js +49 -35
  61. package/core/static/defaultConfig.yaml +3 -0
  62. package/core/unitTests/apiTests/RESTProperties-test.mjs +2 -2
  63. package/core/unitTests/apiTests/basicREST-test.mjs +2 -2
  64. package/core/unitTests/components/Scope.test.js +54 -16
  65. package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/circular.js +4 -0
  66. package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/in-child-dir.js +4 -0
  67. package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/typestrip.ts +2 -0
  68. package/core/unitTests/components/fixtures/testJSWithDeps/resources.js +43 -0
  69. package/core/unitTests/components/fixtures/testJSWithDeps/test-child-process.js +18 -0
  70. package/core/unitTests/components/globalIsolation.test.js +87 -1
  71. package/core/unitTests/config/configUtils.test.js +1 -260
  72. package/core/unitTests/resources/query.test.js +16 -1
  73. package/core/unitTests/resources/vectorIndex.test.js +1 -1
  74. package/core/unitTests/server/fastifyRoutes/operations.test.js +1 -1
  75. package/core/unitTests/testUtils.js +0 -17
  76. package/core/utility/hdbTerms.ts +3 -0
  77. package/core/utility/installation.ts +2 -5
  78. package/core/utility/lmdb/commonUtility.js +21 -10
  79. package/dist/core/{resources/ResourceInterfaceV2.js → components/Logger.js} +1 -1
  80. package/dist/core/components/Logger.js.map +1 -0
  81. package/dist/core/components/Scope.js +18 -10
  82. package/dist/core/components/Scope.js.map +1 -1
  83. package/dist/core/components/componentLoader.js +17 -10
  84. package/dist/core/components/componentLoader.js.map +1 -1
  85. package/dist/core/components/operations.js +2 -2
  86. package/dist/core/components/operations.js.map +1 -1
  87. package/dist/core/config/configUtils.js +1 -1
  88. package/dist/core/config/configUtils.js.map +1 -1
  89. package/dist/core/dataLayer/CreateTableObject.js +2 -2
  90. package/dist/core/dataLayer/CreateTableObject.js.map +1 -1
  91. package/dist/core/dataLayer/schema.js +6 -5
  92. package/dist/core/dataLayer/schema.js.map +1 -1
  93. package/dist/core/dataLayer/schemaDescribe.js +1 -1
  94. package/dist/core/dataLayer/schemaDescribe.js.map +1 -1
  95. package/dist/core/index.js +2 -0
  96. package/dist/core/index.js.map +1 -1
  97. package/dist/core/resources/Table.js +12 -4
  98. package/dist/core/resources/Table.js.map +1 -1
  99. package/dist/core/resources/analytics/read.js +32 -22
  100. package/dist/core/resources/analytics/read.js.map +1 -1
  101. package/dist/core/resources/analytics/write.js +3 -6
  102. package/dist/core/resources/analytics/write.js.map +1 -1
  103. package/dist/core/resources/databases.js +22 -19
  104. package/dist/core/resources/databases.js.map +1 -1
  105. package/dist/core/resources/search.js +11 -5
  106. package/dist/core/resources/search.js.map +1 -1
  107. package/dist/core/security/auth.js +1 -1
  108. package/dist/core/security/auth.js.map +1 -1
  109. package/dist/core/security/jsLoader.js +265 -73
  110. package/dist/core/security/jsLoader.js.map +1 -1
  111. package/dist/core/security/keys.js +11 -12
  112. package/dist/core/security/keys.js.map +1 -1
  113. package/dist/core/security/user.js +3 -3
  114. package/dist/core/security/user.js.map +1 -1
  115. package/dist/core/server/REST.js +16 -2
  116. package/dist/core/server/REST.js.map +1 -1
  117. package/dist/core/server/Server.js.map +1 -1
  118. package/dist/core/server/fastifyRoutes.js +2 -0
  119. package/dist/core/server/fastifyRoutes.js.map +1 -1
  120. package/dist/core/server/http.js +12 -6
  121. package/dist/core/server/http.js.map +1 -1
  122. package/dist/core/server/loadRootComponents.js +1 -0
  123. package/dist/core/server/loadRootComponents.js.map +1 -1
  124. package/dist/core/server/operationsServer.js +3 -1
  125. package/dist/core/server/operationsServer.js.map +1 -1
  126. package/dist/core/server/threads/manageThreads.js +50 -35
  127. package/dist/core/server/threads/manageThreads.js.map +1 -1
  128. package/dist/core/utility/hdbTerms.js +3 -0
  129. package/dist/core/utility/hdbTerms.js.map +1 -1
  130. package/dist/core/utility/installation.js.map +1 -1
  131. package/dist/core/utility/lmdb/commonUtility.js +20 -13
  132. package/dist/core/utility/lmdb/commonUtility.js.map +1 -1
  133. package/dist/licensing/usageLicensing.js.map +1 -1
  134. package/dist/replication/knownNodes.js +5 -37
  135. package/dist/replication/knownNodes.js.map +1 -1
  136. package/dist/replication/nodeIdMapping.js +2 -35
  137. package/dist/replication/nodeIdMapping.js.map +1 -1
  138. package/dist/replication/replicationConnection.js +15 -6
  139. package/dist/replication/replicationConnection.js.map +1 -1
  140. package/dist/replication/replicator.js +3 -2
  141. package/dist/replication/replicator.js.map +1 -1
  142. package/dist/replication/setNode.js +1 -1
  143. package/dist/replication/setNode.js.map +1 -1
  144. package/dist/security/certificate.js.map +1 -1
  145. package/licensing/usageLicensing.ts +3 -2
  146. package/npm-shrinkwrap.json +303 -282
  147. package/package.json +4 -3
  148. package/replication/knownNodes.ts +3 -2
  149. package/replication/nodeIdMapping.ts +1 -1
  150. package/replication/replicationConnection.ts +33 -8
  151. package/replication/replicator.ts +7 -2
  152. package/replication/setNode.ts +1 -1
  153. package/security/certificate.ts +2 -1
  154. package/studio/web/assets/{index-v3wIpSYx.js → index-CWN9Wp5V.js} +2 -2
  155. package/studio/web/assets/{index-v3wIpSYx.js.map → index-CWN9Wp5V.js.map} +1 -1
  156. package/studio/web/assets/{index-ChCctErQ.js → index-CzghSAn2.js} +2 -2
  157. package/studio/web/assets/{index-ChCctErQ.js.map → index-CzghSAn2.js.map} +1 -1
  158. package/studio/web/assets/{index-Qu8D43wo.js → index-DMDhGP7N.js} +5 -5
  159. package/studio/web/assets/{index-Qu8D43wo.js.map → index-DMDhGP7N.js.map} +1 -1
  160. package/studio/web/assets/{index.lazy-tVSPM7bX.js → index.lazy-C-yDTGUy.js} +2 -2
  161. package/studio/web/assets/{index.lazy-tVSPM7bX.js.map → index.lazy-C-yDTGUy.js.map} +1 -1
  162. package/studio/web/assets/{profiler-C9as4sv-.js → profiler-0fZAOscv.js} +2 -2
  163. package/studio/web/assets/{profiler-C9as4sv-.js.map → profiler-0fZAOscv.js.map} +1 -1
  164. package/studio/web/assets/{react-redux-RRIhZnM6.js → react-redux-BIxqK8O6.js} +2 -2
  165. package/studio/web/assets/{react-redux-RRIhZnM6.js.map → react-redux-BIxqK8O6.js.map} +1 -1
  166. package/studio/web/assets/{startRecording-DYa4zCXV.js → startRecording-Ca3Gf2MY.js} +2 -2
  167. package/studio/web/assets/{startRecording-DYa4zCXV.js.map → startRecording-Ca3Gf2MY.js.map} +1 -1
  168. package/studio/web/index.html +1 -1
  169. package/core/resources/ResourceInterfaceV2.ts +0 -53
  170. package/core/resources/ResourceV2.ts +0 -67
  171. package/core/resources/analytics/profile.ts +0 -109
  172. package/core/unitTests/apiTests/analytics-test.mjs +0 -38
  173. package/core/v1.d.ts +0 -47
  174. package/core/v1.js +0 -38
  175. package/core/v2.d.ts +0 -47
  176. package/core/v2.js +0 -38
  177. package/dist/core/resources/ResourceInterfaceV2.js.map +0 -1
  178. package/dist/core/resources/ResourceV2.js +0 -27
  179. package/dist/core/resources/ResourceV2.js.map +0 -1
  180. package/dist/core/resources/analytics/profile.js +0 -144
  181. package/dist/core/resources/analytics/profile.js.map +0 -1
@@ -19,7 +19,7 @@ import { appendHeader, Headers } from './serverHelpers/Headers.ts';
19
19
  import { Blob } from '../resources/blob.ts';
20
20
  import { recordAction, recordActionBinary } from '../resources/analytics/write.ts';
21
21
  import { Readable } from 'node:stream';
22
- import { type HttpOptions, server } from './Server.ts';
22
+ import { server, type ServerOptions, type HttpOptions } from './Server.ts';
23
23
  import { setPortServerMap, SERVERS } from './serverRegistry.ts';
24
24
  import { getComponentName } from '../components/componentLoader.ts';
25
25
  import { throttle } from './throttle.ts';
@@ -179,7 +179,7 @@ function getPorts(options) {
179
179
  ports.push({ port: env.get(terms.CONFIG_PARAMS.HTTP_SECUREPORT), secure: true });
180
180
  }
181
181
 
182
- if (options?.isOperationsServer && env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)) {
182
+ if (options?.usageType === 'operations-api' && env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)) {
183
183
  ports.push({
184
184
  port: resolvePath(env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)),
185
185
  secure: false,
@@ -191,7 +191,7 @@ export function httpServer(listener, options) {
191
191
  const servers = [];
192
192
 
193
193
  for (const { port, secure } of getPorts(options)) {
194
- servers.push(getHTTPServer(port, secure, options?.isOperationsServer, options?.mtls));
194
+ servers.push(getHTTPServer(port, secure, options));
195
195
  if (typeof listener === 'function') {
196
196
  httpResponders[options?.runFirst ? 'unshift' : 'push']({ listener, port: options?.port || port });
197
197
  } else {
@@ -203,8 +203,9 @@ export function httpServer(listener, options) {
203
203
 
204
204
  return servers;
205
205
  }
206
-
207
- function getHTTPServer(port, secure, isOperationsServer, isMtls) {
206
+ function getHTTPServer(port: number, secure: boolean, options: ServerOptions) {
207
+ const { mtls: isMtls, usageType } = options || {};
208
+ const isOperationsServer = usageType === 'operations-api';
208
209
  setPortServerMap(port, { protocol_name: secure ? 'HTTPS' : 'HTTP', name: getComponentName() });
209
210
  if (!httpServers[port]) {
210
211
  // TODO: These should all come from httpOptions or operationsApiOptions
@@ -243,7 +244,7 @@ function getHTTPServer(port, secure, isOperationsServer, isMtls) {
243
244
  rejectUnauthorized: Boolean(mtlsRequired),
244
245
  requestCert: Boolean(mtls || isMtls),
245
246
  ticketKeys: getTicketKeys(),
246
- SNICallback: createTLSSelector(isOperationsServer ? 'operations-api' : 'server', mtls),
247
+ SNICallback: createTLSSelector(usageType ?? 'server', mtls),
247
248
  ciphers: tlsConfig.ciphers ?? tlsConfig[0]?.ciphers,
248
249
  });
249
250
  }
@@ -290,7 +291,9 @@ function getHTTPServer(port, secure, isOperationsServer, isMtls) {
290
291
  if (!response.handlesHeaders) {
291
292
  const headers = response.headers || new Headers();
292
293
  if (!body) {
293
- headers.set('Content-Length', '0');
294
+ if (request.method !== 'HEAD') {
295
+ headers.set('Content-Length', '0');
296
+ }
294
297
  sentBody = true;
295
298
  } else if (body.length >= 0) {
296
299
  if (typeof body === 'string') headers.set('Content-Length', Buffer.byteLength(body));
@@ -517,7 +520,7 @@ type OnWebSocketOptions = {
517
520
  port?: number;
518
521
  securePort?: number;
519
522
  maxPayload?: number;
520
- isOperationsServer?: boolean;
523
+ usageType?: string;
521
524
  mtls?: boolean;
522
525
  };
523
526
  const websocketListeners = [],
@@ -537,7 +540,7 @@ function onWebSocket(listener: (ws: WebSocket) => void, options: OnWebSocketOpti
537
540
  name: getComponentName(),
538
541
  });
539
542
 
540
- const server = getHTTPServer(port, secure, options?.isOperationsServer, options?.mtls);
543
+ const server = getHTTPServer(port, secure, options);
541
544
 
542
545
  if (!websocketServers[port]) {
543
546
  websocketServers[port] = new WebSocketServer({
@@ -597,6 +600,7 @@ function onWebSocket(listener: (ws: WebSocket) => void, options: OnWebSocketOpti
597
600
  }
598
601
 
599
602
  function defaultNotFound(request, response) {
603
+ if (response.headersSent || response.writableEnded) return;
600
604
  response.writeHead(404);
601
605
  response.end('Not found\n');
602
606
  logRequest(request, 404, 0, request.requestId);
@@ -28,6 +28,7 @@ async function loadRootComponents(isWorkerThread = false) {
28
28
  await loadComponent(dirname(configUtils.getConfigFilePath()), resources, 'hdb', {
29
29
  isRoot: true,
30
30
  providedLoadedComponents: loadedComponents,
31
+ autoReload: false,
31
32
  });
32
33
  if (!process.env.HARPER_SAFE_MODE) {
33
34
  // once the global plugins are loaded, we now load all the CF and run applications (and their components)
@@ -61,7 +61,7 @@ async function operationsServer(options: ServerOptions & { resources?: Resources
61
61
  //make sure the process waits for the server to be fully instantiated before moving forward
62
62
  await server.ready();
63
63
  if (!options) options = {};
64
- options.isOperationsServer = true;
64
+ options.usageType = 'operations-api';
65
65
  // fastify can't clean up properly
66
66
  try {
67
67
  // now that server is fully loaded/ready, start listening on port provided in config settings or just use
@@ -155,6 +155,7 @@ function buildServer(isHttps: boolean, resources: Resources): FastifyInstance {
155
155
 
156
156
  app.register(function (instance, options, done) {
157
157
  instance.setNotFoundHandler(function (request, reply) {
158
+ if (reply.sent || reply.raw.headersSent || reply.raw.writableEnded) return;
158
159
  app.server.emit('unhandled', request.raw, reply.raw);
159
160
  });
160
161
  done();
@@ -26,6 +26,9 @@ const ACKNOWLEDGEMENT = 'ack';
26
26
  let getThreadInfo;
27
27
  _assignPackageExport('threads', connectedPorts);
28
28
 
29
+ const listenersByType = new Map();
30
+ const messagesQueuedByType = new Map();
31
+
29
32
  module.exports = {
30
33
  startWorker,
31
34
  restartWorkers,
@@ -37,7 +40,6 @@ module.exports = {
37
40
  onMessageByType,
38
41
  broadcast,
39
42
  broadcastWithAcknowledgement,
40
- setChildListenerByType,
41
43
  getWorkerIndex,
42
44
  getWorkerCount,
43
45
  getTicketKeys,
@@ -94,14 +96,20 @@ Object.defineProperty(server, 'workerCount', {
94
96
  return getWorkerCount();
95
97
  },
96
98
  });
97
- let childListenerByType = {
98
- [REQUEST_THREAD_INFO](message, worker) {
99
- sendThreadInfo(worker);
100
- },
101
- [RESOURCE_REPORT](message, worker) {
102
- recordResourceReport(worker, message);
103
- },
104
- };
99
+ if (!parentPort) {
100
+ onMessageByType(REQUEST_THREAD_INFO, (message, worker) => {
101
+ if (worker) sendThreadInfo(worker);
102
+ });
103
+ onMessageByType(RESOURCE_REPORT, (message, worker) => {
104
+ if (worker) recordResourceReport(worker, message);
105
+ });
106
+ }
107
+ // postMessage type listeners that are registered in other ways or can be registered later
108
+ listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED, null);
109
+ listenersByType.set(hdbTerms.ITC_EVENT_TYPES.SCHEMA, null);
110
+ listenersByType.set(hdbTerms.ITC_EVENT_TYPES.USER, null);
111
+ listenersByType.set(hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST, null);
112
+
105
113
  function startWorker(path, options = {}) {
106
114
  // Take a percentage of total memory to determine the max memory for each thread. The percentage is based
107
115
  // on the thread count. Generally, it is unrealistic to efficiently use the majority of total memory for a single
@@ -200,9 +208,6 @@ function startWorker(path, options = {}) {
200
208
  } else harperLogger.error(`Thread has been restarted ${worker.restarts} times and will not be restarted`);
201
209
  }
202
210
  });
203
- worker.on('message', (message) => {
204
- childListenerByType[message.type]?.(message, worker);
205
- });
206
211
  workers.push(worker);
207
212
  startMonitoring();
208
213
  if (options.onStarted) options.onStarted(worker); // notify that it is ready
@@ -316,9 +321,6 @@ async function restartWorkers(
316
321
  });
317
322
  }
318
323
  }
319
- function setChildListenerByType(type, listener) {
320
- childListenerByType[type] = listener;
321
- }
322
324
  function shutdownWorkers(name) {
323
325
  return restartWorkers(name, Infinity, false);
324
326
  }
@@ -331,11 +333,16 @@ const messageListeners = [];
331
333
  function onMessageFromWorkers(listener) {
332
334
  messageListeners.push(listener);
333
335
  }
334
- const listenersByType = new Map();
335
336
  function onMessageByType(type, listener) {
336
337
  let listeners = listenersByType.get(type);
337
338
  if (!listeners) listenersByType.set(type, (listeners = []));
338
339
  listeners.push(listener);
340
+ if (messagesQueuedByType.has(type)) {
341
+ for (let message of messagesQueuedByType.get(type)) {
342
+ listener(message);
343
+ }
344
+ messagesQueuedByType.delete(type);
345
+ }
339
346
  }
340
347
 
341
348
  const MAX_SYNC_BROADCAST = 10;
@@ -526,14 +533,24 @@ function notifyMessageListeners(message, port) {
526
533
  for (let listener of messageListeners) {
527
534
  listener(message, port);
528
535
  }
529
- let listeners = listenersByType.get(message.type);
530
- if (listeners) {
531
- for (let listener of listeners) {
532
- try {
533
- listener(message, port);
534
- } catch (error) {
535
- harperLogger.error(error);
536
+ if (message.type) {
537
+ let listeners = listenersByType.get(message.type);
538
+ if (listeners) {
539
+ for (let listener of listeners) {
540
+ try {
541
+ listener(message, port);
542
+ } catch (error) {
543
+ harperLogger.error(error);
544
+ }
536
545
  }
546
+ } else if (listeners !== null) {
547
+ // null means it is registered for a later listener
548
+ harperLogger.warn?.(`No listener registered for worker message type ${message.type}, queuing message`);
549
+ let messages = messagesQueuedByType.get(message.type);
550
+ if (!messages) {
551
+ messagesQueuedByType.set(message.type, (messages = []));
552
+ }
553
+ messages.push(message);
537
554
  }
538
555
  }
539
556
  }
@@ -564,17 +581,14 @@ if (isMainThread) {
564
581
  module.exports.watchDir = watchDir;
565
582
  if (process.env.WATCH_DIR) watchDir(process.env.WATCH_DIR);
566
583
  } else {
567
- parentPort.on('message', async (message) => {
568
- const { type } = message;
569
- if (type === hdbTerms.ITC_EVENT_TYPES.SHUTDOWN) {
570
- module.exports.restartNumber = message.restartNumber;
571
- parentPort.unref(); // remove this handle
572
- setTimeout(() => {
573
- harperLogger.warn('Thread did not voluntarily terminate', threadId);
574
- // Note that if this occurs, you may want to use this to debug what is currently running:
575
- // require('why-is-node-running')();
576
- process.exit(0);
577
- }, threadTerminationTimeout).unref(); // don't block the shutdown
578
- }
584
+ onMessageByType(hdbTerms.ITC_EVENT_TYPES.SHUTDOWN, async (message) => {
585
+ module.exports.restartNumber = message.restartNumber;
586
+ parentPort.unref(); // remove this handle
587
+ setTimeout(() => {
588
+ harperLogger.warn('Thread did not voluntarily terminate', threadId);
589
+ // Note that if this occurs, you may want to use this to debug what is currently running:
590
+ // require('why-is-node-running')();
591
+ process.exit(0);
592
+ }, threadTerminationTimeout).unref(); // don't block the shutdown
579
593
  });
580
594
  }
@@ -27,6 +27,9 @@ applications:
27
27
  lockdown: freeze
28
28
  containment: vm
29
29
  dependencyContainment: false
30
+ allowedSpawnCommands:
31
+ - npm
32
+ - node
30
33
  componentsRoot: null
31
34
  localStudio:
32
35
  enabled: true
@@ -81,8 +81,8 @@ describe('test REST with property updates', function () {
81
81
  }
82
82
  );
83
83
  assert.equal(response.status, 400);
84
- assert(response.data.includes('property name must be a string'));
85
- assert(response.data.includes('property age must be an integer'));
84
+ assert(response.data.title.includes('property name must be a string'));
85
+ assert(response.data.title.includes('property age must be an integer'));
86
86
  });
87
87
 
88
88
  it('put with nested path', async () => {
@@ -531,7 +531,7 @@ describe('test REST calls', () => {
531
531
  },
532
532
  });
533
533
  assert.equal(response.status, 500);
534
- assert(response.data.includes('Conflicting paths'));
534
+ assert(response.data.title.includes('Conflicting paths'));
535
535
  });
536
536
  it('Returns thrown plain object', async () => {
537
537
  const response = await axios.get('http://localhost:9926/Echo/error-plain-object', {
@@ -541,7 +541,7 @@ describe('test REST calls', () => {
541
541
  },
542
542
  });
543
543
  assert.equal(response.status, 500);
544
- assert(response.data.message, 'Test error');
544
+ assert(response.data.title, 'Test error');
545
545
  });
546
546
  it('Returns correct error for bad body', async () => {
547
547
  const response = await axios.get('http://localhost:9926/Echo/error-bad-body', {
@@ -19,7 +19,8 @@ describe('Scope', () => {
19
19
  this.resources = new Resources();
20
20
  this.server = {};
21
21
  this.directory = mkdtempSync(join(tmpdir(), 'harper.unit-test.scope-'));
22
- this.name = basename(this.directory);
22
+ this.appName = basename(this.directory);
23
+ this.pluginName = 'plugin';
23
24
  this.configFilePath = join(this.directory, 'config.yaml');
24
25
  this.testFilePath = join(this.directory, 'test.js');
25
26
  writeFileSync(this.testFilePath, '"foo";');
@@ -37,10 +38,11 @@ describe('Scope', () => {
37
38
  });
38
39
 
39
40
  it('should create a default entry handler', async () => {
40
- writeFileSync(this.configFilePath, stringify({ [this.name]: { files: 'test.js' } }));
41
+ writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: 'test.js' } }));
41
42
 
42
43
  const scope = new Scope(
43
- this.name,
44
+ this.appName,
45
+ this.pluginName,
44
46
  this.directory,
45
47
  this.configFilePath,
46
48
  new ApplicationScope('test', this.resources, this.server)
@@ -92,10 +94,11 @@ describe('Scope', () => {
92
94
  });
93
95
 
94
96
  it('should create a default entry handler with urlPath', async () => {
95
- writeFileSync(this.configFilePath, stringify({ [this.name]: { files: 'test.js', urlPath: 'abc' } }));
97
+ writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: 'test.js', urlPath: 'abc' } }));
96
98
 
97
99
  const scope = new Scope(
98
- this.name,
100
+ this.appName,
101
+ this.pluginName,
99
102
  this.directory,
100
103
  this.configFilePath,
101
104
  new ApplicationScope('test', this.resources, this.server)
@@ -148,9 +151,16 @@ describe('Scope', () => {
148
151
  });
149
152
 
150
153
  it('should call requestRestart if no entry handler is provided', async () => {
151
- writeFileSync(this.configFilePath, stringify({ [this.name]: { files: '.' } }));
154
+ writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: '.' } }));
152
155
 
153
- const scope = new Scope(this.name, this.directory, this.configFilePath, this.resources, this.server);
156
+ const scope = new Scope(
157
+ this.appName,
158
+ this.pluginName,
159
+ this.directory,
160
+ this.configFilePath,
161
+ this.resources,
162
+ this.server
163
+ );
154
164
 
155
165
  await scope.ready;
156
166
 
@@ -165,9 +175,16 @@ describe('Scope', () => {
165
175
  });
166
176
 
167
177
  it('should call requestRestart if no options handler is provided', async () => {
168
- writeFileSync(this.configFilePath, stringify({ [this.name]: { files: '.' } }));
178
+ writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: '.' } }));
169
179
 
170
- const scope = new Scope(this.name, this.directory, this.configFilePath, this.resources, this.server);
180
+ const scope = new Scope(
181
+ this.appName,
182
+ this.pluginName,
183
+ this.directory,
184
+ this.configFilePath,
185
+ this.resources,
186
+ this.server
187
+ );
171
188
 
172
189
  await scope.ready;
173
190
 
@@ -175,7 +192,7 @@ describe('Scope', () => {
175
192
 
176
193
  assert.equal(restartNeeded(), false, 'requestRestart was not called');
177
194
 
178
- await writeFile(this.configFilePath, stringify({ [this.name]: { files: '.', foo: 'bar' } }));
195
+ await writeFile(this.configFilePath, stringify({ [this.pluginName]: { files: '.', foo: 'bar' } }));
179
196
 
180
197
  await waitFor(() => restartNeeded());
181
198
 
@@ -185,9 +202,16 @@ describe('Scope', () => {
185
202
  });
186
203
 
187
204
  it('should emit error for missing default entry handler', async () => {
188
- writeFileSync(this.configFilePath, stringify({ [this.name]: { foo: 'bar' } }));
205
+ writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { foo: 'bar' } }));
189
206
 
190
- const scope = new Scope(this.name, this.directory, this.configFilePath, this.resources, this.server);
207
+ const scope = new Scope(
208
+ this.appName,
209
+ this.pluginName,
210
+ this.directory,
211
+ this.configFilePath,
212
+ this.resources,
213
+ this.server
214
+ );
191
215
 
192
216
  await scope.ready;
193
217
 
@@ -219,9 +243,16 @@ describe('Scope', () => {
219
243
  });
220
244
 
221
245
  it('should support custom entry handlers', async () => {
222
- writeFileSync(this.configFilePath, stringify({ [this.name]: { foo: 'bar' } }));
246
+ writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { foo: 'bar' } }));
223
247
 
224
- const scope = new Scope(this.name, this.directory, this.configFilePath, this.resources, this.server);
248
+ const scope = new Scope(
249
+ this.appName,
250
+ this.pluginName,
251
+ this.directory,
252
+ this.configFilePath,
253
+ this.resources,
254
+ this.server
255
+ );
225
256
 
226
257
  await scope.ready;
227
258
 
@@ -249,9 +280,16 @@ describe('Scope', () => {
249
280
  });
250
281
 
251
282
  it('should support synchronous handleEntry with event-based initial load tracking', async () => {
252
- writeFileSync(this.configFilePath, stringify({ [this.name]: { files: 'test.js' } }));
283
+ writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: 'test.js' } }));
253
284
 
254
- const scope = new Scope(this.name, this.directory, this.configFilePath, this.resources, this.server);
285
+ const scope = new Scope(
286
+ this.appName,
287
+ this.pluginName,
288
+ this.directory,
289
+ this.configFilePath,
290
+ this.resources,
291
+ this.server
292
+ );
255
293
 
256
294
  await scope.ready;
257
295
 
@@ -0,0 +1,4 @@
1
+ import { MyComponent } from './in-child-dir.js';
2
+ export function testCircularExport() {
3
+ return MyComponent;
4
+ }
@@ -3,10 +3,14 @@ import { Resource } from 'harperdb';
3
3
  import { connect } from 'mqtt'; // verify we can import from node_modules packages
4
4
  import 'micromatch';
5
5
  import 'needle';
6
+ import { testCircularExport } from './circular.js';
6
7
  // TODO: Verify/support circular dependencies
7
8
  console.log('Verifying we can access console.log in transitive module in application');
9
+ assert(testCircularExport);
8
10
  // verify we can't access parent global variables
9
11
  assert(typeof globalVariableFromParent, 'undefined', 'Global variable from parent value should not be present');
10
12
 
11
13
  export class MyComponent extends Resource {}
12
14
  export { connect };
15
+
16
+ assert.equal(testCircularExport(), MyComponent);
@@ -0,0 +1,2 @@
1
+ import type { Resource } from 'harper';
2
+ export const isTyped: Resource = {}!;
@@ -2,6 +2,7 @@ import assert from 'node:assert'; // verify we can access safe node built-in mod
2
2
  import { Resource, getContext } from 'harper';
3
3
  import { MyComponent, connect as connectFromChild } from './child-dir/in-child-dir.js';
4
4
  import { connect } from 'mqtt'; // verify we can import from node_modules packages
5
+ import { fork, spawn } from 'node:child_process';
5
6
 
6
7
  console.log('Verifying we can access console.log in application');
7
8
  // verify we can't access parent global variables
@@ -20,7 +21,49 @@ export const testExport = {
20
21
  let _a = MyComponent;
21
22
  return 'hello world';
22
23
  },
24
+ async testLoadTypeScript() {
25
+ return await import('./child-dir/typestrip.ts');
26
+ },
23
27
  };
24
28
  export class TestComponent extends Resource {}
25
29
 
26
30
  export { MyComponent as 'my-component' };
31
+
32
+ export const processSpawnTest = {
33
+ get() {}, // make it look like a resource
34
+ testFork() {
35
+ // Fork should work (allowed command)
36
+ const child = fork('npm', ['--version'], { name: 'test-npm-process' });
37
+ assert(child.pid, 'Fork should return a process with a PID');
38
+ return child;
39
+ },
40
+ testSpawnDisallowed() {
41
+ // Spawn with disallowed command should throw
42
+ try {
43
+ spawn('curl', ['https://example.com'], { name: 'test-curl-process' });
44
+ throw new Error('Should have thrown an error for disallowed command');
45
+ } catch (err) {
46
+ assert(err.message.includes('not allowed'), 'Should throw error about disallowed command');
47
+ }
48
+ },
49
+ testSpawnWithoutName() {
50
+ // Spawn without name should throw
51
+ try {
52
+ spawn('npm', ['build']);
53
+ throw new Error('Should have thrown an error for missing name');
54
+ } catch (err) {
55
+ assert(err.message.includes('name'), 'Should throw error about missing name');
56
+ }
57
+ },
58
+ testProcessReuse(childProcessPath) {
59
+ // First call should fork a new process
60
+ const child1 = fork(childProcessPath, [], { name: 'test-reuse-process' });
61
+ assert(child1.pid, 'First fork should return a process with a PID');
62
+
63
+ // Second call with same name should return wrapper for existing process
64
+ const child2 = fork(childProcessPath, [], { name: 'test-reuse-process' });
65
+ assert.equal(child1.pid, child2.pid, 'Second fork should return same PID');
66
+
67
+ return { child1, child2 };
68
+ },
69
+ };
@@ -0,0 +1,18 @@
1
+ // Simple child process that stays alive until killed
2
+ // Used for testing process reuse functionality
3
+
4
+ let isRunning = true;
5
+
6
+ process.on('SIGTERM', () => {
7
+ isRunning = false;
8
+ process.exit(0);
9
+ });
10
+
11
+ // Keep process alive
12
+ const interval = setInterval(() => {
13
+ if (!isRunning) {
14
+ clearInterval(interval);
15
+ }
16
+ }, 100);
17
+
18
+ console.log('Child process started with PID:', process.pid);
@@ -2,10 +2,13 @@ const assert = require('node:assert/strict');
2
2
  const path = require('node:path');
3
3
  const { loadComponent, loadedPaths } = require('#src/components/componentLoader');
4
4
  const { PACKAGE_ROOT } = require('#js/utility/packageUtils');
5
+ const fs = require('node:fs');
6
+ const env = require('#src/utility/environment/environmentManager');
5
7
  const { ApplicationScope } = require('#js/components/ApplicationScope');
6
8
 
7
9
  describe('Global Variable Isolation in testJSWithDeps', function () {
8
10
  let mockResources;
11
+ let pidsDir;
9
12
 
10
13
  beforeEach(function () {
11
14
  // Create mock resources
@@ -17,12 +20,24 @@ describe('Global Variable Isolation in testJSWithDeps', function () {
17
20
  // Set a global variable in the parent context
18
21
  global.globalVariableFromParent = 'parent-value';
19
22
  loadedPaths.clear();
23
+
24
+ // Clean up pids directory
25
+ const basePath = env.getHdbBasePath();
26
+ pidsDir = path.join(basePath, 'pids');
27
+ if (fs.existsSync(pidsDir)) {
28
+ fs.rmSync(pidsDir, { recursive: true, force: true });
29
+ }
20
30
  });
21
31
 
22
32
  afterEach(function () {
23
33
  // Clean up global variables
24
34
  delete global.globalVariableFromParent;
25
35
  delete global.globalVariableFromComponent;
36
+
37
+ // Clean up pids directory
38
+ if (fs.existsSync(pidsDir)) {
39
+ fs.rmSync(pidsDir, { recursive: true, force: true });
40
+ }
26
41
  });
27
42
  const componentDir = path.join(__dirname, 'fixtures', 'testJSWithDeps');
28
43
 
@@ -39,7 +54,7 @@ describe('Global Variable Isolation in testJSWithDeps', function () {
39
54
 
40
55
  // The component's resources.js file asserts that globalVariableFromParent is undefined
41
56
  // If the component loaded without throwing, it means global variables are properly isolated
42
-
57
+ assert.equal(mockResources.get('/'), undefined); // this will contain an error if it failed to load
43
58
  // Verify the component's global variable didn't leak into our context
44
59
  assert.equal(
45
60
  typeof global.globalVariableFromComponent,
@@ -52,6 +67,7 @@ describe('Global Variable Isolation in testJSWithDeps', function () {
52
67
 
53
68
  // verify the exported resource works
54
69
  assert.equal(mockResources.get('/testExport').get(), 'hello world');
70
+ assert((await mockResources.get('/testExport').testLoadTypeScript()).isTyped, 'TypeScript exports');
55
71
  assert.equal(typeof mockResources.get('/TestComponent').get, 'function');
56
72
  assert.equal(typeof mockResources.get('/my-component').get, 'function');
57
73
  });
@@ -102,4 +118,74 @@ describe('Global Variable Isolation in testJSWithDeps', function () {
102
118
  // assert(typeof mockResources.get('/my-component').get === 'function'); // this syntax doesn't seem to work
103
119
  // with SES Compartments
104
120
  });
121
+
122
+ it('should enforce process spawning restrictions', async function () {
123
+ let applicationScope = new ApplicationScope('test', mockResources, server);
124
+ Object.assign(applicationScope, {
125
+ mode: 'vm',
126
+ dependencyContainment: false,
127
+ verifyPath: PACKAGE_ROOT,
128
+ });
129
+ await loadComponent(componentDir, mockResources, 'test-origin', {
130
+ applicationScope,
131
+ });
132
+
133
+ const processSpawnTest = mockResources.get('/processSpawnTest');
134
+
135
+ // Test that disallowed commands throw
136
+ processSpawnTest.testSpawnDisallowed();
137
+
138
+ // Test that spawn without name throws
139
+ processSpawnTest.testSpawnWithoutName();
140
+ });
141
+
142
+ it('should allow fork with allowed commands', async function () {
143
+ let applicationScope = new ApplicationScope('test', mockResources, server);
144
+ Object.assign(applicationScope, {
145
+ mode: 'vm',
146
+ dependencyContainment: false,
147
+ verifyPath: PACKAGE_ROOT,
148
+ });
149
+ await loadComponent(componentDir, mockResources, 'test-origin', {
150
+ applicationScope,
151
+ });
152
+
153
+ const processSpawnTest = mockResources.get('/processSpawnTest');
154
+
155
+ // Test that fork works
156
+ const child = processSpawnTest.testFork();
157
+ assert(child.pid, 'Should return a child process with PID');
158
+
159
+ // Clean up
160
+ child.kill();
161
+ });
162
+
163
+ it('should reuse existing processes with same name', async function () {
164
+ this.timeout(10000); // Increase timeout for process spawning
165
+
166
+ let applicationScope = new ApplicationScope('test', mockResources, server);
167
+ Object.assign(applicationScope, {
168
+ mode: 'vm',
169
+ dependencyContainment: false,
170
+ verifyPath: PACKAGE_ROOT,
171
+ });
172
+ await loadComponent(componentDir, mockResources, 'test-origin', {
173
+ applicationScope,
174
+ });
175
+
176
+ const processSpawnTest = mockResources.get('/processSpawnTest');
177
+ const childProcessPath = path.join(componentDir, 'test-child-process.js');
178
+
179
+ // Test process reuse
180
+ const { child1, child2 } = processSpawnTest.testProcessReuse(childProcessPath);
181
+
182
+ // Verify both have same PID
183
+ assert.equal(child1.pid, child2.pid, 'Should reuse existing process');
184
+
185
+ // Verify exit event is emitted on wrapper
186
+ await new Promise((resolve) => {
187
+ child2.on('exit', resolve);
188
+ child1.kill();
189
+ });
190
+ });
105
191
  });