@harperfast/harper-pro 5.0.0-alpha.9 → 5.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/.dockerignore +9 -0
- package/core/.git-blame-ignore-revs +2 -0
- package/core/.github/workflows/create-release.yaml +4 -4
- package/core/.github/workflows/integration-tests.yml +12 -10
- package/core/.github/workflows/notify-release-published.yaml +1 -1
- package/core/.github/workflows/publish-docker.yaml +2 -2
- package/core/.github/workflows/publish-npm.yaml +4 -4
- package/core/CONTRIBUTING.md +1 -1
- package/core/Dockerfile +62 -0
- package/core/build-tools/build-studio.sh +12 -0
- package/core/build-tools/build.sh +22 -0
- package/core/build-tools/download-prebuilds.js +13 -0
- package/core/components/Logger.ts +14 -0
- package/core/components/Scope.ts +35 -11
- package/core/components/componentLoader.ts +27 -10
- package/core/components/operations.js +10 -2
- package/core/config/configUtils.js +1 -1
- package/core/dataLayer/CreateTableObject.js +2 -2
- package/core/dataLayer/schema.js +7 -5
- package/core/dataLayer/schemaDescribe.js +1 -1
- package/core/index.d.ts +11 -6
- package/core/index.js +2 -0
- package/core/integrationTests/README.md +24 -0
- package/core/integrationTests/apiTests/tests/10_otherRoleTests.mjs +6 -6
- package/core/integrationTests/apiTests/tests/12_configuration.mjs +1 -1
- package/core/integrationTests/apiTests/tests/14_tokenAuth.mjs +2 -2
- package/core/integrationTests/apiTests/tests/16_terminologyUpdates.mjs +4 -4
- package/core/integrationTests/apiTests/tests/1_environmentSetup.mjs +1 -1
- package/core/integrationTests/apiTests/tests/2_dataLoad.mjs +4 -4
- package/core/integrationTests/apiTests/tests/3_sqlTests.mjs +3 -3
- package/core/integrationTests/apiTests/tests/4_noSqlTests.mjs +12 -12
- package/core/integrationTests/apiTests/tests/5_noSqlRoleTesting.mjs +8 -8
- package/core/integrationTests/apiTests/tests/7_jobsAndJobRoleTesting.mjs +10 -12
- package/core/integrationTests/apiTests/tests/8_deleteTests.mjs +8 -8
- package/core/integrationTests/apiTests/tests/9_transactions.mjs +2 -2
- package/core/integrationTests/apiTests/utils/search.mjs +1 -1
- package/core/integrationTests/apiTests/utils/table.mjs +1 -1
- package/core/integrationTests/server/operation-user-rbac.test.ts +1 -1
- package/core/integrationTests/server/operations-server.test.ts +1 -1
- package/core/integrationTests/server/storage-reclamation.test.ts +1 -1
- package/core/integrationTests/utils/README.md +1 -15
- package/core/integrationTests/utils/harperLifecycle.ts +33 -21
- package/core/package.json +23 -5
- package/core/resources/ResourceInterface.ts +1 -1
- package/core/resources/Table.ts +26 -11
- package/core/resources/analytics/read.ts +33 -26
- package/core/resources/analytics/write.ts +3 -7
- package/core/resources/databases.ts +29 -18
- package/core/resources/search.ts +10 -5
- package/core/security/auth.ts +1 -1
- package/core/security/jsLoader.ts +302 -83
- package/core/security/keys.js +11 -12
- package/core/security/user.ts +3 -3
- package/core/server/REST.ts +18 -2
- package/core/server/Server.ts +2 -1
- package/core/server/fastifyRoutes.ts +1 -0
- package/core/server/http.ts +13 -9
- package/core/server/loadRootComponents.js +1 -0
- package/core/server/operationsServer.ts +2 -1
- package/core/server/threads/manageThreads.js +49 -35
- package/core/static/defaultConfig.yaml +3 -0
- package/core/unitTests/apiTests/RESTProperties-test.mjs +2 -2
- package/core/unitTests/apiTests/basicREST-test.mjs +2 -2
- package/core/unitTests/components/Scope.test.js +54 -16
- package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/circular.js +4 -0
- package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/in-child-dir.js +4 -0
- package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/typestrip.ts +2 -0
- package/core/unitTests/components/fixtures/testJSWithDeps/resources.js +43 -0
- package/core/unitTests/components/fixtures/testJSWithDeps/test-child-process.js +18 -0
- package/core/unitTests/components/globalIsolation.test.js +87 -1
- package/core/unitTests/config/configUtils.test.js +1 -260
- package/core/unitTests/resources/query.test.js +16 -1
- package/core/unitTests/resources/vectorIndex.test.js +1 -1
- package/core/unitTests/server/fastifyRoutes/operations.test.js +1 -1
- package/core/unitTests/testUtils.js +0 -17
- package/core/utility/hdbTerms.ts +3 -0
- package/core/utility/installation.ts +2 -5
- package/core/utility/lmdb/commonUtility.js +21 -10
- package/dist/core/{resources/ResourceInterfaceV2.js → components/Logger.js} +1 -1
- package/dist/core/components/Logger.js.map +1 -0
- package/dist/core/components/Scope.js +18 -10
- package/dist/core/components/Scope.js.map +1 -1
- package/dist/core/components/componentLoader.js +17 -10
- package/dist/core/components/componentLoader.js.map +1 -1
- package/dist/core/components/operations.js +2 -2
- package/dist/core/components/operations.js.map +1 -1
- package/dist/core/config/configUtils.js +1 -1
- package/dist/core/config/configUtils.js.map +1 -1
- package/dist/core/dataLayer/CreateTableObject.js +2 -2
- package/dist/core/dataLayer/CreateTableObject.js.map +1 -1
- package/dist/core/dataLayer/schema.js +6 -5
- package/dist/core/dataLayer/schema.js.map +1 -1
- package/dist/core/dataLayer/schemaDescribe.js +1 -1
- package/dist/core/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/core/index.js +2 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/resources/Table.js +12 -4
- package/dist/core/resources/Table.js.map +1 -1
- package/dist/core/resources/analytics/read.js +32 -22
- package/dist/core/resources/analytics/read.js.map +1 -1
- package/dist/core/resources/analytics/write.js +3 -6
- package/dist/core/resources/analytics/write.js.map +1 -1
- package/dist/core/resources/databases.js +22 -19
- package/dist/core/resources/databases.js.map +1 -1
- package/dist/core/resources/search.js +11 -5
- package/dist/core/resources/search.js.map +1 -1
- package/dist/core/security/auth.js +1 -1
- package/dist/core/security/auth.js.map +1 -1
- package/dist/core/security/jsLoader.js +265 -73
- package/dist/core/security/jsLoader.js.map +1 -1
- package/dist/core/security/keys.js +11 -12
- package/dist/core/security/keys.js.map +1 -1
- package/dist/core/security/user.js +3 -3
- package/dist/core/security/user.js.map +1 -1
- package/dist/core/server/REST.js +16 -2
- package/dist/core/server/REST.js.map +1 -1
- package/dist/core/server/Server.js.map +1 -1
- package/dist/core/server/fastifyRoutes.js +2 -0
- package/dist/core/server/fastifyRoutes.js.map +1 -1
- package/dist/core/server/http.js +12 -6
- package/dist/core/server/http.js.map +1 -1
- package/dist/core/server/loadRootComponents.js +1 -0
- package/dist/core/server/loadRootComponents.js.map +1 -1
- package/dist/core/server/operationsServer.js +3 -1
- package/dist/core/server/operationsServer.js.map +1 -1
- package/dist/core/server/threads/manageThreads.js +50 -35
- package/dist/core/server/threads/manageThreads.js.map +1 -1
- package/dist/core/utility/hdbTerms.js +3 -0
- package/dist/core/utility/hdbTerms.js.map +1 -1
- package/dist/core/utility/installation.js.map +1 -1
- package/dist/core/utility/lmdb/commonUtility.js +20 -13
- package/dist/core/utility/lmdb/commonUtility.js.map +1 -1
- package/dist/licensing/usageLicensing.js.map +1 -1
- package/dist/replication/knownNodes.js +5 -37
- package/dist/replication/knownNodes.js.map +1 -1
- package/dist/replication/nodeIdMapping.js +2 -35
- package/dist/replication/nodeIdMapping.js.map +1 -1
- package/dist/replication/replicationConnection.js +15 -6
- package/dist/replication/replicationConnection.js.map +1 -1
- package/dist/replication/replicator.js +3 -2
- package/dist/replication/replicator.js.map +1 -1
- package/dist/replication/setNode.js +1 -1
- package/dist/replication/setNode.js.map +1 -1
- package/dist/security/certificate.js.map +1 -1
- package/licensing/usageLicensing.ts +3 -2
- package/npm-shrinkwrap.json +303 -282
- package/package.json +4 -3
- package/replication/knownNodes.ts +3 -2
- package/replication/nodeIdMapping.ts +1 -1
- package/replication/replicationConnection.ts +33 -8
- package/replication/replicator.ts +7 -2
- package/replication/setNode.ts +1 -1
- package/security/certificate.ts +2 -1
- package/studio/web/assets/{index-v3wIpSYx.js → index-CWN9Wp5V.js} +2 -2
- package/studio/web/assets/{index-v3wIpSYx.js.map → index-CWN9Wp5V.js.map} +1 -1
- package/studio/web/assets/{index-ChCctErQ.js → index-CzghSAn2.js} +2 -2
- package/studio/web/assets/{index-ChCctErQ.js.map → index-CzghSAn2.js.map} +1 -1
- package/studio/web/assets/{index-Qu8D43wo.js → index-DMDhGP7N.js} +5 -5
- package/studio/web/assets/{index-Qu8D43wo.js.map → index-DMDhGP7N.js.map} +1 -1
- package/studio/web/assets/{index.lazy-tVSPM7bX.js → index.lazy-C-yDTGUy.js} +2 -2
- package/studio/web/assets/{index.lazy-tVSPM7bX.js.map → index.lazy-C-yDTGUy.js.map} +1 -1
- package/studio/web/assets/{profiler-C9as4sv-.js → profiler-0fZAOscv.js} +2 -2
- package/studio/web/assets/{profiler-C9as4sv-.js.map → profiler-0fZAOscv.js.map} +1 -1
- package/studio/web/assets/{react-redux-RRIhZnM6.js → react-redux-BIxqK8O6.js} +2 -2
- package/studio/web/assets/{react-redux-RRIhZnM6.js.map → react-redux-BIxqK8O6.js.map} +1 -1
- package/studio/web/assets/{startRecording-DYa4zCXV.js → startRecording-Ca3Gf2MY.js} +2 -2
- package/studio/web/assets/{startRecording-DYa4zCXV.js.map → startRecording-Ca3Gf2MY.js.map} +1 -1
- package/studio/web/index.html +1 -1
- package/core/resources/ResourceInterfaceV2.ts +0 -53
- package/core/resources/ResourceV2.ts +0 -67
- package/core/resources/analytics/profile.ts +0 -109
- package/core/unitTests/apiTests/analytics-test.mjs +0 -38
- package/core/v1.d.ts +0 -47
- package/core/v1.js +0 -38
- package/core/v2.d.ts +0 -47
- package/core/v2.js +0 -38
- package/dist/core/resources/ResourceInterfaceV2.js.map +0 -1
- package/dist/core/resources/ResourceV2.js +0 -27
- package/dist/core/resources/ResourceV2.js.map +0 -1
- package/dist/core/resources/analytics/profile.js +0 -144
- package/dist/core/resources/analytics/profile.js.map +0 -1
package/core/server/http.ts
CHANGED
|
@@ -19,7 +19,7 @@ import { appendHeader, Headers } from './serverHelpers/Headers.ts';
|
|
|
19
19
|
import { Blob } from '../resources/blob.ts';
|
|
20
20
|
import { recordAction, recordActionBinary } from '../resources/analytics/write.ts';
|
|
21
21
|
import { Readable } from 'node:stream';
|
|
22
|
-
import { type
|
|
22
|
+
import { server, type ServerOptions, type HttpOptions } from './Server.ts';
|
|
23
23
|
import { setPortServerMap, SERVERS } from './serverRegistry.ts';
|
|
24
24
|
import { getComponentName } from '../components/componentLoader.ts';
|
|
25
25
|
import { throttle } from './throttle.ts';
|
|
@@ -179,7 +179,7 @@ function getPorts(options) {
|
|
|
179
179
|
ports.push({ port: env.get(terms.CONFIG_PARAMS.HTTP_SECUREPORT), secure: true });
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
-
if (options?.
|
|
182
|
+
if (options?.usageType === 'operations-api' && env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)) {
|
|
183
183
|
ports.push({
|
|
184
184
|
port: resolvePath(env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)),
|
|
185
185
|
secure: false,
|
|
@@ -191,7 +191,7 @@ export function httpServer(listener, options) {
|
|
|
191
191
|
const servers = [];
|
|
192
192
|
|
|
193
193
|
for (const { port, secure } of getPorts(options)) {
|
|
194
|
-
servers.push(getHTTPServer(port, secure, options
|
|
194
|
+
servers.push(getHTTPServer(port, secure, options));
|
|
195
195
|
if (typeof listener === 'function') {
|
|
196
196
|
httpResponders[options?.runFirst ? 'unshift' : 'push']({ listener, port: options?.port || port });
|
|
197
197
|
} else {
|
|
@@ -203,8 +203,9 @@ export function httpServer(listener, options) {
|
|
|
203
203
|
|
|
204
204
|
return servers;
|
|
205
205
|
}
|
|
206
|
-
|
|
207
|
-
|
|
206
|
+
function getHTTPServer(port: number, secure: boolean, options: ServerOptions) {
|
|
207
|
+
const { mtls: isMtls, usageType } = options || {};
|
|
208
|
+
const isOperationsServer = usageType === 'operations-api';
|
|
208
209
|
setPortServerMap(port, { protocol_name: secure ? 'HTTPS' : 'HTTP', name: getComponentName() });
|
|
209
210
|
if (!httpServers[port]) {
|
|
210
211
|
// TODO: These should all come from httpOptions or operationsApiOptions
|
|
@@ -243,7 +244,7 @@ function getHTTPServer(port, secure, isOperationsServer, isMtls) {
|
|
|
243
244
|
rejectUnauthorized: Boolean(mtlsRequired),
|
|
244
245
|
requestCert: Boolean(mtls || isMtls),
|
|
245
246
|
ticketKeys: getTicketKeys(),
|
|
246
|
-
SNICallback: createTLSSelector(
|
|
247
|
+
SNICallback: createTLSSelector(usageType ?? 'server', mtls),
|
|
247
248
|
ciphers: tlsConfig.ciphers ?? tlsConfig[0]?.ciphers,
|
|
248
249
|
});
|
|
249
250
|
}
|
|
@@ -290,7 +291,9 @@ function getHTTPServer(port, secure, isOperationsServer, isMtls) {
|
|
|
290
291
|
if (!response.handlesHeaders) {
|
|
291
292
|
const headers = response.headers || new Headers();
|
|
292
293
|
if (!body) {
|
|
293
|
-
|
|
294
|
+
if (request.method !== 'HEAD') {
|
|
295
|
+
headers.set('Content-Length', '0');
|
|
296
|
+
}
|
|
294
297
|
sentBody = true;
|
|
295
298
|
} else if (body.length >= 0) {
|
|
296
299
|
if (typeof body === 'string') headers.set('Content-Length', Buffer.byteLength(body));
|
|
@@ -517,7 +520,7 @@ type OnWebSocketOptions = {
|
|
|
517
520
|
port?: number;
|
|
518
521
|
securePort?: number;
|
|
519
522
|
maxPayload?: number;
|
|
520
|
-
|
|
523
|
+
usageType?: string;
|
|
521
524
|
mtls?: boolean;
|
|
522
525
|
};
|
|
523
526
|
const websocketListeners = [],
|
|
@@ -537,7 +540,7 @@ function onWebSocket(listener: (ws: WebSocket) => void, options: OnWebSocketOpti
|
|
|
537
540
|
name: getComponentName(),
|
|
538
541
|
});
|
|
539
542
|
|
|
540
|
-
const server = getHTTPServer(port, secure, options
|
|
543
|
+
const server = getHTTPServer(port, secure, options);
|
|
541
544
|
|
|
542
545
|
if (!websocketServers[port]) {
|
|
543
546
|
websocketServers[port] = new WebSocketServer({
|
|
@@ -597,6 +600,7 @@ function onWebSocket(listener: (ws: WebSocket) => void, options: OnWebSocketOpti
|
|
|
597
600
|
}
|
|
598
601
|
|
|
599
602
|
function defaultNotFound(request, response) {
|
|
603
|
+
if (response.headersSent || response.writableEnded) return;
|
|
600
604
|
response.writeHead(404);
|
|
601
605
|
response.end('Not found\n');
|
|
602
606
|
logRequest(request, 404, 0, request.requestId);
|
|
@@ -28,6 +28,7 @@ async function loadRootComponents(isWorkerThread = false) {
|
|
|
28
28
|
await loadComponent(dirname(configUtils.getConfigFilePath()), resources, 'hdb', {
|
|
29
29
|
isRoot: true,
|
|
30
30
|
providedLoadedComponents: loadedComponents,
|
|
31
|
+
autoReload: false,
|
|
31
32
|
});
|
|
32
33
|
if (!process.env.HARPER_SAFE_MODE) {
|
|
33
34
|
// once the global plugins are loaded, we now load all the CF and run applications (and their components)
|
|
@@ -61,7 +61,7 @@ async function operationsServer(options: ServerOptions & { resources?: Resources
|
|
|
61
61
|
//make sure the process waits for the server to be fully instantiated before moving forward
|
|
62
62
|
await server.ready();
|
|
63
63
|
if (!options) options = {};
|
|
64
|
-
options.
|
|
64
|
+
options.usageType = 'operations-api';
|
|
65
65
|
// fastify can't clean up properly
|
|
66
66
|
try {
|
|
67
67
|
// now that server is fully loaded/ready, start listening on port provided in config settings or just use
|
|
@@ -155,6 +155,7 @@ function buildServer(isHttps: boolean, resources: Resources): FastifyInstance {
|
|
|
155
155
|
|
|
156
156
|
app.register(function (instance, options, done) {
|
|
157
157
|
instance.setNotFoundHandler(function (request, reply) {
|
|
158
|
+
if (reply.sent || reply.raw.headersSent || reply.raw.writableEnded) return;
|
|
158
159
|
app.server.emit('unhandled', request.raw, reply.raw);
|
|
159
160
|
});
|
|
160
161
|
done();
|
|
@@ -26,6 +26,9 @@ const ACKNOWLEDGEMENT = 'ack';
|
|
|
26
26
|
let getThreadInfo;
|
|
27
27
|
_assignPackageExport('threads', connectedPorts);
|
|
28
28
|
|
|
29
|
+
const listenersByType = new Map();
|
|
30
|
+
const messagesQueuedByType = new Map();
|
|
31
|
+
|
|
29
32
|
module.exports = {
|
|
30
33
|
startWorker,
|
|
31
34
|
restartWorkers,
|
|
@@ -37,7 +40,6 @@ module.exports = {
|
|
|
37
40
|
onMessageByType,
|
|
38
41
|
broadcast,
|
|
39
42
|
broadcastWithAcknowledgement,
|
|
40
|
-
setChildListenerByType,
|
|
41
43
|
getWorkerIndex,
|
|
42
44
|
getWorkerCount,
|
|
43
45
|
getTicketKeys,
|
|
@@ -94,14 +96,20 @@ Object.defineProperty(server, 'workerCount', {
|
|
|
94
96
|
return getWorkerCount();
|
|
95
97
|
},
|
|
96
98
|
});
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
sendThreadInfo(worker);
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
recordResourceReport(worker, message);
|
|
103
|
-
}
|
|
104
|
-
}
|
|
99
|
+
if (!parentPort) {
|
|
100
|
+
onMessageByType(REQUEST_THREAD_INFO, (message, worker) => {
|
|
101
|
+
if (worker) sendThreadInfo(worker);
|
|
102
|
+
});
|
|
103
|
+
onMessageByType(RESOURCE_REPORT, (message, worker) => {
|
|
104
|
+
if (worker) recordResourceReport(worker, message);
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
// postMessage type listeners that are registered in other ways or can be registered later
|
|
108
|
+
listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED, null);
|
|
109
|
+
listenersByType.set(hdbTerms.ITC_EVENT_TYPES.SCHEMA, null);
|
|
110
|
+
listenersByType.set(hdbTerms.ITC_EVENT_TYPES.USER, null);
|
|
111
|
+
listenersByType.set(hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST, null);
|
|
112
|
+
|
|
105
113
|
function startWorker(path, options = {}) {
|
|
106
114
|
// Take a percentage of total memory to determine the max memory for each thread. The percentage is based
|
|
107
115
|
// on the thread count. Generally, it is unrealistic to efficiently use the majority of total memory for a single
|
|
@@ -200,9 +208,6 @@ function startWorker(path, options = {}) {
|
|
|
200
208
|
} else harperLogger.error(`Thread has been restarted ${worker.restarts} times and will not be restarted`);
|
|
201
209
|
}
|
|
202
210
|
});
|
|
203
|
-
worker.on('message', (message) => {
|
|
204
|
-
childListenerByType[message.type]?.(message, worker);
|
|
205
|
-
});
|
|
206
211
|
workers.push(worker);
|
|
207
212
|
startMonitoring();
|
|
208
213
|
if (options.onStarted) options.onStarted(worker); // notify that it is ready
|
|
@@ -316,9 +321,6 @@ async function restartWorkers(
|
|
|
316
321
|
});
|
|
317
322
|
}
|
|
318
323
|
}
|
|
319
|
-
function setChildListenerByType(type, listener) {
|
|
320
|
-
childListenerByType[type] = listener;
|
|
321
|
-
}
|
|
322
324
|
function shutdownWorkers(name) {
|
|
323
325
|
return restartWorkers(name, Infinity, false);
|
|
324
326
|
}
|
|
@@ -331,11 +333,16 @@ const messageListeners = [];
|
|
|
331
333
|
function onMessageFromWorkers(listener) {
|
|
332
334
|
messageListeners.push(listener);
|
|
333
335
|
}
|
|
334
|
-
const listenersByType = new Map();
|
|
335
336
|
function onMessageByType(type, listener) {
|
|
336
337
|
let listeners = listenersByType.get(type);
|
|
337
338
|
if (!listeners) listenersByType.set(type, (listeners = []));
|
|
338
339
|
listeners.push(listener);
|
|
340
|
+
if (messagesQueuedByType.has(type)) {
|
|
341
|
+
for (let message of messagesQueuedByType.get(type)) {
|
|
342
|
+
listener(message);
|
|
343
|
+
}
|
|
344
|
+
messagesQueuedByType.delete(type);
|
|
345
|
+
}
|
|
339
346
|
}
|
|
340
347
|
|
|
341
348
|
const MAX_SYNC_BROADCAST = 10;
|
|
@@ -526,14 +533,24 @@ function notifyMessageListeners(message, port) {
|
|
|
526
533
|
for (let listener of messageListeners) {
|
|
527
534
|
listener(message, port);
|
|
528
535
|
}
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
+
if (message.type) {
|
|
537
|
+
let listeners = listenersByType.get(message.type);
|
|
538
|
+
if (listeners) {
|
|
539
|
+
for (let listener of listeners) {
|
|
540
|
+
try {
|
|
541
|
+
listener(message, port);
|
|
542
|
+
} catch (error) {
|
|
543
|
+
harperLogger.error(error);
|
|
544
|
+
}
|
|
536
545
|
}
|
|
546
|
+
} else if (listeners !== null) {
|
|
547
|
+
// null means it is registered for a later listener
|
|
548
|
+
harperLogger.warn?.(`No listener registered for worker message type ${message.type}, queuing message`);
|
|
549
|
+
let messages = messagesQueuedByType.get(message.type);
|
|
550
|
+
if (!messages) {
|
|
551
|
+
messagesQueuedByType.set(message.type, (messages = []));
|
|
552
|
+
}
|
|
553
|
+
messages.push(message);
|
|
537
554
|
}
|
|
538
555
|
}
|
|
539
556
|
}
|
|
@@ -564,17 +581,14 @@ if (isMainThread) {
|
|
|
564
581
|
module.exports.watchDir = watchDir;
|
|
565
582
|
if (process.env.WATCH_DIR) watchDir(process.env.WATCH_DIR);
|
|
566
583
|
} else {
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
process.exit(0);
|
|
577
|
-
}, threadTerminationTimeout).unref(); // don't block the shutdown
|
|
578
|
-
}
|
|
584
|
+
onMessageByType(hdbTerms.ITC_EVENT_TYPES.SHUTDOWN, async (message) => {
|
|
585
|
+
module.exports.restartNumber = message.restartNumber;
|
|
586
|
+
parentPort.unref(); // remove this handle
|
|
587
|
+
setTimeout(() => {
|
|
588
|
+
harperLogger.warn('Thread did not voluntarily terminate', threadId);
|
|
589
|
+
// Note that if this occurs, you may want to use this to debug what is currently running:
|
|
590
|
+
// require('why-is-node-running')();
|
|
591
|
+
process.exit(0);
|
|
592
|
+
}, threadTerminationTimeout).unref(); // don't block the shutdown
|
|
579
593
|
});
|
|
580
594
|
}
|
|
@@ -81,8 +81,8 @@ describe('test REST with property updates', function () {
|
|
|
81
81
|
}
|
|
82
82
|
);
|
|
83
83
|
assert.equal(response.status, 400);
|
|
84
|
-
assert(response.data.includes('property name must be a string'));
|
|
85
|
-
assert(response.data.includes('property age must be an integer'));
|
|
84
|
+
assert(response.data.title.includes('property name must be a string'));
|
|
85
|
+
assert(response.data.title.includes('property age must be an integer'));
|
|
86
86
|
});
|
|
87
87
|
|
|
88
88
|
it('put with nested path', async () => {
|
|
@@ -531,7 +531,7 @@ describe('test REST calls', () => {
|
|
|
531
531
|
},
|
|
532
532
|
});
|
|
533
533
|
assert.equal(response.status, 500);
|
|
534
|
-
assert(response.data.includes('Conflicting paths'));
|
|
534
|
+
assert(response.data.title.includes('Conflicting paths'));
|
|
535
535
|
});
|
|
536
536
|
it('Returns thrown plain object', async () => {
|
|
537
537
|
const response = await axios.get('http://localhost:9926/Echo/error-plain-object', {
|
|
@@ -541,7 +541,7 @@ describe('test REST calls', () => {
|
|
|
541
541
|
},
|
|
542
542
|
});
|
|
543
543
|
assert.equal(response.status, 500);
|
|
544
|
-
assert(response.data.
|
|
544
|
+
assert(response.data.title, 'Test error');
|
|
545
545
|
});
|
|
546
546
|
it('Returns correct error for bad body', async () => {
|
|
547
547
|
const response = await axios.get('http://localhost:9926/Echo/error-bad-body', {
|
|
@@ -19,7 +19,8 @@ describe('Scope', () => {
|
|
|
19
19
|
this.resources = new Resources();
|
|
20
20
|
this.server = {};
|
|
21
21
|
this.directory = mkdtempSync(join(tmpdir(), 'harper.unit-test.scope-'));
|
|
22
|
-
this.
|
|
22
|
+
this.appName = basename(this.directory);
|
|
23
|
+
this.pluginName = 'plugin';
|
|
23
24
|
this.configFilePath = join(this.directory, 'config.yaml');
|
|
24
25
|
this.testFilePath = join(this.directory, 'test.js');
|
|
25
26
|
writeFileSync(this.testFilePath, '"foo";');
|
|
@@ -37,10 +38,11 @@ describe('Scope', () => {
|
|
|
37
38
|
});
|
|
38
39
|
|
|
39
40
|
it('should create a default entry handler', async () => {
|
|
40
|
-
writeFileSync(this.configFilePath, stringify({ [this.
|
|
41
|
+
writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: 'test.js' } }));
|
|
41
42
|
|
|
42
43
|
const scope = new Scope(
|
|
43
|
-
this.
|
|
44
|
+
this.appName,
|
|
45
|
+
this.pluginName,
|
|
44
46
|
this.directory,
|
|
45
47
|
this.configFilePath,
|
|
46
48
|
new ApplicationScope('test', this.resources, this.server)
|
|
@@ -92,10 +94,11 @@ describe('Scope', () => {
|
|
|
92
94
|
});
|
|
93
95
|
|
|
94
96
|
it('should create a default entry handler with urlPath', async () => {
|
|
95
|
-
writeFileSync(this.configFilePath, stringify({ [this.
|
|
97
|
+
writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: 'test.js', urlPath: 'abc' } }));
|
|
96
98
|
|
|
97
99
|
const scope = new Scope(
|
|
98
|
-
this.
|
|
100
|
+
this.appName,
|
|
101
|
+
this.pluginName,
|
|
99
102
|
this.directory,
|
|
100
103
|
this.configFilePath,
|
|
101
104
|
new ApplicationScope('test', this.resources, this.server)
|
|
@@ -148,9 +151,16 @@ describe('Scope', () => {
|
|
|
148
151
|
});
|
|
149
152
|
|
|
150
153
|
it('should call requestRestart if no entry handler is provided', async () => {
|
|
151
|
-
writeFileSync(this.configFilePath, stringify({ [this.
|
|
154
|
+
writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: '.' } }));
|
|
152
155
|
|
|
153
|
-
const scope = new Scope(
|
|
156
|
+
const scope = new Scope(
|
|
157
|
+
this.appName,
|
|
158
|
+
this.pluginName,
|
|
159
|
+
this.directory,
|
|
160
|
+
this.configFilePath,
|
|
161
|
+
this.resources,
|
|
162
|
+
this.server
|
|
163
|
+
);
|
|
154
164
|
|
|
155
165
|
await scope.ready;
|
|
156
166
|
|
|
@@ -165,9 +175,16 @@ describe('Scope', () => {
|
|
|
165
175
|
});
|
|
166
176
|
|
|
167
177
|
it('should call requestRestart if no options handler is provided', async () => {
|
|
168
|
-
writeFileSync(this.configFilePath, stringify({ [this.
|
|
178
|
+
writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: '.' } }));
|
|
169
179
|
|
|
170
|
-
const scope = new Scope(
|
|
180
|
+
const scope = new Scope(
|
|
181
|
+
this.appName,
|
|
182
|
+
this.pluginName,
|
|
183
|
+
this.directory,
|
|
184
|
+
this.configFilePath,
|
|
185
|
+
this.resources,
|
|
186
|
+
this.server
|
|
187
|
+
);
|
|
171
188
|
|
|
172
189
|
await scope.ready;
|
|
173
190
|
|
|
@@ -175,7 +192,7 @@ describe('Scope', () => {
|
|
|
175
192
|
|
|
176
193
|
assert.equal(restartNeeded(), false, 'requestRestart was not called');
|
|
177
194
|
|
|
178
|
-
await writeFile(this.configFilePath, stringify({ [this.
|
|
195
|
+
await writeFile(this.configFilePath, stringify({ [this.pluginName]: { files: '.', foo: 'bar' } }));
|
|
179
196
|
|
|
180
197
|
await waitFor(() => restartNeeded());
|
|
181
198
|
|
|
@@ -185,9 +202,16 @@ describe('Scope', () => {
|
|
|
185
202
|
});
|
|
186
203
|
|
|
187
204
|
it('should emit error for missing default entry handler', async () => {
|
|
188
|
-
writeFileSync(this.configFilePath, stringify({ [this.
|
|
205
|
+
writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { foo: 'bar' } }));
|
|
189
206
|
|
|
190
|
-
const scope = new Scope(
|
|
207
|
+
const scope = new Scope(
|
|
208
|
+
this.appName,
|
|
209
|
+
this.pluginName,
|
|
210
|
+
this.directory,
|
|
211
|
+
this.configFilePath,
|
|
212
|
+
this.resources,
|
|
213
|
+
this.server
|
|
214
|
+
);
|
|
191
215
|
|
|
192
216
|
await scope.ready;
|
|
193
217
|
|
|
@@ -219,9 +243,16 @@ describe('Scope', () => {
|
|
|
219
243
|
});
|
|
220
244
|
|
|
221
245
|
it('should support custom entry handlers', async () => {
|
|
222
|
-
writeFileSync(this.configFilePath, stringify({ [this.
|
|
246
|
+
writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { foo: 'bar' } }));
|
|
223
247
|
|
|
224
|
-
const scope = new Scope(
|
|
248
|
+
const scope = new Scope(
|
|
249
|
+
this.appName,
|
|
250
|
+
this.pluginName,
|
|
251
|
+
this.directory,
|
|
252
|
+
this.configFilePath,
|
|
253
|
+
this.resources,
|
|
254
|
+
this.server
|
|
255
|
+
);
|
|
225
256
|
|
|
226
257
|
await scope.ready;
|
|
227
258
|
|
|
@@ -249,9 +280,16 @@ describe('Scope', () => {
|
|
|
249
280
|
});
|
|
250
281
|
|
|
251
282
|
it('should support synchronous handleEntry with event-based initial load tracking', async () => {
|
|
252
|
-
writeFileSync(this.configFilePath, stringify({ [this.
|
|
283
|
+
writeFileSync(this.configFilePath, stringify({ [this.pluginName]: { files: 'test.js' } }));
|
|
253
284
|
|
|
254
|
-
const scope = new Scope(
|
|
285
|
+
const scope = new Scope(
|
|
286
|
+
this.appName,
|
|
287
|
+
this.pluginName,
|
|
288
|
+
this.directory,
|
|
289
|
+
this.configFilePath,
|
|
290
|
+
this.resources,
|
|
291
|
+
this.server
|
|
292
|
+
);
|
|
255
293
|
|
|
256
294
|
await scope.ready;
|
|
257
295
|
|
|
@@ -3,10 +3,14 @@ import { Resource } from 'harperdb';
|
|
|
3
3
|
import { connect } from 'mqtt'; // verify we can import from node_modules packages
|
|
4
4
|
import 'micromatch';
|
|
5
5
|
import 'needle';
|
|
6
|
+
import { testCircularExport } from './circular.js';
|
|
6
7
|
// TODO: Verify/support circular dependencies
|
|
7
8
|
console.log('Verifying we can access console.log in transitive module in application');
|
|
9
|
+
assert(testCircularExport);
|
|
8
10
|
// verify we can't access parent global variables
|
|
9
11
|
assert(typeof globalVariableFromParent, 'undefined', 'Global variable from parent value should not be present');
|
|
10
12
|
|
|
11
13
|
export class MyComponent extends Resource {}
|
|
12
14
|
export { connect };
|
|
15
|
+
|
|
16
|
+
assert.equal(testCircularExport(), MyComponent);
|
|
@@ -2,6 +2,7 @@ import assert from 'node:assert'; // verify we can access safe node built-in mod
|
|
|
2
2
|
import { Resource, getContext } from 'harper';
|
|
3
3
|
import { MyComponent, connect as connectFromChild } from './child-dir/in-child-dir.js';
|
|
4
4
|
import { connect } from 'mqtt'; // verify we can import from node_modules packages
|
|
5
|
+
import { fork, spawn } from 'node:child_process';
|
|
5
6
|
|
|
6
7
|
console.log('Verifying we can access console.log in application');
|
|
7
8
|
// verify we can't access parent global variables
|
|
@@ -20,7 +21,49 @@ export const testExport = {
|
|
|
20
21
|
let _a = MyComponent;
|
|
21
22
|
return 'hello world';
|
|
22
23
|
},
|
|
24
|
+
async testLoadTypeScript() {
|
|
25
|
+
return await import('./child-dir/typestrip.ts');
|
|
26
|
+
},
|
|
23
27
|
};
|
|
24
28
|
export class TestComponent extends Resource {}
|
|
25
29
|
|
|
26
30
|
export { MyComponent as 'my-component' };
|
|
31
|
+
|
|
32
|
+
export const processSpawnTest = {
|
|
33
|
+
get() {}, // make it look like a resource
|
|
34
|
+
testFork() {
|
|
35
|
+
// Fork should work (allowed command)
|
|
36
|
+
const child = fork('npm', ['--version'], { name: 'test-npm-process' });
|
|
37
|
+
assert(child.pid, 'Fork should return a process with a PID');
|
|
38
|
+
return child;
|
|
39
|
+
},
|
|
40
|
+
testSpawnDisallowed() {
|
|
41
|
+
// Spawn with disallowed command should throw
|
|
42
|
+
try {
|
|
43
|
+
spawn('curl', ['https://example.com'], { name: 'test-curl-process' });
|
|
44
|
+
throw new Error('Should have thrown an error for disallowed command');
|
|
45
|
+
} catch (err) {
|
|
46
|
+
assert(err.message.includes('not allowed'), 'Should throw error about disallowed command');
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
testSpawnWithoutName() {
|
|
50
|
+
// Spawn without name should throw
|
|
51
|
+
try {
|
|
52
|
+
spawn('npm', ['build']);
|
|
53
|
+
throw new Error('Should have thrown an error for missing name');
|
|
54
|
+
} catch (err) {
|
|
55
|
+
assert(err.message.includes('name'), 'Should throw error about missing name');
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
testProcessReuse(childProcessPath) {
|
|
59
|
+
// First call should fork a new process
|
|
60
|
+
const child1 = fork(childProcessPath, [], { name: 'test-reuse-process' });
|
|
61
|
+
assert(child1.pid, 'First fork should return a process with a PID');
|
|
62
|
+
|
|
63
|
+
// Second call with same name should return wrapper for existing process
|
|
64
|
+
const child2 = fork(childProcessPath, [], { name: 'test-reuse-process' });
|
|
65
|
+
assert.equal(child1.pid, child2.pid, 'Second fork should return same PID');
|
|
66
|
+
|
|
67
|
+
return { child1, child2 };
|
|
68
|
+
},
|
|
69
|
+
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// Simple child process that stays alive until killed
|
|
2
|
+
// Used for testing process reuse functionality
|
|
3
|
+
|
|
4
|
+
let isRunning = true;
|
|
5
|
+
|
|
6
|
+
process.on('SIGTERM', () => {
|
|
7
|
+
isRunning = false;
|
|
8
|
+
process.exit(0);
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
// Keep process alive
|
|
12
|
+
const interval = setInterval(() => {
|
|
13
|
+
if (!isRunning) {
|
|
14
|
+
clearInterval(interval);
|
|
15
|
+
}
|
|
16
|
+
}, 100);
|
|
17
|
+
|
|
18
|
+
console.log('Child process started with PID:', process.pid);
|
|
@@ -2,10 +2,13 @@ const assert = require('node:assert/strict');
|
|
|
2
2
|
const path = require('node:path');
|
|
3
3
|
const { loadComponent, loadedPaths } = require('#src/components/componentLoader');
|
|
4
4
|
const { PACKAGE_ROOT } = require('#js/utility/packageUtils');
|
|
5
|
+
const fs = require('node:fs');
|
|
6
|
+
const env = require('#src/utility/environment/environmentManager');
|
|
5
7
|
const { ApplicationScope } = require('#js/components/ApplicationScope');
|
|
6
8
|
|
|
7
9
|
describe('Global Variable Isolation in testJSWithDeps', function () {
|
|
8
10
|
let mockResources;
|
|
11
|
+
let pidsDir;
|
|
9
12
|
|
|
10
13
|
beforeEach(function () {
|
|
11
14
|
// Create mock resources
|
|
@@ -17,12 +20,24 @@ describe('Global Variable Isolation in testJSWithDeps', function () {
|
|
|
17
20
|
// Set a global variable in the parent context
|
|
18
21
|
global.globalVariableFromParent = 'parent-value';
|
|
19
22
|
loadedPaths.clear();
|
|
23
|
+
|
|
24
|
+
// Clean up pids directory
|
|
25
|
+
const basePath = env.getHdbBasePath();
|
|
26
|
+
pidsDir = path.join(basePath, 'pids');
|
|
27
|
+
if (fs.existsSync(pidsDir)) {
|
|
28
|
+
fs.rmSync(pidsDir, { recursive: true, force: true });
|
|
29
|
+
}
|
|
20
30
|
});
|
|
21
31
|
|
|
22
32
|
afterEach(function () {
|
|
23
33
|
// Clean up global variables
|
|
24
34
|
delete global.globalVariableFromParent;
|
|
25
35
|
delete global.globalVariableFromComponent;
|
|
36
|
+
|
|
37
|
+
// Clean up pids directory
|
|
38
|
+
if (fs.existsSync(pidsDir)) {
|
|
39
|
+
fs.rmSync(pidsDir, { recursive: true, force: true });
|
|
40
|
+
}
|
|
26
41
|
});
|
|
27
42
|
const componentDir = path.join(__dirname, 'fixtures', 'testJSWithDeps');
|
|
28
43
|
|
|
@@ -39,7 +54,7 @@ describe('Global Variable Isolation in testJSWithDeps', function () {
|
|
|
39
54
|
|
|
40
55
|
// The component's resources.js file asserts that globalVariableFromParent is undefined
|
|
41
56
|
// If the component loaded without throwing, it means global variables are properly isolated
|
|
42
|
-
|
|
57
|
+
assert.equal(mockResources.get('/'), undefined); // this will contain an error if it failed to load
|
|
43
58
|
// Verify the component's global variable didn't leak into our context
|
|
44
59
|
assert.equal(
|
|
45
60
|
typeof global.globalVariableFromComponent,
|
|
@@ -52,6 +67,7 @@ describe('Global Variable Isolation in testJSWithDeps', function () {
|
|
|
52
67
|
|
|
53
68
|
// verify the exported resource works
|
|
54
69
|
assert.equal(mockResources.get('/testExport').get(), 'hello world');
|
|
70
|
+
assert((await mockResources.get('/testExport').testLoadTypeScript()).isTyped, 'TypeScript exports');
|
|
55
71
|
assert.equal(typeof mockResources.get('/TestComponent').get, 'function');
|
|
56
72
|
assert.equal(typeof mockResources.get('/my-component').get, 'function');
|
|
57
73
|
});
|
|
@@ -102,4 +118,74 @@ describe('Global Variable Isolation in testJSWithDeps', function () {
|
|
|
102
118
|
// assert(typeof mockResources.get('/my-component').get === 'function'); // this syntax doesn't seem to work
|
|
103
119
|
// with SES Compartments
|
|
104
120
|
});
|
|
121
|
+
|
|
122
|
+
it('should enforce process spawning restrictions', async function () {
|
|
123
|
+
let applicationScope = new ApplicationScope('test', mockResources, server);
|
|
124
|
+
Object.assign(applicationScope, {
|
|
125
|
+
mode: 'vm',
|
|
126
|
+
dependencyContainment: false,
|
|
127
|
+
verifyPath: PACKAGE_ROOT,
|
|
128
|
+
});
|
|
129
|
+
await loadComponent(componentDir, mockResources, 'test-origin', {
|
|
130
|
+
applicationScope,
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
const processSpawnTest = mockResources.get('/processSpawnTest');
|
|
134
|
+
|
|
135
|
+
// Test that disallowed commands throw
|
|
136
|
+
processSpawnTest.testSpawnDisallowed();
|
|
137
|
+
|
|
138
|
+
// Test that spawn without name throws
|
|
139
|
+
processSpawnTest.testSpawnWithoutName();
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it('should allow fork with allowed commands', async function () {
|
|
143
|
+
let applicationScope = new ApplicationScope('test', mockResources, server);
|
|
144
|
+
Object.assign(applicationScope, {
|
|
145
|
+
mode: 'vm',
|
|
146
|
+
dependencyContainment: false,
|
|
147
|
+
verifyPath: PACKAGE_ROOT,
|
|
148
|
+
});
|
|
149
|
+
await loadComponent(componentDir, mockResources, 'test-origin', {
|
|
150
|
+
applicationScope,
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
const processSpawnTest = mockResources.get('/processSpawnTest');
|
|
154
|
+
|
|
155
|
+
// Test that fork works
|
|
156
|
+
const child = processSpawnTest.testFork();
|
|
157
|
+
assert(child.pid, 'Should return a child process with PID');
|
|
158
|
+
|
|
159
|
+
// Clean up
|
|
160
|
+
child.kill();
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it('should reuse existing processes with same name', async function () {
|
|
164
|
+
this.timeout(10000); // Increase timeout for process spawning
|
|
165
|
+
|
|
166
|
+
let applicationScope = new ApplicationScope('test', mockResources, server);
|
|
167
|
+
Object.assign(applicationScope, {
|
|
168
|
+
mode: 'vm',
|
|
169
|
+
dependencyContainment: false,
|
|
170
|
+
verifyPath: PACKAGE_ROOT,
|
|
171
|
+
});
|
|
172
|
+
await loadComponent(componentDir, mockResources, 'test-origin', {
|
|
173
|
+
applicationScope,
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
const processSpawnTest = mockResources.get('/processSpawnTest');
|
|
177
|
+
const childProcessPath = path.join(componentDir, 'test-child-process.js');
|
|
178
|
+
|
|
179
|
+
// Test process reuse
|
|
180
|
+
const { child1, child2 } = processSpawnTest.testProcessReuse(childProcessPath);
|
|
181
|
+
|
|
182
|
+
// Verify both have same PID
|
|
183
|
+
assert.equal(child1.pid, child2.pid, 'Should reuse existing process');
|
|
184
|
+
|
|
185
|
+
// Verify exit event is emitted on wrapper
|
|
186
|
+
await new Promise((resolve) => {
|
|
187
|
+
child2.on('exit', resolve);
|
|
188
|
+
child1.kill();
|
|
189
|
+
});
|
|
190
|
+
});
|
|
105
191
|
});
|