@harperfast/harper-pro 5.0.0-alpha.9 → 5.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/.dockerignore +9 -0
- package/core/.git-blame-ignore-revs +2 -0
- package/core/.github/workflows/create-release.yaml +4 -4
- package/core/.github/workflows/integration-tests.yml +12 -10
- package/core/.github/workflows/notify-release-published.yaml +1 -1
- package/core/.github/workflows/publish-docker.yaml +2 -2
- package/core/.github/workflows/publish-npm.yaml +4 -4
- package/core/CONTRIBUTING.md +1 -1
- package/core/Dockerfile +62 -0
- package/core/build-tools/build-studio.sh +12 -0
- package/core/build-tools/build.sh +22 -0
- package/core/build-tools/download-prebuilds.js +13 -0
- package/core/components/Logger.ts +14 -0
- package/core/components/Scope.ts +35 -11
- package/core/components/componentLoader.ts +27 -10
- package/core/components/operations.js +10 -2
- package/core/config/configUtils.js +1 -1
- package/core/dataLayer/CreateTableObject.js +2 -2
- package/core/dataLayer/schema.js +7 -5
- package/core/dataLayer/schemaDescribe.js +1 -1
- package/core/index.d.ts +11 -6
- package/core/index.js +2 -0
- package/core/integrationTests/README.md +24 -0
- package/core/integrationTests/apiTests/tests/10_otherRoleTests.mjs +6 -6
- package/core/integrationTests/apiTests/tests/12_configuration.mjs +1 -1
- package/core/integrationTests/apiTests/tests/14_tokenAuth.mjs +2 -2
- package/core/integrationTests/apiTests/tests/16_terminologyUpdates.mjs +4 -4
- package/core/integrationTests/apiTests/tests/1_environmentSetup.mjs +1 -1
- package/core/integrationTests/apiTests/tests/2_dataLoad.mjs +4 -4
- package/core/integrationTests/apiTests/tests/3_sqlTests.mjs +3 -3
- package/core/integrationTests/apiTests/tests/4_noSqlTests.mjs +12 -12
- package/core/integrationTests/apiTests/tests/5_noSqlRoleTesting.mjs +8 -8
- package/core/integrationTests/apiTests/tests/7_jobsAndJobRoleTesting.mjs +10 -12
- package/core/integrationTests/apiTests/tests/8_deleteTests.mjs +8 -8
- package/core/integrationTests/apiTests/tests/9_transactions.mjs +2 -2
- package/core/integrationTests/apiTests/utils/search.mjs +1 -1
- package/core/integrationTests/apiTests/utils/table.mjs +1 -1
- package/core/integrationTests/server/operation-user-rbac.test.ts +1 -1
- package/core/integrationTests/server/operations-server.test.ts +1 -1
- package/core/integrationTests/server/storage-reclamation.test.ts +1 -1
- package/core/integrationTests/utils/README.md +1 -15
- package/core/integrationTests/utils/harperLifecycle.ts +33 -21
- package/core/package.json +23 -5
- package/core/resources/ResourceInterface.ts +1 -1
- package/core/resources/Table.ts +26 -11
- package/core/resources/analytics/read.ts +33 -26
- package/core/resources/analytics/write.ts +3 -7
- package/core/resources/databases.ts +29 -18
- package/core/resources/search.ts +10 -5
- package/core/security/auth.ts +1 -1
- package/core/security/jsLoader.ts +302 -83
- package/core/security/keys.js +11 -12
- package/core/security/user.ts +3 -3
- package/core/server/REST.ts +18 -2
- package/core/server/Server.ts +2 -1
- package/core/server/fastifyRoutes.ts +1 -0
- package/core/server/http.ts +13 -9
- package/core/server/loadRootComponents.js +1 -0
- package/core/server/operationsServer.ts +2 -1
- package/core/server/threads/manageThreads.js +49 -35
- package/core/static/defaultConfig.yaml +3 -0
- package/core/unitTests/apiTests/RESTProperties-test.mjs +2 -2
- package/core/unitTests/apiTests/basicREST-test.mjs +2 -2
- package/core/unitTests/components/Scope.test.js +54 -16
- package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/circular.js +4 -0
- package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/in-child-dir.js +4 -0
- package/core/unitTests/components/fixtures/testJSWithDeps/child-dir/typestrip.ts +2 -0
- package/core/unitTests/components/fixtures/testJSWithDeps/resources.js +43 -0
- package/core/unitTests/components/fixtures/testJSWithDeps/test-child-process.js +18 -0
- package/core/unitTests/components/globalIsolation.test.js +87 -1
- package/core/unitTests/config/configUtils.test.js +1 -260
- package/core/unitTests/resources/query.test.js +16 -1
- package/core/unitTests/resources/vectorIndex.test.js +1 -1
- package/core/unitTests/server/fastifyRoutes/operations.test.js +1 -1
- package/core/unitTests/testUtils.js +0 -17
- package/core/utility/hdbTerms.ts +3 -0
- package/core/utility/installation.ts +2 -5
- package/core/utility/lmdb/commonUtility.js +21 -10
- package/dist/core/{resources/ResourceInterfaceV2.js → components/Logger.js} +1 -1
- package/dist/core/components/Logger.js.map +1 -0
- package/dist/core/components/Scope.js +18 -10
- package/dist/core/components/Scope.js.map +1 -1
- package/dist/core/components/componentLoader.js +17 -10
- package/dist/core/components/componentLoader.js.map +1 -1
- package/dist/core/components/operations.js +2 -2
- package/dist/core/components/operations.js.map +1 -1
- package/dist/core/config/configUtils.js +1 -1
- package/dist/core/config/configUtils.js.map +1 -1
- package/dist/core/dataLayer/CreateTableObject.js +2 -2
- package/dist/core/dataLayer/CreateTableObject.js.map +1 -1
- package/dist/core/dataLayer/schema.js +6 -5
- package/dist/core/dataLayer/schema.js.map +1 -1
- package/dist/core/dataLayer/schemaDescribe.js +1 -1
- package/dist/core/dataLayer/schemaDescribe.js.map +1 -1
- package/dist/core/index.js +2 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/resources/Table.js +12 -4
- package/dist/core/resources/Table.js.map +1 -1
- package/dist/core/resources/analytics/read.js +32 -22
- package/dist/core/resources/analytics/read.js.map +1 -1
- package/dist/core/resources/analytics/write.js +3 -6
- package/dist/core/resources/analytics/write.js.map +1 -1
- package/dist/core/resources/databases.js +22 -19
- package/dist/core/resources/databases.js.map +1 -1
- package/dist/core/resources/search.js +11 -5
- package/dist/core/resources/search.js.map +1 -1
- package/dist/core/security/auth.js +1 -1
- package/dist/core/security/auth.js.map +1 -1
- package/dist/core/security/jsLoader.js +265 -73
- package/dist/core/security/jsLoader.js.map +1 -1
- package/dist/core/security/keys.js +11 -12
- package/dist/core/security/keys.js.map +1 -1
- package/dist/core/security/user.js +3 -3
- package/dist/core/security/user.js.map +1 -1
- package/dist/core/server/REST.js +16 -2
- package/dist/core/server/REST.js.map +1 -1
- package/dist/core/server/Server.js.map +1 -1
- package/dist/core/server/fastifyRoutes.js +2 -0
- package/dist/core/server/fastifyRoutes.js.map +1 -1
- package/dist/core/server/http.js +12 -6
- package/dist/core/server/http.js.map +1 -1
- package/dist/core/server/loadRootComponents.js +1 -0
- package/dist/core/server/loadRootComponents.js.map +1 -1
- package/dist/core/server/operationsServer.js +3 -1
- package/dist/core/server/operationsServer.js.map +1 -1
- package/dist/core/server/threads/manageThreads.js +50 -35
- package/dist/core/server/threads/manageThreads.js.map +1 -1
- package/dist/core/utility/hdbTerms.js +3 -0
- package/dist/core/utility/hdbTerms.js.map +1 -1
- package/dist/core/utility/installation.js.map +1 -1
- package/dist/core/utility/lmdb/commonUtility.js +20 -13
- package/dist/core/utility/lmdb/commonUtility.js.map +1 -1
- package/dist/licensing/usageLicensing.js.map +1 -1
- package/dist/replication/knownNodes.js +5 -37
- package/dist/replication/knownNodes.js.map +1 -1
- package/dist/replication/nodeIdMapping.js +2 -35
- package/dist/replication/nodeIdMapping.js.map +1 -1
- package/dist/replication/replicationConnection.js +15 -6
- package/dist/replication/replicationConnection.js.map +1 -1
- package/dist/replication/replicator.js +3 -2
- package/dist/replication/replicator.js.map +1 -1
- package/dist/replication/setNode.js +1 -1
- package/dist/replication/setNode.js.map +1 -1
- package/dist/security/certificate.js.map +1 -1
- package/licensing/usageLicensing.ts +3 -2
- package/npm-shrinkwrap.json +303 -282
- package/package.json +4 -3
- package/replication/knownNodes.ts +3 -2
- package/replication/nodeIdMapping.ts +1 -1
- package/replication/replicationConnection.ts +33 -8
- package/replication/replicator.ts +7 -2
- package/replication/setNode.ts +1 -1
- package/security/certificate.ts +2 -1
- package/studio/web/assets/{index-v3wIpSYx.js → index-CWN9Wp5V.js} +2 -2
- package/studio/web/assets/{index-v3wIpSYx.js.map → index-CWN9Wp5V.js.map} +1 -1
- package/studio/web/assets/{index-ChCctErQ.js → index-CzghSAn2.js} +2 -2
- package/studio/web/assets/{index-ChCctErQ.js.map → index-CzghSAn2.js.map} +1 -1
- package/studio/web/assets/{index-Qu8D43wo.js → index-DMDhGP7N.js} +5 -5
- package/studio/web/assets/{index-Qu8D43wo.js.map → index-DMDhGP7N.js.map} +1 -1
- package/studio/web/assets/{index.lazy-tVSPM7bX.js → index.lazy-C-yDTGUy.js} +2 -2
- package/studio/web/assets/{index.lazy-tVSPM7bX.js.map → index.lazy-C-yDTGUy.js.map} +1 -1
- package/studio/web/assets/{profiler-C9as4sv-.js → profiler-0fZAOscv.js} +2 -2
- package/studio/web/assets/{profiler-C9as4sv-.js.map → profiler-0fZAOscv.js.map} +1 -1
- package/studio/web/assets/{react-redux-RRIhZnM6.js → react-redux-BIxqK8O6.js} +2 -2
- package/studio/web/assets/{react-redux-RRIhZnM6.js.map → react-redux-BIxqK8O6.js.map} +1 -1
- package/studio/web/assets/{startRecording-DYa4zCXV.js → startRecording-Ca3Gf2MY.js} +2 -2
- package/studio/web/assets/{startRecording-DYa4zCXV.js.map → startRecording-Ca3Gf2MY.js.map} +1 -1
- package/studio/web/index.html +1 -1
- package/core/resources/ResourceInterfaceV2.ts +0 -53
- package/core/resources/ResourceV2.ts +0 -67
- package/core/resources/analytics/profile.ts +0 -109
- package/core/unitTests/apiTests/analytics-test.mjs +0 -38
- package/core/v1.d.ts +0 -47
- package/core/v1.js +0 -38
- package/core/v2.d.ts +0 -47
- package/core/v2.js +0 -38
- package/dist/core/resources/ResourceInterfaceV2.js.map +0 -1
- package/dist/core/resources/ResourceV2.js +0 -27
- package/dist/core/resources/ResourceV2.js.map +0 -1
- package/dist/core/resources/analytics/profile.js +0 -144
- package/dist/core/resources/analytics/profile.js.map +0 -1
|
@@ -3,7 +3,6 @@ import { contextStorage, transaction } from '../resources/transaction.ts';
|
|
|
3
3
|
import { RequestTarget } from '../resources/RequestTarget.ts';
|
|
4
4
|
import { tables, databases } from '../resources/databases.ts';
|
|
5
5
|
import { readFile } from 'node:fs/promises';
|
|
6
|
-
import { readFileSync } from 'node:fs';
|
|
7
6
|
import { dirname, isAbsolute } from 'node:path';
|
|
8
7
|
import { pathToFileURL, fileURLToPath } from 'node:url';
|
|
9
8
|
import { SourceTextModule, SyntheticModule, createContext, runInContext } from 'node:vm';
|
|
@@ -11,9 +10,13 @@ import { ApplicationScope } from '../components/ApplicationScope.ts';
|
|
|
11
10
|
import logger from '../utility/logging/harper_logger.js';
|
|
12
11
|
import { createRequire } from 'node:module';
|
|
13
12
|
import * as env from '../utility/environment/environmentManager';
|
|
13
|
+
import * as child_process from 'node:child_process';
|
|
14
14
|
import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
|
|
15
15
|
import { contentTypes } from '../server/serverHelpers/contentTypes.ts';
|
|
16
16
|
import type { CompartmentOptions } from 'ses';
|
|
17
|
+
import { mkdirSync, readFileSync, writeFileSync, unlinkSync, openSync, closeSync, statSync } from 'node:fs';
|
|
18
|
+
import { join } from 'node:path';
|
|
19
|
+
import { EventEmitter } from 'node:events';
|
|
17
20
|
|
|
18
21
|
type Lockdown = 'none' | 'freeze' | 'ses';
|
|
19
22
|
const APPLICATIONS_LOCKDOWN: Lockdown = env.get(CONFIG_PARAMS.APPLICATIONS_LOCKDOWN);
|
|
@@ -86,12 +89,13 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
|
|
|
86
89
|
if (!scope.compartment) scope.compartment = getCompartment(scope, globals);
|
|
87
90
|
const result = await (await scope.compartment).import(moduleUrl);
|
|
88
91
|
return result.namespace;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
return await import(moduleUrl);
|
|
92
|
+
} else if (SourceTextModule) {
|
|
93
|
+
// else use standard node:vm module to do containment (if it is available)
|
|
94
|
+
return await loadModuleWithVM(moduleUrl, scope);
|
|
95
|
+
}
|
|
94
96
|
}
|
|
97
|
+
// important! we need to await the import, otherwise the error will not be caught
|
|
98
|
+
return await import(moduleUrl);
|
|
95
99
|
} catch (err) {
|
|
96
100
|
try {
|
|
97
101
|
// the actual parse error (internally known as the "arrow message")
|
|
@@ -107,11 +111,23 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
|
|
|
107
111
|
}
|
|
108
112
|
}
|
|
109
113
|
|
|
114
|
+
let amaro: typeof import('amaro') | undefined;
|
|
115
|
+
/**
|
|
116
|
+
* Strip TypeScript types using the amaro library (what Node.js uses internally)
|
|
117
|
+
* Falls back to regex-based stripping if amaro is not available
|
|
118
|
+
*/
|
|
119
|
+
async function stripTypeScriptTypes(source: string): Promise<string> {
|
|
120
|
+
// Use amaro - the library that Node.js uses internally for type stripping
|
|
121
|
+
amaro = await import('amaro');
|
|
122
|
+
return amaro.transformSync(source, { mode: 'strip-only' }).code;
|
|
123
|
+
}
|
|
124
|
+
|
|
110
125
|
/**
|
|
111
126
|
* Load a module using Node's vm.Module API with (not really secure) sandboxing
|
|
112
127
|
*/
|
|
113
128
|
async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
114
|
-
const moduleCache = new Map<string,
|
|
129
|
+
const moduleCache = new Map<string, SourceTextModule | SyntheticModule>();
|
|
130
|
+
const linkingPromises = new Map<string, Promise<void>>();
|
|
115
131
|
|
|
116
132
|
// Create a secure context with limited globals
|
|
117
133
|
const contextObject = getGlobalObject(scope);
|
|
@@ -128,11 +144,16 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
128
144
|
if (specifier.startsWith('file://')) {
|
|
129
145
|
return specifier;
|
|
130
146
|
}
|
|
131
|
-
|
|
132
|
-
if (
|
|
133
|
-
|
|
147
|
+
// For relative paths, resolve to absolute file URL
|
|
148
|
+
if (specifier.startsWith('.')) {
|
|
149
|
+
const resolved = createRequire(referrer).resolve(specifier);
|
|
150
|
+
if (isAbsolute(resolved)) {
|
|
151
|
+
return pathToFileURL(resolved).toString();
|
|
152
|
+
}
|
|
153
|
+
return resolved;
|
|
134
154
|
}
|
|
135
|
-
|
|
155
|
+
// For package names and node: specifiers, keep as-is for proper require() handling
|
|
156
|
+
return specifier;
|
|
136
157
|
}
|
|
137
158
|
|
|
138
159
|
/**
|
|
@@ -208,33 +229,54 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
208
229
|
async function linker(specifier: string, referencingModule: SourceTextModule | SyntheticModule) {
|
|
209
230
|
const resolvedUrl = resolveModule(specifier, referencingModule.identifier);
|
|
210
231
|
|
|
211
|
-
// Check cache first
|
|
212
|
-
if (moduleCache.has(resolvedUrl)) {
|
|
213
|
-
return moduleCache.get(resolvedUrl)!;
|
|
214
|
-
}
|
|
215
|
-
|
|
216
232
|
const useContainment = specifier.startsWith('.') || scope.dependencyContainment;
|
|
217
|
-
//
|
|
218
|
-
return await
|
|
233
|
+
// Return the module immediately (even if not yet linked) to support circular dependencies
|
|
234
|
+
return await getOrCreateModule(resolvedUrl, useContainment);
|
|
219
235
|
}
|
|
220
236
|
|
|
221
|
-
function
|
|
222
|
-
|
|
237
|
+
async function getOrCreateModule(
|
|
238
|
+
url: string,
|
|
239
|
+
usePrivateGlobal: boolean
|
|
240
|
+
): Promise<SourceTextModule | SyntheticModule> {
|
|
241
|
+
// Check cache first - return cached module immediately (even if not linked yet)
|
|
223
242
|
if (moduleCache.has(url)) {
|
|
224
243
|
return moduleCache.get(url)!;
|
|
225
244
|
}
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
245
|
+
|
|
246
|
+
// Create the module and cache it immediately (before linking)
|
|
247
|
+
const module = await createModule(url, usePrivateGlobal);
|
|
248
|
+
moduleCache.set(url, module);
|
|
249
|
+
|
|
250
|
+
return module;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async function loadModuleWithCache(
|
|
254
|
+
url: string,
|
|
255
|
+
usePrivateGlobal: boolean
|
|
256
|
+
): Promise<SourceTextModule | SyntheticModule> {
|
|
257
|
+
const module = await getOrCreateModule(url, usePrivateGlobal);
|
|
258
|
+
|
|
259
|
+
// Only link/evaluate once per module
|
|
260
|
+
if (!linkingPromises.has(url)) {
|
|
261
|
+
linkingPromises.set(
|
|
262
|
+
url,
|
|
263
|
+
module.link(linker).then(() => module.evaluate())
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// Wait for linking to complete
|
|
268
|
+
await linkingPromises.get(url);
|
|
269
|
+
|
|
270
|
+
return module;
|
|
229
271
|
}
|
|
230
272
|
/**
|
|
231
|
-
*
|
|
273
|
+
* Create a module from URL without linking or evaluating
|
|
232
274
|
*/
|
|
233
|
-
async function
|
|
275
|
+
async function createModule(url: string, usePrivateGlobal: boolean): Promise<SourceTextModule | SyntheticModule> {
|
|
234
276
|
let module: SourceTextModule | SyntheticModule;
|
|
235
277
|
|
|
236
278
|
// Handle special built-in modules
|
|
237
|
-
if (url === 'harper') {
|
|
279
|
+
if (url === 'harper' || url === 'harperdb') {
|
|
238
280
|
let harperExports = getHarperExports(scope);
|
|
239
281
|
module = new SyntheticModule(
|
|
240
282
|
Object.keys(harperExports),
|
|
@@ -245,10 +287,15 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
245
287
|
},
|
|
246
288
|
{ identifier: url, context }
|
|
247
289
|
);
|
|
248
|
-
} else if (
|
|
290
|
+
} else if (url.startsWith('file://')) {
|
|
249
291
|
checkAllowedModulePath(url, scope.verifyPath);
|
|
250
292
|
// Load source text from file
|
|
251
|
-
|
|
293
|
+
let source = await readFile(new URL(url), { encoding: 'utf-8' });
|
|
294
|
+
|
|
295
|
+
// Strip TypeScript types if this is a .ts file
|
|
296
|
+
if (url.endsWith('.ts') || url.endsWith('.tsx')) {
|
|
297
|
+
source = await stripTypeScriptTypes(source);
|
|
298
|
+
}
|
|
252
299
|
|
|
253
300
|
// Try to parse as ESM first
|
|
254
301
|
try {
|
|
@@ -264,17 +311,8 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
264
311
|
return dynamicModule;
|
|
265
312
|
},
|
|
266
313
|
});
|
|
267
|
-
// Cache the module
|
|
268
|
-
moduleCache.set(url, module);
|
|
269
|
-
// Link the module (resolve all imports)
|
|
270
|
-
await module.link(linker);
|
|
271
|
-
|
|
272
|
-
// Evaluate the module
|
|
273
|
-
await module.evaluate();
|
|
274
|
-
return module;
|
|
275
314
|
} catch (err) {
|
|
276
315
|
// If ESM parsing fails, try to load as CommonJS
|
|
277
|
-
// but first try the cache again
|
|
278
316
|
if (
|
|
279
317
|
err.message?.includes('require is not defined') ||
|
|
280
318
|
source.includes('module.exports') ||
|
|
@@ -286,28 +324,41 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
286
324
|
}
|
|
287
325
|
}
|
|
288
326
|
} else {
|
|
289
|
-
checkAllowedModulePath(url, scope.verifyPath);
|
|
290
|
-
// For Node.js built-in modules (node:) and npm packages
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
exportNames
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
327
|
+
const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
|
|
328
|
+
// For Node.js built-in modules (node:) and npm packages
|
|
329
|
+
// Always try require first to properly handle CJS modules with named exports
|
|
330
|
+
try {
|
|
331
|
+
const cjsExports = replacedModule ?? require(url);
|
|
332
|
+
// It's a CJS module - expose all properties as named exports
|
|
333
|
+
const exportNames = Object.keys(cjsExports);
|
|
334
|
+
module = new SyntheticModule(
|
|
335
|
+
exportNames.length > 0 ? [...exportNames, 'default'] : ['default'],
|
|
336
|
+
function () {
|
|
337
|
+
if (exportNames.length > 0) {
|
|
338
|
+
for (const key of exportNames) {
|
|
339
|
+
this.setExport(key, cjsExports[key]);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
this.setExport('default', cjsExports);
|
|
343
|
+
},
|
|
344
|
+
{ identifier: url, context }
|
|
345
|
+
);
|
|
346
|
+
} catch {
|
|
347
|
+
// Fall back to dynamic import for ESM packages
|
|
348
|
+
const importedModule = await import(url);
|
|
349
|
+
const exportNames = Object.keys(importedModule);
|
|
350
|
+
module = new SyntheticModule(
|
|
351
|
+
exportNames,
|
|
352
|
+
function () {
|
|
353
|
+
for (const key of exportNames) {
|
|
354
|
+
this.setExport(key, importedModule[key]);
|
|
355
|
+
}
|
|
356
|
+
},
|
|
357
|
+
{ identifier: url, context }
|
|
358
|
+
);
|
|
359
|
+
}
|
|
303
360
|
}
|
|
304
361
|
|
|
305
|
-
// Link the module (resolve all imports)
|
|
306
|
-
await module.link(linker);
|
|
307
|
-
|
|
308
|
-
// Evaluate the module
|
|
309
|
-
await module.evaluate();
|
|
310
|
-
|
|
311
362
|
return module;
|
|
312
363
|
}
|
|
313
364
|
|
|
@@ -439,40 +490,208 @@ function getHarperExports(scope: ApplicationScope) {
|
|
|
439
490
|
contentTypes,
|
|
440
491
|
};
|
|
441
492
|
}
|
|
442
|
-
const ALLOWED_NODE_BUILTIN_MODULES =
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
493
|
+
const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES)
|
|
494
|
+
? new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES))
|
|
495
|
+
: {
|
|
496
|
+
// if we don't have a list of allowed modules, allow everything
|
|
497
|
+
has() {
|
|
498
|
+
return true;
|
|
499
|
+
},
|
|
500
|
+
};
|
|
501
|
+
const ALLOWED_COMMANDS = new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDSPAWNCOMMANDS) ?? []);
|
|
502
|
+
const REPLACED_BUILTIN_MODULES = {
|
|
503
|
+
child_process: {
|
|
504
|
+
exec: createSpawn(child_process.exec),
|
|
505
|
+
execFile: createSpawn(child_process.execFile),
|
|
506
|
+
fork: createSpawn(child_process.fork, true), // this is launching node, so deemed safe
|
|
507
|
+
spawn: createSpawn(child_process.spawn),
|
|
508
|
+
},
|
|
509
|
+
};
|
|
510
|
+
/**
|
|
511
|
+
* Creates a ChildProcess-like object for an existing process
|
|
512
|
+
*/
|
|
513
|
+
class ExistingProcessWrapper extends EventEmitter {
|
|
514
|
+
pid: number;
|
|
515
|
+
private checkInterval: NodeJS.Timeout;
|
|
516
|
+
|
|
517
|
+
constructor(pid: number) {
|
|
518
|
+
super();
|
|
519
|
+
this.pid = pid;
|
|
520
|
+
|
|
521
|
+
// Monitor process and emit exit event when it terminates
|
|
522
|
+
this.checkInterval = setInterval(() => {
|
|
523
|
+
try {
|
|
524
|
+
// Signal 0 checks if process exists without actually killing it
|
|
525
|
+
process.kill(pid, 0);
|
|
526
|
+
} catch {
|
|
527
|
+
// Process no longer exists
|
|
528
|
+
clearInterval(this.checkInterval);
|
|
529
|
+
this.emit('exit', null, null);
|
|
530
|
+
}
|
|
531
|
+
}, 1000);
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
// Kill the process
|
|
535
|
+
kill(signal?: NodeJS.Signals | number) {
|
|
536
|
+
try {
|
|
537
|
+
process.kill(this.pid, signal);
|
|
538
|
+
return true;
|
|
539
|
+
} catch {
|
|
540
|
+
return false;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// Clean up interval when wrapper is no longer needed
|
|
545
|
+
unref() {
|
|
546
|
+
clearInterval(this.checkInterval);
|
|
547
|
+
return this;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* Checks if a process with the given PID is running
|
|
553
|
+
*/
|
|
554
|
+
function isProcessRunning(pid: number): boolean {
|
|
555
|
+
try {
|
|
556
|
+
// Signal 0 checks existence without killing
|
|
557
|
+
process.kill(pid, 0);
|
|
558
|
+
return true;
|
|
559
|
+
} catch {
|
|
560
|
+
return false;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* Acquires an exclusive lock using the PID file itself (synchronously with busy-wait)
|
|
566
|
+
* Returns 0 if lock was acquired (need to spawn new process), or the existing PID if process is running
|
|
567
|
+
*/
|
|
568
|
+
function acquirePidFileLock(pidFilePath: string, maxRetries = 100, retryDelay = 5): number {
|
|
569
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
570
|
+
try {
|
|
571
|
+
// Try to open exclusively - 'wx' fails if file exists
|
|
572
|
+
const fd = openSync(pidFilePath, 'wx');
|
|
573
|
+
closeSync(fd);
|
|
574
|
+
return 0; // Successfully acquired lock (file created), caller should spawn process
|
|
575
|
+
} catch (err) {
|
|
576
|
+
if (err.code === 'EEXIST') {
|
|
577
|
+
// File exists - check if it contains a valid running process
|
|
578
|
+
try {
|
|
579
|
+
const pidContent = readFileSync(pidFilePath, 'utf-8');
|
|
580
|
+
const existingPid = parseInt(pidContent.trim(), 10);
|
|
581
|
+
|
|
582
|
+
if (!isNaN(existingPid) && isProcessRunning(existingPid)) {
|
|
583
|
+
// Valid process is running, return its PID immediately
|
|
584
|
+
return existingPid;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// Invalid/empty PID - check file age to determine if it's stale or being written
|
|
588
|
+
const stats = statSync(pidFilePath);
|
|
589
|
+
const fileAge = Date.now() - stats.mtimeMs;
|
|
590
|
+
|
|
591
|
+
// If file is very new (less than 100ms) and empty/invalid, another thread is likely still writing to it
|
|
592
|
+
if (fileAge < 100) {
|
|
593
|
+
// Just wait and retry, don't try to remove
|
|
594
|
+
} else {
|
|
595
|
+
// Stale PID file (old and invalid), try to remove it
|
|
596
|
+
try {
|
|
597
|
+
unlinkSync(pidFilePath);
|
|
598
|
+
} catch {
|
|
599
|
+
// Another thread may have removed it, retry
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
} catch {
|
|
603
|
+
// Couldn't read/stat file, another thread might be modifying it, retry
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// Wait a bit before retrying
|
|
607
|
+
const start = Date.now();
|
|
608
|
+
while (Date.now() - start < retryDelay) {
|
|
609
|
+
// Busy wait
|
|
610
|
+
}
|
|
611
|
+
} else {
|
|
612
|
+
throw err;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
throw new Error(`Failed to acquire PID file lock after ${maxRetries} attempts`);
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess, alwaysAllow?: boolean) {
|
|
621
|
+
const basePath = env.getHdbBasePath();
|
|
622
|
+
return function (command: string, args?: any, options?: any, callback?: (...args: any[]) => void) {
|
|
623
|
+
if (!ALLOWED_COMMANDS.has(command.split(' ')[0]) && !alwaysAllow) {
|
|
624
|
+
throw new Error(`Command ${command} is not allowed`);
|
|
625
|
+
}
|
|
626
|
+
const processName = options?.name;
|
|
627
|
+
if (!processName)
|
|
628
|
+
throw new Error(
|
|
629
|
+
`Calling ${spawnFunction.name} in Harper must have a process "name" in the options to ensure that a single process is started and reused`
|
|
630
|
+
);
|
|
631
|
+
|
|
632
|
+
// Ensure PID directory exists
|
|
633
|
+
const pidDir = join(basePath, 'pids');
|
|
634
|
+
mkdirSync(pidDir, { recursive: true });
|
|
635
|
+
|
|
636
|
+
const pidFilePath = join(pidDir, `${processName}.pid`);
|
|
637
|
+
|
|
638
|
+
// Try to acquire lock - returns 0 if acquired, or existing PID
|
|
639
|
+
const existingPid = acquirePidFileLock(pidFilePath);
|
|
640
|
+
|
|
641
|
+
if (existingPid !== 0) {
|
|
642
|
+
// Existing process is running, return wrapper
|
|
643
|
+
return new ExistingProcessWrapper(existingPid);
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// We acquired the lock (file was created), spawn new process
|
|
647
|
+
const childProcess = spawnFunction(command, args, options, callback);
|
|
648
|
+
|
|
649
|
+
// Write PID to the file we just created
|
|
650
|
+
try {
|
|
651
|
+
writeFileSync(pidFilePath, childProcess.pid.toString(), 'utf-8');
|
|
652
|
+
} catch (err) {
|
|
653
|
+
// Failed to write PID, clean up
|
|
654
|
+
try {
|
|
655
|
+
childProcess.kill();
|
|
656
|
+
unlinkSync(pidFilePath);
|
|
657
|
+
} catch {}
|
|
658
|
+
throw err;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
// Clean up PID file when process exits
|
|
662
|
+
childProcess.on('exit', () => {
|
|
663
|
+
try {
|
|
664
|
+
unlinkSync(pidFilePath);
|
|
665
|
+
} catch {
|
|
666
|
+
// File may already be removed
|
|
667
|
+
}
|
|
668
|
+
});
|
|
669
|
+
|
|
670
|
+
return childProcess;
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* Validates whether a module can be loaded based on security restrictions and returns the module path or replacement.
|
|
676
|
+
* For file URLs, ensures the module is within the containing folder.
|
|
677
|
+
* For node built-in modules, checks against an allowlist and returns any replacements.
|
|
678
|
+
*
|
|
679
|
+
* @param {string} moduleUrl - The URL or identifier of the module to be loaded, which may be a file: URL, node: URL, or bare module specifier.
|
|
680
|
+
* @param {string} containingFolder - The absolute path of the folder that contains the application, used to validate file: URLs are within bounds.
|
|
681
|
+
* @return {any} Returns undefined for allowed file paths, or a replacement module identifier for allowed node built-in modules.
|
|
682
|
+
* @throws {Error} Throws an error if the module is outside the application folder or if the module is not in the allowed list.
|
|
683
|
+
*/
|
|
684
|
+
function checkAllowedModulePath(moduleUrl: string, containingFolder?: string): boolean {
|
|
466
685
|
if (moduleUrl.startsWith('file:')) {
|
|
467
686
|
const path = moduleUrl.slice(7);
|
|
468
687
|
if (!containingFolder || path.startsWith(containingFolder)) {
|
|
469
|
-
return
|
|
688
|
+
return;
|
|
470
689
|
}
|
|
471
690
|
throw new Error(`Can not load module outside of application folder ${containingFolder}`);
|
|
472
691
|
}
|
|
473
692
|
let simpleName = moduleUrl.startsWith('node:') ? moduleUrl.slice(5) : moduleUrl;
|
|
474
693
|
simpleName = simpleName.split('/')[0];
|
|
475
|
-
if (ALLOWED_NODE_BUILTIN_MODULES.has(simpleName)) return
|
|
694
|
+
if (ALLOWED_NODE_BUILTIN_MODULES.has(simpleName)) return REPLACED_BUILTIN_MODULES[simpleName];
|
|
476
695
|
throw new Error(`Module ${moduleUrl} is not allowed to be imported`);
|
|
477
696
|
}
|
|
478
697
|
|
package/core/security/keys.js
CHANGED
|
@@ -202,7 +202,7 @@ function loadCertificates() {
|
|
|
202
202
|
const x509Cert = new X509Certificate(certificatePem);
|
|
203
203
|
let certCn;
|
|
204
204
|
try {
|
|
205
|
-
certCn = getPrimaryHostName(x509Cert);
|
|
205
|
+
certCn = (!ca && config.name) || getPrimaryHostName(x509Cert);
|
|
206
206
|
} catch (err) {
|
|
207
207
|
logger.error?.('error extracting host name from certificate', err);
|
|
208
208
|
return;
|
|
@@ -238,7 +238,7 @@ function loadCertificates() {
|
|
|
238
238
|
|
|
239
239
|
promise = certificateTable.put({
|
|
240
240
|
name: certCn,
|
|
241
|
-
uses: ['https', ...(configKey.includes('operations') ? ['operations'] : [])],
|
|
241
|
+
uses: config.uses ?? ['https', ...(configKey.includes('operations') ? ['operations'] : [])],
|
|
242
242
|
ciphers: config.ciphers,
|
|
243
243
|
certificate: certificatePem,
|
|
244
244
|
private_key_name,
|
|
@@ -459,7 +459,7 @@ async function getCertAuthority() {
|
|
|
459
459
|
let match;
|
|
460
460
|
for (let cert of allCerts) {
|
|
461
461
|
if (!cert.is_authority) continue;
|
|
462
|
-
const matchingPrivateKey =
|
|
462
|
+
const matchingPrivateKey = getPrivateKeyByName(cert.private_key_name);
|
|
463
463
|
if (cert.private_key_name && matchingPrivateKey) {
|
|
464
464
|
const keyCheck = new X509Certificate(cert.certificate).checkPrivateKey(createPrivateKey(matchingPrivateKey));
|
|
465
465
|
if (keyCheck) {
|
|
@@ -724,7 +724,7 @@ function createTLSSelector(type, mtlsOptions) {
|
|
|
724
724
|
server.secureContextsListeners = [];
|
|
725
725
|
}
|
|
726
726
|
return (SNICallback.ready = new Promise((resolve, reject) => {
|
|
727
|
-
|
|
727
|
+
function updateTLS() {
|
|
728
728
|
try {
|
|
729
729
|
secureContexts.clear();
|
|
730
730
|
caCerts.clear();
|
|
@@ -733,7 +733,7 @@ function createTLSSelector(type, mtlsOptions) {
|
|
|
733
733
|
resolve();
|
|
734
734
|
return;
|
|
735
735
|
}
|
|
736
|
-
for
|
|
736
|
+
for (const cert of databases.system.hdb_certificate.search([])) {
|
|
737
737
|
const certificate = cert.certificate;
|
|
738
738
|
const certParsed = new X509Certificate(certificate);
|
|
739
739
|
if (cert.is_authority) {
|
|
@@ -742,17 +742,16 @@ function createTLSSelector(type, mtlsOptions) {
|
|
|
742
742
|
}
|
|
743
743
|
}
|
|
744
744
|
|
|
745
|
-
for
|
|
745
|
+
for (const cert of databases.system.hdb_certificate.search([])) {
|
|
746
746
|
try {
|
|
747
747
|
if (cert.is_authority) {
|
|
748
748
|
continue;
|
|
749
749
|
}
|
|
750
|
-
let isOperations = type === 'operations-api';
|
|
751
750
|
let quality = cert.is_self_signed ? 1 : 3;
|
|
752
751
|
// prefer operations certificates for operations API
|
|
753
|
-
if (
|
|
752
|
+
if (cert.uses?.includes(type)) quality += 1;
|
|
754
753
|
|
|
755
|
-
const private_key =
|
|
754
|
+
const private_key = getPrivateKeyByName(cert.private_key_name);
|
|
756
755
|
|
|
757
756
|
let certificate = cert.certificate;
|
|
758
757
|
const certParsed = new X509Certificate(certificate);
|
|
@@ -776,7 +775,7 @@ function createTLSSelector(type, mtlsOptions) {
|
|
|
776
775
|
let hostnames = cert.hostnames ?? hostnamesFromCert(certParsed);
|
|
777
776
|
if (!Array.isArray(hostnames)) hostnames = [hostnames];
|
|
778
777
|
for (let hostname of hostnames) {
|
|
779
|
-
if (hostname === getHost()) quality += 1; // prefer a certificate that has our hostname in the SANs
|
|
778
|
+
if (hostname === getHost()) quality += 0.1; // prefer a certificate that has our hostname in the SANs
|
|
780
779
|
}
|
|
781
780
|
let secureContext = tls.createSecureContext(secureOptions);
|
|
782
781
|
secureContext.name = cert.name;
|
|
@@ -876,10 +875,10 @@ function createTLSSelector(type, mtlsOptions) {
|
|
|
876
875
|
}
|
|
877
876
|
}
|
|
878
877
|
|
|
879
|
-
|
|
878
|
+
function getPrivateKeyByName(private_key_name) {
|
|
880
879
|
const private_key = privateKeys.get(private_key_name);
|
|
881
880
|
if (!private_key && private_key_name) {
|
|
882
|
-
return
|
|
881
|
+
return fs.readFileSync(
|
|
883
882
|
path.join(envManager.get(CONFIG_PARAMS.ROOTPATH), hdbTerms.LICENSE_KEY_DIR_NAME, private_key_name),
|
|
884
883
|
'utf8'
|
|
885
884
|
);
|
package/core/security/user.ts
CHANGED
|
@@ -165,7 +165,7 @@ async function addUser(user: User | any): Promise<string> {
|
|
|
165
165
|
throw new ClientError(HDB_ERROR_MSGS.USER_ALREADY_EXISTS(cleanUser.username), HTTP_STATUS_CODES.CONFLICT);
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
-
signalling.signalUserChange(new UserEventMsg(process.pid));
|
|
168
|
+
await signalling.signalUserChange(new UserEventMsg(process.pid));
|
|
169
169
|
return `${cleanUser.username} successfully added`;
|
|
170
170
|
}
|
|
171
171
|
|
|
@@ -228,7 +228,7 @@ async function alterUser(jsonMessage) {
|
|
|
228
228
|
});
|
|
229
229
|
|
|
230
230
|
await setUsersWithRolesCache();
|
|
231
|
-
signalling.signalUserChange(new UserEventMsg(process.pid));
|
|
231
|
+
await signalling.signalUserChange(new UserEventMsg(process.pid));
|
|
232
232
|
|
|
233
233
|
return updateResponse;
|
|
234
234
|
}
|
|
@@ -248,7 +248,7 @@ async function dropUser(user: User | any): Promise<string> {
|
|
|
248
248
|
|
|
249
249
|
logger.debug(deleteResponse);
|
|
250
250
|
await setUsersWithRolesCache();
|
|
251
|
-
signalling.signalUserChange(new UserEventMsg(process.pid));
|
|
251
|
+
await signalling.signalUserChange(new UserEventMsg(process.pid));
|
|
252
252
|
return `${user.username} successfully deleted`;
|
|
253
253
|
}
|
|
254
254
|
|
package/core/server/REST.ts
CHANGED
|
@@ -220,6 +220,7 @@ async function http(request: Context & Request, nextHandler) {
|
|
|
220
220
|
}
|
|
221
221
|
return responseObject;
|
|
222
222
|
} catch (error) {
|
|
223
|
+
error ??= new Error('Unknown error occurred');
|
|
223
224
|
let statusCode = error.statusCode ?? request.response.status;
|
|
224
225
|
if (statusCode) {
|
|
225
226
|
if (statusCode === 500) harperLogger.warn(error);
|
|
@@ -232,12 +233,27 @@ async function http(request: Context & Request, nextHandler) {
|
|
|
232
233
|
}
|
|
233
234
|
}
|
|
234
235
|
} else harperLogger.error(error);
|
|
236
|
+
|
|
237
|
+
// RFC 9457 Problem Details
|
|
238
|
+
const status = statusCode || 500;
|
|
239
|
+
// we prefer to use error classes for error codes (constructor.name), but if there is a code, it is probably a node.js
|
|
240
|
+
// error that denotes error codes with a separate property
|
|
241
|
+
const code = error.code ?? error.constructor.name;
|
|
242
|
+
const problemDetail = {
|
|
243
|
+
type: `error:${code}`, // eventually we want this to be a resolvable URI to our docs
|
|
244
|
+
code,
|
|
245
|
+
title: error.message ?? error.toString(),
|
|
246
|
+
status,
|
|
247
|
+
detail: error.detail,
|
|
248
|
+
instance: request.url,
|
|
249
|
+
};
|
|
250
|
+
|
|
235
251
|
const responseObject = {
|
|
236
|
-
status
|
|
252
|
+
status,
|
|
237
253
|
headers,
|
|
238
254
|
body: undefined,
|
|
239
255
|
};
|
|
240
|
-
responseObject.body = serialize(
|
|
256
|
+
responseObject.body = serialize(problemDetail, request, responseObject);
|
|
241
257
|
return responseObject;
|
|
242
258
|
}
|
|
243
259
|
}
|
package/core/server/Server.ts
CHANGED
|
@@ -179,6 +179,7 @@ async function buildServer(isHttps) {
|
|
|
179
179
|
|
|
180
180
|
app.register(function (instance, options, done) {
|
|
181
181
|
instance.setNotFoundHandler(function (request, reply) {
|
|
182
|
+
if (reply.sent || reply.raw.headersSent || reply.raw.writableEnded) return;
|
|
182
183
|
app.server.emit('unhandled', request.raw, reply.raw);
|
|
183
184
|
});
|
|
184
185
|
done();
|