@harperfast/harper-pro 5.0.0-beta.6 → 5.0.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/bin/cliOperations.js +3 -3
- package/core/bin/harper.js +1 -1
- package/core/bin/run.js +2 -2
- package/core/components/Application.ts +6 -3
- package/core/components/ApplicationScope.ts +1 -0
- package/core/components/componentLoader.ts +23 -12
- package/core/components/operations.js +13 -13
- package/core/components/operationsValidation.js +3 -3
- package/core/config/configUtils.js +20 -4
- package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +3 -2
- package/core/resources/DatabaseTransaction.ts +67 -32
- package/core/resources/Resource.ts +17 -6
- package/core/resources/RocksTransactionLogStore.ts +8 -1
- package/core/resources/Table.ts +29 -10
- package/core/resources/analytics/write.ts +6 -0
- package/core/resources/databases.ts +3 -2
- package/core/security/jsLoader.ts +258 -129
- package/core/server/REST.ts +32 -13
- package/core/server/http.ts +6 -3
- package/core/server/itc/serverHandlers.js +1 -1
- package/core/static/defaultConfig.yaml +1 -1
- package/core/utility/hdbTerms.ts +1 -0
- package/core/utility/logging/harper_logger.js +22 -1
- package/core/utility/logging/readLog.js +2 -2
- package/core/utility/npmUtilities.js +2 -2
- package/core/validation/configValidator.js +16 -8
- package/core/validation/readLogValidator.js +2 -2
- package/dist/core/bin/cliOperations.js +3 -3
- package/dist/core/bin/cliOperations.js.map +1 -1
- package/dist/core/bin/harper.js +1 -1
- package/dist/core/bin/run.js +2 -2
- package/dist/core/bin/run.js.map +1 -1
- package/dist/core/components/Application.js +7 -2
- package/dist/core/components/Application.js.map +1 -1
- package/dist/core/components/ApplicationScope.js +1 -0
- package/dist/core/components/ApplicationScope.js.map +1 -1
- package/dist/core/components/componentLoader.js +21 -10
- package/dist/core/components/componentLoader.js.map +1 -1
- package/dist/core/components/operations.js +13 -13
- package/dist/core/components/operations.js.map +1 -1
- package/dist/core/components/operationsValidation.js +3 -3
- package/dist/core/components/operationsValidation.js.map +1 -1
- package/dist/core/config/configUtils.js +23 -3
- package/dist/core/config/configUtils.js.map +1 -1
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +3 -2
- package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js.map +1 -1
- package/dist/core/resources/DatabaseTransaction.js +71 -35
- package/dist/core/resources/DatabaseTransaction.js.map +1 -1
- package/dist/core/resources/Resource.js +11 -4
- package/dist/core/resources/Resource.js.map +1 -1
- package/dist/core/resources/RocksTransactionLogStore.js +8 -1
- package/dist/core/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/core/resources/Table.js +15 -10
- package/dist/core/resources/Table.js.map +1 -1
- package/dist/core/resources/analytics/write.js +6 -0
- package/dist/core/resources/analytics/write.js.map +1 -1
- package/dist/core/resources/databases.js +3 -2
- package/dist/core/resources/databases.js.map +1 -1
- package/dist/core/security/jsLoader.js +223 -116
- package/dist/core/security/jsLoader.js.map +1 -1
- package/dist/core/server/REST.js +30 -14
- package/dist/core/server/REST.js.map +1 -1
- package/dist/core/server/http.js +6 -1
- package/dist/core/server/http.js.map +1 -1
- package/dist/core/server/itc/serverHandlers.js +1 -1
- package/dist/core/server/itc/serverHandlers.js.map +1 -1
- package/dist/core/utility/hdbTerms.js +1 -0
- package/dist/core/utility/hdbTerms.js.map +1 -1
- package/dist/core/utility/logging/harper_logger.js +24 -1
- package/dist/core/utility/logging/harper_logger.js.map +1 -1
- package/dist/core/utility/logging/readLog.js +2 -2
- package/dist/core/utility/logging/readLog.js.map +1 -1
- package/dist/core/utility/npmUtilities.js +2 -2
- package/dist/core/utility/npmUtilities.js.map +1 -1
- package/dist/core/validation/configValidator.js +18 -8
- package/dist/core/validation/configValidator.js.map +1 -1
- package/dist/core/validation/readLogValidator.js +2 -2
- package/dist/core/validation/readLogValidator.js.map +1 -1
- package/dist/licensing/usageLicensing.js +16 -26
- package/dist/licensing/usageLicensing.js.map +1 -1
- package/dist/replication/nodeIdMapping.js +1 -1
- package/dist/replication/nodeIdMapping.js.map +1 -1
- package/dist/replication/replicationConnection.js +2 -2
- package/dist/replication/replicationConnection.js.map +1 -1
- package/licensing/usageLicensing.ts +22 -32
- package/npm-shrinkwrap.json +615 -614
- package/package.json +9 -6
- package/replication/nodeIdMapping.ts +1 -1
- package/replication/replicationConnection.ts +2 -2
- package/static/defaultConfig.yaml +3 -0
- package/studio/web/assets/{index-BckVDix4.js → index-CXQsBaYq.js} +5 -5
- package/studio/web/assets/{index-BckVDix4.js.map → index-CXQsBaYq.js.map} +1 -1
- package/studio/web/assets/{index.lazy-iG1_8dzm.js → index.lazy-C3Ejfvna.js} +2 -2
- package/studio/web/assets/{index.lazy-iG1_8dzm.js.map → index.lazy-C3Ejfvna.js.map} +1 -1
- package/studio/web/assets/{profile-CzjslUXv.js → profile-BbbbWJCN.js} +2 -2
- package/studio/web/assets/{profile-CzjslUXv.js.map → profile-BbbbWJCN.js.map} +1 -1
- package/studio/web/assets/{status-BP4TQJDR.js → status-CFe85l8C.js} +2 -2
- package/studio/web/assets/{status-BP4TQJDR.js.map → status-CFe85l8C.js.map} +1 -1
- package/studio/web/index.html +1 -1
package/core/resources/Table.ts
CHANGED
|
@@ -580,10 +580,16 @@ export function makeTable(options) {
|
|
|
580
580
|
// dictates not to go to source
|
|
581
581
|
if (!this.doesExist()) throw new ServerError('Entry is not cached', 504);
|
|
582
582
|
} else if (resourceOptions?.ensureLoaded) {
|
|
583
|
-
const loadingFromSource = ensureLoadedFromSource(
|
|
583
|
+
const loadingFromSource = ensureLoadedFromSource(
|
|
584
|
+
this.constructor.source,
|
|
585
|
+
id,
|
|
586
|
+
entry,
|
|
587
|
+
request,
|
|
588
|
+
this,
|
|
589
|
+
target
|
|
590
|
+
);
|
|
584
591
|
if (loadingFromSource) {
|
|
585
592
|
txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
|
|
586
|
-
target.loadedFromSource = true;
|
|
587
593
|
return when(loadingFromSource, (entry) => {
|
|
588
594
|
TableResource._updateResource(this, entry);
|
|
589
595
|
return this;
|
|
@@ -988,10 +994,16 @@ export function makeTable(options) {
|
|
|
988
994
|
// dictates not to go to source
|
|
989
995
|
if (!entry?.value) throw new ServerError('Entry is not cached', 504);
|
|
990
996
|
} else if (ensureLoaded) {
|
|
991
|
-
const loadingFromSource = ensureLoadedFromSource(
|
|
997
|
+
const loadingFromSource = ensureLoadedFromSource(
|
|
998
|
+
constructor.source,
|
|
999
|
+
id,
|
|
1000
|
+
entry,
|
|
1001
|
+
context,
|
|
1002
|
+
this,
|
|
1003
|
+
target
|
|
1004
|
+
);
|
|
992
1005
|
if (loadingFromSource) {
|
|
993
1006
|
txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
|
|
994
|
-
target.loadedFromSource = true;
|
|
995
1007
|
return loadingFromSource.then((entry) => entry?.value);
|
|
996
1008
|
}
|
|
997
1009
|
}
|
|
@@ -3732,7 +3744,7 @@ export function makeTable(options) {
|
|
|
3732
3744
|
}
|
|
3733
3745
|
}
|
|
3734
3746
|
|
|
3735
|
-
function ensureLoadedFromSource(source: typeof TableResource, id, entry, context, resource?) {
|
|
3747
|
+
function ensureLoadedFromSource(source: typeof TableResource, id, entry, context, resource?, target?) {
|
|
3736
3748
|
if (hasSourceGet) {
|
|
3737
3749
|
let needsSourceData = false;
|
|
3738
3750
|
if (context.noCache) needsSourceData = true;
|
|
@@ -3751,7 +3763,7 @@ export function makeTable(options) {
|
|
|
3751
3763
|
recordActionBinary(!needsSourceData, 'cache-hit', tableName);
|
|
3752
3764
|
}
|
|
3753
3765
|
if (needsSourceData) {
|
|
3754
|
-
const loadingFromSource = getFromSource(source, id, entry, context).then((entry) => {
|
|
3766
|
+
const loadingFromSource = getFromSource(source, id, entry, context, target).then((entry) => {
|
|
3755
3767
|
if (entry?.value && entry?.value.getRecord?.())
|
|
3756
3768
|
logger.error?.('Can not assign a record that is already a resource');
|
|
3757
3769
|
if (context) {
|
|
@@ -3921,7 +3933,8 @@ export function makeTable(options) {
|
|
|
3921
3933
|
source: typeof TableResource,
|
|
3922
3934
|
id: Id,
|
|
3923
3935
|
existingEntry: Entry,
|
|
3924
|
-
context: Context
|
|
3936
|
+
context: Context,
|
|
3937
|
+
target?
|
|
3925
3938
|
): Promise<Entry> {
|
|
3926
3939
|
const metadataFlags = existingEntry?.metadataFlags;
|
|
3927
3940
|
|
|
@@ -3943,9 +3956,13 @@ export function makeTable(options) {
|
|
|
3943
3956
|
entry.metadataFlags & (INVALIDATED | EVICTED) ||
|
|
3944
3957
|
(entry.expiresAt != undefined && entry.expiresAt < Date.now())
|
|
3945
3958
|
)
|
|
3946
|
-
// try again
|
|
3947
|
-
whenResolved(getFromSource(source, id, primaryStore.getEntry(id), context));
|
|
3948
|
-
else
|
|
3959
|
+
// try again — entry still not valid, need to actually fetch from source
|
|
3960
|
+
whenResolved(getFromSource(source, id, primaryStore.getEntry(id), context, target));
|
|
3961
|
+
else {
|
|
3962
|
+
// served from cache after waiting for another request to resolve
|
|
3963
|
+
if (target) target.loadedFromSource = false;
|
|
3964
|
+
whenResolved(entry);
|
|
3965
|
+
}
|
|
3949
3966
|
};
|
|
3950
3967
|
const lockAcquired = primaryStore.tryLock(id, callback);
|
|
3951
3968
|
|
|
@@ -3957,6 +3974,8 @@ export function makeTable(options) {
|
|
|
3957
3974
|
}, LOCK_TIMEOUT);
|
|
3958
3975
|
});
|
|
3959
3976
|
}
|
|
3977
|
+
// lock acquired — this request will actually load from source
|
|
3978
|
+
if (target) target.loadedFromSource = true;
|
|
3960
3979
|
|
|
3961
3980
|
const existingRecord = existingEntry?.value;
|
|
3962
3981
|
// it is important to remember that this is _NOT_ part of the current transaction; nothing is changing
|
|
@@ -402,7 +402,12 @@ export async function getDirectorySizeAsync(dirPath: string): Promise<number> {
|
|
|
402
402
|
}
|
|
403
403
|
}
|
|
404
404
|
|
|
405
|
+
const DEFAULT_STORAGE_INTERVAL = 10;
|
|
406
|
+
let nodeStorageInterval = DEFAULT_STORAGE_INTERVAL;
|
|
407
|
+
let nodeStorageCycleCount = 0;
|
|
408
|
+
|
|
405
409
|
async function storeNodeStorageMetric(analyticsTable: Table) {
|
|
410
|
+
if (nodeStorageInterval <= 0 || ++nodeStorageCycleCount % nodeStorageInterval !== 1) return;
|
|
406
411
|
try {
|
|
407
412
|
const size = await getDirectorySizeAsync(getHdbBasePath());
|
|
408
413
|
storeMetric(analyticsTable, {
|
|
@@ -689,6 +694,7 @@ if (!parentPort) onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics);
|
|
|
689
694
|
let scheduledTasksRunning;
|
|
690
695
|
function startScheduledTasks() {
|
|
691
696
|
scheduledTasksRunning = true;
|
|
697
|
+
nodeStorageInterval = envGet(CONFIG_PARAMS.ANALYTICS_STORAGEINTERVAL) ?? DEFAULT_STORAGE_INTERVAL;
|
|
692
698
|
const AGGREGATE_PERIOD = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) * 1000;
|
|
693
699
|
if (AGGREGATE_PERIOD) {
|
|
694
700
|
setInterval(
|
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
import { makeTable } from './Table.ts';
|
|
13
13
|
import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
|
|
14
14
|
import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
|
|
15
|
+
import { getConfigPath } from '../config/configUtils.js';
|
|
15
16
|
import { _assignPackageExport } from '../globals.js';
|
|
16
17
|
import { getIndexedValues } from '../utility/lmdb/commonUtility.js';
|
|
17
18
|
import * as signalling from '../utility/signalling.js';
|
|
@@ -176,7 +177,7 @@ export function getDatabases(): Databases {
|
|
|
176
177
|
if (process.env.SCHEMAS_DATA_PATH) schemaConfigs.data = { path: process.env.SCHEMAS_DATA_PATH };
|
|
177
178
|
databasePath =
|
|
178
179
|
process.env.STORAGE_PATH ||
|
|
179
|
-
|
|
180
|
+
getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
|
|
180
181
|
(databasePath && (existsSync(databasePath) ? databasePath : join(getHdbBasePath(), LEGACY_DATABASES_DIR_NAME)));
|
|
181
182
|
if (!databasePath) return;
|
|
182
183
|
|
|
@@ -694,7 +695,7 @@ export function database({ database: databaseName, table: tableName }) {
|
|
|
694
695
|
tablePath ||
|
|
695
696
|
databaseConfig[databaseName]?.path ||
|
|
696
697
|
process.env.STORAGE_PATH ||
|
|
697
|
-
|
|
698
|
+
getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
|
|
698
699
|
(existsSync(join(hdbBasePath, DATABASES_DIR_NAME))
|
|
699
700
|
? join(hdbBasePath, DATABASES_DIR_NAME)
|
|
700
701
|
: join(hdbBasePath, LEGACY_DATABASES_DIR_NAME));
|
|
@@ -56,6 +56,9 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
|
|
|
56
56
|
}
|
|
57
57
|
overridableProperty(Promise.prototype, 'then');
|
|
58
58
|
overridableProperty(Date, 'now');
|
|
59
|
+
for (let name of ['get', 'set', 'has', 'delete', 'clear', 'forEach', 'entries', 'keys', 'values']) {
|
|
60
|
+
overridableProperty(Map.prototype, name);
|
|
61
|
+
}
|
|
59
62
|
for (let Intrinsic of [
|
|
60
63
|
Object,
|
|
61
64
|
Array,
|
|
@@ -121,12 +124,13 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
|
|
|
121
124
|
|
|
122
125
|
let amaro: typeof import('amaro') | undefined;
|
|
123
126
|
/**
|
|
124
|
-
* Strip TypeScript types using the amaro library (what Node.js uses internally)
|
|
125
|
-
* Falls back to regex-based stripping if amaro is not available
|
|
127
|
+
* Strip TypeScript types synchronously using the amaro library (what Node.js uses internally)
|
|
126
128
|
*/
|
|
127
|
-
|
|
129
|
+
function stripTypeScriptTypes(source: string): string {
|
|
128
130
|
// Use amaro - the library that Node.js uses internally for type stripping
|
|
129
|
-
|
|
131
|
+
if (!amaro) {
|
|
132
|
+
amaro = require('amaro');
|
|
133
|
+
}
|
|
130
134
|
return amaro.transformSync(source, { mode: 'strip-only' }).code;
|
|
131
135
|
}
|
|
132
136
|
|
|
@@ -146,12 +150,27 @@ function parseJsonModule(source: string, url: string): any {
|
|
|
146
150
|
* Load a module using Node's vm.Module API with (not really secure) sandboxing
|
|
147
151
|
*/
|
|
148
152
|
async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
153
|
+
// we want to retain the same module caches across any loading with the application scope
|
|
154
|
+
let moduleCaches = scope.moduleCache as {
|
|
155
|
+
moduleCache: Map<string, SourceTextModule | SyntheticModule | Promise<SourceTextModule | SyntheticModule>>;
|
|
156
|
+
linkingPromises: Map<string, Promise<void>>;
|
|
157
|
+
cjsCache: Map<string, { exports: any }>;
|
|
158
|
+
contextObject: any;
|
|
159
|
+
context: any;
|
|
160
|
+
};
|
|
161
|
+
if (!moduleCaches) {
|
|
162
|
+
// if they haven't been initialized, do so now
|
|
163
|
+
const contextObject = getGlobalObject(scope, true);
|
|
164
|
+
moduleCaches = scope.moduleCache = {
|
|
165
|
+
moduleCache: new Map(),
|
|
166
|
+
linkingPromises: new Map(),
|
|
167
|
+
cjsCache: new Map(),
|
|
168
|
+
// Create a secure context with limited globals
|
|
169
|
+
contextObject,
|
|
170
|
+
context: createContext(contextObject),
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
const { moduleCache, linkingPromises, cjsCache, contextObject, context } = moduleCaches;
|
|
155
174
|
|
|
156
175
|
/**
|
|
157
176
|
* Resolve module specifier to absolute URL
|
|
@@ -165,6 +184,9 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
165
184
|
// block harper/* for now (reserving for potential future use)
|
|
166
185
|
throw new Error(`Module ${specifier} is not allowed, may only access the 'harper' module`);
|
|
167
186
|
}
|
|
187
|
+
if (parts[0] === 'file:') {
|
|
188
|
+
return specifier;
|
|
189
|
+
}
|
|
168
190
|
const resolved = createRequire(referrer).resolve(specifier);
|
|
169
191
|
if (isAbsolute(resolved)) {
|
|
170
192
|
return pathToFileURL(resolved).toString();
|
|
@@ -176,7 +198,16 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
176
198
|
* Load a CommonJS module in our private context
|
|
177
199
|
*/
|
|
178
200
|
function loadCJS(url: string, source: string): { exports: any } {
|
|
201
|
+
// Check cache first to handle circular dependencies
|
|
202
|
+
if (cjsCache.has(url)) {
|
|
203
|
+
return cjsCache.get(url)!;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Create module object and cache it immediately (before execution)
|
|
207
|
+
// This allows circular dependencies to get a reference to the incomplete module
|
|
179
208
|
const cjsModule = { exports: {} };
|
|
209
|
+
cjsCache.set(url, cjsModule);
|
|
210
|
+
|
|
180
211
|
if (url.endsWith('.json')) {
|
|
181
212
|
cjsModule.exports = parseJsonModule(source, url);
|
|
182
213
|
return cjsModule;
|
|
@@ -204,7 +235,7 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
204
235
|
filename: url,
|
|
205
236
|
async importModuleDynamically(specifier: string, script) {
|
|
206
237
|
const resolvedUrl = resolveModule(specifier, script.sourceURL);
|
|
207
|
-
const useContainment = specifier.startsWith('.') || scope.dependencyContainment;
|
|
238
|
+
const useContainment = specifier.startsWith('.') || scope.dependencyContainment !== false;
|
|
208
239
|
const dynamicModule = await loadModuleWithCache(resolvedUrl, useContainment);
|
|
209
240
|
return dynamicModule;
|
|
210
241
|
},
|
|
@@ -221,16 +252,18 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
221
252
|
}
|
|
222
253
|
function loadCJSModule(url: string, source: string, usePrivateGlobal: boolean): SyntheticModule {
|
|
223
254
|
const cjsModule = usePrivateGlobal ? loadCJS(url, source) : { exports: require(url) };
|
|
224
|
-
|
|
255
|
+
let exports = cjsModule.exports;
|
|
256
|
+
if (exports.default === undefined) {
|
|
257
|
+
// provide the default export for compatibility
|
|
258
|
+
exports = { default: exports, ...exports };
|
|
259
|
+
}
|
|
260
|
+
const exportNames = Object.keys(exports);
|
|
261
|
+
|
|
225
262
|
const synModule = new SyntheticModule(
|
|
226
|
-
exportNames
|
|
263
|
+
exportNames,
|
|
227
264
|
function () {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
this.setExport(key, cjsModule.exports[key]);
|
|
231
|
-
}
|
|
232
|
-
} else {
|
|
233
|
-
this.setExport('default', cjsModule.exports);
|
|
265
|
+
for (const key of exportNames) {
|
|
266
|
+
this.setExport(key, exports[key]);
|
|
234
267
|
}
|
|
235
268
|
},
|
|
236
269
|
{ identifier: url, context }
|
|
@@ -240,21 +273,78 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
240
273
|
}
|
|
241
274
|
|
|
242
275
|
/**
|
|
243
|
-
*
|
|
276
|
+
* Check if a package (or any of its dependencies) depends on harper
|
|
277
|
+
* Expects a file URL like: file:///path/to/node_modules/package-name/dist/index.js
|
|
278
|
+
*/
|
|
279
|
+
function packageDependsOnHarper(fileUrl: string): boolean {
|
|
280
|
+
try {
|
|
281
|
+
// Convert file:// URL to path
|
|
282
|
+
const filePath = fileURLToPath(fileUrl);
|
|
283
|
+
|
|
284
|
+
// Find the node_modules directory and package name
|
|
285
|
+
// Example: /path/to/node_modules/package-name/dist/index.js
|
|
286
|
+
// or: /path/to/node_modules/@scope/package-name/dist/index.js
|
|
287
|
+
const nodeModulesMarker = '/node_modules/';
|
|
288
|
+
const nodeModulesIndex = filePath.lastIndexOf(nodeModulesMarker);
|
|
289
|
+
if (nodeModulesIndex === -1) return false;
|
|
290
|
+
|
|
291
|
+
// Get the part after /node_modules/
|
|
292
|
+
const afterNodeModules = filePath.substring(nodeModulesIndex + nodeModulesMarker.length);
|
|
293
|
+
const parts = afterNodeModules.split('/');
|
|
294
|
+
|
|
295
|
+
// Handle scoped packages (@scope/package-name) vs regular packages (package-name)
|
|
296
|
+
const beforeNodeModules = filePath.substring(0, nodeModulesIndex);
|
|
297
|
+
const packageRoot = parts[0].startsWith('@')
|
|
298
|
+
? join(beforeNodeModules, 'node_modules', parts[0], parts[1])
|
|
299
|
+
: join(beforeNodeModules, 'node_modules', parts[0]);
|
|
300
|
+
|
|
301
|
+
// Read package.json from the package root
|
|
302
|
+
const packageJsonPath = join(packageRoot, 'package.json');
|
|
303
|
+
const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8'));
|
|
304
|
+
|
|
305
|
+
const deps = {
|
|
306
|
+
...packageJson.dependencies,
|
|
307
|
+
...packageJson.devDependencies,
|
|
308
|
+
...packageJson.peerDependencies,
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
// Check if harper is a direct dependency
|
|
312
|
+
return Object.keys(deps).some((dep) => HARPER_MODULE_IDS.has(dep));
|
|
313
|
+
} catch {
|
|
314
|
+
return false;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Linker function for module resolution during instantiation.
|
|
320
|
+
* This is synchronous because Node's module.link() requires the linker
|
|
321
|
+
* to return modules synchronously.
|
|
244
322
|
*/
|
|
245
|
-
|
|
323
|
+
function linker(specifier: string, referencingModule: SourceTextModule | SyntheticModule) {
|
|
246
324
|
const resolvedUrl = resolveModule(specifier, referencingModule.identifier);
|
|
247
325
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
326
|
+
// Determine if we should use VM containment for this module
|
|
327
|
+
let useContainment = specifier.startsWith('.'); // Always contain relative imports
|
|
328
|
+
|
|
329
|
+
if (!useContainment && scope.dependencyContainment !== false) {
|
|
330
|
+
// For npm packages, check if they depend on harper
|
|
331
|
+
if (resolvedUrl.startsWith('file://') && resolvedUrl.includes('node_modules')) {
|
|
332
|
+
useContainment = packageDependsOnHarper(resolvedUrl);
|
|
333
|
+
} else {
|
|
334
|
+
// Non-file URLs (bare specifiers) - use default behavior
|
|
335
|
+
useContainment = scope.dependencyContainment === true;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// Return the module
|
|
340
|
+
return getOrCreateModule(resolvedUrl, useContainment);
|
|
251
341
|
}
|
|
252
342
|
|
|
253
|
-
|
|
343
|
+
function getOrCreateModule(
|
|
254
344
|
url: string,
|
|
255
345
|
usePrivateGlobal: boolean
|
|
256
|
-
): Promise<SourceTextModule | SyntheticModule> {
|
|
257
|
-
// Check
|
|
346
|
+
): SourceTextModule | SyntheticModule | Promise<SourceTextModule | SyntheticModule> {
|
|
347
|
+
// Check if module is already created
|
|
258
348
|
if (moduleCache.has(url)) {
|
|
259
349
|
return moduleCache.get(url)!;
|
|
260
350
|
}
|
|
@@ -275,8 +365,15 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
275
365
|
// Only link/evaluate once per module
|
|
276
366
|
if (!linkingPromises.has(url)) {
|
|
277
367
|
const linkingPromise = (async () => {
|
|
278
|
-
|
|
279
|
-
|
|
368
|
+
// Check module status - only link if it's 'unlinked'
|
|
369
|
+
// Status can be: 'unlinked', 'linking', 'linked', 'evaluating', 'evaluated'
|
|
370
|
+
if (module.status === 'unlinked') {
|
|
371
|
+
await module.link(linker);
|
|
372
|
+
}
|
|
373
|
+
// Only evaluate if not already evaluated
|
|
374
|
+
if (module.status === 'linked') {
|
|
375
|
+
await module.evaluate();
|
|
376
|
+
}
|
|
280
377
|
})();
|
|
281
378
|
linkingPromises.set(url, linkingPromise);
|
|
282
379
|
}
|
|
@@ -287,107 +384,107 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
|
|
|
287
384
|
return module;
|
|
288
385
|
}
|
|
289
386
|
/**
|
|
290
|
-
* Create a
|
|
387
|
+
* Create a SyntheticModule from exported object
|
|
291
388
|
*/
|
|
292
|
-
|
|
293
|
-
|
|
389
|
+
function createSyntheticModule(url: string, exportedObject: any): SyntheticModule {
|
|
390
|
+
const exportNames = Object.keys(exportedObject);
|
|
391
|
+
return new SyntheticModule(
|
|
392
|
+
exportNames,
|
|
393
|
+
function () {
|
|
394
|
+
for (const key of exportNames) {
|
|
395
|
+
this.setExport(key, exportedObject[key]);
|
|
396
|
+
}
|
|
397
|
+
},
|
|
398
|
+
{ identifier: url, context }
|
|
399
|
+
);
|
|
400
|
+
}
|
|
294
401
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
402
|
+
/**
|
|
403
|
+
* Normalize imported module to ensure it has proper exports including default
|
|
404
|
+
*/
|
|
405
|
+
function normalizeImportedModule(importedModule: any): any {
|
|
406
|
+
const cjsModule = importedModule['module.exports'];
|
|
407
|
+
if (cjsModule) {
|
|
408
|
+
// back-compat import
|
|
409
|
+
importedModule = importedModule.default ? { default: importedModule.default, ...cjsModule } : cjsModule;
|
|
410
|
+
}
|
|
411
|
+
// Ensure there's a default export for ESM imports that expect it
|
|
412
|
+
if (!importedModule.default) {
|
|
413
|
+
importedModule = { default: importedModule, ...importedModule };
|
|
414
|
+
}
|
|
415
|
+
return importedModule;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Create a SourceTextModule or SyntheticModule from source code
|
|
420
|
+
*/
|
|
421
|
+
function createModuleFromSource(
|
|
422
|
+
url: string,
|
|
423
|
+
source: string,
|
|
424
|
+
usePrivateGlobal: boolean
|
|
425
|
+
): SourceTextModule | SyntheticModule {
|
|
426
|
+
// Handle JSON modules
|
|
427
|
+
if (url.endsWith('.json')) {
|
|
428
|
+
const jsonData = parseJsonModule(source, url);
|
|
429
|
+
return new SyntheticModule(
|
|
430
|
+
['default'],
|
|
300
431
|
function () {
|
|
301
|
-
|
|
302
|
-
this.setExport(key, harperExports[key]);
|
|
303
|
-
}
|
|
432
|
+
this.setExport('default', jsonData);
|
|
304
433
|
},
|
|
305
434
|
{ identifier: url, context }
|
|
306
435
|
);
|
|
307
|
-
}
|
|
308
|
-
checkAllowedModulePath(url, scope.verifyPath);
|
|
309
|
-
let source = await readFile(new URL(url), { encoding: 'utf-8' });
|
|
310
|
-
|
|
311
|
-
// Handle JSON modules as a SyntheticModule with a default export.
|
|
312
|
-
// JSON imports only support default exports per the ESM spec.
|
|
313
|
-
if (url.endsWith('.json')) {
|
|
314
|
-
const jsonData = parseJsonModule(source, url);
|
|
315
|
-
module = new SyntheticModule(
|
|
316
|
-
['default'],
|
|
317
|
-
function () {
|
|
318
|
-
this.setExport('default', jsonData);
|
|
319
|
-
},
|
|
320
|
-
{ identifier: url, context }
|
|
321
|
-
);
|
|
322
|
-
} else {
|
|
323
|
-
// Strip TypeScript types if this is a .ts file
|
|
324
|
-
if (url.endsWith('.ts') || url.endsWith('.tsx')) {
|
|
325
|
-
source = await stripTypeScriptTypes(source);
|
|
326
|
-
}
|
|
436
|
+
}
|
|
327
437
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
} catch {
|
|
332
|
-
// If CJS loading fails (likely due to ESM syntax like import/export), try ESM
|
|
333
|
-
try {
|
|
334
|
-
module = new SourceTextModule(source, {
|
|
335
|
-
identifier: url,
|
|
336
|
-
context,
|
|
337
|
-
initializeImportMeta(meta) {
|
|
338
|
-
meta.url = url;
|
|
339
|
-
},
|
|
340
|
-
async importModuleDynamically(specifier: string) {
|
|
341
|
-
const resolvedUrl = resolveModule(specifier, url);
|
|
342
|
-
const dynamicModule = await loadModuleWithCache(resolvedUrl, true);
|
|
343
|
-
return dynamicModule;
|
|
344
|
-
},
|
|
345
|
-
});
|
|
346
|
-
} catch (esmErr) {
|
|
347
|
-
// Both failed - throw the ESM error as it's likely more relevant
|
|
348
|
-
throw esmErr;
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
} else {
|
|
353
|
-
const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
|
|
354
|
-
// For Node.js built-in modules (node:) and npm packages
|
|
355
|
-
// Always try require first to properly handle CJS modules with named exports
|
|
356
|
-
try {
|
|
357
|
-
const cjsExports = replacedModule ?? require(url);
|
|
358
|
-
// It's a CJS module - expose all properties as named exports
|
|
359
|
-
const exportNames = Object.keys(cjsExports);
|
|
360
|
-
module = new SyntheticModule(
|
|
361
|
-
exportNames.length > 0 ? [...exportNames, 'default'] : ['default'],
|
|
362
|
-
function () {
|
|
363
|
-
if (exportNames.length > 0) {
|
|
364
|
-
for (const key of exportNames) {
|
|
365
|
-
this.setExport(key, cjsExports[key]);
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
this.setExport('default', cjsExports);
|
|
369
|
-
},
|
|
370
|
-
{ identifier: url, context }
|
|
371
|
-
);
|
|
372
|
-
} catch {
|
|
373
|
-
// Fall back to dynamic import for ESM packages
|
|
374
|
-
const importedModule = await import(url);
|
|
375
|
-
const exportNames = Object.keys(importedModule);
|
|
376
|
-
module = new SyntheticModule(
|
|
377
|
-
exportNames,
|
|
378
|
-
function () {
|
|
379
|
-
for (const key of exportNames) {
|
|
380
|
-
this.setExport(key, importedModule[key]);
|
|
381
|
-
}
|
|
382
|
-
},
|
|
383
|
-
{ identifier: url, context }
|
|
384
|
-
);
|
|
385
|
-
}
|
|
438
|
+
// Strip TypeScript types if this is a .ts file
|
|
439
|
+
if (url.endsWith('.ts') || url.endsWith('.tsx')) {
|
|
440
|
+
source = stripTypeScriptTypes(source);
|
|
386
441
|
}
|
|
387
442
|
|
|
388
|
-
|
|
443
|
+
// Try CJS first since it will fail fast with clear syntax errors on ESM syntax
|
|
444
|
+
try {
|
|
445
|
+
return loadCJSModule(url, source, usePrivateGlobal);
|
|
446
|
+
} catch {
|
|
447
|
+
// If CJS loading fails (likely due to ESM syntax like import/export), try ESM
|
|
448
|
+
return new SourceTextModule(source, {
|
|
449
|
+
identifier: url,
|
|
450
|
+
context,
|
|
451
|
+
initializeImportMeta(meta) {
|
|
452
|
+
meta.url = url;
|
|
453
|
+
},
|
|
454
|
+
importModuleDynamically(specifier: string) {
|
|
455
|
+
const resolvedUrl = resolveModule(specifier, url);
|
|
456
|
+
const useContainment = specifier.startsWith('.') || scope.dependencyContainment !== false;
|
|
457
|
+
return loadModuleWithCache(resolvedUrl, useContainment);
|
|
458
|
+
},
|
|
459
|
+
});
|
|
460
|
+
}
|
|
389
461
|
}
|
|
390
462
|
|
|
463
|
+
/**
|
|
464
|
+
* Create a module from URL without linking or evaluating (async version for initial load)
|
|
465
|
+
*/
|
|
466
|
+
function createModule(
|
|
467
|
+
url: string,
|
|
468
|
+
usePrivateGlobal: boolean
|
|
469
|
+
): SourceTextModule | SyntheticModule | Promise<SourceTextModule | SyntheticModule> {
|
|
470
|
+
// Handle special built-in modules
|
|
471
|
+
if (url === 'harper') {
|
|
472
|
+
return createSyntheticModule(url, getHarperExports(scope));
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
if (url.startsWith('file://') && usePrivateGlobal) {
|
|
476
|
+
checkAllowedModulePath(url, scope.verifyPath);
|
|
477
|
+
const source = readFileSync(new URL(url), { encoding: 'utf-8' });
|
|
478
|
+
return createModuleFromSource(url, source, usePrivateGlobal);
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// For Node.js built-in modules (node:) and npm packages without dependency containment
|
|
482
|
+
const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
|
|
483
|
+
if (replacedModule) {
|
|
484
|
+
return createSyntheticModule(url, normalizeImportedModule(replacedModule));
|
|
485
|
+
}
|
|
486
|
+
return import(url).then((importedModule) => createSyntheticModule(url, normalizeImportedModule(importedModule)));
|
|
487
|
+
}
|
|
391
488
|
// Load the entry module
|
|
392
489
|
const entryModule = await loadModuleWithCache(moduleUrl, true);
|
|
393
490
|
|
|
@@ -433,8 +530,8 @@ async function getCompartment(scope: ApplicationScope, globals) {
|
|
|
433
530
|
},
|
|
434
531
|
};
|
|
435
532
|
} else if (moduleSpecifier.startsWith('file:') && !moduleSpecifier.includes('node_modules')) {
|
|
436
|
-
|
|
437
|
-
// Handle JSON files in
|
|
533
|
+
let moduleText = await readFile(new URL(moduleSpecifier), { encoding: 'utf-8' });
|
|
534
|
+
// Handle JSON files in compartment mode the same way as in VM mode
|
|
438
535
|
if (moduleSpecifier.endsWith('.json')) {
|
|
439
536
|
const jsonData = parseJsonModule(moduleText, moduleSpecifier);
|
|
440
537
|
return {
|
|
@@ -445,6 +542,10 @@ async function getCompartment(scope: ApplicationScope, globals) {
|
|
|
445
542
|
},
|
|
446
543
|
};
|
|
447
544
|
}
|
|
545
|
+
// Strip TypeScript types if this is a .ts file
|
|
546
|
+
if (moduleSpecifier.endsWith('.ts') || moduleSpecifier.endsWith('.tsx')) {
|
|
547
|
+
moduleText = stripTypeScriptTypes(moduleText);
|
|
548
|
+
}
|
|
448
549
|
return new StaticModuleRecord(moduleText, moduleSpecifier);
|
|
449
550
|
} else {
|
|
450
551
|
checkAllowedModulePath(moduleSpecifier, scope.verifyPath);
|
|
@@ -479,6 +580,9 @@ function secureOnlyFetch(resource, options) {
|
|
|
479
580
|
return fetch(resource, options);
|
|
480
581
|
}
|
|
481
582
|
|
|
583
|
+
// These globals need to match the literals produced in the VM context
|
|
584
|
+
const contextualizedJSGlobals = ['Object', 'Array', 'Function', 'globalThis'];
|
|
585
|
+
|
|
482
586
|
let defaultJSGlobalNames: string[];
|
|
483
587
|
// get the global variable names that are intrinsically present in a VM context (so we don't override them)
|
|
484
588
|
function getDefaultJSGlobalNames() {
|
|
@@ -494,12 +598,13 @@ function getDefaultJSGlobalNames() {
|
|
|
494
598
|
/**
|
|
495
599
|
* Get the set of global variables that should be available to modules that run in scoped compartments/contexts.
|
|
496
600
|
*/
|
|
497
|
-
function getGlobalObject(scope: ApplicationScope) {
|
|
601
|
+
function getGlobalObject(scope: ApplicationScope, copyIntrinsics = false) {
|
|
498
602
|
const appGlobal = {};
|
|
499
603
|
// create the new global object, assigning all the global variables from this global
|
|
500
604
|
// except those that will be natural intrinsics of the new VM
|
|
605
|
+
const globalsToExclude = copyIntrinsics ? contextualizedJSGlobals : getDefaultJSGlobalNames();
|
|
501
606
|
for (let name of Object.getOwnPropertyNames(global)) {
|
|
502
|
-
if (
|
|
607
|
+
if (globalsToExclude.includes(name)) continue;
|
|
503
608
|
appGlobal[name] = global[name];
|
|
504
609
|
}
|
|
505
610
|
// now assign Harper scope-specific variables
|
|
@@ -533,6 +638,25 @@ function getHarperExports(scope: ApplicationScope) {
|
|
|
533
638
|
authenticateUser: server.authenticateUser,
|
|
534
639
|
operation: server.operation,
|
|
535
640
|
contentTypes,
|
|
641
|
+
Attribute: undefined,
|
|
642
|
+
Config: undefined,
|
|
643
|
+
ConfigValue: undefined,
|
|
644
|
+
Context: undefined,
|
|
645
|
+
FileAndURLPathConfig: undefined,
|
|
646
|
+
FilesOption: undefined,
|
|
647
|
+
FilesOptionObject: undefined,
|
|
648
|
+
IterableEventQueue: undefined,
|
|
649
|
+
Logger: undefined,
|
|
650
|
+
Query: undefined,
|
|
651
|
+
RecordObject: undefined,
|
|
652
|
+
RequestTargetOrId: undefined,
|
|
653
|
+
ResourceInterface: undefined,
|
|
654
|
+
Scope: undefined,
|
|
655
|
+
Session: undefined,
|
|
656
|
+
SourceContext: undefined,
|
|
657
|
+
SubscriptionRequest: undefined,
|
|
658
|
+
Table: undefined,
|
|
659
|
+
User: undefined,
|
|
536
660
|
};
|
|
537
661
|
}
|
|
538
662
|
const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES)
|
|
@@ -544,14 +668,19 @@ const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDB
|
|
|
544
668
|
},
|
|
545
669
|
};
|
|
546
670
|
const ALLOWED_COMMANDS = new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDSPAWNCOMMANDS) ?? []);
|
|
547
|
-
const
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
671
|
+
const child_processConstrained = {
|
|
672
|
+
exec: createSpawn(child_process.exec),
|
|
673
|
+
execFile: createSpawn(child_process.execFile),
|
|
674
|
+
fork: createSpawn(child_process.fork, true), // this is launching node, so deemed safe
|
|
675
|
+
spawn: createSpawn(child_process.spawn),
|
|
676
|
+
execSync: function () {
|
|
677
|
+
throw new Error('execSync is not allowed');
|
|
553
678
|
},
|
|
554
679
|
};
|
|
680
|
+
child_processConstrained.default = child_processConstrained;
|
|
681
|
+
const REPLACED_BUILTIN_MODULES = {
|
|
682
|
+
child_process: child_processConstrained,
|
|
683
|
+
};
|
|
555
684
|
/**
|
|
556
685
|
* Creates a ChildProcess-like object for an existing process
|
|
557
686
|
*/
|