@harperfast/harper-pro 5.0.0-beta.6 → 5.0.0-beta.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/core/bin/cliOperations.js +3 -3
  2. package/core/bin/harper.js +1 -1
  3. package/core/bin/run.js +2 -2
  4. package/core/components/Application.ts +6 -3
  5. package/core/components/ApplicationScope.ts +1 -0
  6. package/core/components/componentLoader.ts +23 -12
  7. package/core/components/operations.js +13 -13
  8. package/core/components/operationsValidation.js +3 -3
  9. package/core/config/configUtils.js +20 -4
  10. package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +3 -2
  11. package/core/resources/DatabaseTransaction.ts +67 -32
  12. package/core/resources/Resource.ts +17 -6
  13. package/core/resources/RocksTransactionLogStore.ts +8 -1
  14. package/core/resources/Table.ts +29 -10
  15. package/core/resources/analytics/write.ts +6 -0
  16. package/core/resources/databases.ts +3 -2
  17. package/core/security/jsLoader.ts +258 -129
  18. package/core/server/REST.ts +32 -13
  19. package/core/server/http.ts +6 -3
  20. package/core/server/itc/serverHandlers.js +1 -1
  21. package/core/static/defaultConfig.yaml +1 -1
  22. package/core/utility/hdbTerms.ts +1 -0
  23. package/core/utility/logging/harper_logger.js +22 -1
  24. package/core/utility/logging/readLog.js +2 -2
  25. package/core/utility/npmUtilities.js +2 -2
  26. package/core/validation/configValidator.js +16 -8
  27. package/core/validation/readLogValidator.js +2 -2
  28. package/dist/core/bin/cliOperations.js +3 -3
  29. package/dist/core/bin/cliOperations.js.map +1 -1
  30. package/dist/core/bin/harper.js +1 -1
  31. package/dist/core/bin/run.js +2 -2
  32. package/dist/core/bin/run.js.map +1 -1
  33. package/dist/core/components/Application.js +7 -2
  34. package/dist/core/components/Application.js.map +1 -1
  35. package/dist/core/components/ApplicationScope.js +1 -0
  36. package/dist/core/components/ApplicationScope.js.map +1 -1
  37. package/dist/core/components/componentLoader.js +21 -10
  38. package/dist/core/components/componentLoader.js.map +1 -1
  39. package/dist/core/components/operations.js +13 -13
  40. package/dist/core/components/operations.js.map +1 -1
  41. package/dist/core/components/operationsValidation.js +3 -3
  42. package/dist/core/components/operationsValidation.js.map +1 -1
  43. package/dist/core/config/configUtils.js +23 -3
  44. package/dist/core/config/configUtils.js.map +1 -1
  45. package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +3 -2
  46. package/dist/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js.map +1 -1
  47. package/dist/core/resources/DatabaseTransaction.js +71 -35
  48. package/dist/core/resources/DatabaseTransaction.js.map +1 -1
  49. package/dist/core/resources/Resource.js +11 -4
  50. package/dist/core/resources/Resource.js.map +1 -1
  51. package/dist/core/resources/RocksTransactionLogStore.js +8 -1
  52. package/dist/core/resources/RocksTransactionLogStore.js.map +1 -1
  53. package/dist/core/resources/Table.js +15 -10
  54. package/dist/core/resources/Table.js.map +1 -1
  55. package/dist/core/resources/analytics/write.js +6 -0
  56. package/dist/core/resources/analytics/write.js.map +1 -1
  57. package/dist/core/resources/databases.js +3 -2
  58. package/dist/core/resources/databases.js.map +1 -1
  59. package/dist/core/security/jsLoader.js +223 -116
  60. package/dist/core/security/jsLoader.js.map +1 -1
  61. package/dist/core/server/REST.js +30 -14
  62. package/dist/core/server/REST.js.map +1 -1
  63. package/dist/core/server/http.js +6 -1
  64. package/dist/core/server/http.js.map +1 -1
  65. package/dist/core/server/itc/serverHandlers.js +1 -1
  66. package/dist/core/server/itc/serverHandlers.js.map +1 -1
  67. package/dist/core/utility/hdbTerms.js +1 -0
  68. package/dist/core/utility/hdbTerms.js.map +1 -1
  69. package/dist/core/utility/logging/harper_logger.js +24 -1
  70. package/dist/core/utility/logging/harper_logger.js.map +1 -1
  71. package/dist/core/utility/logging/readLog.js +2 -2
  72. package/dist/core/utility/logging/readLog.js.map +1 -1
  73. package/dist/core/utility/npmUtilities.js +2 -2
  74. package/dist/core/utility/npmUtilities.js.map +1 -1
  75. package/dist/core/validation/configValidator.js +18 -8
  76. package/dist/core/validation/configValidator.js.map +1 -1
  77. package/dist/core/validation/readLogValidator.js +2 -2
  78. package/dist/core/validation/readLogValidator.js.map +1 -1
  79. package/dist/licensing/usageLicensing.js +16 -26
  80. package/dist/licensing/usageLicensing.js.map +1 -1
  81. package/dist/replication/nodeIdMapping.js +1 -1
  82. package/dist/replication/nodeIdMapping.js.map +1 -1
  83. package/dist/replication/replicationConnection.js +2 -2
  84. package/dist/replication/replicationConnection.js.map +1 -1
  85. package/licensing/usageLicensing.ts +22 -32
  86. package/npm-shrinkwrap.json +615 -614
  87. package/package.json +9 -6
  88. package/replication/nodeIdMapping.ts +1 -1
  89. package/replication/replicationConnection.ts +2 -2
  90. package/static/defaultConfig.yaml +3 -0
  91. package/studio/web/assets/{index-BckVDix4.js → index-CXQsBaYq.js} +5 -5
  92. package/studio/web/assets/{index-BckVDix4.js.map → index-CXQsBaYq.js.map} +1 -1
  93. package/studio/web/assets/{index.lazy-iG1_8dzm.js → index.lazy-C3Ejfvna.js} +2 -2
  94. package/studio/web/assets/{index.lazy-iG1_8dzm.js.map → index.lazy-C3Ejfvna.js.map} +1 -1
  95. package/studio/web/assets/{profile-CzjslUXv.js → profile-BbbbWJCN.js} +2 -2
  96. package/studio/web/assets/{profile-CzjslUXv.js.map → profile-BbbbWJCN.js.map} +1 -1
  97. package/studio/web/assets/{status-BP4TQJDR.js → status-CFe85l8C.js} +2 -2
  98. package/studio/web/assets/{status-BP4TQJDR.js.map → status-CFe85l8C.js.map} +1 -1
  99. package/studio/web/index.html +1 -1
@@ -580,10 +580,16 @@ export function makeTable(options) {
580
580
  // dictates not to go to source
581
581
  if (!this.doesExist()) throw new ServerError('Entry is not cached', 504);
582
582
  } else if (resourceOptions?.ensureLoaded) {
583
- const loadingFromSource = ensureLoadedFromSource(this.constructor.source, id, entry, request, this);
583
+ const loadingFromSource = ensureLoadedFromSource(
584
+ this.constructor.source,
585
+ id,
586
+ entry,
587
+ request,
588
+ this,
589
+ target
590
+ );
584
591
  if (loadingFromSource) {
585
592
  txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
586
- target.loadedFromSource = true;
587
593
  return when(loadingFromSource, (entry) => {
588
594
  TableResource._updateResource(this, entry);
589
595
  return this;
@@ -988,10 +994,16 @@ export function makeTable(options) {
988
994
  // dictates not to go to source
989
995
  if (!entry?.value) throw new ServerError('Entry is not cached', 504);
990
996
  } else if (ensureLoaded) {
991
- const loadingFromSource = ensureLoadedFromSource(constructor.source, id, entry, context, this);
997
+ const loadingFromSource = ensureLoadedFromSource(
998
+ constructor.source,
999
+ id,
1000
+ entry,
1001
+ context,
1002
+ this,
1003
+ target
1004
+ );
992
1005
  if (loadingFromSource) {
993
1006
  txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
994
- target.loadedFromSource = true;
995
1007
  return loadingFromSource.then((entry) => entry?.value);
996
1008
  }
997
1009
  }
@@ -3732,7 +3744,7 @@ export function makeTable(options) {
3732
3744
  }
3733
3745
  }
3734
3746
 
3735
- function ensureLoadedFromSource(source: typeof TableResource, id, entry, context, resource?) {
3747
+ function ensureLoadedFromSource(source: typeof TableResource, id, entry, context, resource?, target?) {
3736
3748
  if (hasSourceGet) {
3737
3749
  let needsSourceData = false;
3738
3750
  if (context.noCache) needsSourceData = true;
@@ -3751,7 +3763,7 @@ export function makeTable(options) {
3751
3763
  recordActionBinary(!needsSourceData, 'cache-hit', tableName);
3752
3764
  }
3753
3765
  if (needsSourceData) {
3754
- const loadingFromSource = getFromSource(source, id, entry, context).then((entry) => {
3766
+ const loadingFromSource = getFromSource(source, id, entry, context, target).then((entry) => {
3755
3767
  if (entry?.value && entry?.value.getRecord?.())
3756
3768
  logger.error?.('Can not assign a record that is already a resource');
3757
3769
  if (context) {
@@ -3921,7 +3933,8 @@ export function makeTable(options) {
3921
3933
  source: typeof TableResource,
3922
3934
  id: Id,
3923
3935
  existingEntry: Entry,
3924
- context: Context
3936
+ context: Context,
3937
+ target?
3925
3938
  ): Promise<Entry> {
3926
3939
  const metadataFlags = existingEntry?.metadataFlags;
3927
3940
 
@@ -3943,9 +3956,13 @@ export function makeTable(options) {
3943
3956
  entry.metadataFlags & (INVALIDATED | EVICTED) ||
3944
3957
  (entry.expiresAt != undefined && entry.expiresAt < Date.now())
3945
3958
  )
3946
- // try again
3947
- whenResolved(getFromSource(source, id, primaryStore.getEntry(id), context));
3948
- else whenResolved(entry);
3959
+ // try again — entry still not valid, need to actually fetch from source
3960
+ whenResolved(getFromSource(source, id, primaryStore.getEntry(id), context, target));
3961
+ else {
3962
+ // served from cache after waiting for another request to resolve
3963
+ if (target) target.loadedFromSource = false;
3964
+ whenResolved(entry);
3965
+ }
3949
3966
  };
3950
3967
  const lockAcquired = primaryStore.tryLock(id, callback);
3951
3968
 
@@ -3957,6 +3974,8 @@ export function makeTable(options) {
3957
3974
  }, LOCK_TIMEOUT);
3958
3975
  });
3959
3976
  }
3977
+ // lock acquired — this request will actually load from source
3978
+ if (target) target.loadedFromSource = true;
3960
3979
 
3961
3980
  const existingRecord = existingEntry?.value;
3962
3981
  // it is important to remember that this is _NOT_ part of the current transaction; nothing is changing
@@ -402,7 +402,12 @@ export async function getDirectorySizeAsync(dirPath: string): Promise<number> {
402
402
  }
403
403
  }
404
404
 
405
+ const DEFAULT_STORAGE_INTERVAL = 10;
406
+ let nodeStorageInterval = DEFAULT_STORAGE_INTERVAL;
407
+ let nodeStorageCycleCount = 0;
408
+
405
409
  async function storeNodeStorageMetric(analyticsTable: Table) {
410
+ if (nodeStorageInterval <= 0 || ++nodeStorageCycleCount % nodeStorageInterval !== 1) return;
406
411
  try {
407
412
  const size = await getDirectorySizeAsync(getHdbBasePath());
408
413
  storeMetric(analyticsTable, {
@@ -689,6 +694,7 @@ if (!parentPort) onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics);
689
694
  let scheduledTasksRunning;
690
695
  function startScheduledTasks() {
691
696
  scheduledTasksRunning = true;
697
+ nodeStorageInterval = envGet(CONFIG_PARAMS.ANALYTICS_STORAGEINTERVAL) ?? DEFAULT_STORAGE_INTERVAL;
692
698
  const AGGREGATE_PERIOD = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) * 1000;
693
699
  if (AGGREGATE_PERIOD) {
694
700
  setInterval(
@@ -12,6 +12,7 @@ import {
12
12
  import { makeTable } from './Table.ts';
13
13
  import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
14
14
  import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
15
+ import { getConfigPath } from '../config/configUtils.js';
15
16
  import { _assignPackageExport } from '../globals.js';
16
17
  import { getIndexedValues } from '../utility/lmdb/commonUtility.js';
17
18
  import * as signalling from '../utility/signalling.js';
@@ -176,7 +177,7 @@ export function getDatabases(): Databases {
176
177
  if (process.env.SCHEMAS_DATA_PATH) schemaConfigs.data = { path: process.env.SCHEMAS_DATA_PATH };
177
178
  databasePath =
178
179
  process.env.STORAGE_PATH ||
179
- envGet(CONFIG_PARAMS.STORAGE_PATH) ||
180
+ getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
180
181
  (databasePath && (existsSync(databasePath) ? databasePath : join(getHdbBasePath(), LEGACY_DATABASES_DIR_NAME)));
181
182
  if (!databasePath) return;
182
183
 
@@ -694,7 +695,7 @@ export function database({ database: databaseName, table: tableName }) {
694
695
  tablePath ||
695
696
  databaseConfig[databaseName]?.path ||
696
697
  process.env.STORAGE_PATH ||
697
- envGet(CONFIG_PARAMS.STORAGE_PATH) ||
698
+ getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
698
699
  (existsSync(join(hdbBasePath, DATABASES_DIR_NAME))
699
700
  ? join(hdbBasePath, DATABASES_DIR_NAME)
700
701
  : join(hdbBasePath, LEGACY_DATABASES_DIR_NAME));
@@ -56,6 +56,9 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
56
56
  }
57
57
  overridableProperty(Promise.prototype, 'then');
58
58
  overridableProperty(Date, 'now');
59
+ for (let name of ['get', 'set', 'has', 'delete', 'clear', 'forEach', 'entries', 'keys', 'values']) {
60
+ overridableProperty(Map.prototype, name);
61
+ }
59
62
  for (let Intrinsic of [
60
63
  Object,
61
64
  Array,
@@ -121,12 +124,13 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
121
124
 
122
125
  let amaro: typeof import('amaro') | undefined;
123
126
  /**
124
- * Strip TypeScript types using the amaro library (what Node.js uses internally)
125
- * Falls back to regex-based stripping if amaro is not available
127
+ * Strip TypeScript types synchronously using the amaro library (what Node.js uses internally)
126
128
  */
127
- async function stripTypeScriptTypes(source: string): Promise<string> {
129
+ function stripTypeScriptTypes(source: string): string {
128
130
  // Use amaro - the library that Node.js uses internally for type stripping
129
- amaro = await import('amaro');
131
+ if (!amaro) {
132
+ amaro = require('amaro');
133
+ }
130
134
  return amaro.transformSync(source, { mode: 'strip-only' }).code;
131
135
  }
132
136
 
@@ -146,12 +150,27 @@ function parseJsonModule(source: string, url: string): any {
146
150
  * Load a module using Node's vm.Module API with (not really secure) sandboxing
147
151
  */
148
152
  async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
149
- const moduleCache = new Map<string, Promise<SourceTextModule | SyntheticModule>>();
150
- const linkingPromises = new Map<string, Promise<void>>();
151
-
152
- // Create a secure context with limited globals
153
- const contextObject = getGlobalObject(scope);
154
- const context = createContext(contextObject);
153
+ // we want to retain the same module caches across any loading with the application scope
154
+ let moduleCaches = scope.moduleCache as {
155
+ moduleCache: Map<string, SourceTextModule | SyntheticModule | Promise<SourceTextModule | SyntheticModule>>;
156
+ linkingPromises: Map<string, Promise<void>>;
157
+ cjsCache: Map<string, { exports: any }>;
158
+ contextObject: any;
159
+ context: any;
160
+ };
161
+ if (!moduleCaches) {
162
+ // if they haven't been initialized, do so now
163
+ const contextObject = getGlobalObject(scope, true);
164
+ moduleCaches = scope.moduleCache = {
165
+ moduleCache: new Map(),
166
+ linkingPromises: new Map(),
167
+ cjsCache: new Map(),
168
+ // Create a secure context with limited globals
169
+ contextObject,
170
+ context: createContext(contextObject),
171
+ };
172
+ }
173
+ const { moduleCache, linkingPromises, cjsCache, contextObject, context } = moduleCaches;
155
174
 
156
175
  /**
157
176
  * Resolve module specifier to absolute URL
@@ -165,6 +184,9 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
165
184
  // block harper/* for now (reserving for potential future use)
166
185
  throw new Error(`Module ${specifier} is not allowed, may only access the 'harper' module`);
167
186
  }
187
+ if (parts[0] === 'file:') {
188
+ return specifier;
189
+ }
168
190
  const resolved = createRequire(referrer).resolve(specifier);
169
191
  if (isAbsolute(resolved)) {
170
192
  return pathToFileURL(resolved).toString();
@@ -176,7 +198,16 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
176
198
  * Load a CommonJS module in our private context
177
199
  */
178
200
  function loadCJS(url: string, source: string): { exports: any } {
201
+ // Check cache first to handle circular dependencies
202
+ if (cjsCache.has(url)) {
203
+ return cjsCache.get(url)!;
204
+ }
205
+
206
+ // Create module object and cache it immediately (before execution)
207
+ // This allows circular dependencies to get a reference to the incomplete module
179
208
  const cjsModule = { exports: {} };
209
+ cjsCache.set(url, cjsModule);
210
+
180
211
  if (url.endsWith('.json')) {
181
212
  cjsModule.exports = parseJsonModule(source, url);
182
213
  return cjsModule;
@@ -204,7 +235,7 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
204
235
  filename: url,
205
236
  async importModuleDynamically(specifier: string, script) {
206
237
  const resolvedUrl = resolveModule(specifier, script.sourceURL);
207
- const useContainment = specifier.startsWith('.') || scope.dependencyContainment;
238
+ const useContainment = specifier.startsWith('.') || scope.dependencyContainment !== false;
208
239
  const dynamicModule = await loadModuleWithCache(resolvedUrl, useContainment);
209
240
  return dynamicModule;
210
241
  },
@@ -221,16 +252,18 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
221
252
  }
222
253
  function loadCJSModule(url: string, source: string, usePrivateGlobal: boolean): SyntheticModule {
223
254
  const cjsModule = usePrivateGlobal ? loadCJS(url, source) : { exports: require(url) };
224
- const exportNames = Object.keys(cjsModule.exports);
255
+ let exports = cjsModule.exports;
256
+ if (exports.default === undefined) {
257
+ // provide the default export for compatibility
258
+ exports = { default: exports, ...exports };
259
+ }
260
+ const exportNames = Object.keys(exports);
261
+
225
262
  const synModule = new SyntheticModule(
226
- exportNames.length > 0 ? exportNames : ['default'],
263
+ exportNames,
227
264
  function () {
228
- if (exportNames.length > 0) {
229
- for (const key of exportNames) {
230
- this.setExport(key, cjsModule.exports[key]);
231
- }
232
- } else {
233
- this.setExport('default', cjsModule.exports);
265
+ for (const key of exportNames) {
266
+ this.setExport(key, exports[key]);
234
267
  }
235
268
  },
236
269
  { identifier: url, context }
@@ -240,21 +273,78 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
240
273
  }
241
274
 
242
275
  /**
243
- * Linker function for module resolution during instantiation
276
+ * Check if a package (or any of its dependencies) depends on harper
277
+ * Expects a file URL like: file:///path/to/node_modules/package-name/dist/index.js
278
+ */
279
+ function packageDependsOnHarper(fileUrl: string): boolean {
280
+ try {
281
+ // Convert file:// URL to path
282
+ const filePath = fileURLToPath(fileUrl);
283
+
284
+ // Find the node_modules directory and package name
285
+ // Example: /path/to/node_modules/package-name/dist/index.js
286
+ // or: /path/to/node_modules/@scope/package-name/dist/index.js
287
+ const nodeModulesMarker = '/node_modules/';
288
+ const nodeModulesIndex = filePath.lastIndexOf(nodeModulesMarker);
289
+ if (nodeModulesIndex === -1) return false;
290
+
291
+ // Get the part after /node_modules/
292
+ const afterNodeModules = filePath.substring(nodeModulesIndex + nodeModulesMarker.length);
293
+ const parts = afterNodeModules.split('/');
294
+
295
+ // Handle scoped packages (@scope/package-name) vs regular packages (package-name)
296
+ const beforeNodeModules = filePath.substring(0, nodeModulesIndex);
297
+ const packageRoot = parts[0].startsWith('@')
298
+ ? join(beforeNodeModules, 'node_modules', parts[0], parts[1])
299
+ : join(beforeNodeModules, 'node_modules', parts[0]);
300
+
301
+ // Read package.json from the package root
302
+ const packageJsonPath = join(packageRoot, 'package.json');
303
+ const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8'));
304
+
305
+ const deps = {
306
+ ...packageJson.dependencies,
307
+ ...packageJson.devDependencies,
308
+ ...packageJson.peerDependencies,
309
+ };
310
+
311
+ // Check if harper is a direct dependency
312
+ return Object.keys(deps).some((dep) => HARPER_MODULE_IDS.has(dep));
313
+ } catch {
314
+ return false;
315
+ }
316
+ }
317
+
318
+ /**
319
+ * Linker function for module resolution during instantiation.
320
+ * This is synchronous because Node's module.link() requires the linker
321
+ * to return modules synchronously.
244
322
  */
245
- async function linker(specifier: string, referencingModule: SourceTextModule | SyntheticModule) {
323
+ function linker(specifier: string, referencingModule: SourceTextModule | SyntheticModule) {
246
324
  const resolvedUrl = resolveModule(specifier, referencingModule.identifier);
247
325
 
248
- const useContainment = specifier.startsWith('.') || scope.dependencyContainment;
249
- // Return the module immediately (even if not yet linked) to support circular dependencies
250
- return await getOrCreateModule(resolvedUrl, useContainment);
326
+ // Determine if we should use VM containment for this module
327
+ let useContainment = specifier.startsWith('.'); // Always contain relative imports
328
+
329
+ if (!useContainment && scope.dependencyContainment !== false) {
330
+ // For npm packages, check if they depend on harper
331
+ if (resolvedUrl.startsWith('file://') && resolvedUrl.includes('node_modules')) {
332
+ useContainment = packageDependsOnHarper(resolvedUrl);
333
+ } else {
334
+ // Non-file URLs (bare specifiers) - use default behavior
335
+ useContainment = scope.dependencyContainment === true;
336
+ }
337
+ }
338
+
339
+ // Return the module
340
+ return getOrCreateModule(resolvedUrl, useContainment);
251
341
  }
252
342
 
253
- async function getOrCreateModule(
343
+ function getOrCreateModule(
254
344
  url: string,
255
345
  usePrivateGlobal: boolean
256
- ): Promise<SourceTextModule | SyntheticModule> {
257
- // Check cache first - return cached module immediately (even if not linked yet)
346
+ ): SourceTextModule | SyntheticModule | Promise<SourceTextModule | SyntheticModule> {
347
+ // Check if module is already created
258
348
  if (moduleCache.has(url)) {
259
349
  return moduleCache.get(url)!;
260
350
  }
@@ -275,8 +365,15 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
275
365
  // Only link/evaluate once per module
276
366
  if (!linkingPromises.has(url)) {
277
367
  const linkingPromise = (async () => {
278
- await module.link(linker);
279
- await module.evaluate();
368
+ // Check module status - only link if it's 'unlinked'
369
+ // Status can be: 'unlinked', 'linking', 'linked', 'evaluating', 'evaluated'
370
+ if (module.status === 'unlinked') {
371
+ await module.link(linker);
372
+ }
373
+ // Only evaluate if not already evaluated
374
+ if (module.status === 'linked') {
375
+ await module.evaluate();
376
+ }
280
377
  })();
281
378
  linkingPromises.set(url, linkingPromise);
282
379
  }
@@ -287,107 +384,107 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
287
384
  return module;
288
385
  }
289
386
  /**
290
- * Create a module from URL without linking or evaluating
387
+ * Create a SyntheticModule from exported object
291
388
  */
292
- async function createModule(url: string, usePrivateGlobal: boolean): Promise<SourceTextModule | SyntheticModule> {
293
- let module: SourceTextModule | SyntheticModule;
389
+ function createSyntheticModule(url: string, exportedObject: any): SyntheticModule {
390
+ const exportNames = Object.keys(exportedObject);
391
+ return new SyntheticModule(
392
+ exportNames,
393
+ function () {
394
+ for (const key of exportNames) {
395
+ this.setExport(key, exportedObject[key]);
396
+ }
397
+ },
398
+ { identifier: url, context }
399
+ );
400
+ }
294
401
 
295
- // Handle special built-in modules
296
- if (url === 'harper') {
297
- let harperExports = getHarperExports(scope);
298
- module = new SyntheticModule(
299
- Object.keys(harperExports),
402
+ /**
403
+ * Normalize imported module to ensure it has proper exports including default
404
+ */
405
+ function normalizeImportedModule(importedModule: any): any {
406
+ const cjsModule = importedModule['module.exports'];
407
+ if (cjsModule) {
408
+ // back-compat import
409
+ importedModule = importedModule.default ? { default: importedModule.default, ...cjsModule } : cjsModule;
410
+ }
411
+ // Ensure there's a default export for ESM imports that expect it
412
+ if (!importedModule.default) {
413
+ importedModule = { default: importedModule, ...importedModule };
414
+ }
415
+ return importedModule;
416
+ }
417
+
418
+ /**
419
+ * Create a SourceTextModule or SyntheticModule from source code
420
+ */
421
+ function createModuleFromSource(
422
+ url: string,
423
+ source: string,
424
+ usePrivateGlobal: boolean
425
+ ): SourceTextModule | SyntheticModule {
426
+ // Handle JSON modules
427
+ if (url.endsWith('.json')) {
428
+ const jsonData = parseJsonModule(source, url);
429
+ return new SyntheticModule(
430
+ ['default'],
300
431
  function () {
301
- for (let key in harperExports) {
302
- this.setExport(key, harperExports[key]);
303
- }
432
+ this.setExport('default', jsonData);
304
433
  },
305
434
  { identifier: url, context }
306
435
  );
307
- } else if (url.startsWith('file://') && usePrivateGlobal) {
308
- checkAllowedModulePath(url, scope.verifyPath);
309
- let source = await readFile(new URL(url), { encoding: 'utf-8' });
310
-
311
- // Handle JSON modules as a SyntheticModule with a default export.
312
- // JSON imports only support default exports per the ESM spec.
313
- if (url.endsWith('.json')) {
314
- const jsonData = parseJsonModule(source, url);
315
- module = new SyntheticModule(
316
- ['default'],
317
- function () {
318
- this.setExport('default', jsonData);
319
- },
320
- { identifier: url, context }
321
- );
322
- } else {
323
- // Strip TypeScript types if this is a .ts file
324
- if (url.endsWith('.ts') || url.endsWith('.tsx')) {
325
- source = await stripTypeScriptTypes(source);
326
- }
436
+ }
327
437
 
328
- // Try CJS first since it will fail fast with clear syntax errors on ESM syntax
329
- try {
330
- module = loadCJSModule(url, source, usePrivateGlobal);
331
- } catch {
332
- // If CJS loading fails (likely due to ESM syntax like import/export), try ESM
333
- try {
334
- module = new SourceTextModule(source, {
335
- identifier: url,
336
- context,
337
- initializeImportMeta(meta) {
338
- meta.url = url;
339
- },
340
- async importModuleDynamically(specifier: string) {
341
- const resolvedUrl = resolveModule(specifier, url);
342
- const dynamicModule = await loadModuleWithCache(resolvedUrl, true);
343
- return dynamicModule;
344
- },
345
- });
346
- } catch (esmErr) {
347
- // Both failed - throw the ESM error as it's likely more relevant
348
- throw esmErr;
349
- }
350
- }
351
- }
352
- } else {
353
- const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
354
- // For Node.js built-in modules (node:) and npm packages
355
- // Always try require first to properly handle CJS modules with named exports
356
- try {
357
- const cjsExports = replacedModule ?? require(url);
358
- // It's a CJS module - expose all properties as named exports
359
- const exportNames = Object.keys(cjsExports);
360
- module = new SyntheticModule(
361
- exportNames.length > 0 ? [...exportNames, 'default'] : ['default'],
362
- function () {
363
- if (exportNames.length > 0) {
364
- for (const key of exportNames) {
365
- this.setExport(key, cjsExports[key]);
366
- }
367
- }
368
- this.setExport('default', cjsExports);
369
- },
370
- { identifier: url, context }
371
- );
372
- } catch {
373
- // Fall back to dynamic import for ESM packages
374
- const importedModule = await import(url);
375
- const exportNames = Object.keys(importedModule);
376
- module = new SyntheticModule(
377
- exportNames,
378
- function () {
379
- for (const key of exportNames) {
380
- this.setExport(key, importedModule[key]);
381
- }
382
- },
383
- { identifier: url, context }
384
- );
385
- }
438
+ // Strip TypeScript types if this is a .ts file
439
+ if (url.endsWith('.ts') || url.endsWith('.tsx')) {
440
+ source = stripTypeScriptTypes(source);
386
441
  }
387
442
 
388
- return module;
443
+ // Try CJS first since it will fail fast with clear syntax errors on ESM syntax
444
+ try {
445
+ return loadCJSModule(url, source, usePrivateGlobal);
446
+ } catch {
447
+ // If CJS loading fails (likely due to ESM syntax like import/export), try ESM
448
+ return new SourceTextModule(source, {
449
+ identifier: url,
450
+ context,
451
+ initializeImportMeta(meta) {
452
+ meta.url = url;
453
+ },
454
+ importModuleDynamically(specifier: string) {
455
+ const resolvedUrl = resolveModule(specifier, url);
456
+ const useContainment = specifier.startsWith('.') || scope.dependencyContainment !== false;
457
+ return loadModuleWithCache(resolvedUrl, useContainment);
458
+ },
459
+ });
460
+ }
389
461
  }
390
462
 
463
+ /**
464
+ * Create a module from URL without linking or evaluating (async version for initial load)
465
+ */
466
+ function createModule(
467
+ url: string,
468
+ usePrivateGlobal: boolean
469
+ ): SourceTextModule | SyntheticModule | Promise<SourceTextModule | SyntheticModule> {
470
+ // Handle special built-in modules
471
+ if (url === 'harper') {
472
+ return createSyntheticModule(url, getHarperExports(scope));
473
+ }
474
+
475
+ if (url.startsWith('file://') && usePrivateGlobal) {
476
+ checkAllowedModulePath(url, scope.verifyPath);
477
+ const source = readFileSync(new URL(url), { encoding: 'utf-8' });
478
+ return createModuleFromSource(url, source, usePrivateGlobal);
479
+ }
480
+
481
+ // For Node.js built-in modules (node:) and npm packages without dependency containment
482
+ const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
483
+ if (replacedModule) {
484
+ return createSyntheticModule(url, normalizeImportedModule(replacedModule));
485
+ }
486
+ return import(url).then((importedModule) => createSyntheticModule(url, normalizeImportedModule(importedModule)));
487
+ }
391
488
  // Load the entry module
392
489
  const entryModule = await loadModuleWithCache(moduleUrl, true);
393
490
 
@@ -433,8 +530,8 @@ async function getCompartment(scope: ApplicationScope, globals) {
433
530
  },
434
531
  };
435
532
  } else if (moduleSpecifier.startsWith('file:') && !moduleSpecifier.includes('node_modules')) {
436
- const moduleText = await readFile(new URL(moduleSpecifier), { encoding: 'utf-8' });
437
- // Handle JSON files in comparttment mode the same way as in VM mode
533
+ let moduleText = await readFile(new URL(moduleSpecifier), { encoding: 'utf-8' });
534
+ // Handle JSON files in compartment mode the same way as in VM mode
438
535
  if (moduleSpecifier.endsWith('.json')) {
439
536
  const jsonData = parseJsonModule(moduleText, moduleSpecifier);
440
537
  return {
@@ -445,6 +542,10 @@ async function getCompartment(scope: ApplicationScope, globals) {
445
542
  },
446
543
  };
447
544
  }
545
+ // Strip TypeScript types if this is a .ts file
546
+ if (moduleSpecifier.endsWith('.ts') || moduleSpecifier.endsWith('.tsx')) {
547
+ moduleText = stripTypeScriptTypes(moduleText);
548
+ }
448
549
  return new StaticModuleRecord(moduleText, moduleSpecifier);
449
550
  } else {
450
551
  checkAllowedModulePath(moduleSpecifier, scope.verifyPath);
@@ -479,6 +580,9 @@ function secureOnlyFetch(resource, options) {
479
580
  return fetch(resource, options);
480
581
  }
481
582
 
583
+ // These globals need to match the literals produced in the VM context
584
+ const contextualizedJSGlobals = ['Object', 'Array', 'Function', 'globalThis'];
585
+
482
586
  let defaultJSGlobalNames: string[];
483
587
  // get the global variable names that are intrinsically present in a VM context (so we don't override them)
484
588
  function getDefaultJSGlobalNames() {
@@ -494,12 +598,13 @@ function getDefaultJSGlobalNames() {
494
598
  /**
495
599
  * Get the set of global variables that should be available to modules that run in scoped compartments/contexts.
496
600
  */
497
- function getGlobalObject(scope: ApplicationScope) {
601
+ function getGlobalObject(scope: ApplicationScope, copyIntrinsics = false) {
498
602
  const appGlobal = {};
499
603
  // create the new global object, assigning all the global variables from this global
500
604
  // except those that will be natural intrinsics of the new VM
605
+ const globalsToExclude = copyIntrinsics ? contextualizedJSGlobals : getDefaultJSGlobalNames();
501
606
  for (let name of Object.getOwnPropertyNames(global)) {
502
- if (getDefaultJSGlobalNames().includes(name)) continue;
607
+ if (globalsToExclude.includes(name)) continue;
503
608
  appGlobal[name] = global[name];
504
609
  }
505
610
  // now assign Harper scope-specific variables
@@ -533,6 +638,25 @@ function getHarperExports(scope: ApplicationScope) {
533
638
  authenticateUser: server.authenticateUser,
534
639
  operation: server.operation,
535
640
  contentTypes,
641
+ Attribute: undefined,
642
+ Config: undefined,
643
+ ConfigValue: undefined,
644
+ Context: undefined,
645
+ FileAndURLPathConfig: undefined,
646
+ FilesOption: undefined,
647
+ FilesOptionObject: undefined,
648
+ IterableEventQueue: undefined,
649
+ Logger: undefined,
650
+ Query: undefined,
651
+ RecordObject: undefined,
652
+ RequestTargetOrId: undefined,
653
+ ResourceInterface: undefined,
654
+ Scope: undefined,
655
+ Session: undefined,
656
+ SourceContext: undefined,
657
+ SubscriptionRequest: undefined,
658
+ Table: undefined,
659
+ User: undefined,
536
660
  };
537
661
  }
538
662
  const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES)
@@ -544,14 +668,19 @@ const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDB
544
668
  },
545
669
  };
546
670
  const ALLOWED_COMMANDS = new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDSPAWNCOMMANDS) ?? []);
547
- const REPLACED_BUILTIN_MODULES = {
548
- child_process: {
549
- exec: createSpawn(child_process.exec),
550
- execFile: createSpawn(child_process.execFile),
551
- fork: createSpawn(child_process.fork, true), // this is launching node, so deemed safe
552
- spawn: createSpawn(child_process.spawn),
671
+ const child_processConstrained = {
672
+ exec: createSpawn(child_process.exec),
673
+ execFile: createSpawn(child_process.execFile),
674
+ fork: createSpawn(child_process.fork, true), // this is launching node, so deemed safe
675
+ spawn: createSpawn(child_process.spawn),
676
+ execSync: function () {
677
+ throw new Error('execSync is not allowed');
553
678
  },
554
679
  };
680
+ child_processConstrained.default = child_processConstrained;
681
+ const REPLACED_BUILTIN_MODULES = {
682
+ child_process: child_processConstrained,
683
+ };
555
684
  /**
556
685
  * Creates a ChildProcess-like object for an existing process
557
686
  */