@harperfast/harper-pro 5.0.0-alpha.7 → 5.0.0-alpha.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/core/components/Application.ts +10 -9
  2. package/core/components/ApplicationScope.ts +49 -0
  3. package/core/components/EntryHandler.ts +5 -4
  4. package/core/components/OptionsWatcher.ts +6 -5
  5. package/core/components/Scope.ts +17 -33
  6. package/core/components/componentLoader.ts +24 -7
  7. package/core/components/status/crossThread.ts +1 -1
  8. package/core/config/harperConfigEnvVars.ts +3 -2
  9. package/core/index.d.ts +5 -0
  10. package/core/index.js +1 -0
  11. package/core/integrationTests/utils/harperLifecycle.ts +5 -6
  12. package/core/package.json +3 -1
  13. package/core/resources/RecordEncoder.ts +13 -4
  14. package/core/resources/RocksTransactionLogStore.ts +1 -0
  15. package/core/resources/Table.ts +3 -3
  16. package/core/resources/blob.ts +1 -1
  17. package/core/resources/databases.ts +26 -16
  18. package/core/resources/indexes/HierarchicalNavigableSmallWorld.ts +1 -1
  19. package/core/resources/jsResource.ts +2 -2
  20. package/core/security/certificateVerification/crlVerification.ts +1 -1
  21. package/core/security/certificateVerification/index.ts +1 -1
  22. package/core/security/certificateVerification/ocspVerification.ts +1 -1
  23. package/core/security/certificateVerification/verificationConfig.ts +1 -1
  24. package/core/security/certificateVerification/verificationUtils.ts +1 -1
  25. package/core/security/jsLoader.ts +16 -16
  26. package/core/security/tokenAuthentication.ts +30 -19
  27. package/core/server/mqtt.ts +1 -1
  28. package/core/server/serverHelpers/contentTypes.ts +2 -2
  29. package/core/server/status/index.ts +1 -1
  30. package/core/server/storageReclamation.ts +1 -1
  31. package/core/server/throttle.ts +1 -1
  32. package/core/unitTests/components/Scope.test.js +13 -2
  33. package/core/unitTests/components/fixtures/testJSWithDeps/resources.js +1 -1
  34. package/core/unitTests/components/globalIsolation.test.js +22 -22
  35. package/core/utility/lmdb/environmentUtility.js +1 -67
  36. package/core/utility/lmdb/searchUtility.js +0 -1
  37. package/core/utility/logging/logger.ts +24 -0
  38. package/core/utility/password.ts +2 -2
  39. package/core/utility/when.ts +3 -1
  40. package/dist/bin/harper.js +3 -1
  41. package/dist/bin/harper.js.map +1 -1
  42. package/dist/core/components/Application.js +8 -8
  43. package/dist/core/components/Application.js.map +1 -1
  44. package/dist/core/components/ApplicationScope.js +81 -0
  45. package/dist/core/components/ApplicationScope.js.map +1 -0
  46. package/dist/core/components/EntryHandler.js +2 -2
  47. package/dist/core/components/EntryHandler.js.map +1 -1
  48. package/dist/core/components/OptionsWatcher.js +3 -3
  49. package/dist/core/components/OptionsWatcher.js.map +1 -1
  50. package/dist/core/components/Scope.js +18 -52
  51. package/dist/core/components/Scope.js.map +1 -1
  52. package/dist/core/components/componentLoader.js +12 -7
  53. package/dist/core/components/componentLoader.js.map +1 -1
  54. package/dist/core/components/status/crossThread.js +2 -2
  55. package/dist/core/config/harperConfigEnvVars.js +1 -1
  56. package/dist/core/config/harperConfigEnvVars.js.map +1 -1
  57. package/dist/core/index.js +1 -0
  58. package/dist/core/index.js.map +1 -1
  59. package/dist/core/resources/RecordEncoder.js +19 -4
  60. package/dist/core/resources/RecordEncoder.js.map +1 -1
  61. package/dist/core/resources/RocksTransactionLogStore.js +2 -0
  62. package/dist/core/resources/RocksTransactionLogStore.js.map +1 -1
  63. package/dist/core/resources/Table.js +48 -48
  64. package/dist/core/resources/Table.js.map +1 -1
  65. package/dist/core/resources/blob.js +22 -25
  66. package/dist/core/resources/blob.js.map +1 -1
  67. package/dist/core/resources/databases.js +29 -18
  68. package/dist/core/resources/databases.js.map +1 -1
  69. package/dist/core/resources/indexes/HierarchicalNavigableSmallWorld.js +2 -2
  70. package/dist/core/resources/jsResource.js +2 -2
  71. package/dist/core/resources/jsResource.js.map +1 -1
  72. package/dist/core/security/certificateVerification/crlVerification.js +2 -2
  73. package/dist/core/security/certificateVerification/index.js +2 -2
  74. package/dist/core/security/certificateVerification/ocspVerification.js +2 -2
  75. package/dist/core/security/certificateVerification/verificationConfig.js +2 -2
  76. package/dist/core/security/certificateVerification/verificationUtils.js +2 -2
  77. package/dist/core/security/jsLoader.js +10 -10
  78. package/dist/core/security/jsLoader.js.map +1 -1
  79. package/dist/core/security/tokenAuthentication.js +15 -7
  80. package/dist/core/security/tokenAuthentication.js.map +1 -1
  81. package/dist/core/server/mqtt.js +2 -2
  82. package/dist/core/server/serverHelpers/contentTypes.js +3 -3
  83. package/dist/core/server/serverHelpers/contentTypes.js.map +1 -1
  84. package/dist/core/server/status/index.js +2 -2
  85. package/dist/core/server/storageReclamation.js +3 -3
  86. package/dist/core/server/storageReclamation.js.map +1 -1
  87. package/dist/core/server/throttle.js +2 -5
  88. package/dist/core/server/throttle.js.map +1 -1
  89. package/dist/core/utility/lmdb/environmentUtility.js +1 -63
  90. package/dist/core/utility/lmdb/environmentUtility.js.map +1 -1
  91. package/dist/core/utility/lmdb/searchUtility.js +0 -1
  92. package/dist/core/utility/lmdb/searchUtility.js.map +1 -1
  93. package/dist/core/utility/logging/logger.js +14 -7
  94. package/dist/core/utility/logging/logger.js.map +1 -1
  95. package/dist/core/utility/password.js.map +1 -1
  96. package/dist/core/utility/when.js.map +1 -1
  97. package/dist/replication/replicationConnection.js +2 -2
  98. package/dist/replication/replicationConnection.js.map +1 -1
  99. package/npm-shrinkwrap.json +455 -411
  100. package/package.json +4 -2
  101. package/replication/knownNodes.ts +1 -1
  102. package/replication/nodeIdMapping.ts +1 -1
  103. package/replication/replicationConnection.ts +2 -2
  104. package/static/defaultConfig.yaml +4 -0
  105. package/studio/web/assets/{index-Cs_ZP9GJ.js → index-ChCctErQ.js} +2 -2
  106. package/studio/web/assets/{index-Cs_ZP9GJ.js.map → index-ChCctErQ.js.map} +1 -1
  107. package/studio/web/assets/{index-vG4Utcyg.js → index-Qu8D43wo.js} +5 -5
  108. package/studio/web/assets/{index-vG4Utcyg.js.map → index-Qu8D43wo.js.map} +1 -1
  109. package/studio/web/assets/{index-Cd4R_BKr.js → index-v3wIpSYx.js} +2 -2
  110. package/studio/web/assets/{index-Cd4R_BKr.js.map → index-v3wIpSYx.js.map} +1 -1
  111. package/studio/web/assets/{index.lazy-CqinvCXH.js → index.lazy-tVSPM7bX.js} +2 -2
  112. package/studio/web/assets/{index.lazy-CqinvCXH.js.map → index.lazy-tVSPM7bX.js.map} +1 -1
  113. package/studio/web/assets/{profiler-hYHhy-KP.js → profiler-C9as4sv-.js} +2 -2
  114. package/studio/web/assets/{profiler-hYHhy-KP.js.map → profiler-C9as4sv-.js.map} +1 -1
  115. package/studio/web/assets/{react-redux-zvetwa5O.js → react-redux-RRIhZnM6.js} +2 -2
  116. package/studio/web/assets/{react-redux-zvetwa5O.js.map → react-redux-RRIhZnM6.js.map} +1 -1
  117. package/studio/web/assets/{startRecording-lV1Mo15A.js → startRecording-DYa4zCXV.js} +2 -2
  118. package/studio/web/assets/{startRecording-lV1Mo15A.js.map → startRecording-DYa4zCXV.js.map} +1 -1
  119. package/studio/web/index.html +1 -1
  120. package/core/utility/logging/logger.js +0 -11
@@ -1,3 +1,4 @@
1
+ import { type Logger } from '../utility/logging/logger.ts';
1
2
  import { getConfigObj, getConfigValue } from '../config/configUtils.js';
2
3
  import { CONFIG_PARAMS } from '../utility/hdbTerms.js';
3
4
  import logger from '../utility/logging/harper_logger.js';
@@ -392,7 +393,7 @@ export class Application {
392
393
  packageIdentifier?: string;
393
394
  install?: { command?: string; timeout?: number };
394
395
  dirPath: string;
395
- logger: any;
396
+ logger: Logger;
396
397
  packageManagerPrefix: string; // can be used to configure a package manager prefix, specifically "sfw".
397
398
 
398
399
  constructor({ name, payload, packageIdentifier, install }: ApplicationOptions) {
@@ -503,7 +504,7 @@ export async function installApplications() {
503
504
  harperApplicationLock.applications[name] &&
504
505
  JSON.stringify(harperApplicationLock.applications[name]) === JSON.stringify(applicationConfig)
505
506
  ) {
506
- logger.info(`Application ${name} is already installed with matching configuration; skipping installation`);
507
+ logger.info?.(`Application ${name} is already installed with matching configuration; skipping installation`);
507
508
  continue;
508
509
  }
509
510
 
@@ -511,13 +512,13 @@ export async function installApplications() {
511
512
 
512
513
  harperApplicationLock.applications[name] = applicationConfig;
513
514
  } catch (error) {
514
- logger.error(`Skipping installation of application ${name} due to invalid configuration: ${error.message}`);
515
+ logger.error?.(`Skipping installation of application ${name} due to invalid configuration: ${error.message}`);
515
516
  }
516
517
  }
517
518
 
518
519
  const applicationInstallationStatuses = await Promise.allSettled(applicationInstallationPromises);
519
- logger.debug(applicationInstallationStatuses);
520
- logger.info('All root applications loaded');
520
+ logger.debug?.(applicationInstallationStatuses);
521
+ logger.info?.('All root applications loaded');
521
522
 
522
523
  // Finally, write the lock file
523
524
  await writeFile(harperApplicationLockPath, JSON.stringify(harperApplicationLock, null, 2), 'utf8');
@@ -558,7 +559,7 @@ export function nonInteractiveSpawn(
558
559
  return new Promise((resolve, reject) => {
559
560
  logger
560
561
  .loggerWithTag(`${applicationName}:spawn:${command}`)
561
- .debug(`Executing \`${command} ${args.join(' ')}\` in ${cwd}`);
562
+ .debug?.(`Executing \`${command} ${args.join(' ')}\` in ${cwd}`);
562
563
 
563
564
  const env = { ...process.env };
564
565
 
@@ -586,7 +587,7 @@ export function nonInteractiveSpawn(
586
587
  // log stdout lines immediately
587
588
  // TODO: Technically nothing guarantees that a chunk will be a complete line so need to implement
588
589
  // something here to buffer until a newline character, then log the complete line
589
- logger.loggerWithTag(`${applicationName}:spawn:${command}:stdout`).debug(chunk.toString());
590
+ logger.loggerWithTag(`${applicationName}:spawn:${command}:stdout`).debug?.(chunk.toString());
590
591
  });
591
592
 
592
593
  // buffer stderr
@@ -609,7 +610,7 @@ export function nonInteractiveSpawn(
609
610
  if (stderr) {
610
611
  printStd(applicationName, command, stderr, 'stderr');
611
612
  }
612
- logger.loggerWithTag(`${applicationName}:spawn:${command}`).debug(`Process exited with code ${code}`);
613
+ logger.loggerWithTag(`${applicationName}:spawn:${command}`).debug?.(`Process exited with code ${code}`);
613
614
  resolve({
614
615
  stdout,
615
616
  stderr,
@@ -640,6 +641,6 @@ function printStd(
640
641
  ) {
641
642
  const stdLogger = logger.loggerWithTag(`${applicationName}:spawn:${command}:${stdStreamLabel}`);
642
643
  for (const line of stdString.split('\n')) {
643
- stdLogger[level](line);
644
+ stdLogger[level]?.(line);
644
645
  }
645
646
  }
@@ -0,0 +1,49 @@
1
+ import type { Resources } from '../resources/Resources.ts';
2
+ import { type Server } from '../server/Server.ts';
3
+ import { loggerWithTag } from '../utility/logging/harper_logger.js';
4
+ import { scopedImport } from '../security/jsLoader.ts';
5
+ import * as env from '../utility/environment/environmentManager.js';
6
+ import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
7
+
8
+ export class MissingDefaultFilesOptionError extends Error {
9
+ constructor() {
10
+ super('No default files option exists. Ensure `files` is specified in config.yaml');
11
+ this.name = 'MissingDefaultFilesOptionError';
12
+ }
13
+ }
14
+
15
+ /**
16
+ * This class is used to represent the application scope for the VM context used for loading modules within an application
17
+ */
18
+ export class ApplicationScope {
19
+ logger: any;
20
+ resources: Resources;
21
+ server: Server;
22
+ mode?: 'none' | 'vm' | 'compartment'; // option to set this from the scope
23
+ dependencyContainment?: boolean; // option to set this from the scope
24
+ verifyPath?: string;
25
+ config: any;
26
+ constructor(name: string, resources: Resources, server: Server, verifyPath?: string) {
27
+ this.logger = loggerWithTag(name);
28
+
29
+ this.resources = resources;
30
+ this.server = server;
31
+
32
+ this.mode = env.get(CONFIG_PARAMS.APPLICATIONS_CONTAINMENT) ?? 'vm';
33
+ this.dependencyContainment = Boolean(env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYCONTAINMENT));
34
+ this.verifyPath = verifyPath;
35
+ }
36
+
37
+ /**
38
+ * The compartment that is used for this scope and any imports that it makes
39
+ */
40
+ compartment?: Promise<any>;
41
+ /**
42
+ * Import a file into the scope's sandbox.
43
+ * @param filePath - The path of the file to import.
44
+ * @returns A promise that resolves with the imported module or value.
45
+ */
46
+ async import(filePath: string): Promise<unknown> {
47
+ return scopedImport(filePath, this);
48
+ }
49
+ }
@@ -1,7 +1,8 @@
1
+ import { type Logger } from '../utility/logging/logger.ts';
2
+ import { loggerWithTag } from '../utility/logging/harper_logger.js';
1
3
  import type { Stats } from 'node:fs';
2
4
  import { EventEmitter, once } from 'node:events';
3
5
  import { Component, FileAndURLPathConfig } from './Component.js';
4
- import harperLogger from '../utility/logging/harper_logger.js';
5
6
  import chokidar, { FSWatcher, FSWatcherEventMap } from 'chokidar';
6
7
  import { join } from 'node:path';
7
8
  import { readFile } from 'node:fs/promises';
@@ -71,16 +72,16 @@ export type EntryHandlerEventMap = {
71
72
  export class EntryHandler extends EventEmitter<EntryHandlerEventMap> {
72
73
  #component: Component;
73
74
  #watcher?: FSWatcher;
74
- #logger: any;
75
+ #logger: Logger;
75
76
  #pendingFileReads: Set<Promise<void>>;
76
77
  #isInitialScanComplete: boolean;
77
78
  ready: Promise<any[]>;
78
79
 
79
- constructor(name: string, directory: string, config: FilesOption | FileAndURLPathConfig, logger?: any) {
80
+ constructor(name: string, directory: string, config: FilesOption | FileAndURLPathConfig, logger?: Logger) {
80
81
  super();
81
82
 
82
83
  this.#component = new Component(name, directory, castConfig(config));
83
- this.#logger = logger || harperLogger.loggerWithTag(name);
84
+ this.#logger = logger || loggerWithTag(name);
84
85
  this.#pendingFileReads = new Set();
85
86
  this.#isInitialScanComplete = false;
86
87
  this.ready = once(this, 'ready');
@@ -1,9 +1,10 @@
1
+ import { type Logger } from '../utility/logging/logger.ts';
2
+ import { loggerWithTag } from '../utility/logging/harper_logger.js';
1
3
  import { EventEmitter, once } from 'events';
2
4
  import yaml from 'yaml';
3
5
  import chokidar, { type FSWatcher } from 'chokidar';
4
6
  import { readFile } from 'node:fs/promises';
5
7
  import { isDeepStrictEqual } from 'util';
6
- import harperLogger from '../utility/logging/harper_logger.js';
7
8
  import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.js';
8
9
  import { cloneDeep } from 'lodash';
9
10
 
@@ -85,14 +86,14 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
85
86
  #scopedConfig?: ConfigValue;
86
87
  #rootConfig?: Config;
87
88
  #name: string;
88
- #logger: any;
89
+ #logger: Logger;
89
90
  ready: Promise<any[]>;
90
91
 
91
- constructor(name: string, filePath: string, logger?: any) {
92
+ constructor(name: string, filePath: string, logger?: Logger) {
92
93
  super();
93
94
  this.#name = name;
94
95
  this.#filePath = filePath;
95
- this.#logger = logger || harperLogger.loggerWithTag(name);
96
+ this.#logger = logger || loggerWithTag(name);
96
97
  this.ready = once(this, 'ready');
97
98
  this.#watcher = chokidar
98
99
  .watch(filePath, { persistent: false })
@@ -153,7 +154,7 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
153
154
  }
154
155
 
155
156
  #handleUnlink(path: string) {
156
- this.#logger.warn(
157
+ this.#logger.warn?.(
157
158
  `Configuration file ${path} was deleted. Reverting to default configuration. Recreate it to restore the options watcher.`
158
159
  );
159
160
  this.#resetConfig();
@@ -1,15 +1,14 @@
1
+ import { type Logger } from '../utility/logging/logger.ts';
2
+ import { loggerWithTag } from '../utility/logging/harper_logger.js';
1
3
  import { EventEmitter, once } from 'node:events';
2
- import { type Server } from '../server/Server.ts';
4
+ import { server, type Server } from '../server/Server.ts';
3
5
  import { EntryHandler, type EntryHandlerEventMap, type onEntryEventHandler } from './EntryHandler.ts';
4
6
  import { OptionsWatcher, OptionsWatcherEventMap } from './OptionsWatcher.ts';
5
- import { loggerWithTag } from '../utility/logging/harper_logger.js';
6
- import type { Resources } from '../resources/Resources.ts';
7
+ import { resources, type Resources } from '../resources/Resources.ts';
7
8
  import type { FileAndURLPathConfig } from './Component.ts';
8
9
  import { FilesOption } from './deriveGlobOptions.ts';
9
10
  import { requestRestart } from './requestRestart.ts';
10
- import { scopedImport } from '../security/jsLoader.ts';
11
- import * as env from '../utility/environment/environmentManager.js';
12
- import { CONFIG_PARAMS } from '../utility/hdbTerms';
11
+ import { ApplicationScope } from './ApplicationScope.ts';
13
12
 
14
13
  export class MissingDefaultFilesOptionError extends Error {
15
14
  constructor() {
@@ -18,12 +17,6 @@ export class MissingDefaultFilesOptionError extends Error {
18
17
  }
19
18
  }
20
19
 
21
- export interface ApplicationContainment {
22
- mode?: 'none' | 'vm' | 'compartment'; // option to set this from the scope
23
- dependencyContainment?: boolean; // option to set this from the scope
24
- verifyPath?: string;
25
- }
26
-
27
20
  /**
28
21
  * This class is what is passed to the `handleApplication` function of an extension.
29
22
  *
@@ -37,15 +30,15 @@ export class Scope extends EventEmitter {
37
30
  #name: string;
38
31
  #entryHandler?: EntryHandler;
39
32
  #entryHandlers: EntryHandler[];
40
- #logger: any;
33
+ #logger: Logger;
41
34
  #pendingInitialLoads: Set<Promise<void>>;
35
+ applicationScope?: ApplicationScope;
42
36
 
43
37
  options: OptionsWatcher;
44
- resources: Resources;
45
- server: Server;
38
+ resources?: Resources;
39
+ server?: Server;
46
40
  ready: Promise<any[]>;
47
- applicationContainment?: ApplicationContainment;
48
- constructor(name: string, directory: string, configFilePath: string, resources: Resources, server: Server) {
41
+ constructor(name: string, directory: string, configFilePath: string, applicationScope: ApplicationScope) {
49
42
  super();
50
43
 
51
44
  this.#name = name;
@@ -53,8 +46,9 @@ export class Scope extends EventEmitter {
53
46
  this.#configFilePath = configFilePath;
54
47
  this.#logger = loggerWithTag(this.#name);
55
48
 
56
- this.resources = resources;
57
- this.server = server;
49
+ this.applicationScope = applicationScope;
50
+ this.resources = applicationScope?.resources ?? resources;
51
+ this.server = applicationScope?.server ?? server;
58
52
 
59
53
  this.#entryHandlers = [];
60
54
  this.#pendingInitialLoads = new Set();
@@ -66,15 +60,9 @@ export class Scope extends EventEmitter {
66
60
  .on('error', this.#handleError.bind(this))
67
61
  .on('change', this.#optionsWatcherChangeListener.bind(this)())
68
62
  .on('ready', this.#handleOptionsWatcherReady.bind(this));
69
-
70
- this.applicationContainment = {
71
- mode: env.get(CONFIG_PARAMS.APPLICATIONS_CONTAINMENT) ?? 'vm',
72
- dependencyContainment: Boolean(env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYCONTAINMENT)),
73
- verifyPath: directory,
74
- };
75
63
  }
76
64
 
77
- get logger(): any {
65
+ get logger(): Logger {
78
66
  return this.#logger;
79
67
  }
80
68
 
@@ -174,7 +162,7 @@ export class Scope extends EventEmitter {
174
162
  return;
175
163
  }
176
164
 
177
- scope.#logger.debug(`Options changed: ${key.join('.')}, requesting restart`);
165
+ scope.#logger.debug?.(`Options changed: ${key.join('.')}, requesting restart`);
178
166
  scope.requestRestart();
179
167
  };
180
168
  }
@@ -291,7 +279,7 @@ export class Scope extends EventEmitter {
291
279
  }
292
280
 
293
281
  requestRestart() {
294
- this.#logger.debug(`Restart requested from ${this.name} scope for ${this.directory}`);
282
+ this.#logger.debug?.(`Restart requested from ${this.name} scope for ${this.directory}`);
295
283
  requestRestart();
296
284
  }
297
285
 
@@ -306,16 +294,12 @@ export class Scope extends EventEmitter {
306
294
  }
307
295
  }
308
296
 
309
- /**
310
- * The compartment that is used for this scope and any imports that it makes
311
- */
312
- compartment?: Promise<any>;
313
297
  /**
314
298
  * Import a file into the scope's sandbox.
315
299
  * @param filePath - The path of the file to import.
316
300
  * @returns A promise that resolves with the imported module or value.
317
301
  */
318
302
  async import(filePath: string): Promise<unknown> {
319
- return scopedImport(filePath, this);
303
+ return this.applicationScope ? this.applicationScope.import(filePath) : import(filePath);
320
304
  }
321
305
  }
@@ -29,7 +29,8 @@ import * as mqtt from '../server/mqtt.ts';
29
29
  import { getConfigObj, resolvePath } from '../config/configUtils.js';
30
30
  import { createReuseportFd } from '../server/serverHelpers/Request.ts';
31
31
  import { ErrorResource } from '../resources/ErrorResource.ts';
32
- import { Scope, type ApplicationContainment } from './Scope.ts';
32
+ import { Scope } from './Scope.ts';
33
+ import { ApplicationScope } from './ApplicationScope.ts';
33
34
  import { ComponentV1, processResourceExtensionComponent } from './ComponentV1.ts';
34
35
  import * as httpComponent from '../server/http.ts';
35
36
  import { Status } from '../server/status/index.ts';
@@ -219,6 +220,7 @@ function sequentiallyHandleApplication(scope: Scope, plugin: PluginModule) {
219
220
 
220
221
  export interface LoadComponentOptions {
221
222
  isRoot?: boolean;
223
+ applicationScope?: ApplicationScope;
222
224
  autoReload?: boolean;
223
225
  applicationContainment?: ApplicationContainment;
224
226
  providedLoadedComponents?: Map<any, any>;
@@ -241,7 +243,14 @@ export async function loadComponent(
241
243
  const resolvedFolder = realpathSync(componentDirectory);
242
244
  if (loadedPaths.has(resolvedFolder)) return loadedPaths.get(resolvedFolder);
243
245
  loadedPaths.set(resolvedFolder, true);
244
- const { providedLoadedComponents, isRoot, autoReload } = options;
246
+
247
+ const {
248
+ providedLoadedComponents,
249
+ applicationScope = new ApplicationScope(basename(componentDirectory), resources, server),
250
+ isRoot,
251
+ autoReload,
252
+ } = options;
253
+ applicationScope.verifyPath ??= componentDirectory;
245
254
  if (providedLoadedComponents) loadedComponents = providedLoadedComponents;
246
255
  try {
247
256
  let config;
@@ -257,6 +266,7 @@ export async function loadComponent(
257
266
  } else {
258
267
  config = DEFAULT_CONFIG;
259
268
  }
269
+ applicationScope.config ??= config;
260
270
 
261
271
  if (!isRoot) {
262
272
  try {
@@ -286,6 +296,8 @@ export async function loadComponent(
286
296
  // Initialize loading status for all components (applications and extensions)
287
297
  componentLifecycle.loading(componentStatusName);
288
298
 
299
+ const subApplicationScope = isRoot ? new ApplicationScope(componentName, resources, server) : applicationScope;
300
+
289
301
  let extensionModule: any;
290
302
  const pkg = componentConfig.package;
291
303
  try {
@@ -306,8 +318,11 @@ export async function loadComponent(
306
318
  }
307
319
  }
308
320
  if (componentPath) {
321
+ subApplicationScope.verifyPath = componentPath;
309
322
  if (!process.env.HARPER_SAFE_MODE) {
310
- extensionModule = await loadComponent(componentPath, resources, origin);
323
+ extensionModule = await loadComponent(componentPath, resources, origin, {
324
+ applicationScope: subApplicationScope,
325
+ });
311
326
  componentFunctionality[componentName] = true;
312
327
  }
313
328
  } else {
@@ -317,7 +332,7 @@ export async function loadComponent(
317
332
  const plugin = TRUSTED_RESOURCE_PLUGINS[componentName];
318
333
  extensionModule =
319
334
  typeof plugin === 'string'
320
- ? await scopedImport(plugin.startsWith('@/') ? join(PACKAGE_ROOT, plugin.slice(1)) : plugin)
335
+ ? await import(plugin.startsWith('@/') ? join(PACKAGE_ROOT, plugin.slice(1)) : plugin)
321
336
  : plugin;
322
337
  }
323
338
 
@@ -358,8 +373,7 @@ export async function loadComponent(
358
373
 
359
374
  // New Plugin API (`handleApplication`)
360
375
  if (resources.isWorker && extensionModule.handleApplication) {
361
- const scope = new Scope(componentName, componentDirectory, configPath, resources, server);
362
- if (options.applicationContainment) scope.applicationContainment = options.applicationContainment;
376
+ const scope = new Scope(componentName, componentDirectory, configPath, applicationScope);
363
377
 
364
378
  await sequentiallyHandleApplication(scope, extensionModule);
365
379
 
@@ -464,7 +478,10 @@ export async function loadComponent(
464
478
  });
465
479
  }
466
480
  if (config.extensionModule || config.pluginModule) {
467
- const extensionModule = await import(join(componentDirectory, config.extensionModule || config.pluginModule));
481
+ const extensionModule = await scopedImport(
482
+ join(componentDirectory, config.extensionModule || config.pluginModule),
483
+ applicationScope
484
+ );
468
485
  loadedPaths.set(resolvedFolder, extensionModule);
469
486
  return extensionModule;
470
487
  }
@@ -8,7 +8,7 @@
8
8
  import { sendItcEvent } from '../../server/threads/itc.js';
9
9
  import { getWorkerIndex, onMessageByType, getWorkerCount } from '../../server/threads/manageThreads.js';
10
10
  import { ITC_EVENT_TYPES } from '../../utility/hdbTerms.ts';
11
- import { loggerWithTag } from '../../utility/logging/logger.js';
11
+ import { loggerWithTag } from '../../utility/logging/logger.ts';
12
12
  import { ComponentStatusRegistry } from './ComponentStatusRegistry.ts';
13
13
  import {
14
14
  type ComponentStatusSummary,
@@ -11,6 +11,7 @@
11
11
  * - Snapshot-based deletion (remove values when omitted from env var)
12
12
  */
13
13
 
14
+ import type { Logger } from '../utility/logging/logger.ts';
14
15
  import * as fs from 'fs-extra';
15
16
  import * as path from 'node:path';
16
17
  import * as crypto from 'node:crypto';
@@ -22,8 +23,8 @@ const STATE_FILE_NAME = '.harper-config-state.json';
22
23
  * Get logger instance with tag - lazy loaded to avoid circular dependencies
23
24
  * and ensure logger is initialized before use
24
25
  */
25
- function getLogger() {
26
- const { loggerWithTag } = require('../utility/logging/logger.js');
26
+ function getLogger(): Logger {
27
+ const { loggerWithTag } = require('../utility/logging/harper_logger.js');
27
28
  return loggerWithTag('env-config');
28
29
  }
29
30
 
package/core/index.d.ts CHANGED
@@ -17,6 +17,7 @@ import { server as serverImport } from './server/Server.ts';
17
17
  export { tables, databases } from './resources/databases.ts';
18
18
  import { tables as dbTables, databases as dbDatabases } from './resources/databases.ts';
19
19
  import { BlobCreationOptions } from './resources/blob.ts';
20
+ import { Logger } from './utility/logging/logger.ts';
20
21
  export { Scope } from './components/Scope.ts';
21
22
  export type { FilesOption, FilesOptionObject } from './components/deriveGlobOptions.ts';
22
23
  export type { FileAndURLPathConfig } from './components/Component.ts';
@@ -34,6 +35,10 @@ export {
34
35
  type UnlinkDirectoryEvent,
35
36
  type DirectoryEntryEvent,
36
37
  } from './components/EntryHandler.ts';
38
+
39
+ declare const logger: Logger;
40
+ export { type Logger, logger };
41
+
37
42
  declare global {
38
43
  const tables: typeof dbTables;
39
44
  const databases: typeof dbDatabases;
package/core/index.js CHANGED
@@ -14,6 +14,7 @@ exports.FileAndURLPathConfig = undefined;
14
14
  exports.FilesOption = undefined;
15
15
  exports.FilesOptionObject = undefined;
16
16
  exports.IterableEventQueue = undefined;
17
+ exports.Logger = undefined;
17
18
  exports.Query = undefined;
18
19
  exports.RecordObject = undefined;
19
20
  exports.RequestTarget = undefined;
@@ -153,11 +153,6 @@ function runHarperCommand({ args, env, completionMessage, logDir }: RunHarperCom
153
153
  stderr += data.toString();
154
154
  });
155
155
 
156
- proc.on('exit', () => {
157
- stdoutStream?.end();
158
- stderrStream?.end();
159
- });
160
-
161
156
  proc.on('error', (error) => {
162
157
  reject(error);
163
158
  });
@@ -167,11 +162,14 @@ function runHarperCommand({ args, env, completionMessage, logDir }: RunHarperCom
167
162
  resolve(proc);
168
163
  } else {
169
164
  let errorMessage = `Harper process failed with exit code ${statusCode}`;
165
+ stderrStream?.write(errorMessage);
170
166
  if (stderr) {
171
167
  errorMessage += `\n\nstderr:\n${stderr}`;
172
168
  }
173
169
  reject(errorMessage);
174
170
  }
171
+ stdoutStream?.end();
172
+ stderrStream?.end();
175
173
  });
176
174
  });
177
175
  }
@@ -236,7 +234,7 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
236
234
  }
237
235
 
238
236
  // Point Harper's log directory to the suite log dir so hdb.log is preserved for upload
239
- const config = { ...(options?.config || {}) };
237
+ const config = { ...options?.config };
240
238
  if (logDir) {
241
239
  config.logging = { ...config.logging, root: logDir };
242
240
 
@@ -262,6 +260,7 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
262
260
  `--HTTP_PORT=${loopbackAddress}:${HTTP_PORT}`,
263
261
  `--OPERATIONSAPI_NETWORK_PORT=${loopbackAddress}:${OPERATIONS_API_PORT}`,
264
262
  '--LOGGING_LEVEL=debug',
263
+ '--LOGGING_STDSTREAMS=false',
265
264
  '--HARPER_SET_CONFIG=' + JSON.stringify(config),
266
265
  ],
267
266
  env: options?.env || {},
package/core/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "harper",
3
3
  "description": "Harper is an open-source Node.js performance platform that unifies database, cache, application, and messaging layers into one in-memory process.",
4
- "version": "5.0.0-unreleased",
4
+ "version": "5.0.0-alpha.8",
5
5
  "private": true,
6
6
  "license": "Apache-2.0",
7
7
  "homepage": "https://harper.fast",
@@ -99,7 +99,9 @@
99
99
  },
100
100
  "devDependencies": {
101
101
  "@harperdb/code-guidelines": "^0.0.6",
102
+ "@types/fs-extra": "^11.0.4",
102
103
  "@types/gunzip-maybe": "^1.4.3",
104
+ "@types/jsonwebtoken": "^9.0.10",
103
105
  "@types/lodash": "^4.17.23",
104
106
  "@types/micromatch": "^4.0.9",
105
107
  "@types/mocha": "^10.0.10",
@@ -75,6 +75,7 @@ export let lastMetadata: Entry | null = null;
75
75
  export class RecordEncoder extends Encoder {
76
76
  structureUpdate?: any;
77
77
  isRocksDB: boolean;
78
+ name: string;
78
79
  constructor(options) {
79
80
  options.useBigIntExtension = true;
80
81
  /**
@@ -116,14 +117,23 @@ export class RecordEncoder extends Encoder {
116
117
  if (expiresAt >= 0) {
117
118
  valueStart += 8; // make room for expiration timestamp
118
119
  expiresAtNextEncoding = -1; // reset indicator to mean no expiration
120
+ if (!(metadata & HAS_EXPIRATION)) {
121
+ throw new Error('Expiration included, but not in metadata flags');
122
+ }
119
123
  }
120
124
  if (residencyId) {
121
125
  valueStart += 4; // make room for residency id
122
126
  residencyIdAtNextEncoding = 0; // reset indicator to mean no residency id
127
+ if (!(metadata & HAS_RESIDENCY_ID)) {
128
+ throw new Error('Residency id included, but not in metadata flags');
129
+ }
123
130
  }
124
131
  if (nodeId >= 0) {
125
132
  valueStart += 4; // make room for node id
126
133
  nodeIdAtNextEncoding = -1; // reset indicator to mean no node id
134
+ if (!(metadata & HAS_NODE_ID)) {
135
+ throw new Error('Node id included, but not in metadata flags');
136
+ }
127
137
  }
128
138
  if (additionalAuditRefs && additionalAuditRefs.length > 0) {
129
139
  valueStart += 1 + additionalAuditRefs.length * 12; // 1 byte for count + 8 bytes version + 4 bytes nodeId per ref
@@ -553,17 +563,17 @@ export function recordUpdater(store, tableId, auditStore) {
553
563
  residencyIdAtNextEncoding = residencyId;
554
564
  metadataInNextEncoding |= HAS_RESIDENCY_ID;
555
565
  extendedType |= HAS_CURRENT_RESIDENCY_ID;
556
- } // else residencyIdAtNextEncoding = 0;
566
+ } else residencyIdAtNextEncoding = 0;
557
567
  const nodeId = options?.nodeId;
558
568
  if (nodeId >= 0) {
559
569
  nodeIdAtNextEncoding = nodeId;
560
570
  metadataInNextEncoding |= HAS_NODE_ID;
561
- } // else nodeIdAtNextEncoding = -1;
571
+ } else nodeIdAtNextEncoding = -1;
562
572
  const additionalAuditRefs = options?.additionalAuditRefs;
563
573
  if (additionalAuditRefs && additionalAuditRefs.length > 0) {
564
574
  additionalAuditRefsNextEncoding = additionalAuditRefs;
565
575
  metadataInNextEncoding |= HAS_ADDITIONAL_AUDIT_REFS;
566
- } // else additionalAuditRefsNextEncoding = undefined;
576
+ } else additionalAuditRefsNextEncoding = undefined;
567
577
  const previousAdditionalAuditRefs = existingEntry?.additionalAuditRefs;
568
578
  if (previousAdditionalAuditRefs && previousAdditionalAuditRefs.length > 0) {
569
579
  extendedType |= HAS_ADDITIONAL_AUDIT_REFS_AUDIT;
@@ -606,7 +616,6 @@ export function recordUpdater(store, tableId, auditStore) {
606
616
  const structureVersion = store.encoder.structures.length + (store.encoder.typedStructs?.length ?? 0);
607
617
  const nodeId = options?.nodeId ?? server.replication?.getThisNodeId(auditStore) ?? 0;
608
618
  const viaNodeId = options?.viaNodeId ?? nodeId;
609
- logger.debug('recording audit entry', { id, newVersion, previousVersion: existingEntry?.version, nodeId });
610
619
  if (resolveRecord && existingEntry?.localTime) {
611
620
  const replacingId = existingEntry?.localTime;
612
621
  const replacingEntry = auditStore.get(replacingId, tableId, id);
@@ -317,6 +317,7 @@ export class RocksTransactionLogStore extends EventEmitter {
317
317
  let totalSize = 0;
318
318
  const logs = [];
319
319
  for (const log of this.loadLogs()) {
320
+ if (!log) continue;
320
321
  const size = log.getLogFileSize();
321
322
  totalSize += size;
322
323
  logs.push({ name: log.name, size });
@@ -38,7 +38,7 @@ import {
38
38
  COERCIBLE_OPERATORS,
39
39
  executeConditions,
40
40
  } from './search.ts';
41
- import logger from '../utility/logging/logger.js';
41
+ import { logger } from '../utility/logging/logger.ts';
42
42
  import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericTrackedObject } from './tracked.ts';
43
43
  import { transaction, contextStorage } from './transaction.ts';
44
44
  import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary';
@@ -2693,7 +2693,7 @@ export function makeTable(options) {
2693
2693
  if (--count <= 0) break;
2694
2694
  }
2695
2695
  } catch (error) {
2696
- logger.error('Error getting history entry', auditRecord.localTime, error);
2696
+ logger.error?.('Error getting history entry', auditRecord.localTime, error);
2697
2697
  }
2698
2698
  // TODO: Would like to do this asynchronously, but would need to catch up on anything published during iteration
2699
2699
  //await rest(); // yield for fairness
@@ -3305,7 +3305,7 @@ export function makeTable(options) {
3305
3305
  const userResolver = this.userResolvers[attribute.name];
3306
3306
  if (userResolver) return userResolver(value, context, entry);
3307
3307
  else {
3308
- logger.warn(
3308
+ logger.warn?.(
3309
3309
  `Computed attribute "${attribute.name}" does not have a function assigned to it. Please use setComputedAttribute('${attribute.name}', resolver) to assign a resolver function.`
3310
3310
  );
3311
3311
  // silence future warnings but just returning undefined
@@ -37,7 +37,7 @@ import { ensureDirSync } from 'fs-extra';
37
37
  import { get as envGet, getHdbBasePath } from '../utility/environment/environmentManager.js';
38
38
  import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
39
39
  import { join, dirname } from 'path';
40
- import logger from '../utility/logging/logger.js';
40
+ import { logger } from '../utility/logging/logger.ts';
41
41
  import type { LMDBStore } from 'lmdb';
42
42
  import { asyncSerialization, hasAsyncSerialization } from '../server/serverHelpers/contentTypes.ts';
43
43
  import { HAS_BLOBS } from './auditStore.ts';