@harperfast/harper-pro 5.0.0-alpha.7 → 5.0.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/components/Application.ts +10 -9
- package/core/components/ApplicationScope.ts +49 -0
- package/core/components/EntryHandler.ts +5 -4
- package/core/components/OptionsWatcher.ts +6 -5
- package/core/components/Scope.ts +17 -33
- package/core/components/componentLoader.ts +24 -7
- package/core/components/status/crossThread.ts +1 -1
- package/core/config/harperConfigEnvVars.ts +3 -2
- package/core/index.d.ts +5 -0
- package/core/index.js +1 -0
- package/core/integrationTests/utils/harperLifecycle.ts +5 -6
- package/core/package.json +3 -1
- package/core/resources/RecordEncoder.ts +13 -4
- package/core/resources/RocksTransactionLogStore.ts +1 -0
- package/core/resources/Table.ts +3 -3
- package/core/resources/blob.ts +1 -1
- package/core/resources/databases.ts +26 -16
- package/core/resources/indexes/HierarchicalNavigableSmallWorld.ts +1 -1
- package/core/resources/jsResource.ts +2 -2
- package/core/security/certificateVerification/crlVerification.ts +1 -1
- package/core/security/certificateVerification/index.ts +1 -1
- package/core/security/certificateVerification/ocspVerification.ts +1 -1
- package/core/security/certificateVerification/verificationConfig.ts +1 -1
- package/core/security/certificateVerification/verificationUtils.ts +1 -1
- package/core/security/jsLoader.ts +16 -16
- package/core/security/tokenAuthentication.ts +30 -19
- package/core/server/mqtt.ts +1 -1
- package/core/server/serverHelpers/contentTypes.ts +2 -2
- package/core/server/status/index.ts +1 -1
- package/core/server/storageReclamation.ts +1 -1
- package/core/server/throttle.ts +1 -1
- package/core/unitTests/components/Scope.test.js +13 -2
- package/core/unitTests/components/fixtures/testJSWithDeps/resources.js +1 -1
- package/core/unitTests/components/globalIsolation.test.js +22 -22
- package/core/utility/lmdb/environmentUtility.js +1 -67
- package/core/utility/lmdb/searchUtility.js +0 -1
- package/core/utility/logging/logger.ts +24 -0
- package/core/utility/password.ts +2 -2
- package/core/utility/when.ts +3 -1
- package/dist/bin/harper.js +3 -1
- package/dist/bin/harper.js.map +1 -1
- package/dist/core/components/Application.js +8 -8
- package/dist/core/components/Application.js.map +1 -1
- package/dist/core/components/ApplicationScope.js +81 -0
- package/dist/core/components/ApplicationScope.js.map +1 -0
- package/dist/core/components/EntryHandler.js +2 -2
- package/dist/core/components/EntryHandler.js.map +1 -1
- package/dist/core/components/OptionsWatcher.js +3 -3
- package/dist/core/components/OptionsWatcher.js.map +1 -1
- package/dist/core/components/Scope.js +18 -52
- package/dist/core/components/Scope.js.map +1 -1
- package/dist/core/components/componentLoader.js +12 -7
- package/dist/core/components/componentLoader.js.map +1 -1
- package/dist/core/components/status/crossThread.js +2 -2
- package/dist/core/config/harperConfigEnvVars.js +1 -1
- package/dist/core/config/harperConfigEnvVars.js.map +1 -1
- package/dist/core/index.js +1 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/resources/RecordEncoder.js +19 -4
- package/dist/core/resources/RecordEncoder.js.map +1 -1
- package/dist/core/resources/RocksTransactionLogStore.js +2 -0
- package/dist/core/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/core/resources/Table.js +48 -48
- package/dist/core/resources/Table.js.map +1 -1
- package/dist/core/resources/blob.js +22 -25
- package/dist/core/resources/blob.js.map +1 -1
- package/dist/core/resources/databases.js +29 -18
- package/dist/core/resources/databases.js.map +1 -1
- package/dist/core/resources/indexes/HierarchicalNavigableSmallWorld.js +2 -2
- package/dist/core/resources/jsResource.js +2 -2
- package/dist/core/resources/jsResource.js.map +1 -1
- package/dist/core/security/certificateVerification/crlVerification.js +2 -2
- package/dist/core/security/certificateVerification/index.js +2 -2
- package/dist/core/security/certificateVerification/ocspVerification.js +2 -2
- package/dist/core/security/certificateVerification/verificationConfig.js +2 -2
- package/dist/core/security/certificateVerification/verificationUtils.js +2 -2
- package/dist/core/security/jsLoader.js +10 -10
- package/dist/core/security/jsLoader.js.map +1 -1
- package/dist/core/security/tokenAuthentication.js +15 -7
- package/dist/core/security/tokenAuthentication.js.map +1 -1
- package/dist/core/server/mqtt.js +2 -2
- package/dist/core/server/serverHelpers/contentTypes.js +3 -3
- package/dist/core/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/core/server/status/index.js +2 -2
- package/dist/core/server/storageReclamation.js +3 -3
- package/dist/core/server/storageReclamation.js.map +1 -1
- package/dist/core/server/throttle.js +2 -5
- package/dist/core/server/throttle.js.map +1 -1
- package/dist/core/utility/lmdb/environmentUtility.js +1 -63
- package/dist/core/utility/lmdb/environmentUtility.js.map +1 -1
- package/dist/core/utility/lmdb/searchUtility.js +0 -1
- package/dist/core/utility/lmdb/searchUtility.js.map +1 -1
- package/dist/core/utility/logging/logger.js +14 -7
- package/dist/core/utility/logging/logger.js.map +1 -1
- package/dist/core/utility/password.js.map +1 -1
- package/dist/core/utility/when.js.map +1 -1
- package/dist/replication/replicationConnection.js +2 -2
- package/dist/replication/replicationConnection.js.map +1 -1
- package/npm-shrinkwrap.json +455 -411
- package/package.json +4 -2
- package/replication/knownNodes.ts +1 -1
- package/replication/nodeIdMapping.ts +1 -1
- package/replication/replicationConnection.ts +2 -2
- package/static/defaultConfig.yaml +4 -0
- package/studio/web/assets/{index-Cs_ZP9GJ.js → index-ChCctErQ.js} +2 -2
- package/studio/web/assets/{index-Cs_ZP9GJ.js.map → index-ChCctErQ.js.map} +1 -1
- package/studio/web/assets/{index-vG4Utcyg.js → index-Qu8D43wo.js} +5 -5
- package/studio/web/assets/{index-vG4Utcyg.js.map → index-Qu8D43wo.js.map} +1 -1
- package/studio/web/assets/{index-Cd4R_BKr.js → index-v3wIpSYx.js} +2 -2
- package/studio/web/assets/{index-Cd4R_BKr.js.map → index-v3wIpSYx.js.map} +1 -1
- package/studio/web/assets/{index.lazy-CqinvCXH.js → index.lazy-tVSPM7bX.js} +2 -2
- package/studio/web/assets/{index.lazy-CqinvCXH.js.map → index.lazy-tVSPM7bX.js.map} +1 -1
- package/studio/web/assets/{profiler-hYHhy-KP.js → profiler-C9as4sv-.js} +2 -2
- package/studio/web/assets/{profiler-hYHhy-KP.js.map → profiler-C9as4sv-.js.map} +1 -1
- package/studio/web/assets/{react-redux-zvetwa5O.js → react-redux-RRIhZnM6.js} +2 -2
- package/studio/web/assets/{react-redux-zvetwa5O.js.map → react-redux-RRIhZnM6.js.map} +1 -1
- package/studio/web/assets/{startRecording-lV1Mo15A.js → startRecording-DYa4zCXV.js} +2 -2
- package/studio/web/assets/{startRecording-lV1Mo15A.js.map → startRecording-DYa4zCXV.js.map} +1 -1
- package/studio/web/index.html +1 -1
- package/core/utility/logging/logger.js +0 -11
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type Logger } from '../utility/logging/logger.ts';
|
|
1
2
|
import { getConfigObj, getConfigValue } from '../config/configUtils.js';
|
|
2
3
|
import { CONFIG_PARAMS } from '../utility/hdbTerms.js';
|
|
3
4
|
import logger from '../utility/logging/harper_logger.js';
|
|
@@ -392,7 +393,7 @@ export class Application {
|
|
|
392
393
|
packageIdentifier?: string;
|
|
393
394
|
install?: { command?: string; timeout?: number };
|
|
394
395
|
dirPath: string;
|
|
395
|
-
logger:
|
|
396
|
+
logger: Logger;
|
|
396
397
|
packageManagerPrefix: string; // can be used to configure a package manager prefix, specifically "sfw".
|
|
397
398
|
|
|
398
399
|
constructor({ name, payload, packageIdentifier, install }: ApplicationOptions) {
|
|
@@ -503,7 +504,7 @@ export async function installApplications() {
|
|
|
503
504
|
harperApplicationLock.applications[name] &&
|
|
504
505
|
JSON.stringify(harperApplicationLock.applications[name]) === JSON.stringify(applicationConfig)
|
|
505
506
|
) {
|
|
506
|
-
logger.info(`Application ${name} is already installed with matching configuration; skipping installation`);
|
|
507
|
+
logger.info?.(`Application ${name} is already installed with matching configuration; skipping installation`);
|
|
507
508
|
continue;
|
|
508
509
|
}
|
|
509
510
|
|
|
@@ -511,13 +512,13 @@ export async function installApplications() {
|
|
|
511
512
|
|
|
512
513
|
harperApplicationLock.applications[name] = applicationConfig;
|
|
513
514
|
} catch (error) {
|
|
514
|
-
logger.error(`Skipping installation of application ${name} due to invalid configuration: ${error.message}`);
|
|
515
|
+
logger.error?.(`Skipping installation of application ${name} due to invalid configuration: ${error.message}`);
|
|
515
516
|
}
|
|
516
517
|
}
|
|
517
518
|
|
|
518
519
|
const applicationInstallationStatuses = await Promise.allSettled(applicationInstallationPromises);
|
|
519
|
-
logger.debug(applicationInstallationStatuses);
|
|
520
|
-
logger.info('All root applications loaded');
|
|
520
|
+
logger.debug?.(applicationInstallationStatuses);
|
|
521
|
+
logger.info?.('All root applications loaded');
|
|
521
522
|
|
|
522
523
|
// Finally, write the lock file
|
|
523
524
|
await writeFile(harperApplicationLockPath, JSON.stringify(harperApplicationLock, null, 2), 'utf8');
|
|
@@ -558,7 +559,7 @@ export function nonInteractiveSpawn(
|
|
|
558
559
|
return new Promise((resolve, reject) => {
|
|
559
560
|
logger
|
|
560
561
|
.loggerWithTag(`${applicationName}:spawn:${command}`)
|
|
561
|
-
.debug(`Executing \`${command} ${args.join(' ')}\` in ${cwd}`);
|
|
562
|
+
.debug?.(`Executing \`${command} ${args.join(' ')}\` in ${cwd}`);
|
|
562
563
|
|
|
563
564
|
const env = { ...process.env };
|
|
564
565
|
|
|
@@ -586,7 +587,7 @@ export function nonInteractiveSpawn(
|
|
|
586
587
|
// log stdout lines immediately
|
|
587
588
|
// TODO: Technically nothing guarantees that a chunk will be a complete line so need to implement
|
|
588
589
|
// something here to buffer until a newline character, then log the complete line
|
|
589
|
-
logger.loggerWithTag(`${applicationName}:spawn:${command}:stdout`).debug(chunk.toString());
|
|
590
|
+
logger.loggerWithTag(`${applicationName}:spawn:${command}:stdout`).debug?.(chunk.toString());
|
|
590
591
|
});
|
|
591
592
|
|
|
592
593
|
// buffer stderr
|
|
@@ -609,7 +610,7 @@ export function nonInteractiveSpawn(
|
|
|
609
610
|
if (stderr) {
|
|
610
611
|
printStd(applicationName, command, stderr, 'stderr');
|
|
611
612
|
}
|
|
612
|
-
logger.loggerWithTag(`${applicationName}:spawn:${command}`).debug(`Process exited with code ${code}`);
|
|
613
|
+
logger.loggerWithTag(`${applicationName}:spawn:${command}`).debug?.(`Process exited with code ${code}`);
|
|
613
614
|
resolve({
|
|
614
615
|
stdout,
|
|
615
616
|
stderr,
|
|
@@ -640,6 +641,6 @@ function printStd(
|
|
|
640
641
|
) {
|
|
641
642
|
const stdLogger = logger.loggerWithTag(`${applicationName}:spawn:${command}:${stdStreamLabel}`);
|
|
642
643
|
for (const line of stdString.split('\n')) {
|
|
643
|
-
stdLogger[level](line);
|
|
644
|
+
stdLogger[level]?.(line);
|
|
644
645
|
}
|
|
645
646
|
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { Resources } from '../resources/Resources.ts';
|
|
2
|
+
import { type Server } from '../server/Server.ts';
|
|
3
|
+
import { loggerWithTag } from '../utility/logging/harper_logger.js';
|
|
4
|
+
import { scopedImport } from '../security/jsLoader.ts';
|
|
5
|
+
import * as env from '../utility/environment/environmentManager.js';
|
|
6
|
+
import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
|
|
7
|
+
|
|
8
|
+
export class MissingDefaultFilesOptionError extends Error {
|
|
9
|
+
constructor() {
|
|
10
|
+
super('No default files option exists. Ensure `files` is specified in config.yaml');
|
|
11
|
+
this.name = 'MissingDefaultFilesOptionError';
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* This class is used to represent the application scope for the VM context used for loading modules within an application
|
|
17
|
+
*/
|
|
18
|
+
export class ApplicationScope {
|
|
19
|
+
logger: any;
|
|
20
|
+
resources: Resources;
|
|
21
|
+
server: Server;
|
|
22
|
+
mode?: 'none' | 'vm' | 'compartment'; // option to set this from the scope
|
|
23
|
+
dependencyContainment?: boolean; // option to set this from the scope
|
|
24
|
+
verifyPath?: string;
|
|
25
|
+
config: any;
|
|
26
|
+
constructor(name: string, resources: Resources, server: Server, verifyPath?: string) {
|
|
27
|
+
this.logger = loggerWithTag(name);
|
|
28
|
+
|
|
29
|
+
this.resources = resources;
|
|
30
|
+
this.server = server;
|
|
31
|
+
|
|
32
|
+
this.mode = env.get(CONFIG_PARAMS.APPLICATIONS_CONTAINMENT) ?? 'vm';
|
|
33
|
+
this.dependencyContainment = Boolean(env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYCONTAINMENT));
|
|
34
|
+
this.verifyPath = verifyPath;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The compartment that is used for this scope and any imports that it makes
|
|
39
|
+
*/
|
|
40
|
+
compartment?: Promise<any>;
|
|
41
|
+
/**
|
|
42
|
+
* Import a file into the scope's sandbox.
|
|
43
|
+
* @param filePath - The path of the file to import.
|
|
44
|
+
* @returns A promise that resolves with the imported module or value.
|
|
45
|
+
*/
|
|
46
|
+
async import(filePath: string): Promise<unknown> {
|
|
47
|
+
return scopedImport(filePath, this);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { type Logger } from '../utility/logging/logger.ts';
|
|
2
|
+
import { loggerWithTag } from '../utility/logging/harper_logger.js';
|
|
1
3
|
import type { Stats } from 'node:fs';
|
|
2
4
|
import { EventEmitter, once } from 'node:events';
|
|
3
5
|
import { Component, FileAndURLPathConfig } from './Component.js';
|
|
4
|
-
import harperLogger from '../utility/logging/harper_logger.js';
|
|
5
6
|
import chokidar, { FSWatcher, FSWatcherEventMap } from 'chokidar';
|
|
6
7
|
import { join } from 'node:path';
|
|
7
8
|
import { readFile } from 'node:fs/promises';
|
|
@@ -71,16 +72,16 @@ export type EntryHandlerEventMap = {
|
|
|
71
72
|
export class EntryHandler extends EventEmitter<EntryHandlerEventMap> {
|
|
72
73
|
#component: Component;
|
|
73
74
|
#watcher?: FSWatcher;
|
|
74
|
-
#logger:
|
|
75
|
+
#logger: Logger;
|
|
75
76
|
#pendingFileReads: Set<Promise<void>>;
|
|
76
77
|
#isInitialScanComplete: boolean;
|
|
77
78
|
ready: Promise<any[]>;
|
|
78
79
|
|
|
79
|
-
constructor(name: string, directory: string, config: FilesOption | FileAndURLPathConfig, logger?:
|
|
80
|
+
constructor(name: string, directory: string, config: FilesOption | FileAndURLPathConfig, logger?: Logger) {
|
|
80
81
|
super();
|
|
81
82
|
|
|
82
83
|
this.#component = new Component(name, directory, castConfig(config));
|
|
83
|
-
this.#logger = logger ||
|
|
84
|
+
this.#logger = logger || loggerWithTag(name);
|
|
84
85
|
this.#pendingFileReads = new Set();
|
|
85
86
|
this.#isInitialScanComplete = false;
|
|
86
87
|
this.ready = once(this, 'ready');
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { type Logger } from '../utility/logging/logger.ts';
|
|
2
|
+
import { loggerWithTag } from '../utility/logging/harper_logger.js';
|
|
1
3
|
import { EventEmitter, once } from 'events';
|
|
2
4
|
import yaml from 'yaml';
|
|
3
5
|
import chokidar, { type FSWatcher } from 'chokidar';
|
|
4
6
|
import { readFile } from 'node:fs/promises';
|
|
5
7
|
import { isDeepStrictEqual } from 'util';
|
|
6
|
-
import harperLogger from '../utility/logging/harper_logger.js';
|
|
7
8
|
import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.js';
|
|
8
9
|
import { cloneDeep } from 'lodash';
|
|
9
10
|
|
|
@@ -85,14 +86,14 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
|
|
|
85
86
|
#scopedConfig?: ConfigValue;
|
|
86
87
|
#rootConfig?: Config;
|
|
87
88
|
#name: string;
|
|
88
|
-
#logger:
|
|
89
|
+
#logger: Logger;
|
|
89
90
|
ready: Promise<any[]>;
|
|
90
91
|
|
|
91
|
-
constructor(name: string, filePath: string, logger?:
|
|
92
|
+
constructor(name: string, filePath: string, logger?: Logger) {
|
|
92
93
|
super();
|
|
93
94
|
this.#name = name;
|
|
94
95
|
this.#filePath = filePath;
|
|
95
|
-
this.#logger = logger ||
|
|
96
|
+
this.#logger = logger || loggerWithTag(name);
|
|
96
97
|
this.ready = once(this, 'ready');
|
|
97
98
|
this.#watcher = chokidar
|
|
98
99
|
.watch(filePath, { persistent: false })
|
|
@@ -153,7 +154,7 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
|
|
|
153
154
|
}
|
|
154
155
|
|
|
155
156
|
#handleUnlink(path: string) {
|
|
156
|
-
this.#logger.warn(
|
|
157
|
+
this.#logger.warn?.(
|
|
157
158
|
`Configuration file ${path} was deleted. Reverting to default configuration. Recreate it to restore the options watcher.`
|
|
158
159
|
);
|
|
159
160
|
this.#resetConfig();
|
package/core/components/Scope.ts
CHANGED
|
@@ -1,15 +1,14 @@
|
|
|
1
|
+
import { type Logger } from '../utility/logging/logger.ts';
|
|
2
|
+
import { loggerWithTag } from '../utility/logging/harper_logger.js';
|
|
1
3
|
import { EventEmitter, once } from 'node:events';
|
|
2
|
-
import { type Server } from '../server/Server.ts';
|
|
4
|
+
import { server, type Server } from '../server/Server.ts';
|
|
3
5
|
import { EntryHandler, type EntryHandlerEventMap, type onEntryEventHandler } from './EntryHandler.ts';
|
|
4
6
|
import { OptionsWatcher, OptionsWatcherEventMap } from './OptionsWatcher.ts';
|
|
5
|
-
import {
|
|
6
|
-
import type { Resources } from '../resources/Resources.ts';
|
|
7
|
+
import { resources, type Resources } from '../resources/Resources.ts';
|
|
7
8
|
import type { FileAndURLPathConfig } from './Component.ts';
|
|
8
9
|
import { FilesOption } from './deriveGlobOptions.ts';
|
|
9
10
|
import { requestRestart } from './requestRestart.ts';
|
|
10
|
-
import {
|
|
11
|
-
import * as env from '../utility/environment/environmentManager.js';
|
|
12
|
-
import { CONFIG_PARAMS } from '../utility/hdbTerms';
|
|
11
|
+
import { ApplicationScope } from './ApplicationScope.ts';
|
|
13
12
|
|
|
14
13
|
export class MissingDefaultFilesOptionError extends Error {
|
|
15
14
|
constructor() {
|
|
@@ -18,12 +17,6 @@ export class MissingDefaultFilesOptionError extends Error {
|
|
|
18
17
|
}
|
|
19
18
|
}
|
|
20
19
|
|
|
21
|
-
export interface ApplicationContainment {
|
|
22
|
-
mode?: 'none' | 'vm' | 'compartment'; // option to set this from the scope
|
|
23
|
-
dependencyContainment?: boolean; // option to set this from the scope
|
|
24
|
-
verifyPath?: string;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
20
|
/**
|
|
28
21
|
* This class is what is passed to the `handleApplication` function of an extension.
|
|
29
22
|
*
|
|
@@ -37,15 +30,15 @@ export class Scope extends EventEmitter {
|
|
|
37
30
|
#name: string;
|
|
38
31
|
#entryHandler?: EntryHandler;
|
|
39
32
|
#entryHandlers: EntryHandler[];
|
|
40
|
-
#logger:
|
|
33
|
+
#logger: Logger;
|
|
41
34
|
#pendingInitialLoads: Set<Promise<void>>;
|
|
35
|
+
applicationScope?: ApplicationScope;
|
|
42
36
|
|
|
43
37
|
options: OptionsWatcher;
|
|
44
|
-
resources
|
|
45
|
-
server
|
|
38
|
+
resources?: Resources;
|
|
39
|
+
server?: Server;
|
|
46
40
|
ready: Promise<any[]>;
|
|
47
|
-
|
|
48
|
-
constructor(name: string, directory: string, configFilePath: string, resources: Resources, server: Server) {
|
|
41
|
+
constructor(name: string, directory: string, configFilePath: string, applicationScope: ApplicationScope) {
|
|
49
42
|
super();
|
|
50
43
|
|
|
51
44
|
this.#name = name;
|
|
@@ -53,8 +46,9 @@ export class Scope extends EventEmitter {
|
|
|
53
46
|
this.#configFilePath = configFilePath;
|
|
54
47
|
this.#logger = loggerWithTag(this.#name);
|
|
55
48
|
|
|
56
|
-
this.
|
|
57
|
-
this.
|
|
49
|
+
this.applicationScope = applicationScope;
|
|
50
|
+
this.resources = applicationScope?.resources ?? resources;
|
|
51
|
+
this.server = applicationScope?.server ?? server;
|
|
58
52
|
|
|
59
53
|
this.#entryHandlers = [];
|
|
60
54
|
this.#pendingInitialLoads = new Set();
|
|
@@ -66,15 +60,9 @@ export class Scope extends EventEmitter {
|
|
|
66
60
|
.on('error', this.#handleError.bind(this))
|
|
67
61
|
.on('change', this.#optionsWatcherChangeListener.bind(this)())
|
|
68
62
|
.on('ready', this.#handleOptionsWatcherReady.bind(this));
|
|
69
|
-
|
|
70
|
-
this.applicationContainment = {
|
|
71
|
-
mode: env.get(CONFIG_PARAMS.APPLICATIONS_CONTAINMENT) ?? 'vm',
|
|
72
|
-
dependencyContainment: Boolean(env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYCONTAINMENT)),
|
|
73
|
-
verifyPath: directory,
|
|
74
|
-
};
|
|
75
63
|
}
|
|
76
64
|
|
|
77
|
-
get logger():
|
|
65
|
+
get logger(): Logger {
|
|
78
66
|
return this.#logger;
|
|
79
67
|
}
|
|
80
68
|
|
|
@@ -174,7 +162,7 @@ export class Scope extends EventEmitter {
|
|
|
174
162
|
return;
|
|
175
163
|
}
|
|
176
164
|
|
|
177
|
-
scope.#logger.debug(`Options changed: ${key.join('.')}, requesting restart`);
|
|
165
|
+
scope.#logger.debug?.(`Options changed: ${key.join('.')}, requesting restart`);
|
|
178
166
|
scope.requestRestart();
|
|
179
167
|
};
|
|
180
168
|
}
|
|
@@ -291,7 +279,7 @@ export class Scope extends EventEmitter {
|
|
|
291
279
|
}
|
|
292
280
|
|
|
293
281
|
requestRestart() {
|
|
294
|
-
this.#logger.debug(`Restart requested from ${this.name} scope for ${this.directory}`);
|
|
282
|
+
this.#logger.debug?.(`Restart requested from ${this.name} scope for ${this.directory}`);
|
|
295
283
|
requestRestart();
|
|
296
284
|
}
|
|
297
285
|
|
|
@@ -306,16 +294,12 @@ export class Scope extends EventEmitter {
|
|
|
306
294
|
}
|
|
307
295
|
}
|
|
308
296
|
|
|
309
|
-
/**
|
|
310
|
-
* The compartment that is used for this scope and any imports that it makes
|
|
311
|
-
*/
|
|
312
|
-
compartment?: Promise<any>;
|
|
313
297
|
/**
|
|
314
298
|
* Import a file into the scope's sandbox.
|
|
315
299
|
* @param filePath - The path of the file to import.
|
|
316
300
|
* @returns A promise that resolves with the imported module or value.
|
|
317
301
|
*/
|
|
318
302
|
async import(filePath: string): Promise<unknown> {
|
|
319
|
-
return
|
|
303
|
+
return this.applicationScope ? this.applicationScope.import(filePath) : import(filePath);
|
|
320
304
|
}
|
|
321
305
|
}
|
|
@@ -29,7 +29,8 @@ import * as mqtt from '../server/mqtt.ts';
|
|
|
29
29
|
import { getConfigObj, resolvePath } from '../config/configUtils.js';
|
|
30
30
|
import { createReuseportFd } from '../server/serverHelpers/Request.ts';
|
|
31
31
|
import { ErrorResource } from '../resources/ErrorResource.ts';
|
|
32
|
-
import { Scope
|
|
32
|
+
import { Scope } from './Scope.ts';
|
|
33
|
+
import { ApplicationScope } from './ApplicationScope.ts';
|
|
33
34
|
import { ComponentV1, processResourceExtensionComponent } from './ComponentV1.ts';
|
|
34
35
|
import * as httpComponent from '../server/http.ts';
|
|
35
36
|
import { Status } from '../server/status/index.ts';
|
|
@@ -219,6 +220,7 @@ function sequentiallyHandleApplication(scope: Scope, plugin: PluginModule) {
|
|
|
219
220
|
|
|
220
221
|
export interface LoadComponentOptions {
|
|
221
222
|
isRoot?: boolean;
|
|
223
|
+
applicationScope?: ApplicationScope;
|
|
222
224
|
autoReload?: boolean;
|
|
223
225
|
applicationContainment?: ApplicationContainment;
|
|
224
226
|
providedLoadedComponents?: Map<any, any>;
|
|
@@ -241,7 +243,14 @@ export async function loadComponent(
|
|
|
241
243
|
const resolvedFolder = realpathSync(componentDirectory);
|
|
242
244
|
if (loadedPaths.has(resolvedFolder)) return loadedPaths.get(resolvedFolder);
|
|
243
245
|
loadedPaths.set(resolvedFolder, true);
|
|
244
|
-
|
|
246
|
+
|
|
247
|
+
const {
|
|
248
|
+
providedLoadedComponents,
|
|
249
|
+
applicationScope = new ApplicationScope(basename(componentDirectory), resources, server),
|
|
250
|
+
isRoot,
|
|
251
|
+
autoReload,
|
|
252
|
+
} = options;
|
|
253
|
+
applicationScope.verifyPath ??= componentDirectory;
|
|
245
254
|
if (providedLoadedComponents) loadedComponents = providedLoadedComponents;
|
|
246
255
|
try {
|
|
247
256
|
let config;
|
|
@@ -257,6 +266,7 @@ export async function loadComponent(
|
|
|
257
266
|
} else {
|
|
258
267
|
config = DEFAULT_CONFIG;
|
|
259
268
|
}
|
|
269
|
+
applicationScope.config ??= config;
|
|
260
270
|
|
|
261
271
|
if (!isRoot) {
|
|
262
272
|
try {
|
|
@@ -286,6 +296,8 @@ export async function loadComponent(
|
|
|
286
296
|
// Initialize loading status for all components (applications and extensions)
|
|
287
297
|
componentLifecycle.loading(componentStatusName);
|
|
288
298
|
|
|
299
|
+
const subApplicationScope = isRoot ? new ApplicationScope(componentName, resources, server) : applicationScope;
|
|
300
|
+
|
|
289
301
|
let extensionModule: any;
|
|
290
302
|
const pkg = componentConfig.package;
|
|
291
303
|
try {
|
|
@@ -306,8 +318,11 @@ export async function loadComponent(
|
|
|
306
318
|
}
|
|
307
319
|
}
|
|
308
320
|
if (componentPath) {
|
|
321
|
+
subApplicationScope.verifyPath = componentPath;
|
|
309
322
|
if (!process.env.HARPER_SAFE_MODE) {
|
|
310
|
-
extensionModule = await loadComponent(componentPath, resources, origin
|
|
323
|
+
extensionModule = await loadComponent(componentPath, resources, origin, {
|
|
324
|
+
applicationScope: subApplicationScope,
|
|
325
|
+
});
|
|
311
326
|
componentFunctionality[componentName] = true;
|
|
312
327
|
}
|
|
313
328
|
} else {
|
|
@@ -317,7 +332,7 @@ export async function loadComponent(
|
|
|
317
332
|
const plugin = TRUSTED_RESOURCE_PLUGINS[componentName];
|
|
318
333
|
extensionModule =
|
|
319
334
|
typeof plugin === 'string'
|
|
320
|
-
? await
|
|
335
|
+
? await import(plugin.startsWith('@/') ? join(PACKAGE_ROOT, plugin.slice(1)) : plugin)
|
|
321
336
|
: plugin;
|
|
322
337
|
}
|
|
323
338
|
|
|
@@ -358,8 +373,7 @@ export async function loadComponent(
|
|
|
358
373
|
|
|
359
374
|
// New Plugin API (`handleApplication`)
|
|
360
375
|
if (resources.isWorker && extensionModule.handleApplication) {
|
|
361
|
-
const scope = new Scope(componentName, componentDirectory, configPath,
|
|
362
|
-
if (options.applicationContainment) scope.applicationContainment = options.applicationContainment;
|
|
376
|
+
const scope = new Scope(componentName, componentDirectory, configPath, applicationScope);
|
|
363
377
|
|
|
364
378
|
await sequentiallyHandleApplication(scope, extensionModule);
|
|
365
379
|
|
|
@@ -464,7 +478,10 @@ export async function loadComponent(
|
|
|
464
478
|
});
|
|
465
479
|
}
|
|
466
480
|
if (config.extensionModule || config.pluginModule) {
|
|
467
|
-
const extensionModule = await
|
|
481
|
+
const extensionModule = await scopedImport(
|
|
482
|
+
join(componentDirectory, config.extensionModule || config.pluginModule),
|
|
483
|
+
applicationScope
|
|
484
|
+
);
|
|
468
485
|
loadedPaths.set(resolvedFolder, extensionModule);
|
|
469
486
|
return extensionModule;
|
|
470
487
|
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import { sendItcEvent } from '../../server/threads/itc.js';
|
|
9
9
|
import { getWorkerIndex, onMessageByType, getWorkerCount } from '../../server/threads/manageThreads.js';
|
|
10
10
|
import { ITC_EVENT_TYPES } from '../../utility/hdbTerms.ts';
|
|
11
|
-
import { loggerWithTag } from '../../utility/logging/logger.
|
|
11
|
+
import { loggerWithTag } from '../../utility/logging/logger.ts';
|
|
12
12
|
import { ComponentStatusRegistry } from './ComponentStatusRegistry.ts';
|
|
13
13
|
import {
|
|
14
14
|
type ComponentStatusSummary,
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* - Snapshot-based deletion (remove values when omitted from env var)
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
+
import type { Logger } from '../utility/logging/logger.ts';
|
|
14
15
|
import * as fs from 'fs-extra';
|
|
15
16
|
import * as path from 'node:path';
|
|
16
17
|
import * as crypto from 'node:crypto';
|
|
@@ -22,8 +23,8 @@ const STATE_FILE_NAME = '.harper-config-state.json';
|
|
|
22
23
|
* Get logger instance with tag - lazy loaded to avoid circular dependencies
|
|
23
24
|
* and ensure logger is initialized before use
|
|
24
25
|
*/
|
|
25
|
-
function getLogger() {
|
|
26
|
-
const { loggerWithTag } = require('../utility/logging/
|
|
26
|
+
function getLogger(): Logger {
|
|
27
|
+
const { loggerWithTag } = require('../utility/logging/harper_logger.js');
|
|
27
28
|
return loggerWithTag('env-config');
|
|
28
29
|
}
|
|
29
30
|
|
package/core/index.d.ts
CHANGED
|
@@ -17,6 +17,7 @@ import { server as serverImport } from './server/Server.ts';
|
|
|
17
17
|
export { tables, databases } from './resources/databases.ts';
|
|
18
18
|
import { tables as dbTables, databases as dbDatabases } from './resources/databases.ts';
|
|
19
19
|
import { BlobCreationOptions } from './resources/blob.ts';
|
|
20
|
+
import { Logger } from './utility/logging/logger.ts';
|
|
20
21
|
export { Scope } from './components/Scope.ts';
|
|
21
22
|
export type { FilesOption, FilesOptionObject } from './components/deriveGlobOptions.ts';
|
|
22
23
|
export type { FileAndURLPathConfig } from './components/Component.ts';
|
|
@@ -34,6 +35,10 @@ export {
|
|
|
34
35
|
type UnlinkDirectoryEvent,
|
|
35
36
|
type DirectoryEntryEvent,
|
|
36
37
|
} from './components/EntryHandler.ts';
|
|
38
|
+
|
|
39
|
+
declare const logger: Logger;
|
|
40
|
+
export { type Logger, logger };
|
|
41
|
+
|
|
37
42
|
declare global {
|
|
38
43
|
const tables: typeof dbTables;
|
|
39
44
|
const databases: typeof dbDatabases;
|
package/core/index.js
CHANGED
|
@@ -14,6 +14,7 @@ exports.FileAndURLPathConfig = undefined;
|
|
|
14
14
|
exports.FilesOption = undefined;
|
|
15
15
|
exports.FilesOptionObject = undefined;
|
|
16
16
|
exports.IterableEventQueue = undefined;
|
|
17
|
+
exports.Logger = undefined;
|
|
17
18
|
exports.Query = undefined;
|
|
18
19
|
exports.RecordObject = undefined;
|
|
19
20
|
exports.RequestTarget = undefined;
|
|
@@ -153,11 +153,6 @@ function runHarperCommand({ args, env, completionMessage, logDir }: RunHarperCom
|
|
|
153
153
|
stderr += data.toString();
|
|
154
154
|
});
|
|
155
155
|
|
|
156
|
-
proc.on('exit', () => {
|
|
157
|
-
stdoutStream?.end();
|
|
158
|
-
stderrStream?.end();
|
|
159
|
-
});
|
|
160
|
-
|
|
161
156
|
proc.on('error', (error) => {
|
|
162
157
|
reject(error);
|
|
163
158
|
});
|
|
@@ -167,11 +162,14 @@ function runHarperCommand({ args, env, completionMessage, logDir }: RunHarperCom
|
|
|
167
162
|
resolve(proc);
|
|
168
163
|
} else {
|
|
169
164
|
let errorMessage = `Harper process failed with exit code ${statusCode}`;
|
|
165
|
+
stderrStream?.write(errorMessage);
|
|
170
166
|
if (stderr) {
|
|
171
167
|
errorMessage += `\n\nstderr:\n${stderr}`;
|
|
172
168
|
}
|
|
173
169
|
reject(errorMessage);
|
|
174
170
|
}
|
|
171
|
+
stdoutStream?.end();
|
|
172
|
+
stderrStream?.end();
|
|
175
173
|
});
|
|
176
174
|
});
|
|
177
175
|
}
|
|
@@ -236,7 +234,7 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
|
|
|
236
234
|
}
|
|
237
235
|
|
|
238
236
|
// Point Harper's log directory to the suite log dir so hdb.log is preserved for upload
|
|
239
|
-
const config = { ...
|
|
237
|
+
const config = { ...options?.config };
|
|
240
238
|
if (logDir) {
|
|
241
239
|
config.logging = { ...config.logging, root: logDir };
|
|
242
240
|
|
|
@@ -262,6 +260,7 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
|
|
|
262
260
|
`--HTTP_PORT=${loopbackAddress}:${HTTP_PORT}`,
|
|
263
261
|
`--OPERATIONSAPI_NETWORK_PORT=${loopbackAddress}:${OPERATIONS_API_PORT}`,
|
|
264
262
|
'--LOGGING_LEVEL=debug',
|
|
263
|
+
'--LOGGING_STDSTREAMS=false',
|
|
265
264
|
'--HARPER_SET_CONFIG=' + JSON.stringify(config),
|
|
266
265
|
],
|
|
267
266
|
env: options?.env || {},
|
package/core/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "harper",
|
|
3
3
|
"description": "Harper is an open-source Node.js performance platform that unifies database, cache, application, and messaging layers into one in-memory process.",
|
|
4
|
-
"version": "5.0.0-
|
|
4
|
+
"version": "5.0.0-alpha.8",
|
|
5
5
|
"private": true,
|
|
6
6
|
"license": "Apache-2.0",
|
|
7
7
|
"homepage": "https://harper.fast",
|
|
@@ -99,7 +99,9 @@
|
|
|
99
99
|
},
|
|
100
100
|
"devDependencies": {
|
|
101
101
|
"@harperdb/code-guidelines": "^0.0.6",
|
|
102
|
+
"@types/fs-extra": "^11.0.4",
|
|
102
103
|
"@types/gunzip-maybe": "^1.4.3",
|
|
104
|
+
"@types/jsonwebtoken": "^9.0.10",
|
|
103
105
|
"@types/lodash": "^4.17.23",
|
|
104
106
|
"@types/micromatch": "^4.0.9",
|
|
105
107
|
"@types/mocha": "^10.0.10",
|
|
@@ -75,6 +75,7 @@ export let lastMetadata: Entry | null = null;
|
|
|
75
75
|
export class RecordEncoder extends Encoder {
|
|
76
76
|
structureUpdate?: any;
|
|
77
77
|
isRocksDB: boolean;
|
|
78
|
+
name: string;
|
|
78
79
|
constructor(options) {
|
|
79
80
|
options.useBigIntExtension = true;
|
|
80
81
|
/**
|
|
@@ -116,14 +117,23 @@ export class RecordEncoder extends Encoder {
|
|
|
116
117
|
if (expiresAt >= 0) {
|
|
117
118
|
valueStart += 8; // make room for expiration timestamp
|
|
118
119
|
expiresAtNextEncoding = -1; // reset indicator to mean no expiration
|
|
120
|
+
if (!(metadata & HAS_EXPIRATION)) {
|
|
121
|
+
throw new Error('Expiration included, but not in metadata flags');
|
|
122
|
+
}
|
|
119
123
|
}
|
|
120
124
|
if (residencyId) {
|
|
121
125
|
valueStart += 4; // make room for residency id
|
|
122
126
|
residencyIdAtNextEncoding = 0; // reset indicator to mean no residency id
|
|
127
|
+
if (!(metadata & HAS_RESIDENCY_ID)) {
|
|
128
|
+
throw new Error('Residency id included, but not in metadata flags');
|
|
129
|
+
}
|
|
123
130
|
}
|
|
124
131
|
if (nodeId >= 0) {
|
|
125
132
|
valueStart += 4; // make room for node id
|
|
126
133
|
nodeIdAtNextEncoding = -1; // reset indicator to mean no node id
|
|
134
|
+
if (!(metadata & HAS_NODE_ID)) {
|
|
135
|
+
throw new Error('Node id included, but not in metadata flags');
|
|
136
|
+
}
|
|
127
137
|
}
|
|
128
138
|
if (additionalAuditRefs && additionalAuditRefs.length > 0) {
|
|
129
139
|
valueStart += 1 + additionalAuditRefs.length * 12; // 1 byte for count + 8 bytes version + 4 bytes nodeId per ref
|
|
@@ -553,17 +563,17 @@ export function recordUpdater(store, tableId, auditStore) {
|
|
|
553
563
|
residencyIdAtNextEncoding = residencyId;
|
|
554
564
|
metadataInNextEncoding |= HAS_RESIDENCY_ID;
|
|
555
565
|
extendedType |= HAS_CURRENT_RESIDENCY_ID;
|
|
556
|
-
}
|
|
566
|
+
} else residencyIdAtNextEncoding = 0;
|
|
557
567
|
const nodeId = options?.nodeId;
|
|
558
568
|
if (nodeId >= 0) {
|
|
559
569
|
nodeIdAtNextEncoding = nodeId;
|
|
560
570
|
metadataInNextEncoding |= HAS_NODE_ID;
|
|
561
|
-
}
|
|
571
|
+
} else nodeIdAtNextEncoding = -1;
|
|
562
572
|
const additionalAuditRefs = options?.additionalAuditRefs;
|
|
563
573
|
if (additionalAuditRefs && additionalAuditRefs.length > 0) {
|
|
564
574
|
additionalAuditRefsNextEncoding = additionalAuditRefs;
|
|
565
575
|
metadataInNextEncoding |= HAS_ADDITIONAL_AUDIT_REFS;
|
|
566
|
-
}
|
|
576
|
+
} else additionalAuditRefsNextEncoding = undefined;
|
|
567
577
|
const previousAdditionalAuditRefs = existingEntry?.additionalAuditRefs;
|
|
568
578
|
if (previousAdditionalAuditRefs && previousAdditionalAuditRefs.length > 0) {
|
|
569
579
|
extendedType |= HAS_ADDITIONAL_AUDIT_REFS_AUDIT;
|
|
@@ -606,7 +616,6 @@ export function recordUpdater(store, tableId, auditStore) {
|
|
|
606
616
|
const structureVersion = store.encoder.structures.length + (store.encoder.typedStructs?.length ?? 0);
|
|
607
617
|
const nodeId = options?.nodeId ?? server.replication?.getThisNodeId(auditStore) ?? 0;
|
|
608
618
|
const viaNodeId = options?.viaNodeId ?? nodeId;
|
|
609
|
-
logger.debug('recording audit entry', { id, newVersion, previousVersion: existingEntry?.version, nodeId });
|
|
610
619
|
if (resolveRecord && existingEntry?.localTime) {
|
|
611
620
|
const replacingId = existingEntry?.localTime;
|
|
612
621
|
const replacingEntry = auditStore.get(replacingId, tableId, id);
|
|
@@ -317,6 +317,7 @@ export class RocksTransactionLogStore extends EventEmitter {
|
|
|
317
317
|
let totalSize = 0;
|
|
318
318
|
const logs = [];
|
|
319
319
|
for (const log of this.loadLogs()) {
|
|
320
|
+
if (!log) continue;
|
|
320
321
|
const size = log.getLogFileSize();
|
|
321
322
|
totalSize += size;
|
|
322
323
|
logs.push({ name: log.name, size });
|
package/core/resources/Table.ts
CHANGED
|
@@ -38,7 +38,7 @@ import {
|
|
|
38
38
|
COERCIBLE_OPERATORS,
|
|
39
39
|
executeConditions,
|
|
40
40
|
} from './search.ts';
|
|
41
|
-
import logger from '../utility/logging/logger.
|
|
41
|
+
import { logger } from '../utility/logging/logger.ts';
|
|
42
42
|
import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericTrackedObject } from './tracked.ts';
|
|
43
43
|
import { transaction, contextStorage } from './transaction.ts';
|
|
44
44
|
import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary';
|
|
@@ -2693,7 +2693,7 @@ export function makeTable(options) {
|
|
|
2693
2693
|
if (--count <= 0) break;
|
|
2694
2694
|
}
|
|
2695
2695
|
} catch (error) {
|
|
2696
|
-
logger.error('Error getting history entry', auditRecord.localTime, error);
|
|
2696
|
+
logger.error?.('Error getting history entry', auditRecord.localTime, error);
|
|
2697
2697
|
}
|
|
2698
2698
|
// TODO: Would like to do this asynchronously, but would need to catch up on anything published during iteration
|
|
2699
2699
|
//await rest(); // yield for fairness
|
|
@@ -3305,7 +3305,7 @@ export function makeTable(options) {
|
|
|
3305
3305
|
const userResolver = this.userResolvers[attribute.name];
|
|
3306
3306
|
if (userResolver) return userResolver(value, context, entry);
|
|
3307
3307
|
else {
|
|
3308
|
-
logger.warn(
|
|
3308
|
+
logger.warn?.(
|
|
3309
3309
|
`Computed attribute "${attribute.name}" does not have a function assigned to it. Please use setComputedAttribute('${attribute.name}', resolver) to assign a resolver function.`
|
|
3310
3310
|
);
|
|
3311
3311
|
// silence future warnings but just returning undefined
|
package/core/resources/blob.ts
CHANGED
|
@@ -37,7 +37,7 @@ import { ensureDirSync } from 'fs-extra';
|
|
|
37
37
|
import { get as envGet, getHdbBasePath } from '../utility/environment/environmentManager.js';
|
|
38
38
|
import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
|
|
39
39
|
import { join, dirname } from 'path';
|
|
40
|
-
import logger from '../utility/logging/logger.
|
|
40
|
+
import { logger } from '../utility/logging/logger.ts';
|
|
41
41
|
import type { LMDBStore } from 'lmdb';
|
|
42
42
|
import { asyncSerialization, hasAsyncSerialization } from '../server/serverHelpers/contentTypes.ts';
|
|
43
43
|
import { HAS_BLOBS } from './auditStore.ts';
|