@harperfast/harper-pro 5.0.1 → 5.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/analytics/profile.ts +47 -0
  2. package/core/components/Application.ts +24 -9
  3. package/core/components/ApplicationScope.ts +2 -3
  4. package/core/components/componentLoader.ts +13 -2
  5. package/core/config/harperConfigEnvVars.ts +34 -0
  6. package/core/resources/Table.ts +12 -1
  7. package/core/resources/analytics/write.ts +7 -10
  8. package/core/resources/indexes/HierarchicalNavigableSmallWorld.ts +23 -7
  9. package/core/security/jsLoader.ts +68 -22
  10. package/core/security/user.ts +10 -8
  11. package/core/static/defaultConfig.yaml +1 -1
  12. package/dist/analytics/profile.js +47 -0
  13. package/dist/analytics/profile.js.map +1 -1
  14. package/dist/core/components/Application.js +15 -5
  15. package/dist/core/components/Application.js.map +1 -1
  16. package/dist/core/components/ApplicationScope.js +2 -3
  17. package/dist/core/components/ApplicationScope.js.map +1 -1
  18. package/dist/core/components/componentLoader.js +11 -2
  19. package/dist/core/components/componentLoader.js.map +1 -1
  20. package/dist/core/config/harperConfigEnvVars.js +33 -0
  21. package/dist/core/config/harperConfigEnvVars.js.map +1 -1
  22. package/dist/core/resources/Table.js +12 -1
  23. package/dist/core/resources/Table.js.map +1 -1
  24. package/dist/core/resources/analytics/write.js +7 -10
  25. package/dist/core/resources/analytics/write.js.map +1 -1
  26. package/dist/core/resources/indexes/HierarchicalNavigableSmallWorld.js +24 -7
  27. package/dist/core/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
  28. package/dist/core/security/jsLoader.js +54 -21
  29. package/dist/core/security/jsLoader.js.map +1 -1
  30. package/dist/core/security/user.js +9 -8
  31. package/dist/core/security/user.js.map +1 -1
  32. package/dist/replication/setNode.js +5 -2
  33. package/dist/replication/setNode.js.map +1 -1
  34. package/dist/security/certificate.js +28 -6
  35. package/dist/security/certificate.js.map +1 -1
  36. package/npm-shrinkwrap.json +2 -2
  37. package/package.json +1 -1
  38. package/replication/setNode.ts +5 -2
  39. package/security/certificate.ts +28 -6
  40. package/static/defaultConfig.yaml +1 -1
  41. package/studio/web/assets/{index-C0iJWrnF.js → index-f5-e8ocl.js} +5 -5
  42. package/studio/web/assets/{index-C0iJWrnF.js.map → index-f5-e8ocl.js.map} +1 -1
  43. package/studio/web/assets/{index.lazy-C647wC7n.js → index.lazy-CCd1vMot.js} +2 -2
  44. package/studio/web/assets/{index.lazy-C647wC7n.js.map → index.lazy-CCd1vMot.js.map} +1 -1
  45. package/studio/web/assets/{profile-BTS_ZjxV.js → profile-gjpePJuu.js} +2 -2
  46. package/studio/web/assets/{profile-BTS_ZjxV.js.map → profile-gjpePJuu.js.map} +1 -1
  47. package/studio/web/assets/{status-Dc-S5M23.js → status-CmoVx0A5.js} +2 -2
  48. package/studio/web/assets/{status-Dc-S5M23.js.map → status-CmoVx0A5.js.map} +1 -1
  49. package/studio/web/index.html +1 -1
@@ -6,6 +6,10 @@ import { recordAction } from '../core/resources/analytics/write.ts';
6
6
  import { getHdbBasePath } from '../core/utility/environment/environmentManager.js';
7
7
  import { PACKAGE_ROOT } from '../core/utility/packageUtils.js';
8
8
  import { realpathSync, readFileSync, readdirSync } from 'node:fs';
9
+ import { execFile } from 'node:child_process';
10
+ import { promisify } from 'node:util';
11
+
12
+ const execFileAsync = promisify(execFile);
9
13
  import { time as timeProfiler } from '@datadog/pprof';
10
14
  import { getWorkerIndex } from '../core/server/threads/manageThreads.js';
11
15
  import * as log from '../core/utility/logging/harper_logger.js';
@@ -42,6 +46,7 @@ export function handleApplication({ options }: Scope) {
42
46
  }, 1000); // wait for everything to load before we start the profiler
43
47
  }
44
48
  let lastChildCpuTime = 0;
49
+ let gpuAvailable = true;
45
50
 
46
51
  export async function captureProfile(delayToNextCapture = (capturePeriod ?? 60) * 1000): Promise<void> {
47
52
  clearTimeout(profilerTimer);
@@ -58,6 +63,8 @@ export async function captureProfile(delayToNextCapture = (capturePeriod ?? 60)
58
63
  const samplesByLocationId = new Map<number, number>();
59
64
  let totalUserCount = 0;
60
65
  let totalHarperCount = 0;
66
+ // Start GPU measurement early so it runs in parallel with CPU profiling work
67
+ const gpuPromise = getWorkerIndex() === 0 && gpuAvailable ? getGpuUtilization() : null;
61
68
  try {
62
69
  const profile = timeProfiler.stop(true);
63
70
  const strings = profile.stringTable.strings;
@@ -89,6 +96,11 @@ export async function captureProfile(delayToNextCapture = (capturePeriod ?? 60)
89
96
  recordAction(childCpuTimeInInterval, 'cpu-usage', 'user', 'child-processes');
90
97
  lastChildCpuTime = childCpuTime;
91
98
  }
99
+ // Record GPU utilization for this process and child processes
100
+ const gpuSeconds = await gpuPromise;
101
+ if (gpuSeconds !== null) {
102
+ recordAction(gpuSeconds, 'gpu-usage', 'user');
103
+ }
92
104
  }
93
105
  } catch (error) {
94
106
  log.error?.('analytics profiler error:', error);
@@ -177,6 +189,41 @@ function getChildProcessCpuTime(): number | null {
177
189
  }
178
190
  }
179
191
 
192
+ /**
193
+ * Get the total SM (shader/streaming multiprocessor) utilization percentage across all GPUs
194
+ * for this process and all descendant processes.
195
+ * Uses nvidia-smi pmon for per-process GPU utilization.
196
+ * Only works on Linux with NVIDIA GPUs. Returns null if unavailable.
197
+ */
198
+ async function getGpuUtilization(): Promise<number | null> {
199
+ try {
200
+ const currentPid = process.pid;
201
+ const descendants = findAllDescendants(currentPid);
202
+ const pidsToMonitor = new Set([currentPid, ...descendants]);
203
+
204
+ const { stdout } = await execFileAsync('nvidia-smi', ['pmon', '-c', '1', '-s', 'u']);
205
+
206
+ let totalSmPercent = 0;
207
+ for (const line of stdout.split('\n')) {
208
+ if (line.startsWith('#') || !line.trim()) continue;
209
+ const parts = line.trim().split(/\s+/);
210
+ // pmon -s u format: gpu pid type fb sm enc dec jpg ofa command
211
+ if (parts.length < 5) continue;
212
+ const pid = parseInt(parts[1], 10);
213
+ if (isNaN(pid) || !pidsToMonitor.has(pid)) continue;
214
+ const sm = parseInt(parts[4], 10); // SM utilization %
215
+ if (!isNaN(sm)) totalSmPercent += sm;
216
+ }
217
+
218
+ // Convert SM utilization % to GPU-seconds over the capture period
219
+ // e.g. 50% utilization over 60s = 30 GPU-seconds
220
+ return (totalSmPercent / 100) * (capturePeriod / 1000);
221
+ } catch {
222
+ gpuAvailable = false;
223
+ return null;
224
+ }
225
+ }
226
+
180
227
  /**
181
228
  * Recursively find all descendant PIDs of the given parent PID.
182
229
  */
@@ -172,15 +172,30 @@ export async function extractApplication(application: Application) {
172
172
  }
173
173
  }
174
174
  } else {
175
- // Given a package, resolve using `npm pack` (downloads the package as a tarball and writes the path to stdout)
176
- const {
177
- stdout: tarballFilePath,
178
- code,
179
- stderr,
180
- } = await nonInteractiveSpawn(application.name, 'npm', ['pack', application.packageIdentifier], parentDirPath);
181
- if (code !== 0) throw new Error(`Failed to download package ${application.packageIdentifier}: ${stderr}`);
182
- tarballPath = join(parentDirPath, tarballFilePath.trim());
183
- // Create a Readable from the tarball
175
+ // `npm pack --json` writes a JSON array describing the packed tarball(s).
176
+ const { stdout, code, stderr } = await nonInteractiveSpawn(
177
+ application.name,
178
+ 'npm',
179
+ ['pack', '--json', application.packageIdentifier],
180
+ parentDirPath
181
+ );
182
+ if (code !== 0) {
183
+ throw new Error(`Failed to download package ${application.packageIdentifier}: ${stderr}`);
184
+ }
185
+
186
+ let packResult: Array<{ filename: string }>;
187
+ try {
188
+ packResult = JSON.parse(stdout.slice(stdout.indexOf('[')));
189
+ } catch (err) {
190
+ throw new Error(
191
+ `Failed to parse npm pack output for ${application.packageIdentifier}: ${err.message}\nstdout: ${stdout}`
192
+ );
193
+ }
194
+ if (!Array.isArray(packResult) || typeof packResult[0]?.filename !== 'string') {
195
+ throw new Error(`Unexpected npm pack output for ${application.packageIdentifier}:\n${stdout}`);
196
+ }
197
+
198
+ tarballPath = join(parentDirPath, packResult[0].filename);
184
199
  tarball = createReadStream(tarballPath);
185
200
  }
186
201
  }
@@ -21,10 +21,10 @@ export class ApplicationScope {
21
21
  server: Server;
22
22
  mode?: 'native' | 'vm' | 'vm-current-context' | 'compartment'; // option to set this from the scope
23
23
  dependencyLoader?: 'native' | 'app' | 'auto'; // option to set this from the scope
24
- verifyPath?: string;
24
+ allowedPath?: string;
25
25
  config: any;
26
26
  moduleCache: any; // used by the loader to retain a cache of modules, type is an internal detail of the loader
27
- constructor(name: string, resources: Resources, server: Server, isInternal = false, verifyPath?: string) {
27
+ constructor(name: string, resources: Resources, server: Server, isInternal = false) {
28
28
  this.logger = forComponent(name, !isInternal);
29
29
 
30
30
  this.resources = resources;
@@ -32,7 +32,6 @@ export class ApplicationScope {
32
32
 
33
33
  this.mode = env.get(CONFIG_PARAMS.APPLICATIONS_MODULELOADER) ?? 'vm';
34
34
  this.dependencyLoader = env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYLOADER);
35
- this.verifyPath = verifyPath;
36
35
  }
37
36
 
38
37
  /**
@@ -274,7 +274,7 @@ export async function loadComponent(
274
274
  autoReload,
275
275
  appName,
276
276
  } = options;
277
- applicationScope.verifyPath ??= componentDirectory;
277
+ applicationScope.allowedPath ??= realpathSync(componentDirectory);
278
278
  if (providedLoadedComponents) loadedComponents = providedLoadedComponents;
279
279
  try {
280
280
  let config;
@@ -326,8 +326,13 @@ export async function loadComponent(
326
326
 
327
327
  let extensionModule: any;
328
328
  const pkg = componentConfig.package;
329
+ const loadComponentOption = componentConfig.loadComponent ?? 'always';
329
330
  try {
330
331
  if (pkg) {
332
+ if (loadComponentOption === 'dev-only' && !process.env.DEV_MODE) {
333
+ componentLifecycle.loaded(componentStatusName, `Component '${componentStatusName}' skipped (dev-only)`);
334
+ continue;
335
+ }
331
336
  let componentPath: string | null = null;
332
337
  if (isRoot) {
333
338
  componentPath = join(componentDirectory, 'components', componentName);
@@ -344,7 +349,7 @@ export async function loadComponent(
344
349
  }
345
350
  }
346
351
  if (componentPath) {
347
- subApplicationScope.verifyPath ??= componentPath;
352
+ subApplicationScope.allowedPath ??= realpathSync(componentPath);
348
353
  if (!process.env.HARPER_SAFE_MODE) {
349
354
  extensionModule = await loadComponent(componentPath, resources, origin, {
350
355
  isRoot: false,
@@ -354,6 +359,12 @@ export async function loadComponent(
354
359
  });
355
360
  componentFunctionality[componentName] = true;
356
361
  }
362
+ } else if (loadComponentOption === 'if-installed') {
363
+ componentLifecycle.loaded(
364
+ componentStatusName,
365
+ `Component '${componentStatusName}' skipped (not installed)`
366
+ );
367
+ continue;
357
368
  } else {
358
369
  throw new Error(`Unable to find package ${componentName}:${pkg}`);
359
370
  }
@@ -15,6 +15,7 @@ import type { Logger } from '../utility/logging/logger.ts';
15
15
  import * as fs from 'fs-extra';
16
16
  import * as path from 'node:path';
17
17
  import * as crypto from 'node:crypto';
18
+ import { cloneDeep } from 'lodash';
18
19
  import { getBackupDirPath } from './configHelpers.ts';
19
20
 
20
21
  const STATE_FILE_NAME = '.harper-config-state.json';
@@ -590,6 +591,39 @@ function cleanupRemovedEnvVar(
590
591
  logger.debug?.(`${envVarName} removed, cleaned up values`);
591
592
  }
592
593
 
594
+ /**
595
+ * Compose a merged config from HARPER_DEFAULT_CONFIG and HARPER_SET_CONFIG
596
+ * layered with an optional base. Later layers win:
597
+ * HARPER_DEFAULT_CONFIG < base < HARPER_SET_CONFIG
598
+ *
599
+ * HARPER_DEFAULT_CONFIG provides scaffolding defaults, the base (e.g., the
600
+ * user's existing config file) is layered on top, and HARPER_SET_CONFIG
601
+ * force-overrides everything. This matches the precedence applied by the
602
+ * runtime pipeline in applyRuntimeEnvConfig.
603
+ *
604
+ * Unlike applyRuntimeEnvConfig, this does NOT read or write the config state
605
+ * file and does NOT track sources — it returns a fresh object. Use when you
606
+ * need the effective value of a config key before the state/file wiring is in
607
+ * place (e.g., during clone / pre-install).
608
+ */
609
+ export function composeConfigFromEnv(base: ConfigObject = {}): ConfigObject {
610
+ const result: ConfigObject = {};
611
+ const layers: (ConfigObject | null)[] = [
612
+ parseConfigEnvVar(process.env.HARPER_DEFAULT_CONFIG, 'HARPER_DEFAULT_CONFIG'),
613
+ cloneDeep(base),
614
+ parseConfigEnvVar(process.env.HARPER_SET_CONFIG, 'HARPER_SET_CONFIG'),
615
+ ];
616
+
617
+ for (const layer of layers) {
618
+ if (!layer) continue;
619
+ for (const [p, value] of Object.entries(flattenObject(layer))) {
620
+ setNestedValue(result, p, value);
621
+ }
622
+ }
623
+
624
+ return result;
625
+ }
626
+
593
627
  /**
594
628
  * Apply HARPER_DEFAULT_CONFIG and HARPER_SET_CONFIG
595
629
  * Can be used for both install-time and runtime
@@ -2262,7 +2262,9 @@ export function makeTable(options) {
2262
2262
  return (entryA, entryB) => {
2263
2263
  const a = getAttributeValue(entryA, order.attribute, context);
2264
2264
  const b = getAttributeValue(entryB, order.attribute, context);
2265
- const diff = descending ? compareKeys(b, a) : compareKeys(a, b);
2265
+ const diff = descending
2266
+ ? compareKeys(convertToComparableKeys(b), convertToComparableKeys(a))
2267
+ : compareKeys(convertToComparableKeys(a), convertToComparableKeys(b));
2266
2268
  if (diff === 0) return nextComparator?.(entryA, entryB) || 0;
2267
2269
  return diff;
2268
2270
  };
@@ -4572,3 +4574,12 @@ function hasOtherProcesses(store) {
4572
4574
  return +line.match(/\d+/)?.[0] != pid;
4573
4575
  });
4574
4576
  }
4577
+ function convertToComparableKeys(a) {
4578
+ if (a instanceof Date) {
4579
+ return a.getTime();
4580
+ }
4581
+ if (Array.isArray(a)) {
4582
+ return a.map(convertToComparableKeys);
4583
+ }
4584
+ return a;
4585
+ }
@@ -219,15 +219,12 @@ export async function recordHostname() {
219
219
  const nodeId = stableNodeId(hostname);
220
220
  log.trace?.('recordHostname nodeId:', nodeId);
221
221
  const hostnamesTable = getAnalyticsHostnameTable();
222
- const record = await hostnamesTable.get(nodeId);
223
- if (!record) {
224
- const hostnameRecord = {
225
- id: nodeId,
226
- hostname,
227
- };
228
- log.trace?.(`recordHostname storing hostname: ${JSON.stringify(hostnameRecord)}`);
229
- await hostnamesTable.put(hostnameRecord.id, hostnameRecord);
230
- }
222
+ const hostnameRecord = {
223
+ id: nodeId,
224
+ hostname,
225
+ };
226
+ log.trace?.(`recordHostname storing hostname: ${JSON.stringify(hostnameRecord)}`);
227
+ await hostnamesTable.put(hostnameRecord.id, hostnameRecord);
231
228
  }
232
229
 
233
230
  export interface Metric {
@@ -514,7 +511,7 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
514
511
  await rest();
515
512
  }
516
513
  for (const entry of threadsToAverage) {
517
- // eslint-disable-next-line @typescript-eslint/no-unused-vars,prefer-const
514
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
518
515
  let { path, method, type, metric, count, total, distribution, threads, ...measures } = entry;
519
516
  threads = threads.filter((thread) => thread);
520
517
  for (const measureName in measures) {
@@ -130,8 +130,8 @@ export class HierarchicalNavigableSmallWorld {
130
130
  // Generate random level for this new element
131
131
  const level = oldNode.level ?? Math.min(Math.floor(-Math.log(Math.random()) * this.mL), MAX_LEVEL);
132
132
  let currentLevel = entryPoint.level;
133
- if (level >= currentLevel) {
134
- // if we are at this level or higher, make this the new entry point
133
+ if (level > currentLevel) {
134
+ // if we are at a higher level, make this the new entry point
135
135
  if (typeof nodeId !== 'number') {
136
136
  throw new Error('Invalid nodeId: ' + nodeId);
137
137
  }
@@ -232,6 +232,19 @@ export class HierarchicalNavigableSmallWorld {
232
232
  oldNode[l] = oldConnections;
233
233
  }
234
234
  oldConnections.splice(oldPosition, 1);
235
+ // update the distance in the reverse connection if the vector changed
236
+ if (oldConnection.distance !== distance) {
237
+ const neighborNode = updateNode(id, node);
238
+ if (neighborNode[l]) {
239
+ if (Object.isFrozen(neighborNode[l])) {
240
+ neighborNode[l] = neighborNode[l].slice();
241
+ }
242
+ const reverseIdx = neighborNode[l].findIndex(({ id: nid }) => nid === nodeId);
243
+ if (reverseIdx >= 0) {
244
+ neighborNode[l][reverseIdx] = { id: nodeId, distance };
245
+ }
246
+ }
247
+ }
235
248
  } else {
236
249
  // add new connection since this is truly a new connection now
237
250
  this.addConnection(id, updateNode(id, node), nodeId, l, distance, updateNode, options);
@@ -360,7 +373,7 @@ export class HierarchicalNavigableSmallWorld {
360
373
  const candidates = [
361
374
  {
362
375
  id: entryPointId,
363
- distance: this.distance(queryVector, entryPoint.vector),
376
+ distance: distanceFunction(queryVector, entryPoint.vector),
364
377
  node: entryPoint,
365
378
  },
366
379
  ];
@@ -531,10 +544,13 @@ export class HierarchicalNavigableSmallWorld {
531
544
  if (removedNode) {
532
545
  // Remove the reverse connection if it exists
533
546
  if (removedNode[level]) {
534
- removedNode = updateNode(removed.id, removedNode);
535
- removedNode[level] = removedNode[level].filter(({ id }) => id !== fromId);
536
- if (level === 0 && removedNode[level].length === 0) {
537
- logger.info?.('should not remove last connection', fromId, toId);
547
+ const filtered = removedNode[level].filter(({ id }) => id !== fromId);
548
+ if (level === 0 && filtered.length === 0) {
549
+ // don't remove the last connection at level 0 — it would orphan this node
550
+ logger.info?.('skipping removal of last connection', fromId, toId);
551
+ } else {
552
+ removedNode = updateNode(removed.id, removedNode);
553
+ removedNode[level] = filtered;
538
554
  }
539
555
  }
540
556
  }
@@ -14,7 +14,16 @@ import * as child_process from 'node:child_process';
14
14
  import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
15
15
  import { contentTypes } from '../server/serverHelpers/contentTypes.ts';
16
16
  import type { CompartmentOptions } from 'ses';
17
- import { mkdirSync, readFileSync, writeFileSync, unlinkSync, openSync, closeSync, statSync } from 'node:fs';
17
+ import {
18
+ mkdirSync,
19
+ readFileSync,
20
+ writeFileSync,
21
+ unlinkSync,
22
+ openSync,
23
+ closeSync,
24
+ statSync,
25
+ realpathSync,
26
+ } from 'node:fs';
18
27
  import { join } from 'node:path';
19
28
  import { EventEmitter } from 'node:events';
20
29
  import { whenComponentsLoaded } from '../server/threads/threadServer.js';
@@ -495,13 +504,13 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope, useC
495
504
  }
496
505
 
497
506
  if (url.startsWith('file://') && usePrivateGlobal) {
498
- checkAllowedModulePath(url, scope.verifyPath);
507
+ checkAllowedModulePath(url, scope.allowedPath);
499
508
  const source = readFileSync(new URL(url), { encoding: 'utf-8' });
500
509
  return createModuleFromSource(url, source, usePrivateGlobal);
501
510
  }
502
511
 
503
512
  // For Node.js built-in modules (node:) and npm packages without application loader for dependency
504
- const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
513
+ const replacedModule = checkAllowedModulePath(url, scope.allowedPath);
505
514
  if (replacedModule) {
506
515
  return createSyntheticModule(url, normalizeImportedModule(replacedModule));
507
516
  }
@@ -570,7 +579,7 @@ async function getCompartment(scope: ApplicationScope, globals) {
570
579
  }
571
580
  return new StaticModuleRecord(moduleText, moduleSpecifier);
572
581
  } else {
573
- checkAllowedModulePath(moduleSpecifier, scope.verifyPath);
582
+ checkAllowedModulePath(moduleSpecifier, scope.allowedPath);
574
583
  const moduleExports = await import(moduleSpecifier);
575
584
  return {
576
585
  imports: [],
@@ -761,23 +770,54 @@ function isProcessRunning(pid: number): boolean {
761
770
  * Acquires an exclusive lock using the PID file itself (synchronously with busy-wait)
762
771
  * Returns 0 if lock was acquired (need to spawn new process), or the existing PID if process is running
763
772
  */
764
- function acquirePidFileLock(pidFilePath: string, maxRetries = 100, retryDelay = 5): number {
773
+ function parsePidFile(content: string): { pid: number; version: number } {
774
+ const lines = content.trim().split('\n');
775
+ const pid = Number.parseInt(lines[0], 10);
776
+ const version = lines.length > 1 ? parseInt(lines[1], 10) : 0;
777
+ return { pid, version };
778
+ }
779
+
780
+ function acquirePidFileLock(
781
+ pidFilePath: string,
782
+ requestedVersion?: number,
783
+ maxRetries = 100,
784
+ retryDelay = 5
785
+ ): { pid: number; version: number } {
765
786
  for (let attempt = 0; attempt < maxRetries; attempt++) {
766
787
  try {
767
788
  // Try to open exclusively - 'wx' fails if file exists
768
789
  const fd = openSync(pidFilePath, 'wx');
769
790
  closeSync(fd);
770
- return 0; // Successfully acquired lock (file created), caller should spawn process
791
+ return { pid: 0, version: 0 }; // Successfully acquired lock (file created), caller should spawn process
771
792
  } catch (err) {
772
793
  if (err.code === 'EEXIST') {
773
794
  // File exists - check if it contains a valid running process
774
795
  try {
775
796
  const pidContent = readFileSync(pidFilePath, 'utf-8');
776
- const existingPid = parseInt(pidContent.trim(), 10);
797
+ const { pid: existingPid, version: existingVersion } = parsePidFile(pidContent);
777
798
 
778
799
  if (!isNaN(existingPid) && isProcessRunning(existingPid)) {
779
- // Valid process is running, return its PID immediately
780
- return existingPid;
800
+ // If the version isn't the one we want, kill the existing process and re-acquire
801
+ if (requestedVersion != null && requestedVersion !== existingVersion) {
802
+ try {
803
+ process.kill(existingPid);
804
+ } catch {
805
+ // Process may have already exited
806
+ }
807
+ try {
808
+ unlinkSync(pidFilePath);
809
+ } catch {
810
+ // Another thread may have removed it
811
+ }
812
+ // Retry to acquire the lock for the new version
813
+ const start = Date.now();
814
+ while (Date.now() - start < retryDelay) {
815
+ // Busy wait for process cleanup
816
+ }
817
+ continue;
818
+ }
819
+ // Valid process is running at same or higher version, return its PID
820
+ return { pid: existingPid, version: existingVersion };
781
821
  }
782
822
 
783
823
  // Invalid/empty PID - check file age to determine if it's stale or being written
@@ -824,6 +864,7 @@ function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess
824
864
  throw new Error(
825
865
  `Calling ${spawnFunction.name} in Harper must have a process "name" in the options to ensure that a single process is started and reused`
826
866
  );
867
+ const requestedVersion = options?.version;
827
868
 
828
869
  // Ensure PID directory exists
829
870
  const pidDir = join(basePath, 'pids');
@@ -831,20 +872,22 @@ function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess
831
872
 
832
873
  const pidFilePath = join(pidDir, `${processName}.pid`);
833
874
 
834
- // Try to acquire lock - returns 0 if acquired, or existing PID
835
- const existingPid = acquirePidFileLock(pidFilePath);
875
+ // Try to acquire lock - returns pid: 0 if acquired, or existing PID/version
876
+ const existing = acquirePidFileLock(pidFilePath, requestedVersion);
836
877
 
837
- if (existingPid !== 0) {
878
+ if (existing.pid !== 0) {
838
879
  // Existing process is running, return wrapper
839
- return new ExistingProcessWrapper(existingPid);
880
+ return new ExistingProcessWrapper(existing.pid);
840
881
  }
841
882
 
842
883
  // We acquired the lock (file was created), spawn new process
843
884
  const childProcess = spawnFunction(command, args, options, callback);
844
885
 
845
- // Write PID to the file we just created
886
+ // Write PID (and version if provided) to the file we just created
887
+ const pidFileContent =
888
+ requestedVersion != null ? `${childProcess.pid}\n${requestedVersion}` : childProcess.pid.toString();
846
889
  try {
847
- writeFileSync(pidFilePath, childProcess.pid.toString(), 'utf-8');
890
+ writeFileSync(pidFilePath, pidFileContent, 'utf-8');
848
891
  } catch (err) {
849
892
  // Failed to write PID, clean up
850
893
  try {
@@ -869,21 +912,24 @@ function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess
869
912
 
870
913
  /**
871
914
  * Validates whether a module can be loaded based on security restrictions and returns the module path or replacement.
872
- * For file URLs, ensures the module is within the containing folder.
915
+ * For file URLs, ensures the module is within the allowed path.
873
916
  * For node built-in modules, checks against an allowlist and returns any replacements.
874
917
  *
875
918
  * @param {string} moduleUrl - The URL or identifier of the module to be loaded, which may be a file: URL, node: URL, or bare module specifier.
876
- * @param {string} containingFolder - The absolute path of the folder that contains the application, used to validate file: URLs are within bounds.
919
+ * @param {string} allowedPath - The absolute path that the module is allowed to load from.
877
920
  * @return {any} Returns undefined for allowed file paths, or a replacement module identifier for allowed node built-in modules.
878
- * @throws {Error} Throws an error if the module is outside the application folder or if the module is not in the allowed list.
921
+ * @throws {Error} Throws an error if the module is outside the allowed path or if the module is not in the allowed list.
879
922
  */
880
- function checkAllowedModulePath(moduleUrl: string, containingFolder?: string): boolean {
923
+ function checkAllowedModulePath(moduleUrl: string, allowedPath?: string): boolean {
881
924
  if (moduleUrl.startsWith('file:')) {
882
- const path = moduleUrl.slice(7);
883
- if (!containingFolder || path.startsWith(containingFolder)) {
925
+ let path = moduleUrl.slice(7);
926
+ try {
927
+ path = realpathSync(path);
928
+ } catch {}
929
+ if (!allowedPath || path.startsWith(allowedPath)) {
884
930
  return;
885
931
  }
886
- throw new Error(`Can not load module outside of application folder ${containingFolder}`);
932
+ throw new Error(`Can not load module outside of allowed path`);
887
933
  }
888
934
  let simpleName = moduleUrl.startsWith('node:') ? moduleUrl.slice(5) : moduleUrl;
889
935
  simpleName = simpleName.split('/')[0];
@@ -258,14 +258,16 @@ async function userInfo(body): Promise<string | User> {
258
258
  }
259
259
 
260
260
  let user = _.cloneDeep(body.hdb_user);
261
- let roleData = await search.searchByHash({
262
- schema: 'system',
263
- table: 'hdb_role',
264
- hash_values: [user.role.id],
265
- get_attributes: ['*'],
266
- });
261
+ let roleData =
262
+ user.role &&
263
+ (await search.searchByHash({
264
+ schema: 'system',
265
+ table: 'hdb_role',
266
+ hash_values: [user.role.id],
267
+ get_attributes: ['*'],
268
+ }));
267
269
 
268
- user.role = roleData[0];
270
+ user.role = roleData?.[0];
269
271
  delete user.password;
270
272
  delete user.refresh_token;
271
273
  delete user.hash;
@@ -429,7 +431,7 @@ async function getSuperUser(): Promise<User | undefined> {
429
431
  await setUsersWithRolesCache();
430
432
  }
431
433
  for (let [, user] of usersWithRolesMap) {
432
- if (user.role.role === 'super_user') return user;
434
+ if (user.role?.role === 'super_user') return user;
433
435
  }
434
436
  }
435
437
 
@@ -24,7 +24,7 @@ analytics:
24
24
  aggregatePeriod: 60
25
25
  replicate: false
26
26
  applications:
27
- lockdown: freeze
27
+ lockdown: freeze-after-load
28
28
  moduleLoader: vm
29
29
  dependencyLoader: auto
30
30
  allowedSpawnCommands:
@@ -44,6 +44,9 @@ const write_ts_1 = require("../core/resources/analytics/write.js");
44
44
  const environmentManager_js_1 = require("../core/utility/environment/environmentManager.js");
45
45
  const packageUtils_js_1 = require("../core/utility/packageUtils.js");
46
46
  const node_fs_1 = require("node:fs");
47
+ const node_child_process_1 = require("node:child_process");
48
+ const node_util_1 = require("node:util");
49
+ const execFileAsync = (0, node_util_1.promisify)(node_child_process_1.execFile);
47
50
  const pprof_1 = require("@datadog/pprof");
48
51
  const manageThreads_js_1 = require("../core/server/threads/manageThreads.js");
49
52
  const log = __importStar(require("../core/utility/logging/harper_logger.js"));
@@ -77,6 +80,7 @@ function handleApplication({ options }) {
77
80
  }, 1000); // wait for everything to load before we start the profiler
78
81
  }
79
82
  let lastChildCpuTime = 0;
83
+ let gpuAvailable = true;
80
84
  async function captureProfile(delayToNextCapture = (capturePeriod ?? 60) * 1000) {
81
85
  clearTimeout(profilerTimer);
82
86
  if (!profilerStarted) {
@@ -92,6 +96,8 @@ async function captureProfile(delayToNextCapture = (capturePeriod ?? 60) * 1000)
92
96
  const samplesByLocationId = new Map();
93
97
  let totalUserCount = 0;
94
98
  let totalHarperCount = 0;
99
+ // Start GPU measurement early so it runs in parallel with CPU profiling work
100
+ const gpuPromise = (0, manageThreads_js_1.getWorkerIndex)() === 0 && gpuAvailable ? getGpuUtilization() : null;
95
101
  try {
96
102
  const profile = pprof_1.time.stop(true);
97
103
  const strings = profile.stringTable.strings;
@@ -122,6 +128,11 @@ async function captureProfile(delayToNextCapture = (capturePeriod ?? 60) * 1000)
122
128
  (0, write_ts_1.recordAction)(childCpuTimeInInterval, 'cpu-usage', 'user', 'child-processes');
123
129
  lastChildCpuTime = childCpuTime;
124
130
  }
131
+ // Record GPU utilization for this process and child processes
132
+ const gpuSeconds = await gpuPromise;
133
+ if (gpuSeconds !== null) {
134
+ (0, write_ts_1.recordAction)(gpuSeconds, 'gpu-usage', 'user');
135
+ }
125
136
  }
126
137
  }
127
138
  catch (error) {
@@ -211,6 +222,42 @@ function getChildProcessCpuTime() {
211
222
  return null;
212
223
  }
213
224
  }
225
+ /**
226
+ * Get the total SM (shader/streaming multiprocessor) utilization percentage across all GPUs
227
+ * for this process and all descendant processes.
228
+ * Uses nvidia-smi pmon for per-process GPU utilization.
229
+ * Only works on Linux with NVIDIA GPUs. Returns null if unavailable.
230
+ */
231
+ async function getGpuUtilization() {
232
+ try {
233
+ const currentPid = process.pid;
234
+ const descendants = findAllDescendants(currentPid);
235
+ const pidsToMonitor = new Set([currentPid, ...descendants]);
236
+ const { stdout } = await execFileAsync('nvidia-smi', ['pmon', '-c', '1', '-s', 'u']);
237
+ let totalSmPercent = 0;
238
+ for (const line of stdout.split('\n')) {
239
+ if (line.startsWith('#') || !line.trim())
240
+ continue;
241
+ const parts = line.trim().split(/\s+/);
242
+ // pmon -s u format: gpu pid type fb sm enc dec jpg ofa command
243
+ if (parts.length < 5)
244
+ continue;
245
+ const pid = parseInt(parts[1], 10);
246
+ if (isNaN(pid) || !pidsToMonitor.has(pid))
247
+ continue;
248
+ const sm = parseInt(parts[4], 10); // SM utilization %
249
+ if (!isNaN(sm))
250
+ totalSmPercent += sm;
251
+ }
252
+ // Convert SM utilization % to GPU-seconds over the capture period
253
+ // e.g. 50% utilization over 60s = 30 GPU-seconds
254
+ return (totalSmPercent / 100) * (capturePeriod / 1000);
255
+ }
256
+ catch {
257
+ gpuAvailable = false;
258
+ return null;
259
+ }
260
+ }
214
261
  /**
215
262
  * Recursively find all descendant PIDs of the given parent PID.
216
263
  */