@harperfast/harper 5.2.0-alpha.4 → 5.2.0-alpha.6

package/components/secretOperations.ts

@@ -23,6 +23,7 @@ import * as validator from './operationsValidation.js';
 import { getSecretCustody } from '../resources/secretDecryptor.ts';
 import { encryptEnvelope, parseEnvelopeFields } from '../utility/secretEnvelope.ts';
 import { ENV_ENCRYPTED_PREFIX } from '../utility/envFile.ts';
+import { normalizeGitHost, type ResolvedGitCredential } from './gitCredentialServer.ts';
 
 const { HTTP_STATUS_CODES } = hdbErrors;
 const SECRET_TABLE = terms.SYSTEM_TABLE_NAMES.SECRET_TABLE_NAME;
@@ -325,20 +326,41 @@ export function getSecretsPublicKey(req: any) {
 	return { public_key: publicKey, fingerprint };
 }
 
-/** A resolved registry-auth entry as consumed by the transient .npmrc writer. */
-export interface ResolvedRegistryAuthEntry {
+// deploy_component `credentials` is a kind-heterogeneous array: an entry's kind is implied by its
+// identifying key (`registry` = npm registry auth, `host` = git host auth for a git-reference
+// package, #1792), and every kind carries its credential as either a literal `token` or a `secret`
+// reference. Ingest/resolve below are kind-agnostic about that token/secret half and only branch on
+// the identifying key where a kind-specific detail is needed (the derived secret name).
+
+/** A resolved registry credential as consumed by the transient .npmrc writer. */
+export interface ResolvedRegistryCredential {
 	registry: string;
 	token: string;
 	scope?: string;
 }
 
-/** A registry-auth entry after ingestion: a reference into hdb_secret, never a token. */
-export interface RegistryAuthReference {
+/** A registry credential after ingestion: a reference into hdb_secret, never a token. */
+export interface RegistryCredentialReference {
 	registry: string;
 	secret: string;
 	scope?: string;
 }
 
+/** A git-host credential after ingestion: a reference into hdb_secret, never a token. */
+export interface GitCredentialReference {
+	host: string;
+	secret: string;
+	username?: string;
+}
+
+export type CredentialReference = RegistryCredentialReference | GitCredentialReference;
+export type ResolvedCredential = ResolvedRegistryCredential | ResolvedGitCredential;
+
+/** True when a credential entry (in any form) is npm registry auth rather than git host auth. */
+export function isRegistryCredential(entry: any): boolean {
+	return entry?.registry !== undefined;
+}
+
 /**
  * Deterministic name for the auto-minted secret backing a literal registry token: keyed by the
  * deploying component and the registry, so re-supplying (or rotating) the token on a later deploy
@@ -358,9 +380,65 @@ export function deriveRegistrySecretName(component: string, registry: string): s
 }
 
 /**
- * Ingest deploy_component `registryAuth` into the secrets store so a provided registry token lives
- * as ciphertext in the replicated, audited `hdb_secret` store (reference, not embed) rather than
- * travelling in the operation body. Returns the entries in reference form (`{ registry, secret }`).
+ * Deterministic name for the secret backing a literal git-host token, following the registry
+ * convention above but with a `git` kind segment: a registry entry accepts a bare host too, so
+ * without a distinguishing segment a git and a registry credential for the same host would derive
+ * the same name and silently overwrite each other's secret. Keyed by host rather than by
+ * repository: the credential entry identifies itself by host, so the name has to be derivable from
+ * the entry alone for a rotation to overwrite the same row. A per-repository credential is
+ * expressible today by scoping the token itself (a fine-grained PAT) rather than by splitting the
+ * secret name.
+ */
+export function deriveGitSecretName(component: string, host: string): string {
+	const hostKey = normalizeGitHost(host).replace(/[^\w.-]+/g, '_');
+	const componentKey = String(component).replace(/[^\w.-]+/g, '_');
+	return `deploy.${componentKey}.git.${hostKey}`;
+}
+
+/**
+ * An entry's kind, from its identifying key. Every path that rebuilds an entry goes through this, so
+ * an entry of an unrecognized kind fails loudly rather than being rebuilt into a half-empty one: an
+ * operation-body entry is schema-validated, but a `credentials` entry read back from
+ * applicationConfig or an hdb_deployment row is not.
+ */
+function credentialKind(entry: any): 'registry' | 'git' {
+	if (isRegistryCredential(entry)) return 'registry';
+	if (entry?.host !== undefined) return 'git';
+	throw new ClientError(
+		`Unsupported credential entry: expected a 'registry' (npm registry auth) or 'host' (git host auth) entry`
+	);
+}
+
+/** The hdb_secret name a literal-token entry seals into, by kind. */
+function deriveSecretName(entry: any, component: string): string {
+	return credentialKind(entry) === 'registry'
+		? deriveRegistrySecretName(component, entry.registry)
+		: deriveGitSecretName(component, entry.host);
+}
+
+/** Rebuild an entry in reference form, preserving only the fields its kind defines. */
+function toReference(entry: any, secret: string): CredentialReference {
+	if (credentialKind(entry) === 'registry') {
+		return entry.scope === undefined
+			? { registry: entry.registry, secret }
+			: { registry: entry.registry, secret, scope: entry.scope };
+	}
+	return entry.username === undefined
+		? { host: entry.host, secret }
+		: { host: entry.host, secret, username: entry.username };
+}
+
+/** Rebuild an entry in resolved (literal token) form, preserving only the fields its kind defines. */
+function toResolved(entry: any, token: string): ResolvedCredential {
+	return credentialKind(entry) === 'registry'
+		? { registry: entry.registry, token, scope: entry.scope }
+		: { host: entry.host, token, username: entry.username };
+}
+
+/**
+ * Ingest deploy_component `credentials` into the secrets store so a provided token lives as
+ * ciphertext in the replicated, audited `hdb_secret` store (reference, not embed) rather than
+ * travelling in the operation body. Returns the entries in reference form (`{ …, secret }`).
  *
  * - A literal `{ registry, token }` entry is encrypted (via `set_secret`, custody required) under a
  *   derived name granted to `component`, and returned as a reference. Overwrites an existing
@@ -373,17 +451,19 @@ export function deriveRegistrySecretName(component: string, registry: string): s
  *
  * Runs on the deploying main thread where custody is registered.
  */
-export async function ingestRegistryAuth(req: any, registryAuth: any[] | undefined, component: string): Promise<any[]> {
-	if (!Array.isArray(registryAuth) || registryAuth.length === 0) return registryAuth ?? [];
+export async function ingestCredentials(req: any, credentials: any[] | undefined, component: string): Promise<any[]> {
+	if (!Array.isArray(credentials) || credentials.length === 0) return credentials ?? [];
 	const custody = getSecretCustody();
 	const out: any[] = [];
-	for (const entry of registryAuth) {
+	for (const entry of credentials) {
 		// Already a reference (or nothing to seal without custody): leave as-is.
 		if (entry.secret !== undefined || !custody) {
 			out.push(entry);
 			continue;
 		}
-		const name = deriveRegistrySecretName(component, entry.registry);
+		// The derived secret name is the one kind-specific detail on this path; each kind derives its
+		// name from its own identifying key.
+		const name = deriveSecretName(entry, component);
 		// Reuse set_secret's seal-and-store path (encrypt with custody, grant to the component, audit
 		// the mutation). The deploy request is already super_user, which set_secret requires.
 		await setSecret({
@@ -394,11 +474,7 @@ export async function ingestRegistryAuth(req: any, registryAuth: any[] | undefin
 			grants: [component],
 			processEnv: false,
 		});
-		out.push(
-			entry.scope === undefined
-				? { registry: entry.registry, secret: name }
-				: { registry: entry.registry, secret: name, scope: entry.scope }
-		);
+		out.push(toReference(entry, name));
 	}
 	return out;
 }
@@ -416,7 +492,7 @@ async function waitForSecretRow(table: any, name: string, waitMs: number): Promi
 }
 
 /**
- * Resolve deploy_component `registryAuth` entries that reference a stored secret
+ * Resolve deploy_component `credentials` entries that reference a stored secret
  * (`{ registry, secret }`) into concrete token entries (`{ registry, token }`) by decrypting the
  * named hdb_secret row on this thread. Entries that already carry a literal `token` pass through
  * unchanged, so this is a no-op for the token-only fallback form.
@@ -440,36 +516,40 @@ async function waitForSecretRow(table: any, name: string, waitMs: number): Promi
  * (OSS core, or a custody key not held here) a referenced secret cannot be resolved and the deploy
  * fails loudly rather than silently installing without auth.
  */
-export async function resolveRegistryAuth(
-	registryAuth: any[] | undefined,
+export async function resolveCredentials(
+	credentials: any[] | undefined,
 	component: string,
 	options: { waitMs?: number } = {}
-): Promise<ResolvedRegistryAuthEntry[] | undefined> {
-	if (!Array.isArray(registryAuth) || registryAuth.length === 0) return registryAuth;
+): Promise<ResolvedCredential[] | undefined> {
+	if (!Array.isArray(credentials) || credentials.length === 0) return credentials;
+	// Reject an unrecognized kind before anything consumes the array, including on the token-only fast
+	// path below: a caller that silently drops an entry it does not understand would install without
+	// the credential the operator asked for.
+	for (const entry of credentials) if (entry) credentialKind(entry);
 	// Token-only requests must not require custody or a provisioned store — keep the fast path pure.
-	if (!registryAuth.some((entry) => entry && entry.secret !== undefined)) return registryAuth;
+	if (!credentials.some((entry) => entry && entry.secret !== undefined)) return credentials;
 
 	const waitMs = options.waitMs ?? 0;
 	const table = secretTable();
-	const resolved: ResolvedRegistryAuthEntry[] = [];
-	for (const entry of registryAuth) {
+	const resolved: ResolvedCredential[] = [];
+	for (const entry of credentials) {
 		if (!entry) continue; // parity with the fast-path guard above; validated entries are never null
 		if (entry.secret === undefined) {
-			resolved.push({ registry: entry.registry, token: entry.token, scope: entry.scope });
+			resolved.push(toResolved(entry, entry.token));
 			continue;
 		}
 		const name: string = entry.secret;
 		const row = waitMs > 0 ? await waitForSecretRow(table, name, waitMs) : await table.get(name);
 		if (!row) {
 			throw new ClientError(
-				`registryAuth references secret '${name}', which does not exist`,
+				`credential entry references secret '${name}', which does not exist`,
 				HTTP_STATUS_CODES.NOT_FOUND
 			);
 		}
 		const grants = Array.isArray(row.grants) ? row.grants : [];
 		if (!row.processEnv && !grants.includes(component)) {
 			throw new ClientError(
-				`registryAuth secret '${name}' is not granted to component '${component}' ` +
+				`credential secret '${name}' is not granted to component '${component}' ` +
 					`(grant it with grant_secret, or set it processEnv:true)`,
 				HTTP_STATUS_CODES.FORBIDDEN
 			);
@@ -479,7 +559,7 @@ export async function resolveRegistryAuth(
 			// Server-state condition, not a client-fixable request: the same body would resolve once
 			// custody comes up, so report it retryable (503) rather than the ClientError default 400.
 			throw new ClientError(
-				`secrets custody is not initialized on this node; cannot resolve registryAuth secret '${name}'`,
+				`secrets custody is not initialized on this node; cannot resolve credential secret '${name}'`,
 				HTTP_STATUS_CODES.SERVICE_UNAVAILABLE
 			);
 		}
@@ -488,11 +568,11 @@ export async function resolveRegistryAuth(
 			token = custody.decrypt(row.envelope);
 		} catch (error) {
 			throw new ClientError(
-				`Failed to decrypt registryAuth secret '${name}': ${(error as Error).message}`,
+				`Failed to decrypt credential secret '${name}': ${(error as Error).message}`,
 				HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR
 			);
 		}
-		resolved.push({ registry: entry.registry, token, scope: entry.scope });
+		resolved.push(toResolved(entry, token));
 	}
 	return resolved;
 }

package/config/harperConfigEnvVars.ts

@@ -28,6 +28,7 @@ import * as path from 'node:path';
 import * as crypto from 'node:crypto';
 import { cloneDeep } from 'lodash';
 import { getBackupDirPath } from './configHelpers.ts';
+import * as hdbTerms from '../utility/hdbTerms.ts';
 
 const STATE_FILE_NAME = '.harper-config-state.json';
 
@@ -305,7 +306,18 @@ export function filterArgsAgainstRuntimeConfig(args: Record<string, any>): Recor
 /**
  * Flatten nested object to dot-notation paths
  */
-function flattenObject(obj: ConfigObject, prefix = ''): Record<string, any> {
+// One warning per distinct path per process — flattenObject runs on every env apply.
+const warnedEmptyObjectPaths = new Set<string>();
+function warnEmptyObjectDropped(path: string): void {
+	if (warnedEmptyObjectPaths.has(path)) return;
+	warnedEmptyObjectPaths.add(path);
+	getLogger().warn?.(
+		`[env-config] '${path}' is an empty object in env-var config and carries no settings; ` +
+			`it will not appear in the resolved config. Set an explicit value (e.g. '${path}.enabled: true').`
+	);
+}
+
+function flattenObject(obj: ConfigObject, prefix = '', warnOnEmptyDrop = true): Record<string, any> {
 	const result: Record<string, any> = {};
 
 	for (const key in obj) {
@@ -315,8 +327,16 @@ function flattenObject(obj: ConfigObject, prefix = ''): Record<string, any> {
 		const newKey = prefix ? `${prefix}.${key}` : key;
 
 		if (isPlainObject(value) && !isDirectiveObject(value)) {
-			// Recurse for nested objects
-			Object.assign(result, flattenObject(value, newKey));
+			// Recurse for nested objects. An EMPTY object contributes no leaves — that is
+			// load-bearing for removal semantics (`http: {}` in SET_CONFIG means "no
+			// overrides under http", restoring originals) — but it also means a bare
+			// `componentName: {}` carries no signal at all, which has silently confused
+			// users (#1618). Warn once per path so the drop is visible; use an explicit
+			// value (e.g. `enabled: true`) to convey presence. The base config file is
+			// exempt (warnOnEmptyDrop=false): its empty objects are user content and are
+			// restored after composition rather than dropped (#1726 review).
+			if (Object.keys(value).length === 0 && warnOnEmptyDrop) warnEmptyObjectDropped(newKey);
+			Object.assign(result, flattenObject(value, newKey, warnOnEmptyDrop));
 		} else {
 			// Store primitive, array, or directive ({ $union: [...] }) as a leaf
 			result[newKey] = value;
@@ -326,6 +346,34 @@ function flattenObject(obj: ConfigObject, prefix = ''): Record<string, any> {
 	return result;
 }
 
+/**
+ * Re-add empty-object paths from the base config file that `flattenObject` dropped
+ * during composition. The base file is user content, not an override layer: a bare
+ * `componentName: {}` there is a real (empty) scope declaration, unlike an env-layer
+ * `{}` which means "no overrides here". Only paths the composition did not otherwise
+ * populate are restored — an env layer that set or replaced the path wins.
+ */
+function restoreBaseEmptyObjects(source: ConfigObject, target: ConfigObject): void {
+	for (const key in source) {
+		if (!Object.prototype.hasOwnProperty.call(source, key)) continue;
+		const value = source[key];
+		if (!isPlainObject(value) || isDirectiveObject(value)) continue;
+		if (Object.keys(value).length === 0) {
+			if (!(key in target)) target[key] = {};
+		} else if (key in target) {
+			// Recurse only while the composed side is still an object; a non-object env
+			// replacement wins over the base subtree.
+			if (isPlainObject(target[key])) restoreBaseEmptyObjects(value, target[key] as ConfigObject);
+		} else {
+			// The subtree produced no leaves at all (its only content was empty objects,
+			// possibly nested) — rebuild just the empty-object skeleton.
+			const skeleton: ConfigObject = {};
+			restoreBaseEmptyObjects(value, skeleton);
+			if (Object.keys(skeleton).length > 0) target[key] = skeleton;
+		}
+	}
+}
+
 /**
  * Get nested value by dot-notation path
  */
@@ -738,24 +786,61 @@ function cleanupRemovedEnvVar(
  */
 export function composeConfigFromEnv(base: ConfigObject = {}): ConfigObject {
 	const result: ConfigObject = {};
+	const baseLayer = cloneDeep(base);
 	const layers: (ConfigObject | null)[] = [
 		parseConfigEnvVar(process.env.HARPER_DEFAULT_CONFIG, 'HARPER_DEFAULT_CONFIG'),
-		cloneDeep(base),
+		baseLayer,
 		parseConfigEnvVar(process.env.HARPER_CONFIG, 'HARPER_CONFIG'),
 		parseConfigEnvVar(process.env.HARPER_SET_CONFIG, 'HARPER_SET_CONFIG'),
 	];
 
 	for (const layer of layers) {
 		if (!layer) continue;
-		for (const [p, value] of Object.entries(flattenObject(layer))) {
+		for (const [p, value] of Object.entries(flattenObject(layer, '', layer !== baseLayer))) {
 			// directive leaves compose against the value accumulated by prior layers
 			setNestedValue(result, p, resolveLeafValue(getNestedValue(result, p), value, p));
 		}
 	}
 
+	// The base file's empty objects are user content (e.g. a bare `componentName: {}`
+	// scope) — restore the ones composition didn't otherwise populate (#1726 review).
+	restoreBaseEmptyObjects(baseLayer, result);
+
 	return result;
 }
 
+/** True when any config-shaping env var (HARPER_DEFAULT_CONFIG / HARPER_CONFIG / HARPER_SET_CONFIG) is set. */
+export function hasConfigEnvVars(): boolean {
+	return Boolean(process.env.HARPER_DEFAULT_CONFIG || process.env.HARPER_CONFIG || process.env.HARPER_SET_CONFIG);
+}
+
+/**
+ * Overlay runtime env config onto a base root-config object, hiding the env-var names and
+ * the composition rules from callers. Returns `base` unchanged when no config env vars are
+ * set (a true no-op — callers can invoke it on every root-config read without branching).
+ * A missing/non-object `base` (e.g. the install window before the config file is written)
+ * is treated as an empty base. Throws (via composeConfigFromEnv) on malformed env-var JSON.
+ */
+export function overlayRootEnvConfig(base: ConfigObject | undefined): ConfigObject | undefined {
+	if (!hasConfigEnvVars()) return base;
+	return composeConfigFromEnv(isPlainObject(base) ? base : {});
+}
+
+/**
+ * True when `filePath` names THE root Harper config file (current or legacy name), as
+ * opposed to a component/application `config.yaml`. Filename-only by design: an exact
+ * path comparison against the resolved root-config path misclassifies watchers in any
+ * environment where a real config instance is resolved (including the unit harness).
+ * FALLBACK ONLY: real component loads thread the loader's authoritative `isRoot` through
+ * `Scope` → `OptionsWatcher(…, isRootConfig)`, so this heuristic applies just to direct
+ * constructions (tests, ad-hoc callers), where a root-named app config file is a known,
+ * accepted false positive.
+ */
+export function isRootConfigFilename(filePath: string): boolean {
+	const name = path.basename(filePath);
+	return name === hdbTerms.HARPER_CONFIG_FILE || name === hdbTerms.HDB_CONFIG_FILE;
+}
+
 /**
  * True if a config-state file exists with tracked env-var snapshots. Callers use this to
  * decide whether applyRuntimeEnvConfig must run even when no config env vars are currently

package/config-root.schema.json

@@ -649,6 +649,10 @@
 				"componentsScope": {
 					"type": "string",
 					"description": "Filesystem scope for component edits, relative to rootPath. Default: ./components"
+				},
+				"systemPromptAppend": {
+					"type": "string",
+					"description": "Operator text appended to the agent's system prompt, after the built-in grounding and the Harper best-practices overview. Tunable at runtime via set_agent_config."
 				}
 			}
 		},

package/dist/agent/agent.d.ts

@@ -8,16 +8,20 @@
  * is realized lazily on first use, and the loop runs in-process.
  *
  * The component intentionally avoids `handleApplication`: it has nothing
- * worker-thread-shaped to do. Operator-only tools (FS, schedule, fetch) are
- * inline; registry-backed tools (#615/#617/#618) will fold in via toolset.ts
- * once those land.
+ * worker-thread-shaped to do. Two tool sources compose: operator-only tools
+ * (FS, schedule, fetch, and the V8 inspector) that are inline, and RBAC-filtered
+ * registry tools (#615/#617) drained for the agent's configured user via
+ * `registryTools.ts`.
  */
+import type { AuthedUser } from '../components/mcp/toolRegistry.ts';
 interface StartOpts {
     server: {
         registerOperation: (def: {
             name: string;
             execute: (op: any) => any | Promise<any>;
         }) => void;
+        /** Resolve a Harper user (with role/permissions) by username. Password/request unused here. */
+        getUser?: (username: string, password: string | null, request: unknown) => Promise<AuthedUser> | AuthedUser;
     };
     enabled?: boolean;
     provider?: string;
@@ -28,8 +32,25 @@ interface StartOpts {
     allowDestructive?: boolean;
     user?: string;
     componentsScope?: string;
+    systemPromptAppend?: string;
 }
 export declare function startOnMainThread(opts: StartOpts): Promise<void>;
+/**
+ * Resolve the identity the agent's tool calls run as. Registry tools are enforced
+ * (`hdb_user` at call time) against this user, so an operator who sets `agent.user`
+ * to a restricted role gets a narrowed surface — and a later role change is honored,
+ * because this runs per call rather than once at startup.
+ *
+ * Failure policy — fail closed, with one bounded exception:
+ *   - If `agent.user` resolves to a permissioned user, use it.
+ *   - If it can't be resolved AND it's the *default* `hdb_agent` bootstrap user,
+ *     fall back to a super_user identity (the system user isn't provisioned yet —
+ *     #626 defers creating it). This is the documented default-agent behavior.
+ *   - If it can't be resolved and the operator configured a *non-default* user,
+ *     throw. Silently escalating a misconfigured/transient restricted account to
+ *     super_user is the vulnerability we refuse to introduce.
+ */
+export declare function resolveAgentIdentity(server: StartOpts['server'], username: string): Promise<AuthedUser>;
 /** Test-only: reset module state between specs. */
 export declare function _resetForTests(): void;
 export {};