@hypit/hypit 0.1.4 → 0.1.6

package/packages/runtime-host-node/src/index.ts

@@ -16,7 +16,6 @@ import type {
   CredentialRef,
   OperationSnapshot,
   ResourceStore,
-  RuntimeWorkerRunOptions,
 } from "@hypit/runtime";
 import type { RuntimeDoctorDiagnostic } from "@hypit/runtime-kit";
 
@@ -63,6 +62,12 @@ export type BuildView = {
   readonly acceptedRecords: number;
   readonly outstandingCommands: number;
   readonly operations: readonly BuildOperationView[];
+  /** Activity reported by calls running in the local Worker. */
+  readonly commands?: readonly {
+    readonly id: string;
+    readonly endpoint: string;
+    readonly progress: NonNullable<OperationSnapshot["progress"]>;
+  }[];
 };
 
 export type RuntimeHostCredentialStatus = {
@@ -77,6 +82,8 @@ export type RuntimeHostCredentialStatus = {
 };
 
 export type RuntimeHostControl = {
+  /** Build evidence while active. Finished evidence is read through its Result Repository. */
+  logs?(build: string, lines: number): Promise<import("@hypit/runtime").ExecutionLogView | undefined>;
   inspect(build: string): Promise<BuildView | undefined>;
   activity(): Promise<{
     readonly builds: readonly BuildView[];
@@ -127,7 +134,6 @@ export type RuntimeHostExecution = RuntimeHostControl & RuntimeHostCredentialCon
     readonly maxWaitMs?: number;
     readonly signal?: AbortSignal;
   }): Promise<RuntimeHostBuildSubmission>;
-  work(options: RuntimeWorkerRunOptions): Promise<void>;
 };
 
 export type RuntimeHostDoctorResult = {
@@ -214,7 +220,6 @@ export type ManagedProgramReport = {
 
 export type RuntimeWorkerState = {
   readonly state: "running" | "stopped";
-  readonly configuration?: "current" | "changed";
   readonly profile: string;
   readonly pid?: number;
   readonly startedAt?: number;
@@ -235,9 +240,10 @@ export type RuntimeController = {
       readonly maxWaitMs?: number;
       readonly onProgress?: (event: ManagedProgramProgress) => void;
       readonly capabilities?: readonly CapabilityRef[];
+      readonly endpoints?: readonly string[];
     }): Promise<{ readonly dataRoot: string; readonly programs: readonly ManagedProgramReport[] }>;
-    down(): Promise<{ readonly dataRoot: string; readonly programs: readonly ManagedProgramReport[] }>;
-    report(): Promise<{ readonly dataRoot: string; readonly programs: readonly ManagedProgramReport[] }>;
+    down(options?: { readonly endpoints?: readonly string[] }): Promise<{ readonly dataRoot: string; readonly programs: readonly ManagedProgramReport[] }>;
+    report(options?: { readonly endpoints?: readonly string[] }): Promise<{ readonly dataRoot: string; readonly programs: readonly ManagedProgramReport[] }>;
   };
 };
 
@@ -255,7 +261,7 @@ export type NodeRuntimeHost = {
   controller(options?: {
     readonly packageRoot?: string;
   }): Promise<RuntimeController>;
-  createRuntime(): Promise<RuntimeHostExecution>;
+  createRuntime(options?: { readonly endpoints?: readonly string[] }): Promise<RuntimeHostExecution>;
   openControl(options?: { readonly readOnly?: boolean }): Promise<RuntimeHostControl>;
   /** Open only Result-writing dependencies; never construct execution Providers. */
   openResultControl(): Promise<RuntimeHostResultControl>;
@@ -263,6 +269,7 @@ export type NodeRuntimeHost = {
   /** Explicitly prepare upstream packages selected by this Runtime Profile. */
   prepare(options?: {
     readonly onProgress?: (event: import("./packages.js").HostPackageProgress) => void;
+    readonly endpoints?: readonly string[];
   }): Promise<readonly import("./packages.js").HostPackageReport[]>;
   /**
    * Read-only, bounded deployment validation for one demanded Build slice.
@@ -271,10 +278,12 @@ export type NodeRuntimeHost = {
    */
   preflight(options?: {
     readonly capabilities?: readonly CapabilityRef[];
+    readonly endpoints?: readonly string[];
   }): Promise<RuntimeHostDoctorResult>;
   /** Active deployment diagnosis. Unlike preflight, adapters may contact their configured services. */
   doctor(options?: {
     readonly capabilities?: readonly CapabilityRef[];
+    readonly endpoints?: readonly string[];
   }): Promise<RuntimeHostDoctorResult>;
   /**
    * Which selected Endpoint would serve each capability and where its Provider publishes prices.
@@ -288,10 +297,16 @@ export type NodeRuntimeHost = {
    * The creation-time boundary for observation, transcription and other quick capabilities; it
    * creates no Build, Result or state, and refuses asynchronous capabilities.
    */
-  invoke(need: Need, resources: ResourceStore): Promise<{ readonly value: StoredValue }>;
+  invoke(need: Need, resources: ResourceStore, observation?: RuntimeInvocationObservation): Promise<{ readonly value: StoredValue }>;
   /** Open one disposable authoring execution. It creates no Build, Result or recoverable Operation. */
   openTransientExecution(): Promise<RuntimeHostTransientExecution>;
-  runWorker(readyFile: string, owner: string): Promise<void>;
+  runWorker(readyFile: string, owner: string, execution?: { readonly dataRoot: string }): Promise<void>;
+};
+
+/** Progress stays with the selected Provider; immediate calls need no synthetic Build. */
+export type RuntimeInvocationObservation = {
+  readonly reportProgress?: (progress: import("@hypit/runtime").OperationProgress) => Promise<void>;
+  readonly reportDiagnostic?: (diagnostic: import("@hypit/runtime").ExecutionDiagnostic) => Promise<void>;
 };
 
 /**
@@ -321,7 +336,7 @@ export function hypitHostStateRoot(options: {
     : state, "hypit");
 }
 
-/** Upstream npm packages shared by every Hypit project and future session on this machine. */
+/** Home of version-isolated upstream npm installations shared across projects. */
 export function hypitHostPackageRoot(hostStateRoot = hypitHostStateRoot()): string {
   return join(resolve(hostStateRoot), "packages");
 }

package/packages/runtime-host-node/src/packages.ts

@@ -1,6 +1,7 @@
 import { spawn } from "node:child_process";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { mkdir, open, readFile, realpath, writeFile } from "node:fs/promises";
 import { join, resolve } from "node:path";
+import { externalPackageInstallRoot } from "@hypit/package-loader-node";
 
 export type RegistryPackageSpec = {
   readonly name: string;
@@ -10,11 +11,13 @@ export type RegistryPackageSpec = {
 
 export type HostPackageReport = RegistryPackageSpec & {
   readonly root: string;
+  readonly logPath?: string;
   readonly action: "already-installed" | "installed";
 };
 
 export type HostPackageProgress = RegistryPackageSpec & {
   readonly phase: "checking" | "installing" | "ready";
+  readonly logPath?: string;
 };
 
 function exactVersion(value: string): boolean {
@@ -63,18 +66,20 @@ export async function inspectHostPackage(
   root: string,
 ): Promise<HostPackageReport | undefined> {
   const required = parseRegistryPackageSpec(specifier);
-  const version = await installedVersion(resolve(root), required.name);
+  const installation = externalPackageInstallRoot(root, required.name, required.version);
+  const version = await installedVersion(installation, required.name);
   return version === required.version
-    ? { ...required, root: resolve(root), action: "already-installed" }
+    ? { ...required, root: installation, action: "already-installed" }
     : undefined;
 }
 
-function runNpm(root: string, specifiers: readonly string[]): Promise<void> {
+async function runNpm(root: string, specifiers: readonly string[], logPath: string): Promise<void> {
+  // npm's prefix and cwd must denote the same physical project (not /tmp vs /private/tmp).
+  const cwd = await realpath(root);
   const npmArgs = [
     "install",
-    "--prefix", root,
+    "--prefix", cwd,
     "--save-exact",
-    "--package-lock=false",
     "--no-audit",
     "--no-fund",
     "--omit=dev",
@@ -83,27 +88,30 @@ function runNpm(root: string, specifiers: readonly string[]): Promise<void> {
   const windows = process.platform === "win32";
   const command = windows ? (process.env.ComSpec ?? "cmd.exe") : "npm";
   const args = windows ? ["/d", "/s", "/c", "npm.cmd", ...npmArgs] : npmArgs;
-  return new Promise((done, reject) => {
-    const child = spawn(command, args, {
-      shell: false,
-      windowsHide: true,
-      stdio: ["ignore", "pipe", "pipe"],
-    });
-    let output = "";
-    child.stdout.on("data", (chunk: Buffer) => { output = `${output}${chunk.toString()}`.slice(-32_000); });
-    child.stderr.on("data", (chunk: Buffer) => { output = `${output}${chunk.toString()}`.slice(-32_000); });
-    child.on("error", reject);
-    child.on("close", (code) => {
-      if (code === 0) done();
-      else reject(new Error(`npm install failed${output.trim().length === 0 ? "" : `: ${output.trim().split("\n").at(-1)}`}`));
+  const log = await open(logPath, "a");
+  await log.write(`\n${new Date().toISOString()} npm ${npmArgs.join(" ")}\n`);
+  try {
+    await new Promise<void>((done, reject) => {
+      const child = spawn(command, args, {
+        cwd,
+        shell: false,
+        windowsHide: true,
+        stdio: ["ignore", log.fd, log.fd],
+      });
+      child.on("error", (error) => reject(new Error(`Cannot start npm: ${error.message}. Log: ${logPath}`, { cause: error })));
+      child.on("close", (code) => {
+        if (code === 0) done();
+        else reject(new Error(`npm install failed (exit ${code ?? "signal"}). Log: ${logPath}`));
+      });
     });
-  });
+  } finally {
+    await log.close();
+  }
 }
 
 /**
- * Install missing upstream packages into one user/machine npm home.
- * npm's ordinary package.json is the only persistent package-set record; Hypit
- * creates no lock, receipt, hash inventory or project-local copy.
+ * Reuse each exact upstream release in its own npm installation. npm owns its
+ * package.json, lockfile and dependencies; Hypit keeps no parallel inventory.
  */
 export async function prepareHostPackages(
   specifiers: readonly string[],
@@ -113,46 +121,35 @@ export async function prepareHostPackages(
   },
 ): Promise<readonly HostPackageReport[]> {
   const root = resolve(options.root);
-  const byName = new Map<string, RegistryPackageSpec>();
+  const bySpecifier = new Map<string, RegistryPackageSpec>();
   for (const specifier of specifiers) {
     const parsed = parseRegistryPackageSpec(specifier);
-    const previous = byName.get(parsed.name);
-    if (previous !== undefined && previous.version !== parsed.version) {
-      throw new Error(`${parsed.name} is required at both ${previous.version} and ${parsed.version}`);
-    }
-    byName.set(parsed.name, parsed);
+    bySpecifier.set(parsed.specifier, parsed);
   }
-  const required = [...byName.values()].sort((left, right) => left.name.localeCompare(right.name));
-  const missing: RegistryPackageSpec[] = [];
+  const required = [...bySpecifier.values()].sort((left, right) => left.specifier.localeCompare(right.specifier));
+  const reports: HostPackageReport[] = [];
   for (const item of required) {
+    const installation = externalPackageInstallRoot(root, item.name, item.version);
     options.onProgress?.({ ...item, phase: "checking" });
-    if (await installedVersion(root, item.name) !== item.version) missing.push(item);
-  }
-  if (missing.length > 0) {
-    await mkdir(root, { recursive: true });
-    try {
-      await readFile(join(root, "package.json"), "utf8");
-    } catch (error) {
-      if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
-      await writeFile(join(root, "package.json"), `${JSON.stringify({
+    const missing = await installedVersion(installation, item.name) !== item.version;
+    const logPath = join(installation, "install.log");
+    if (missing) {
+      await mkdir(installation, { recursive: true });
+      await writeFile(join(installation, "package.json"), `${JSON.stringify({
         name: "hypit-machine-packages",
         private: true,
         description: "Upstream npm packages used on demand by Hypit",
-        dependencies: {},
+        dependencies: { [item.name]: item.version },
       }, null, 2)}\n`, "utf8");
+      options.onProgress?.({ ...item, phase: "installing", logPath });
+      await runNpm(installation, [item.specifier], logPath);
     }
-    for (const item of missing) options.onProgress?.({ ...item, phase: "installing" });
-    await runNpm(root, missing.map((item) => item.specifier));
-  }
-  const installed = new Set(missing.map((item) => item.name));
-  const reports: HostPackageReport[] = [];
-  for (const item of required) {
-    const version = await installedVersion(root, item.name);
+    const version = await installedVersion(installation, item.name);
     if (version !== item.version) {
-      throw new Error(`npm did not install ${item.specifier} into ${root}`);
+      throw new Error(`npm did not install ${item.specifier} into ${installation}. Log: ${logPath}`);
     }
     options.onProgress?.({ ...item, phase: "ready" });
-    reports.push({ ...item, root, action: installed.has(item.name) ? "installed" : "already-installed" });
+    reports.push({ ...item, root: installation, ...(missing ? { logPath } : {}), action: missing ? "installed" : "already-installed" });
   }
   return reports;
 }

package/packages/runtime-kit/README.md

@@ -40,3 +40,8 @@ WhisperX process. Credential adapters validate configuration before opening a st
 storage belongs to the Runtime implementation, while project Build Result repositories use the
 separate `@hypit/build-result-kit` boundary. Runtime Kit knows no Provider, filesystem, database or
 video package by name.
+
+`ManagedProgram.installation.prepareBeforeStart` lets a Provider reconcile its installed environment
+before a cold start, even when the installation probe already passes. The declared commands use the
+Provider's ordinary package manager. A healthy running Program is reused before this preparation is
+considered; Runtime does not inspect source files or infer implementation versions.

package/packages/runtime-kit/src/index.ts

@@ -42,7 +42,9 @@ export type ManagedProgramCommand = {
 export type ManagedProgramInstallation = {
   /** Truthful inspection of the installed program; no installer-owned receipt. */
   probe(): Promise<ManagedProgramState>;
-  /** Commands run only when the installation probe is not ready. */
+  /** Reconcile an installed environment before a cold start, using its package manager's cache. */
+  readonly prepareBeforeStart?: boolean;
+  /** Commands run when installation is missing, or before a cold start when requested above. */
   readonly commands: readonly ManagedProgramCommand[];
 };
 

package/packages/runtime-local/README.md

@@ -70,7 +70,7 @@ quota; exact-model limits may narrow it further. Build identity is not a capacit
 no second queue.
 Claims may carry `units` (default 1): one render Need can occupy one request slot and four browser
 slots at once. The same admission rule applies to total Provider capacity, exact models, and local
-compute resources. SQLite retains the weighted reservation across Worker turns and restarts. This
+compute resources. SQLite retains the weighted reservation across execution turns; an ended attempt releases its reservations. This
 coordinates Builds sharing this Runtime's Execution Store; it does not enforce a service's quotas
 across other machines. Short-action limits are described below.
 
@@ -95,8 +95,7 @@ becomes available; known Operations are polled from their own lightweight record
 serialized, while network actions and local work remain concurrent under their declared limits.
 
 HypiHub can be the explicitly selected gateway for users without their own service keys. A bound
-Provider's authentication, quota or transport error never changes that selection. Separate Kie and
-HypiHub accounts use separate pools even when they implement the same model.
+Provider's authentication, quota or transport error never changes that selection. Separate accounts use separate pools even when they implement the same model.
 
 Without `hypit.results.json`, the official video Distribution selects the filesystem adapter at the
 project's `.hypit/results` directory, with no cloud account or service. Runtime Local opens that default
@@ -123,12 +122,20 @@ for a submission that never became active and therefore has no Result to save.
 Source imports select author packages. Runtime Profile entries select only code allowed to access files,
 credentials, processes or networks. Installing a package changes neither selection.
 
+Endpoint scoping happens before activation when explicit instance IDs are supplied, or when every
+requested capability has a binding. Otherwise discovery loads the Profile's Endpoint packages to
+find eligible implementations. Thus unused service readiness is not required, but an uninstalled
+declared package can still prevent unbound discovery. `scopedProfile` owns this distinction; package
+names are not used to guess which capabilities they supply.
+
 The lifecycle commands have deliberately narrow meanings:
 
 - `hypit runtime init` writes and selects the Distribution's starter Profile; it performs no setup or
   network access and never overwrites an existing Profile.
 - `hypit runtime up` prepares selected local dependencies, starts declared local Programs and starts
-  the Worker. `hypit runtime down` stops the Worker; `hypit programs down` stops the Programs.
+  the Worker. Repeat `--endpoint <instance>` to limit preparation to chosen services; omission covers
+  the whole Profile. `doctor` and Program operations accept the same scope. Build preflight receives
+  actual resolved Endpoint IDs and checks only their credentials and Programs. `hypit runtime down` stops the Worker; `hypit programs down` stops the Programs.
   Remote services have no lifecycle for Hypit to start or stop.
 - `hypit runtime logs` reads the Worker's output, including separately redirected standard errors
   on Windows. The files are `worker.log` and `worker.err.log` under the Runtime data directory's
@@ -136,3 +143,101 @@ The lifecycle commands have deliberately narrow meanings:
   event order. A failure before readiness also includes the recorded error in its startup message.
 - `hypit doctor` is the active, read-only check. Endpoint-owned diagnostics may authenticate and read a
   remote capability catalog; normal preflight never does.
+
+## Build execution evidence
+
+The Worker records Endpoint calls independently of CLI follow. Local calls use the execution context;
+remote Operations retain their existing receipts and contribute phase changes as they advance.
+`fileExecutionLogs` appends `hypit.execution-log@1` records to `work/<build>/execution.jsonl` under the
+Profile's `dataRoot`. Each record carries time, Command and Endpoint identity. Pure graph evaluation
+and repeated progress counters do not become log entries. The latest counters remain active Runtime
+state; Provider-authored diagnostics and phase changes are durable evidence.
+
+Result finishing streams this log through the selected Repository before publishing the terminal
+manifest or clearing the Build working directory. Complete, failed and cancelled Builds use the same
+finishing path. A failed archive leaves Result attention and preserves the working directory; finishing
+that Result performs no external execution. Log write failures also surface at the Result boundary,
+without converting a successful Provider call into another generation attempt.
+
+`hypit logs <build-id>` reads active evidence through Runtime control or finished evidence through the
+project Result Repository. `runtime logs` remains the Worker's process log. Program installation and
+service logs remain owned by their Program; they are shared service history, not copied into every Build.
+The logger does not capture the process environment, credential values, request bodies or global console.
+
+## Build-scoped execution
+
+The long-lived Worker supervises execution carriers. Each Build has its own module scope,
+component/Provider registries and selected configuration. Editing project code or Profile selections
+applies to the next Build without restarting the Worker; already loaded modules remain bound to their
+original Build. Installed Distribution modules stay process-owned. The Package Loader owns scoped
+JavaScript/TypeScript loading, including transitive ESM and CommonJS dependencies.
+
+One carrier shares a SQLite connection and event loop across active Builds. The database returns
+ready Build IDs; the dispatcher opens that Build's context once, hydrates local work in sequence, then
+lets asynchronous actions proceed concurrently. Pending remote submission does not hold a whole-Build
+admission slot. Accepted Operations retain receipts and wake times; their next poll can advance without
+materializing the whole graph. Endpoint-declared task limits, action concurrency and rates still govern
+actual work. A remote task waiting in an eight-slot pool does not block another Build's unrelated local
+step. CPU-heavy rendering belongs to its execution backend and declared resources.
+
+The active execution request owns its opaque local context: project package root, Distribution root,
+selected Endpoint configurations and bindings, and relevant Credential Store declarations. Source
+compilation and execution use the same project package root. Core does not see these choices. Different
+Profiles selecting the same Runtime data directory share its coordinator and capacity accounting; the
+coordinator's startup directory and recorded Profile path do not select later Builds' project code.
+
+`startedAt` records assignment to an execution carrier. Losing that carrier ends its assigned attempts
+and preserves completed Outputs and external receipts, without resubmitting work. Unstarted queue
+entries may still start. Result-writing failures retain the attention and `result finish` path. A live
+carrier never reclaims another carrier's turns. These are module/registry contexts in trusted code,
+not OS or security isolation: process globals, native libraries and a fatal process failure are shared.
+The carrier has one working directory and inherited environment. Package-relative file access uses
+module URLs; spawned tools receive their own explicit working directory and environment as needed.
+
+After its assigned Builds end, the carrier exits and releases Node's module cache. The coordinator and
+Managed Programs stay available. With continuous submissions, the coordinator can retire a carrier
+from accepting new Builds while its assigned work continues in place. New work uses a fresh carrier;
+all carriers share the same SQLite resource accounting. A live Build is never unloaded or moved.
+
+The coordinator reads process policy from its startup Profile:
+
+```json
+"worker": { "executionMemoryMb": 1024 }
+```
+
+The default is 1024 MiB of execution-process RSS. It is a soft retirement budget, not a memory ceiling
+or a limit on active Builds. Carriers report RSS after opening a Build context and when a Build ends.
+Once a carrier has completed work and its reported RSS reaches the budget, it stops accepting new
+Builds. Its remaining Builds finish before it exits. Live work alone does not trigger rotation, and
+a fresh carrier can accept work even when its startup footprint exceeds the budget. There is no
+forced timeout or replay. Draining carriers can still use substantial memory while long jobs run;
+this policy reclaims finished-work accumulation, not the memory required by unfinished work.
+The Profile that starts the coordinator owns this process policy until that coordinator exits.
+
+Loaded module bindings are scoped, not the filesystem. Later file reads observe ordinary files;
+a first-time dynamic import reads the implementation then available and subsequently remains cached
+in that Build's scope. No source snapshot, filesystem scan, content hash or Build recovery is involved.
+`createLocalRuntime` remains available for embeddings that explicitly supply their implementations.
+Its `workOnce` advances fresh work or work already owned by that instance; an explicit request to
+take over another context's started Build fails before any Provider call. The coordinator supplies
+the assignment when opening each production Build context. Client code calls `worker.up()` on the
+controller returned by `await host.controller()`, and calls `build(...)` on the Runtime returned by
+`await host.createRuntime()`. There is one production process-lifecycle path.
+
+Managed Programs have independent lifetimes. A healthy WhisperX service keeps its model loaded;
+`prepareBeforeStart` reconciles a cold installation through uv's source-aware synchronization.
+`runtime down` stops the coordinator and asks its executors to stop; Programs are stopped separately.
+Distribution changes and shell environment changes still concern the coordinator's bootstrap process.
+Inspect active work before restarting it. Environment-backed credentials use that inherited process
+environment; external writable Credential Stores resolve through their own implementation.
+
+### Concurrency checks
+
+`pnpm test` exercises 24 Builds with an eight-task remote pool and with all 24 submissions pending
+at once. Both cases verify local progress, per-Build module isolation, one submission per Build,
+capacity release and every completed Result. They use the production execution process and SQLite.
+
+`pnpm test:runtime-scale` runs the same checks with 1000 simultaneous remote submissions, on its own.
+This is a load experiment rather than part of every package regression. It reports fixture preparation,
+local progress, completion, executor RSS and cleanup separately; total test time also includes creating
+and removing the temporary Result repositories. Neither suite uses a completion-time performance target.