@apisurf/ngn-core 0.3.0 → 0.3.1

package/dist/core.d.ts

@@ -1,3 +1,5 @@
+import { RunStage } from '@apisurf/ngn-schema/describe';
+export { RunStage } from '@apisurf/ngn-schema/describe';
 import { FsNodeFileDescriptor } from '@apisurf/ngn-os';
 import { Client, InArgs, ResultSet, InStatement, InValue } from '@apisurf/ngn-persistence';
 export { initDbFileIfNotExists } from '@apisurf/ngn-persistence';
@@ -120,6 +122,13 @@ interface TaskSqlite {
  * Close every database opened by a task. Called by the CLI on shutdown.
  */
 declare function closeTaskDatabases(): void;
+/**
+ * Let go of the databases one task file opened, closing those no other task
+ * holds. For a one-off run inside a process that keeps scheduling others (the
+ * editor's): `closeTaskDatabases` there would close the connections of
+ * scheduled runs still in progress, and their next query would fail.
+ */
+declare function releaseTaskDatabases(taskSourcePath: string): void;
 
 /**
  * A length of time as tasks write it: milliseconds, or a string such as
@@ -484,7 +493,8 @@ interface TaskContext {
         file: FsNodeFileDescriptor;
         tasksRootDir: string;
     };
-    env: Record<string, string> | null;
+    /** The variables from the config's `envFile`; empty when there is none. */
+    env: Record<string, string>;
     kv: {
         set(key: string, value: string): Promise<void>;
         get(key: string): Promise<string | null>;
@@ -541,9 +551,6 @@ interface CompiledTaskCallbacks {
     onTaskExportsLoaded?: () => Promise<void>;
 }
 interface TaskRuntimeCallbacks {
-    onNotFound?: (meta: {
-        path: string;
-    }) => Promise<void>;
     /** `reason` is set when ngn skipped the run, rather than the task's `shouldSkip`. */
     onSkip?: (meta: {
         path: string;
@@ -556,10 +563,31 @@ interface TaskRuntimeCallbacks {
         path: string;
         result?: unknown;
     }) => Promise<void>;
+    /** The run failed: `error` was thrown in `stage`. Never throws itself. */
     onFailure?: (meta: {
         path: string;
+        stage: RunStage;
+        error: unknown;
+    }) => Promise<void>;
+    /** A hook threw; the run's outcome stands. Never throws itself. */
+    onHookError?: (meta: {
+        path: string;
+        stage: RunStage;
+        error: unknown;
     }) => Promise<void>;
 }
+/**
+ * Something a run threw, once it is recorded against the run: what the CLI
+ * prints. `fatal` is false for a hook's error, which leaves the run's outcome
+ * as it was.
+ */
+interface RunProblem {
+    path: string;
+    runId: number;
+    stage: RunStage;
+    error: unknown;
+    fatal: boolean;
+}
 /**
  * One run's bookkeeping: its row in `task_runs`, and everything that writes
  * against that row. Made fresh for every execution, so two runs of one task
@@ -607,8 +635,17 @@ interface ExecuteSchedule {
     /** When the next tick is due, epoch ms. */
     nextAt?: number | null;
 }
-/** The `task_runs.id` of the run that threw `error`, when a task run did. */
+/** The `task_runs.id` of the run that threw `error`, when a run recorded it. */
 declare function failedRunId(error: unknown): number | null;
+/**
+ * The run that recorded `error` and the stage it was thrown in; null for an
+ * error no run recorded (a task file that did not compile, a database ngn
+ * could not open), which only its thrower knows about.
+ */
+declare function runFailure(error: unknown): {
+    runId: number;
+    stage: RunStage;
+} | null;
 declare class Task {
     private taskCallbacks;
     private startRun;
@@ -654,8 +691,26 @@ declare class Task {
     hasTaskExport(): boolean;
     hasTimingExport(): boolean;
     loadEntry(): Promise<void>;
+    /**
+     * Load the task file's code and register its version. A failure here has no
+     * run to mark yet, so it is thrown to whoever called execute(). Evaluating
+     * the code comes after the run exists — see `readExports`.
+     */
     private loadAndRegister;
-    private runTask;
+    /**
+     * Evaluate the task file and check the exports a run depends on. Throws
+     * what the module threw, or what is wrong with an export, for the run to
+     * record as failed in `load`.
+     */
+    private readExports;
+    /** Mark the run failed in `stage`, remembering where for whoever reports the error. */
+    private failRun;
+    /**
+     * Call one of the task's hooks. What it throws is recorded on the run and
+     * goes no further: the run's outcome was decided before the hook ran, and
+     * `onError`'s error must not replace the one it was called with.
+     */
+    private runHook;
     /**
      * Run the task for one tick, retrying as its `retry` export allows. A
      * scheduled tick passes when it fired and when the next one is due, which
@@ -682,6 +737,17 @@ declare class Task {
      */
     private holdBack;
     private buildContext;
+    /**
+     * Load the task file, open the attempt's `task_runs` row, then evaluate the
+     * file's code; what that throws is recorded on the row as failed in `load`.
+     */
+    private openRun;
+    /**
+     * One attempt: a `task_runs` row, then the task, with every failure recorded
+     * against that row as the stage it was thrown in. Only a task file whose code
+     * cannot be loaded at all, or a run row that cannot be written, fails before
+     * there is a row to record it on.
+     */
     private executeOnce;
 }
 
@@ -750,9 +816,12 @@ declare function scheduleTask(task: Task, pattern: string, onError: (error: unkn
 declare function setupDbClient(dbPath: string): Promise<Client>;
 declare function getDbClient(): Client;
 
-declare function createControlsGenerator({ dbClient, env, }: {
+declare function createControlsGenerator({ dbClient, env, report, }: {
     dbClient: Client;
-    env: TaskContext["env"];
+    /** A missing env file is null from the readers; tasks always get an object. */
+    env: TaskContext["env"] | null | undefined;
+    /** Told about everything a run throws, after it is recorded. The CLI prints it. */
+    report?: (problem: RunProblem) => void;
 }): BuildCompiledTaskConfigFn;
 
-export { type AddJob, type Backoff, type ClaimOptions, Compiler, type DBInstance, type Duration, type InitDBOptions, type Job, type JobFilter, type JobStatus, LockBusyError, type LockOptions, type Migration, type OverlapPolicy, Queue, type QueueOptions, type QueueStats, type ScheduleReport, Task, type TaskContext, TaskLibrary, TaskLibraryScheduler, TaskOnceRunner, type TaskRun, type TaskSqlite, type WorkHandler, type WorkOptions, type WorkSummary, type WorkerHandle, closeTaskDatabases, createControlsGenerator, executeEsm, failedRunId, getDbClient, isCronPatternValid, scheduleTask, setupDbClient };
+export { type AddJob, type Backoff, type ClaimOptions, Compiler, type DBInstance, type Duration, type InitDBOptions, type Job, type JobFilter, type JobStatus, LockBusyError, type LockOptions, type Migration, type OverlapPolicy, Queue, type QueueOptions, type QueueStats, type RunProblem, type ScheduleReport, Task, type TaskContext, TaskLibrary, TaskLibraryScheduler, TaskOnceRunner, type TaskRun, type TaskSqlite, type WorkHandler, type WorkOptions, type WorkSummary, type WorkerHandle, closeTaskDatabases, createControlsGenerator, executeEsm, failedRunId, getDbClient, isCronPatternValid, releaseTaskDatabases, runFailure, scheduleTask, setupDbClient };

package/dist/core.js

@@ -297,36 +297,36 @@ var Entry = class {
   isLoaded() {
     return this.entry.isLoaded();
   }
+  /**
+   * Evaluate the task file and read its exports. Whatever the module throws
+   * while it is evaluated is thrown from here, so the run it was for can
+   * record it.
+   */
   async getExports() {
     invariant4(this.entry.code, "Entry code not loaded. Load entry first.");
-    try {
-      const taskExports = await executeEsm(this.entry.code);
-      this.callbacks.onTaskCodeLoaded?.(this.entry.codeHash);
-      let taskFunction;
-      if (typeof taskExports.default === "function") {
-        taskFunction = taskExports.default;
-      } else if (taskExports.default && typeof taskExports.default === "object" && "task" in taskExports.default) {
-        taskFunction = taskExports.default.task;
-      } else {
-        taskFunction = taskExports.task;
-      }
-      return {
-        task: taskFunction,
-        timing: taskExports.timing,
-        overlap: taskExports.overlap,
-        budget: taskExports.budget,
-        exclusive: taskExports.exclusive,
-        retry: taskExports.retry,
-        shouldSkip: taskExports.shouldSkip,
-        shouldRetry: taskExports.shouldRetry,
-        onSuccess: taskExports.onSuccess,
-        onError: taskExports.onError,
-        onComplete: taskExports.onComplete
-      };
-    } catch (error) {
-      console.error("Error getting task exports:", error);
-      return null;
+    const taskExports = await executeEsm(this.entry.code);
+    this.callbacks.onTaskCodeLoaded?.(this.entry.codeHash);
+    let taskFunction;
+    if (typeof taskExports.default === "function") {
+      taskFunction = taskExports.default;
+    } else if (taskExports.default && typeof taskExports.default === "object" && "task" in taskExports.default) {
+      taskFunction = taskExports.default.task;
+    } else {
+      taskFunction = taskExports.task;
     }
+    return {
+      task: taskFunction,
+      timing: taskExports.timing,
+      overlap: taskExports.overlap,
+      budget: taskExports.budget,
+      exclusive: taskExports.exclusive,
+      retry: taskExports.retry,
+      shouldSkip: taskExports.shouldSkip,
+      shouldRetry: taskExports.shouldRetry,
+      onSuccess: taskExports.onSuccess,
+      onError: taskExports.onError,
+      onComplete: taskExports.onComplete
+    };
   }
   get paths() {
     return this.entry.path;
@@ -530,27 +530,28 @@ function resolveTaskDbPath(taskDir, file) {
   }
   return resolved;
 }
-function getClient(absolutePath) {
+function getClient(absolutePath, owner) {
   const existing = openClients.get(absolutePath);
   if (existing) {
-    return existing;
+    existing.owners.add(owner);
+    return existing.client;
   }
   mkdirSync(dirname(absolutePath), { recursive: true });
   const client = openDbClient(`file:${absolutePath}`);
-  openClients.set(absolutePath, client);
+  openClients.set(absolutePath, { client, owners: /* @__PURE__ */ new Set([owner]) });
   return client;
 }
 function closeClient(absolutePath) {
-  const client = openClients.get(absolutePath);
-  if (client) {
-    client.close();
+  const open = openClients.get(absolutePath);
+  if (open) {
+    open.client.close();
     openClients.delete(absolutePath);
   }
 }
 function createTaskSqlite(taskSourcePath) {
   const taskDir = dirname(taskSourcePath);
   const defaultDbPath = join(taskDir, `${basename(taskSourcePath, extname(taskSourcePath))}.db`);
-  const defaultClient = () => getClient(defaultDbPath);
+  const defaultClient = () => getClient(defaultDbPath, taskSourcePath);
   return {
     get client() {
       return defaultClient();
@@ -562,7 +563,7 @@ function createTaskSqlite(taskSourcePath) {
     batch: (statements) => defaultClient().batch(statements),
     async initDB(options) {
       const dbPath = resolveTaskDbPath(taskDir, options.file);
-      const client = getClient(dbPath);
+      const client = getClient(dbPath, taskSourcePath);
       if (options.migrations?.length) {
         await runMigrations(client, options.migrations);
       }
@@ -582,11 +583,18 @@ function createTaskSqlite(taskSourcePath) {
   };
 }
 function closeTaskDatabases() {
-  for (const client of openClients.values()) {
+  for (const { client } of openClients.values()) {
     client.close();
   }
   openClients.clear();
 }
+function releaseTaskDatabases(taskSourcePath) {
+  for (const [path, { client, owners }] of openClients) {
+    if (!owners.delete(taskSourcePath) || owners.size > 0) continue;
+    client.close();
+    openClients.delete(path);
+  }
+}
 
 // src/runtime/queue/schema.ts
 var QUEUE_MIGRATIONS = [
@@ -1172,6 +1180,9 @@ function breakerSkips(failedInARow, breaker) {
 var sleep2 = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));
 var failedRuns = /* @__PURE__ */ new WeakMap();
 function failedRunId(error) {
+  return runFailure(error)?.runId ?? null;
+}
+function runFailure(error) {
   return error && typeof error === "object" ? failedRuns.get(error) ?? null : null;
 }
 var Task = class {
@@ -1284,12 +1295,15 @@ var Task = class {
     await this.entry.load();
     this.entryExports = await this.entry.getExports();
   }
+  /**
+   * Load the task file's code and register its version. A failure here has no
+   * run to mark yet, so it is thrown to whoever called execute(). Evaluating
+   * the code comes after the run exists — see `readExports`.
+   */
   async loadAndRegister() {
     invariant5(this.entry, "Entry not initialized.");
-    await this.loadEntry();
+    await this.entry.load();
     invariant5(this.entry.isLoaded(), "Entry failed to load.");
-    invariant5(this.entryExports, "Entry failed to get exports.");
-    this.taskCallbacks?.onTaskExportsLoaded?.();
     const loadResult = await this.taskCallbacks?.onTaskLoadComplete?.({
       paths: this.entry.paths,
       md5Hash: this.entry.codeHash,
@@ -1298,22 +1312,44 @@ var Task = class {
     this.meta.fileTaskId = loadResult?.fileTaskId ?? -1;
     this.meta.fileTaskVersionId = loadResult?.fileTaskVersionId ?? -1;
   }
-  async runTask(task, taskContext, hooks, run) {
+  /**
+   * Evaluate the task file and check the exports a run depends on. Throws
+   * what the module threw, or what is wrong with an export, for the run to
+   * record as failed in `load`.
+   */
+  async readExports(path) {
+    invariant5(this.entry, "Entry not initialized.");
+    const exports = await this.entry.getExports();
+    this.entryExports = exports;
+    this.taskCallbacks?.onTaskExportsLoaded?.();
+    if (typeof exports.task !== "function") {
+      throw new Error(`${path} exports no task function`);
+    }
+    return {
+      exports,
+      task: exports.task,
+      budgetMs: exports.budget == null ? null : toMs(exports.budget, `budget for ${path}`),
+      exclusiveLocks: exclusiveNames(exports.exclusive, path),
+      retry: resolveRetry(exports.retry, path)
+    };
+  }
+  /** Mark the run failed in `stage`, remembering where for whoever reports the error. */
+  async failRun(run, stage, error) {
+    invariant5(this.entry, "Entry not initialized.");
+    if (error && typeof error === "object") failedRuns.set(error, { runId: run.id, stage });
+    await run.callbacks.onFailure?.({ path: this.entry.paths.relativeEntry, stage, error });
+  }
+  /**
+   * Call one of the task's hooks. What it throws is recorded on the run and
+   * goes no further: the run's outcome was decided before the hook ran, and
+   * `onError`'s error must not replace the one it was called with.
+   */
+  async runHook(run, stage, hook) {
     invariant5(this.entry, "Entry not initialized.");
-    const path = this.entry.paths.relativeEntry;
     try {
-      await run.callbacks.onStart?.({ path });
-      const taskReturnValue = await task(taskContext);
-      await run.callbacks.onSuccess?.({ path, result: taskReturnValue });
-      await hooks.onSuccess?.(taskContext);
-      return taskReturnValue;
-    } catch (err) {
-      if (err && typeof err === "object") failedRuns.set(err, run.id);
-      await run.callbacks.onFailure?.({ path });
-      await hooks.onError?.(err, taskContext);
-      throw err;
-    } finally {
-      await hooks.onComplete?.(taskContext);
+      await hook();
+    } catch (error) {
+      await run.callbacks.onHookError?.({ path: this.entry.paths.relativeEntry, stage, error });
     }
   }
   /**
@@ -1403,27 +1439,38 @@ var Task = class {
       queue: (name, options = {}) => openQueue(options.db?.client ?? this.sqlite.client, name, options, { nextAt, signal })
     };
   }
-  async executeOnce({ scheduledAt = null, nextAt = null }, tick2) {
+  /**
+   * Load the task file, open the attempt's `task_runs` row, then evaluate the
+   * file's code; what that throws is recorded on the row as failed in `load`.
+   */
+  async openRun(tick2) {
     invariant5(this.entry, "Entry not initialized.");
     await this.loadAndRegister();
-    invariant5(this.entryExports, "Entry failed to get exports.");
-    const exports = this.entryExports;
     const meta = { ...this.meta };
     const path = this.entry.paths.relativeEntry;
-    const budgetMs = exports.budget == null ? null : toMs(exports.budget, `budget for ${path}`);
-    const exclusiveLocks = exclusiveNames(exports.exclusive, path);
-    const retry = resolveRetry(exports.retry, path);
-    tick2.retry = retry;
     const run = await this.startRun({ retryOf: tick2.firstRunId });
     const firstAttempt = tick2.firstRunId == null;
     tick2.firstRunId ??= run.id;
-    if (!exports.task) {
-      await run.callbacks.onNotFound?.({ path });
-      throw new Error("Task function not found");
+    try {
+      return { run, meta, path, firstAttempt, ...await this.readExports(path) };
+    } catch (error) {
+      await this.failRun(run, "load", error);
+      throw error;
     }
+  }
+  /**
+   * One attempt: a `task_runs` row, then the task, with every failure recorded
+   * against that row as the stage it was thrown in. Only a task file whose code
+   * cannot be loaded at all, or a run row that cannot be written, fails before
+   * there is a row to record it on.
+   */
+  async executeOnce({ scheduledAt = null, nextAt = null }, tick2) {
+    const { run, meta, path, firstAttempt, exports, task, budgetMs, exclusiveLocks, retry } = await this.openRun(tick2);
+    tick2.retry = retry;
     const deadline = runDeadline({ startedAt: Date.now(), scheduledAt, nextAt, budgetMs });
     const { signal, dispose } = deadlineSignal(deadline);
     const taskContext = this.buildContext(run, meta, { scheduledAt, nextAt, deadline }, signal);
+    let stage = "start";
     let exclusiveHeld = null;
     try {
       const gate = await this.holdBack(run, {
@@ -1437,20 +1484,29 @@ var Task = class {
         return;
       }
       exclusiveHeld = gate.held;
+      stage = "shouldSkip";
       if (await exports.shouldSkip?.(taskContext)) {
         await run.callbacks.onSkip?.({ path });
         return;
       }
+      stage = "start";
+      await run.callbacks.onStart?.({ path });
+      stage = "task";
       tick2.failedContext = taskContext;
-      const { onSuccess, onError, onComplete } = exports;
-      const value = await this.runTask(
-        exports.task,
-        taskContext,
-        { onSuccess, onError, onComplete },
-        run
-      );
+      const value = await task(taskContext);
       tick2.failedContext = null;
+      stage = "record";
+      await run.callbacks.onSuccess?.({ path, result: value });
+      await this.runHook(run, "onSuccess", () => exports.onSuccess?.(taskContext));
+      await this.runHook(run, "onComplete", () => exports.onComplete?.(taskContext));
       return value;
+    } catch (error) {
+      await this.failRun(run, stage, error);
+      if (stage === "task" || stage === "record") {
+        await this.runHook(run, "onError", () => exports.onError?.(error, taskContext));
+        await this.runHook(run, "onComplete", () => exports.onComplete?.(taskContext));
+      }
+      throw error;
     } finally {
       dispose();
       await exclusiveHeld?.release();
@@ -1698,6 +1754,7 @@ import {
   logTaskConsoleWarn
 } from "@apisurf/ngn-os";
 import invariant11 from "tiny-invariant";
+import { errorSummary, failureLine } from "@apisurf/ngn-schema/describe";
 
 // src/db/services/fileTask.ts
 import { $fileTaskInsert } from "@apisurf/ngn-schema";
@@ -2089,9 +2146,17 @@ var LockService = class {
 };
 
 // src/runtime/generateControls.ts
+async function recordSafely(runId, write) {
+  try {
+    await write();
+  } catch (error) {
+    logTaskCallback(`Could not record what run ${runId} threw: ${errorSummary(error)}`);
+  }
+}
 function createControlsGenerator({
   dbClient,
-  env
+  env,
+  report
 }) {
   const taskRunService = new TaskRunService(dbClient);
   const logService = new LogService(dbClient);
@@ -2134,10 +2199,6 @@ function createControlsGenerator({
         retry_of: retryOf
       });
       const callbacks = {
-        async onNotFound() {
-          await logService.logError(taskId, runId, "Task not found");
-          logTaskCallback(`Task not found: ${taskId}, run ID: ${runId}`);
-        },
         async onSkip({ reason }) {
           await taskRunService.skipTaskRun(runId, reason ?? null);
           await logService.logInfo(taskId, runId, "Task skipped");
@@ -2149,10 +2210,22 @@ function createControlsGenerator({
           await logService.logInfo(taskId, runId, "Task started");
           logTaskCallback(`Task started: ${taskId}, run ID: ${runId}`);
         },
-        async onFailure() {
-          await taskRunService.endTaskRun(runId, "failure");
-          await logService.logError(taskId, runId, "Task failed");
-          logTaskCallback(`Task failed: ${taskId}, run ID: ${runId}`);
+        async onFailure({ path, stage, error }) {
+          await recordSafely(runId, async () => {
+            await taskRunService.endTaskRun(runId, "failure");
+            await logService.logError(taskId, runId, failureLine(stage, error));
+            await logService.logError(taskId, runId, "Task failed");
+          });
+          logTaskCallback(`Task failed in ${stage}: ${taskId}, run ID: ${runId}`);
+          report?.({ path, runId, stage, error, fatal: true });
+        },
+        async onHookError({ path, stage, error }) {
+          await recordSafely(
+            runId,
+            () => logService.logError(taskId, runId, failureLine(stage, error))
+          );
+          logTaskCallback(`Hook ${stage} threw: ${taskId}, run ID: ${runId}`);
+          report?.({ path, runId, stage, error, fatal: false });
         },
         async onSuccess({ result }) {
           await taskRunService.endTaskRun(runId, "success", serializeResult(result));
@@ -2200,7 +2273,7 @@ function createControlsGenerator({
           return taskRunService.countSkipsSince(fileTaskId, reason, afterId);
         }
       },
-      env,
+      env: env ?? {},
       // Scoped to this task's own folder — see createTaskSqlite
       sqlite: createTaskSqlite(sourcePath),
       kv: {
@@ -2233,6 +2306,8 @@ export {
   getDbClient,
   initDbFileIfNotExists,
   isCronPatternValid,
+  releaseTaskDatabases,
+  runFailure,
   scheduleTask,
   setupDbClient
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apisurf/ngn-core",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "ngn runtime: task compilation, scheduling and the run/log/timing services.",
   "type": "module",
   "author": "Luka Vidakovic",
@@ -38,9 +38,9 @@
     "esbuild": "^0.27.1",
     "tiny-invariant": "^1.3.3",
     "zod": "^3.24.2",
-    "@apisurf/ngn-os": "0.3.0",
-    "@apisurf/ngn-persistence": "0.3.0",
-    "@apisurf/ngn-schema": "0.3.0"
+    "@apisurf/ngn-os": "0.3.1",
+    "@apisurf/ngn-schema": "0.3.1",
+    "@apisurf/ngn-persistence": "0.3.1"
   },
   "devDependencies": {
     "@types/node": "^22.8.1",