@swapai/core 0.2.2 → 0.3.0

package/dist/index.js

@@ -1120,6 +1120,8 @@ var SCHEMA = `
     new_examples_since_training INTEGER NOT NULL DEFAULT 0,
     training_attempts INTEGER NOT NULL DEFAULT 0,
     examples_used_for_training INTEGER NOT NULL DEFAULT 0,
+    last_evaluated_error REAL,
+    last_evaluated_at INTEGER,
     local_classifications_since_retest INTEGER NOT NULL DEFAULT 0,
     total_local_classifications INTEGER NOT NULL DEFAULT 0,
     total_retests INTEGER NOT NULL DEFAULT 0,
@@ -1165,6 +1167,19 @@ var SCHEMA = `
   CREATE INDEX IF NOT EXISTS examples_by_split
     ON examples(classifier_name, generation, split, id);
 
+  CREATE TABLE IF NOT EXISTS classifier_runtimes (
+    classifier_name TEXT NOT NULL,
+    runtime_id TEXT NOT NULL,
+    pid INTEGER NOT NULL,
+    started_at INTEGER NOT NULL,
+    heartbeat_at INTEGER NOT NULL,
+    PRIMARY KEY (classifier_name, runtime_id),
+    FOREIGN KEY (classifier_name) REFERENCES classifiers(name) ON DELETE CASCADE
+  );
+
+  CREATE INDEX IF NOT EXISTS classifier_runtimes_by_heartbeat
+    ON classifier_runtimes(classifier_name, heartbeat_at);
+
   CREATE TRIGGER IF NOT EXISTS swapai_v2_classifiers_insert
     BEFORE INSERT ON classifiers
     WHEN swapai_writer_version() < 2
@@ -1306,6 +1321,8 @@ function openStorage(options) {
           c.new_examples_since_training,
           c.training_attempts,
           c.examples_used_for_training,
+          c.last_evaluated_error,
+          c.last_evaluated_at,
           c.local_classifications_since_retest,
           c.total_local_classifications,
           c.total_retests,
@@ -1341,6 +1358,8 @@ function openStorage(options) {
         newExamplesSinceTraining: row2.new_examples_since_training,
         trainingAttempts: row2.training_attempts,
         examplesUsedForTraining: row2.examples_used_for_training,
+        lastEvaluatedError: row2.last_evaluated_error,
+        lastEvaluatedAt: row2.last_evaluated_at,
         localClassificationsSinceRetest: row2.local_classifications_since_retest,
         totalLocalClassifications: row2.total_local_classifications,
         totalRetests: row2.total_retests,
@@ -1439,6 +1458,21 @@ function openStorage(options) {
       `).run(exampleCount, Date.now(), options.name, epoch);
       return result.changes === 1;
     },
+    recordEvaluation(error, expectedDataEpoch) {
+      assertOpen(closed);
+      if (!Number.isFinite(error) || error < 0 || error > 1) {
+        throw new TypeError("evaluation error must be between 0 and 1");
+      }
+      const epoch = expectedDataEpoch ?? storage.snapshot().dataEpoch;
+      const result = database.prepare(`
+        UPDATE classifiers
+        SET last_evaluated_error = ?,
+            last_evaluated_at = ?,
+            updated_at = ?
+        WHERE name = ? AND data_epoch = ? AND clear_pending = 0
+      `).run(error, Date.now(), Date.now(), options.name, epoch);
+      return result.changes === 1;
+    },
     promoteGeneration(model, expectedDataEpoch) {
       assertOpen(closed);
       let generation = 0;
@@ -1522,6 +1556,52 @@ function openStorage(options) {
         WHERE name = ? AND training_lease_owner = ?
       `).run(Date.now(), options.name, owner);
     },
+    registerRuntime(runtimeId, pid) {
+      assertOpen(closed);
+      if (runtimeId.trim() === "" || !Number.isSafeInteger(pid) || pid <= 0) {
+        throw new TypeError("runtime needs an id and positive process id");
+      }
+      const now = Date.now();
+      database.prepare(`
+        INSERT INTO classifier_runtimes (
+          classifier_name, runtime_id, pid, started_at, heartbeat_at
+        ) VALUES (?, ?, ?, ?, ?)
+        ON CONFLICT(classifier_name, runtime_id) DO UPDATE SET
+          pid = excluded.pid,
+          heartbeat_at = excluded.heartbeat_at
+      `).run(options.name, runtimeId, pid, now, now);
+    },
+    heartbeatRuntime(runtimeId) {
+      assertOpen(closed);
+      database.prepare(`
+        UPDATE classifier_runtimes
+        SET heartbeat_at = ?
+        WHERE classifier_name = ? AND runtime_id = ?
+      `).run(Date.now(), options.name, runtimeId);
+    },
+    removeRuntime(runtimeId) {
+      assertOpen(closed);
+      database.prepare(`
+        DELETE FROM classifier_runtimes
+        WHERE classifier_name = ? AND runtime_id = ?
+      `).run(options.name, runtimeId);
+    },
+    listLiveRuntimes(heartbeatAfter) {
+      assertOpen(closed);
+      if (!Number.isFinite(heartbeatAfter)) {
+        throw new TypeError("heartbeat cutoff must be finite");
+      }
+      return database.prepare(`
+        SELECT
+          runtime_id AS runtimeId,
+          pid,
+          started_at AS startedAt,
+          heartbeat_at AS heartbeatAt
+        FROM classifier_runtimes
+        WHERE classifier_name = ? AND heartbeat_at >= ?
+        ORDER BY started_at, runtime_id
+      `).all(options.name, heartbeatAfter).map((value) => row(value));
+    },
     archiveAndReset(expectedDataEpoch, resetOptions) {
       assertOpen(closed);
       let nextGeneration = 0;
@@ -1563,6 +1643,8 @@ function openStorage(options) {
               new_examples_since_training = ?,
               training_attempts = 0,
               examples_used_for_training = 0,
+              last_evaluated_error = NULL,
+              last_evaluated_at = NULL,
               training_lease_owner = NULL,
               training_lease_until = NULL,
               training_lease_epoch = NULL,
@@ -1630,6 +1712,8 @@ function openStorage(options) {
               new_examples_since_training = 0,
               training_attempts = 0,
               examples_used_for_training = 0,
+              last_evaluated_error = NULL,
+              last_evaluated_at = NULL,
               local_classifications_since_retest = 0,
               total_local_classifications = 0,
               total_retests = 0,
@@ -1792,6 +1876,16 @@ function migrateClassifierColumns(database) {
         "ALTER TABLE classifiers ADD COLUMN training_lease_epoch INTEGER"
       );
     }
+    if (!columns.has("last_evaluated_error")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN last_evaluated_error REAL"
+      );
+    }
+    if (!columns.has("last_evaluated_at")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN last_evaluated_at INTEGER"
+      );
+    }
   });
 }
 function generationByNumber(database, name, generation) {
@@ -1925,6 +2019,7 @@ function requiredRow(value) {
 // src/classifier.ts
 var TRAINING_LEASE_DURATION_MS = 5 * 60 * 1e3;
 var TRAINING_LEASE_RENEWAL_MS = 60 * 1e3;
+var RUNTIME_HEARTBEAT_MS = 2e3;
 var ARTIFACT_LOCK_WAIT_MS = 3e4;
 function init(inputConfig) {
   const config = normalizeConfig(inputConfig);
@@ -1974,6 +2069,18 @@ function createClassifier(config, storage, runtime) {
   let trainingQueue = Promise.resolve();
   let trainingScheduled = false;
   const trainingLeaseOwner = `${process.pid}:${randomUUID2()}`;
+  storage.registerRuntime(trainingLeaseOwner, process.pid);
+  const runtimeHeartbeat = setInterval(() => {
+    try {
+      storage.heartbeatRuntime(trainingLeaseOwner);
+    } catch (error) {
+      reportBackgroundError(
+        config,
+        toSwapAIError(error, "storage_failed", "Could not update classifier status")
+      );
+    }
+  }, RUNTIME_HEARTBEAT_MS);
+  runtimeHeartbeat.unref();
   let classificationQueue = Promise.resolve();
   let queuedFailure = null;
   let clearFailure = null;
@@ -2199,7 +2306,15 @@ function createClassifier(config, storage, runtime) {
             candidate: candidateResult
           });
         }
-        if (averageError(config.result, comparisons) > config.acceptableError) {
+        const measuredError = averageError(config.result, comparisons);
+        if (!storage.recordEvaluation(measuredError, trainingEpoch)) {
+          await runtime.clearClassifierGenerationArtifacts(
+            config.name,
+            snapshot.activeGeneration
+          );
+          return false;
+        }
+        if (measuredError > config.acceptableError) {
           return true;
         }
         if (!renewTrainingLease()) {
@@ -2582,6 +2697,8 @@ function createClassifier(config, storage, runtime) {
           failures.push(error);
         }
         try {
+          clearInterval(runtimeHeartbeat);
+          storage.removeRuntime(trainingLeaseOwner);
           storage.close();
         } catch (error) {
           failures.push(error);