@swapai/core 0.1.0 → 0.2.0

package/dist/index.js

@@ -128,11 +128,12 @@ import {
   mkdir,
   readFile,
   readdir,
+  rm,
   stat,
   writeFile
 } from "fs/promises";
 import { createInterface } from "readline";
-import { dirname, join } from "path";
+import { dirname, join, resolve, sep } from "path";
 import { fileURLToPath } from "url";
 
 // src/needle.ts
@@ -224,8 +225,8 @@ var ManagedNeedleRuntime = class {
   #environment;
   #closed = false;
   constructor(options) {
-    this.#dataDirectory = options.dataDirectory;
-    this.#runtimeDirectory = join(options.dataDirectory, "runtime");
+    this.#dataDirectory = resolve(options.dataDirectory);
+    this.#runtimeDirectory = join(this.#dataDirectory, "runtime");
     this.#runCommand = options.dependencies?.runCommand ?? runCommand;
     this.#spawnProcess = options.dependencies?.spawnProcess ?? spawnProcess;
     this.#fetch = options.dependencies?.fetch ?? globalThis.fetch;
@@ -252,6 +253,12 @@ var ManagedNeedleRuntime = class {
         "Needle generation must be a non-negative integer."
       );
     }
+    if (!Number.isSafeInteger(options.expectedEpoch) || options.expectedEpoch < 0) {
+      throw new NeedleRuntimeError(
+        "service_unavailable",
+        "Needle data epoch must be a non-negative integer."
+      );
+    }
     if (options.examples.length === 0) {
       throw new NeedleRuntimeError(
         "service_unavailable",
@@ -281,8 +288,11 @@ var ManagedNeedleRuntime = class {
     const lines = options.examples.map(
       (example) => createTrainingLine(example.input, example.result, options.resultConfig)
     );
-    await writeFile(trainingPath, `${lines.join("\n")}
-`, "utf8");
+    const lockPath = join(
+      this.#dataDirectory,
+      "locks",
+      `${createHash("sha256").update(options.classifierName).digest("hex")}.sqlite`
+    );
     try {
       await this.#runCommand(
         this.#pythonPath,
@@ -296,12 +306,22 @@ var ManagedNeedleRuntime = class {
           "--checkpoint-dir",
           checkpointDirectory,
           "--epochs",
-          String(options.epochs ?? 10)
+          String(options.epochs ?? 10),
+          "--artifact-lock-database",
+          lockPath,
+          "--main-database",
+          join(this.#dataDirectory, "swapai.sqlite"),
+          "--classifier-name",
+          options.classifierName,
+          "--expected-epoch",
+          String(options.expectedEpoch)
         ],
         {
           cwd: candidateDirectory,
           env: this.#environment,
-          timeoutMs: 6 * 60 * 60 * 1e3
+          timeoutMs: 6 * 60 * 60 * 1e3,
+          stdin: `${lines.join("\n")}
+`
         }
       );
       await stat(modelPath);
@@ -352,6 +372,70 @@ var ManagedNeedleRuntime = class {
       throw toRuntimeError("Needle model could not start.", error);
     }
   }
+  async clearClassifierArtifacts(classifierName) {
+    await rm(
+      join(this.#dataDirectory, "classifiers", classifierKey(classifierName)),
+      { recursive: true, force: true }
+    );
+  }
+  async clearClassifierGenerationArtifacts(classifierName, generation) {
+    if (!Number.isSafeInteger(generation) || generation < 0) {
+      throw new NeedleRuntimeError(
+        "service_unavailable",
+        "Classifier generation must be a non-negative safe integer."
+      );
+    }
+    const classifierDirectory = resolve(
+      this.#dataDirectory,
+      "classifiers",
+      classifierKey(classifierName)
+    );
+    const generationDirectory = resolve(
+      classifierDirectory,
+      `generation-${generation}`
+    );
+    if (!generationDirectory.startsWith(`${classifierDirectory}${sep}`)) {
+      throw new NeedleRuntimeError(
+        "service_unavailable",
+        "Classifier generation path is outside its classifier directory."
+      );
+    }
+    await rm(generationDirectory, { recursive: true, force: true });
+  }
+  async clearClassifierArtifactsThroughGeneration(classifierName, maximumGeneration) {
+    if (!Number.isSafeInteger(maximumGeneration) || maximumGeneration < 0) {
+      throw new NeedleRuntimeError(
+        "service_unavailable",
+        "Maximum classifier generation must be a non-negative safe integer."
+      );
+    }
+    const classifierDirectory = join(
+      this.#dataDirectory,
+      "classifiers",
+      classifierKey(classifierName)
+    );
+    let entries;
+    try {
+      entries = await readdir(classifierDirectory, { withFileTypes: true });
+    } catch (error) {
+      if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
+        return;
+      }
+      throw error;
+    }
+    await Promise.all(
+      entries.map(async (entry) => {
+        const match = /^generation-(\d+)$/.exec(entry.name);
+        if (!entry.isDirectory() || match === null) return;
+        const generation = Number(match[1]);
+        if (generation > maximumGeneration) return;
+        await rm(join(classifierDirectory, entry.name), {
+          recursive: true,
+          force: true
+        });
+      })
+    );
+  }
   async close() {
     if (this.#closed) return;
     this.#closed = true;
@@ -533,12 +617,12 @@ var NeedleModelProcess = class {
     this.#resultConfig = resultConfig;
     this.#onBackgroundError = onBackgroundError;
     this.#onClose = onClose;
-    this.#readyPromise = new Promise((resolve, reject) => {
-      this.#resolveReady = resolve;
+    this.#readyPromise = new Promise((resolve2, reject) => {
+      this.#resolveReady = resolve2;
       this.#rejectReady = reject;
     });
-    this.#exitPromise = new Promise((resolve) => {
-      this.#resolveExit = resolve;
+    this.#exitPromise = new Promise((resolve2) => {
+      this.#resolveExit = resolve2;
     });
     this.#listen();
   }
@@ -556,8 +640,8 @@ var NeedleModelProcess = class {
       );
     }
     const id = ++this.#requestId;
-    const result = new Promise((resolve, reject) => {
-      this.#pending.set(id, { resolve, reject });
+    const result = new Promise((resolve2, reject) => {
+      this.#pending.set(id, { resolve: resolve2, reject });
       this.#process.stdin.write(
         `${JSON.stringify({ id, type: "classify", input })}
 `,
@@ -596,10 +680,7 @@ var NeedleModelProcess = class {
         await withTimeout(this.#exitPromise, 2e3, "Needle ignored SIGTERM.");
       } catch {
         this.#process.kill("SIGKILL");
-        await Promise.race([
-          this.#exitPromise,
-          new Promise((resolve) => setTimeout(resolve, 2e3))
-        ]);
+        await withTimeout(this.#exitPromise, 2e3, "Needle ignored SIGKILL.");
       }
     }
   }
@@ -710,12 +791,16 @@ var spawnProcess = (command, args, options) => spawn(command, [...args], {
   env: options.env,
   stdio: ["pipe", "pipe", "pipe"]
 });
-var runCommand = (command, args, options) => new Promise((resolve, reject) => {
+var runCommand = (command, args, options) => new Promise((resolve2, reject) => {
   const child = spawn(command, [...args], {
     cwd: options?.cwd,
     env: options?.env,
-    stdio: ["ignore", "pipe", "pipe"]
+    stdio: [options?.stdin === void 0 ? "ignore" : "pipe", "pipe", "pipe"]
   });
+  if (options?.stdin !== void 0 && child.stdin !== null) {
+    child.stdin.on("error", () => void 0);
+    child.stdin.end(options.stdin, "utf8");
+  }
   let stdout = "";
   let stderr = "";
   let timedOut = false;
@@ -764,7 +849,7 @@ var runCommand = (command, args, options) => new Promise((resolve, reject) => {
       return;
     }
     if (code === 0) {
-      resolve({ stdout, stderr });
+      resolve2({ stdout, stderr });
       return;
     }
     reject(
@@ -831,7 +916,7 @@ async function findFile(directory, name) {
   return void 0;
 }
 function withTimeout(promise, milliseconds, message, onTimeout) {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     const timer = setTimeout(() => {
       onTimeout?.();
       reject(new NeedleRuntimeError("service_unavailable", message));
@@ -840,7 +925,7 @@ function withTimeout(promise, milliseconds, message, onTimeout) {
     promise.then(
       (value) => {
         clearTimeout(timer);
-        resolve(value);
+        resolve2(value);
       },
       (error) => {
         clearTimeout(timer);
@@ -879,7 +964,12 @@ var SCHEMA = `
     training_lease_owner TEXT,
     training_lease_until INTEGER,
     created_at INTEGER NOT NULL,
-    updated_at INTEGER NOT NULL
+    updated_at INTEGER NOT NULL,
+    data_epoch INTEGER NOT NULL DEFAULT 0,
+    clear_pending INTEGER NOT NULL DEFAULT 0,
+    clear_erased INTEGER NOT NULL DEFAULT 0,
+    clear_artifact_generation_max INTEGER,
+    training_lease_epoch INTEGER
   );
 
   CREATE TABLE IF NOT EXISTS generations (
@@ -911,6 +1001,43 @@ var SCHEMA = `
     ON examples(classifier_name, generation, id);
   CREATE INDEX IF NOT EXISTS examples_by_split
     ON examples(classifier_name, generation, split, id);
+
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_classifiers_insert
+    BEFORE INSERT ON classifiers
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_classifiers_update
+    BEFORE UPDATE ON classifiers
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_classifiers_delete
+    BEFORE DELETE ON classifiers
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_generations_insert
+    BEFORE INSERT ON generations
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_generations_update
+    BEFORE UPDATE ON generations
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_generations_delete
+    BEFORE DELETE ON generations
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_examples_insert
+    BEFORE INSERT ON examples
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_examples_update
+    BEFORE UPDATE ON examples
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_examples_delete
+    BEFORE DELETE ON examples
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
 `;
 function assignExampleSplit(name, input) {
   const hash = createHash2("sha256").update(name).update("\0").update(input).digest();
@@ -924,40 +1051,85 @@ function openStorage(options) {
   chmodSync(options.dataDirectory, 448);
   const databasePath = join2(options.dataDirectory, "swapai.sqlite");
   const database = new DatabaseSync(databasePath);
-  chmodSync(databasePath, 384);
-  database.exec("PRAGMA journal_mode = WAL");
-  database.exec("PRAGMA foreign_keys = ON");
-  database.exec("PRAGMA busy_timeout = 5000");
-  database.exec(SCHEMA);
-  const now = Date.now();
-  const configJson = stringifyJson(options.config, "classifier config");
-  const existing = database.prepare(`
-    SELECT config_json FROM classifiers WHERE name = ?
-  `).get(options.name);
-  if (existing !== void 0 && criticalConfigJson(existing.config_json) !== criticalConfigJson(configJson)) {
-    database.close();
-    throw new TypeError(
-      `Classifier "${options.name}" already exists with different result or behavior settings`
+  let artifactLockDatabase = null;
+  try {
+    database.function(
+      "swapai_writer_version",
+      { deterministic: true },
+      () => 2
     );
+    chmodSync(databasePath, 384);
+    database.exec("PRAGMA journal_mode = WAL");
+    database.exec("PRAGMA foreign_keys = ON");
+    database.exec("PRAGMA secure_delete = ON");
+    database.exec("PRAGMA busy_timeout = 5000");
+    const secureDelete = requiredRow(
+      database.prepare("PRAGMA secure_delete").get()
+    );
+    if (secureDelete.secure_delete !== 1) {
+      throw new Error("SQLite secure deletion could not be enabled");
+    }
+    database.exec(SCHEMA);
+    migrateClassifierColumns(database);
+    const lockDirectory = join2(options.dataDirectory, "locks");
+    mkdirSync(lockDirectory, { recursive: true, mode: 448 });
+    chmodSync(lockDirectory, 448);
+    const lockPath = join2(
+      lockDirectory,
+      `${createHash2("sha256").update(options.name).digest("hex")}.sqlite`
+    );
+    artifactLockDatabase = new DatabaseSync(lockPath);
+    chmodSync(lockPath, 384);
+    artifactLockDatabase.exec("PRAGMA busy_timeout = 0");
+    artifactLockDatabase.exec(
+      "CREATE TABLE IF NOT EXISTS artifact_lock (id INTEGER PRIMARY KEY)"
+    );
+    const now = Date.now();
+    const configJson = stringifyJson(options.config, "classifier config");
+    const existing = database.prepare(`
+      SELECT config_json FROM classifiers WHERE name = ?
+    `).get(options.name);
+    if (existing !== void 0 && criticalConfigJson(existing.config_json) !== criticalConfigJson(configJson)) {
+      throw new TypeError(
+        `Classifier "${options.name}" already exists with different result or behavior settings`
+      );
+    }
+    transaction(database, () => {
+      database.prepare(`
+        INSERT INTO classifiers (
+          name, config_json, max_training_set, created_at, updated_at
+        ) VALUES (?, ?, ?, ?, ?)
+        ON CONFLICT(name) DO UPDATE SET
+          config_json = excluded.config_json,
+          max_training_set = excluded.max_training_set,
+          updated_at = excluded.updated_at
+      `).run(options.name, configJson, options.maxTrainingSet, now, now);
+      database.prepare(`
+        INSERT OR IGNORE INTO generations (
+          classifier_name, generation, status, created_at
+        ) VALUES (?, 1, 'active', ?)
+      `).run(options.name, now);
+      trimExamples(
+        database,
+        options.name,
+        activeGeneration(database, options.name),
+        options.maxTrainingSet
+      );
+    });
+  } catch (error) {
+    try {
+      artifactLockDatabase?.close();
+    } catch {
+    }
+    try {
+      database.close();
+    } catch {
+    }
+    throw error;
   }
-  transaction(database, () => {
-    database.prepare(`
-      INSERT INTO classifiers (
-        name, config_json, max_training_set, created_at, updated_at
-      ) VALUES (?, ?, ?, ?, ?)
-      ON CONFLICT(name) DO UPDATE SET
-        config_json = excluded.config_json,
-        max_training_set = excluded.max_training_set,
-        updated_at = excluded.updated_at
-    `).run(options.name, configJson, options.maxTrainingSet, now, now);
-    database.prepare(`
-      INSERT OR IGNORE INTO generations (
-        classifier_name, generation, status, created_at
-      ) VALUES (?, 1, 'active', ?)
-    `).run(options.name, now);
-    trimExamples(database, options.name, activeGeneration(database, options.name), options.maxTrainingSet);
-  });
+  const initializedArtifactLockDatabase = artifactLockDatabase;
   let closed = false;
+  let artifactLockHeld = false;
   const storage = {
     databasePath,
     snapshot() {
@@ -978,6 +1150,13 @@ function openStorage(options) {
           g.trained,
           g.model_path,
           g.needle_version,
+          c.data_epoch,
+          c.clear_pending,
+          c.clear_erased,
+          c.clear_artifact_generation_max,
+          c.training_lease_owner,
+          c.training_lease_epoch,
+          c.training_lease_until,
           (
             SELECT COUNT(*)
             FROM examples e
@@ -1005,23 +1184,36 @@ function openStorage(options) {
         consecutiveRetestFailures: row2.consecutive_retest_failures,
         trained: row2.trained === 1,
         modelPath: row2.model_path,
-        needleVersion: row2.needle_version
+        needleVersion: row2.needle_version,
+        dataEpoch: row2.data_epoch,
+        clearPending: row2.clear_pending === 1,
+        clearErased: row2.clear_erased === 1,
+        clearArtifactGenerationMax: row2.clear_artifact_generation_max,
+        trainingLeaseOwner: row2.training_lease_owner,
+        trainingLeaseEpoch: row2.training_lease_epoch,
+        trainingLeaseUntil: row2.training_lease_until
       };
     },
-    addExample(input, result) {
+    addExample(input, result, expectedDataEpoch) {
       assertOpen(closed);
-      const generation = activeGeneration(database, options.name);
       const split = assignExampleSplit(options.name, input);
       const createdAt = Date.now();
       const resultJson = stringifyJson(result, "classification result");
       let id = 0;
+      let generation = 0;
+      let accepted = false;
       transaction(database, () => {
+        const state = classifierState(database, options.name);
+        const epoch = expectedDataEpoch ?? state.data_epoch;
+        if (state.data_epoch !== epoch || state.clear_pending === 1) return;
+        generation = state.active_generation;
         const insertion = database.prepare(`
           INSERT INTO examples (
             classifier_name, generation, input, result_json, split, created_at
           ) VALUES (?, ?, ?, ?, ?, ?)
         `).run(options.name, generation, input, resultJson, split, createdAt);
         id = Number(insertion.lastInsertRowid);
+        accepted = true;
         database.prepare(`
           UPDATE classifiers
           SET total_examples_logged = total_examples_logged + 1,
@@ -1031,7 +1223,7 @@ function openStorage(options) {
         `).run(createdAt, options.name);
         trimExamples(database, options.name, generation, options.maxTrainingSet);
       });
-      return { id, generation, input, result, split, createdAt };
+      return accepted ? { id, generation, input, result, split, createdAt } : null;
     },
     listExamples(split, generation) {
       assertOpen(closed);
@@ -1049,44 +1241,76 @@ function openStorage(options) {
           `).all(options.name, selectedGeneration, split);
       return rows.map((value) => mapExample(row(value)));
     },
-    markTrainingAttempted(exampleCount) {
+    listExamplesForTraining(split, generation, expectedDataEpoch) {
+      assertOpen(closed);
+      let examples = null;
+      transaction(database, () => {
+        const state = classifierState(database, options.name);
+        if (state.data_epoch !== expectedDataEpoch || state.clear_pending === 1 || state.active_generation !== generation) {
+          return;
+        }
+        examples = database.prepare(`
+          SELECT id, generation, input, result_json, split, created_at
+          FROM examples
+          WHERE classifier_name = ? AND generation = ? AND split = ?
+          ORDER BY id
+        `).all(options.name, generation, split).map(
+          (value) => mapExample(row(value))
+        );
+      });
+      return examples;
+    },
+    markTrainingAttempted(exampleCount, expectedDataEpoch) {
       assertOpen(closed);
       if (!Number.isSafeInteger(exampleCount) || exampleCount <= 0) {
         throw new TypeError("training example count must be a positive integer");
       }
-      database.prepare(`
+      const epoch = expectedDataEpoch ?? storage.snapshot().dataEpoch;
+      const result = database.prepare(`
         UPDATE classifiers
         SET new_examples_since_training = 0,
             training_attempts = training_attempts + 1,
             examples_used_for_training = ?,
             updated_at = ?
-        WHERE name = ?
-      `).run(exampleCount, Date.now(), options.name);
+        WHERE name = ? AND data_epoch = ? AND clear_pending = 0
+      `).run(exampleCount, Date.now(), options.name, epoch);
+      return result.changes === 1;
     },
-    promoteGeneration(model) {
+    promoteGeneration(model, expectedDataEpoch) {
       assertOpen(closed);
-      const generation = activeGeneration(database, options.name);
-      database.prepare(`
-        UPDATE generations
-        SET trained = 1, model_path = ?, needle_version = ?
-        WHERE classifier_name = ? AND generation = ?
-      `).run(model.modelPath, model.needleVersion, options.name, generation);
-      return generationByNumber(database, options.name, generation);
+      let generation = 0;
+      let promoted = false;
+      transaction(database, () => {
+        const state = classifierState(database, options.name);
+        const epoch = expectedDataEpoch ?? state.data_epoch;
+        if (state.data_epoch !== epoch || state.clear_pending === 1) return;
+        generation = state.active_generation;
+        database.prepare(`
+          UPDATE generations
+          SET trained = 1, model_path = ?, needle_version = ?
+          WHERE classifier_name = ? AND generation = ?
+        `).run(model.modelPath, model.needleVersion, options.name, generation);
+        promoted = true;
+      });
+      return promoted ? generationByNumber(database, options.name, generation) : null;
     },
-    recordLocalClassification() {
+    recordLocalClassification(expectedDataEpoch) {
       assertOpen(closed);
-      database.prepare(`
+      const epoch = expectedDataEpoch ?? storage.snapshot().dataEpoch;
+      const result = database.prepare(`
         UPDATE classifiers
         SET local_classifications_since_retest = local_classifications_since_retest + 1,
             total_local_classifications = total_local_classifications + 1,
             updated_at = ?
-        WHERE name = ?
-      `).run(Date.now(), options.name);
+        WHERE name = ? AND data_epoch = ? AND clear_pending = 0
+      `).run(Date.now(), options.name, epoch);
+      if (result.changes !== 1) return null;
       return storage.snapshot().localClassificationsSinceRetest;
     },
-    recordRetest(passed) {
+    recordRetest(passed, expectedDataEpoch) {
       assertOpen(closed);
-      database.prepare(`
+      const epoch = expectedDataEpoch ?? storage.snapshot().dataEpoch;
+      const result = database.prepare(`
         UPDATE classifiers
         SET local_classifications_since_retest = 0,
             total_retests = total_retests + 1,
@@ -1095,28 +1319,33 @@ function openStorage(options) {
               ELSE consecutive_retest_failures + 1
             END,
             updated_at = ?
-        WHERE name = ?
-      `).run(passed ? 1 : 0, Date.now(), options.name);
+        WHERE name = ? AND data_epoch = ? AND clear_pending = 0
+      `).run(passed ? 1 : 0, Date.now(), options.name, epoch);
+      if (result.changes !== 1) return null;
       return storage.snapshot().consecutiveRetestFailures;
     },
-    claimTrainingLease(owner, durationMs) {
+    claimTrainingLease(owner, durationMs, expectedDataEpoch) {
       assertOpen(closed);
       if (owner.trim() === "" || !Number.isSafeInteger(durationMs) || durationMs <= 0) {
         throw new TypeError("training lease needs an owner and positive duration");
       }
-      const now2 = Date.now();
+      const now = Date.now();
+      const epoch = expectedDataEpoch ?? storage.snapshot().dataEpoch;
       const result = database.prepare(`
         UPDATE classifiers
         SET training_lease_owner = ?,
             training_lease_until = ?,
+            training_lease_epoch = ?,
             updated_at = ?
         WHERE name = ?
+          AND data_epoch = ?
+          AND clear_pending = 0
           AND (
             training_lease_owner IS NULL
             OR training_lease_until <= ?
             OR training_lease_owner = ?
           )
-      `).run(owner, now2 + durationMs, now2, options.name, now2, owner);
+      `).run(owner, now + durationMs, epoch, now, options.name, epoch, now, owner);
       return result.changes === 1;
     },
     releaseTrainingLease(owner) {
@@ -1125,16 +1354,21 @@ function openStorage(options) {
         UPDATE classifiers
         SET training_lease_owner = NULL,
             training_lease_until = NULL,
+            training_lease_epoch = NULL,
             updated_at = ?
         WHERE name = ? AND training_lease_owner = ?
       `).run(Date.now(), options.name, owner);
     },
-    archiveAndReset() {
+    archiveAndReset(expectedDataEpoch) {
       assertOpen(closed);
       let nextGeneration = 0;
       const changedAt = Date.now();
+      let reset = false;
       transaction(database, () => {
-        const currentGeneration = activeGeneration(database, options.name);
+        const state = classifierState(database, options.name);
+        const epoch = expectedDataEpoch ?? state.data_epoch;
+        if (state.data_epoch !== epoch || state.clear_pending === 1) return;
+        const currentGeneration = state.active_generation;
         database.prepare(`
           UPDATE generations
           SET status = 'archived', archived_at = ?
@@ -1154,19 +1388,141 @@ function openStorage(options) {
         database.prepare(`
           UPDATE classifiers
           SET active_generation = ?,
+              data_epoch = data_epoch + 1,
               new_examples_since_training = 0,
               training_attempts = 0,
               examples_used_for_training = 0,
               training_lease_owner = NULL,
               training_lease_until = NULL,
+              training_lease_epoch = NULL,
+              local_classifications_since_retest = 0,
+              consecutive_retest_failures = 0,
+              updated_at = ?
+          WHERE name = ?
+        `).run(nextGeneration, changedAt, options.name);
+        reset = true;
+      });
+      return reset ? generationByNumber(database, options.name, nextGeneration) : null;
+    },
+    beginClearTrainingData() {
+      assertOpen(closed);
+      database.prepare(`
+        UPDATE classifiers
+        SET data_epoch = data_epoch + 1,
+            clear_pending = 1,
+            clear_erased = 0,
+            clear_artifact_generation_max = (
+              SELECT MAX(generation)
+              FROM generations
+              WHERE classifier_name = ?
+            ),
+            updated_at = ?
+        WHERE name = ?
+      `).run(options.name, Date.now(), options.name);
+      return classifierState(database, options.name).data_epoch;
+    },
+    eraseTrainingData(dataEpoch) {
+      assertOpen(closed);
+      let nextGeneration = 0;
+      const changedAt = Date.now();
+      let erased = false;
+      transaction(database, () => {
+        const state = classifierState(database, options.name);
+        if (state.data_epoch !== dataEpoch || state.clear_pending !== 1) return;
+        if (state.clear_erased === 1) return;
+        const maximum = requiredRow(database.prepare(`
+          SELECT MAX(generation) AS maximum
+          FROM generations
+          WHERE classifier_name = ?
+        `).get(options.name));
+        nextGeneration = maximum.maximum + 1;
+        database.prepare(`
+          DELETE FROM examples WHERE classifier_name = ?
+        `).run(options.name);
+        database.prepare(`
+          DELETE FROM generations WHERE classifier_name = ?
+        `).run(options.name);
+        database.prepare(`
+          INSERT INTO generations (
+            classifier_name, generation, status, created_at
+          ) VALUES (?, ?, 'active', ?)
+        `).run(options.name, nextGeneration, changedAt);
+        database.prepare(`
+          UPDATE classifiers
+          SET active_generation = ?,
+              total_examples_logged = 0,
+              new_examples_since_training = 0,
+              training_attempts = 0,
+              examples_used_for_training = 0,
               local_classifications_since_retest = 0,
+              total_local_classifications = 0,
+              total_retests = 0,
               consecutive_retest_failures = 0,
+              training_lease_owner = NULL,
+              training_lease_until = NULL,
+              training_lease_epoch = NULL,
+              clear_erased = 1,
               updated_at = ?
           WHERE name = ?
         `).run(nextGeneration, changedAt, options.name);
+        erased = true;
       });
+      if (!erased) return null;
+      truncateWriteAheadLog(database);
       return generationByNumber(database, options.name, nextGeneration);
     },
+    finishClearTrainingData(dataEpoch) {
+      assertOpen(closed);
+      const state = classifierState(database, options.name);
+      if (state.data_epoch !== dataEpoch || state.clear_pending !== 1 || state.clear_erased !== 1) {
+        return false;
+      }
+      truncateWriteAheadLog(database);
+      const result = database.prepare(`
+        UPDATE classifiers
+        SET clear_pending = 0, updated_at = ?
+        WHERE name = ?
+          AND data_epoch = ?
+          AND clear_pending = 1
+          AND clear_erased = 1
+      `).run(Date.now(), options.name, dataEpoch);
+      return result.changes === 1;
+    },
+    clearTrainingData() {
+      const dataEpoch = storage.beginClearTrainingData();
+      if (!storage.claimArtifactWriteLock()) {
+        throw new Error("Classifier artifacts are currently being written");
+      }
+      try {
+        const generation = storage.eraseTrainingData(dataEpoch);
+        if (generation === null || !storage.finishClearTrainingData(dataEpoch)) {
+          throw new Error("Training data clear was superseded");
+        }
+        return generation;
+      } finally {
+        storage.releaseArtifactWriteLock();
+      }
+    },
+    claimArtifactWriteLock() {
+      assertOpen(closed);
+      if (artifactLockHeld) return true;
+      try {
+        initializedArtifactLockDatabase.exec("BEGIN IMMEDIATE");
+        artifactLockHeld = true;
+        return true;
+      } catch (error) {
+        if (typeof error === "object" && error !== null && "code" in error && error.code === "ERR_SQLITE_ERROR" && "message" in error && typeof error.message === "string" && /locked|busy/i.test(error.message)) {
+          return false;
+        }
+        throw error;
+      }
+    },
+    releaseArtifactWriteLock() {
+      assertOpen(closed);
+      if (!artifactLockHeld) return;
+      initializedArtifactLockDatabase.exec("COMMIT");
+      artifactLockHeld = false;
+    },
     listGenerations() {
       assertOpen(closed);
       return database.prepare(`
@@ -1182,6 +1538,11 @@ function openStorage(options) {
       if (closed) {
         return;
       }
+      if (artifactLockHeld) {
+        initializedArtifactLockDatabase.exec("ROLLBACK");
+        artifactLockHeld = false;
+      }
+      initializedArtifactLockDatabase.close();
       closed = true;
       database.close();
     }
@@ -1194,6 +1555,69 @@ function activeGeneration(database, name) {
   `).get(name));
   return active.active_generation;
 }
+function truncateWriteAheadLog(database) {
+  database.exec("PRAGMA busy_timeout = 0");
+  let checkpoint;
+  try {
+    checkpoint = requiredRow(database.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get());
+  } finally {
+    database.exec("PRAGMA busy_timeout = 5000");
+  }
+  if (checkpoint.busy !== 0) {
+    throw new Error(
+      "SQLite WAL could not be truncated after clearing training data"
+    );
+  }
+}
+function classifierState(database, name) {
+  return requiredRow(database.prepare(`
+    SELECT
+      active_generation,
+      data_epoch,
+      clear_pending,
+      clear_erased,
+      clear_artifact_generation_max,
+      training_lease_owner,
+      training_lease_epoch,
+      training_lease_until
+    FROM classifiers
+    WHERE name = ?
+  `).get(name));
+}
+function migrateClassifierColumns(database) {
+  transaction(database, () => {
+    const columns = new Set(
+      database.prepare("PRAGMA table_info(classifiers)").all().map(
+        (value) => row(value).name
+      )
+    );
+    if (!columns.has("data_epoch")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN data_epoch INTEGER NOT NULL DEFAULT 0"
+      );
+    }
+    if (!columns.has("clear_pending")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN clear_pending INTEGER NOT NULL DEFAULT 0"
+      );
+    }
+    if (!columns.has("clear_erased")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN clear_erased INTEGER NOT NULL DEFAULT 0"
+      );
+    }
+    if (!columns.has("clear_artifact_generation_max")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN clear_artifact_generation_max INTEGER"
+      );
+    }
+    if (!columns.has("training_lease_epoch")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN training_lease_epoch INTEGER"
+      );
+    }
+  });
+}
 function generationByNumber(database, name, generation) {
   const stored = requiredRow(database.prepare(`
     SELECT
@@ -1325,6 +1749,7 @@ function requiredRow(value) {
 // src/classifier.ts
 var TRAINING_LEASE_DURATION_MS = 5 * 60 * 1e3;
 var TRAINING_LEASE_RENEWAL_MS = 60 * 1e3;
+var ARTIFACT_LOCK_WAIT_MS = 3e4;
 function init(inputConfig) {
   const config = normalizeConfig(inputConfig);
   const dataDirectory = config.dataDirectory ?? join3(process.cwd(), ".swapai");
@@ -1362,30 +1787,65 @@ function openClassifierStorage(config, dataDirectory) {
   }
 }
 function createClassifier(config, storage, runtime) {
+  const initialState = storage.snapshot();
   let closed = false;
-  let trained = storage.snapshot().trained;
+  let closing = false;
+  let closePromise = null;
+  let trained = initialState.trained && !initialState.clearPending;
   let loadedModel = null;
+  let loadedModelEpoch = null;
   let operationQueue = Promise.resolve();
   let trainingQueue = Promise.resolve();
   let trainingScheduled = false;
   const trainingLeaseOwner = `${process.pid}:${randomUUID2()}`;
   let classificationQueue = Promise.resolve();
   let queuedFailure = null;
-  const saved = storage.snapshot();
-  if (saved.trained && (saved.modelPath === null || saved.needleVersion !== NEEDLE_VERSION)) {
-    storage.archiveAndReset();
-    trained = false;
+  let clearFailure = null;
+  let dataEpoch = initialState.dataEpoch;
+  let clearedEpoch = initialState.clearPending ? initialState.dataEpoch - 1 : initialState.dataEpoch;
+  const saved = initialState;
+  if (!saved.clearPending && saved.trained && (saved.modelPath === null || saved.needleVersion !== NEEDLE_VERSION)) {
+    if (storage.archiveAndReset(saved.dataEpoch) !== null) {
+      const reset = storage.snapshot();
+      dataEpoch = reset.dataEpoch;
+      clearedEpoch = reset.dataEpoch;
+      trained = false;
+    }
   }
   const startup = startRuntime();
   async function startRuntime() {
+    const startupEpoch = dataEpoch;
+    const beforeRuntime = storage.snapshot();
+    if (beforeRuntime.clearPending) {
+      trained = false;
+      dataEpoch = beforeRuntime.dataEpoch;
+      try {
+        await performDurableClear(beforeRuntime.dataEpoch);
+      } catch (error) {
+        rememberClearFailure(error);
+      }
+    }
     try {
       await runtime.ready();
       const snapshot = storage.snapshot();
+      if (snapshot.clearPending) {
+        trained = false;
+        dataEpoch = snapshot.dataEpoch;
+        return;
+      }
       if (snapshot.trained && snapshot.modelPath !== null) {
-        loadedModel = await runtime.loadModel({
+        const restoredModel = await runtime.loadModel({
           modelPath: snapshot.modelPath,
           resultConfig: config.result
         });
+        const current = storage.snapshot();
+        if (startupEpoch === current.dataEpoch && !current.clearPending && current.trained) {
+          loadedModel = restoredModel;
+          loadedModelEpoch = startupEpoch;
+          trained = true;
+        } else {
+          await restoredModel.close();
+        }
       }
     } catch (error) {
       const swapAIError = toSwapAIError(
@@ -1400,6 +1860,25 @@ function createClassifier(config, storage, runtime) {
     queuedFailure = error;
     reportBackgroundError(config, error);
   }
+  function rememberClearFailure(error) {
+    clearFailure = toSwapAIError(
+      error,
+      "storage_failed",
+      `Could not completely clear classifier "${config.name}"`
+    );
+    reportBackgroundError(config, clearFailure);
+  }
+  function refreshStoredState() {
+    const snapshot = storage.snapshot();
+    dataEpoch = snapshot.dataEpoch;
+    if (snapshot.clearPending) {
+      trained = false;
+    } else {
+      trained = snapshot.trained;
+      clearedEpoch = snapshot.dataEpoch;
+    }
+    return snapshot;
+  }
   function enqueue(operation) {
     const result = operationQueue.then(operation);
     operationQueue = result.catch((error) => {
@@ -1409,14 +1888,15 @@ function createClassifier(config, storage, runtime) {
     });
     return result;
   }
-  function persistExample(input, result) {
+  function persistExample(input, result, epoch = dataEpoch) {
+    if (epoch !== dataEpoch) return Promise.resolve();
     return enqueue(() => {
-      storage.addExample(input, result);
-      scheduleTraining();
+      if (epoch !== dataEpoch) return;
+      if (storage.addExample(input, result, epoch) !== null) scheduleTraining();
     });
   }
   function shouldTrain(snapshot = storage.snapshot()) {
-    return !snapshot.trained && snapshot.examplesUsedForTraining < config.maxTrainingSet && snapshot.newExamplesSinceTraining >= config.retrainOnCount;
+    return clearedEpoch === dataEpoch && snapshot.dataEpoch === dataEpoch && !snapshot.clearPending && !snapshot.trained && snapshot.examplesUsedForTraining < config.maxTrainingSet && snapshot.newExamplesSinceTraining >= config.retrainOnCount;
   }
   function scheduleTraining() {
     if (trainingScheduled || !shouldTrain()) return;
@@ -1435,8 +1915,13 @@ function createClassifier(config, storage, runtime) {
     });
   }
   async function trainWhenDue() {
+    const trainingEpoch = dataEpoch;
     if (!shouldTrain()) return false;
-    if (!storage.claimTrainingLease(trainingLeaseOwner, TRAINING_LEASE_DURATION_MS)) {
+    if (!storage.claimTrainingLease(
+      trainingLeaseOwner,
+      TRAINING_LEASE_DURATION_MS,
+      trainingEpoch
+    )) {
       return false;
     }
     let leaseHeld = true;
@@ -1445,7 +1930,8 @@ function createClassifier(config, storage, runtime) {
       try {
         leaseHeld = storage.claimTrainingLease(
           trainingLeaseOwner,
-          TRAINING_LEASE_DURATION_MS
+          TRAINING_LEASE_DURATION_MS,
+          trainingEpoch
         );
       } catch {
         leaseHeld = false;
@@ -1460,15 +1946,30 @@ function createClassifier(config, storage, runtime) {
     try {
       const snapshot = storage.snapshot();
       if (!shouldTrain(snapshot)) return false;
-      const trainingExamples = storage.listExamples("training");
-      const heldOutExamples = storage.listExamples("held_out");
-      storage.markTrainingAttempted(snapshot.activeExampleCount);
+      if (!storage.markTrainingAttempted(
+        snapshot.activeExampleCount,
+        trainingEpoch
+      )) {
+        return false;
+      }
+      const trainingExamples = storage.listExamplesForTraining(
+        "training",
+        snapshot.activeGeneration,
+        trainingEpoch
+      );
+      const heldOutExamples = storage.listExamplesForTraining(
+        "held_out",
+        snapshot.activeGeneration,
+        trainingEpoch
+      );
+      if (trainingExamples === null || heldOutExamples === null) return false;
       if (trainingExamples.length === 0 || heldOutExamples.length === 0) {
         return true;
       }
       const candidate = await runtime.train({
         classifierName: config.name,
         generation: snapshot.activeGeneration,
+        expectedEpoch: trainingEpoch,
         examples: trainingExamples.map(({ input, result }) => ({ input, result })),
         resultConfig: config.result
       });
@@ -1492,14 +1993,43 @@ function createClassifier(config, storage, runtime) {
           return true;
         }
         if (!renewTrainingLease()) {
+          const current = refreshStoredState();
+          if (current.dataEpoch !== trainingEpoch || current.clearPending) {
+            await runtime.clearClassifierGenerationArtifacts(
+              config.name,
+              snapshot.activeGeneration
+            );
+            return false;
+          }
           throw new SwapAIError(
             "service_unavailable",
             `Classifier "${config.name}" lost its training lease`
           );
         }
-        storage.promoteGeneration(candidate);
+        if (trainingEpoch !== refreshStoredState().dataEpoch) {
+          await runtime.clearClassifierGenerationArtifacts(
+            config.name,
+            snapshot.activeGeneration
+          );
+          return false;
+        }
+        if (storage.promoteGeneration(candidate, trainingEpoch) === null) {
+          await runtime.clearClassifierGenerationArtifacts(
+            config.name,
+            snapshot.activeGeneration
+          );
+          return false;
+        }
         if (loadedModel !== null) await loadedModel.close();
+        if (trainingEpoch !== refreshStoredState().dataEpoch) {
+          await runtime.clearClassifierGenerationArtifacts(
+            config.name,
+            snapshot.activeGeneration
+          );
+          return false;
+        }
         loadedModel = candidateModel;
+        loadedModelEpoch = trainingEpoch;
         trained = true;
         return true;
       } finally {
@@ -1510,34 +2040,49 @@ function createClassifier(config, storage, runtime) {
       storage.releaseTrainingLease(trainingLeaseOwner);
     }
   }
-  async function model() {
+  async function model(expectedEpoch) {
     await startup;
-    if (!trained) {
+    const current = refreshStoredState();
+    if (!trained || current.clearPending || current.dataEpoch !== expectedEpoch) {
       throw new SwapAIError(
         "not_trained",
         `Classifier "${config.name}" has not passed its held-out test`
       );
     }
+    if (loadedModel !== null && loadedModelEpoch !== expectedEpoch) {
+      await loadedModel.close();
+      loadedModel = null;
+      loadedModelEpoch = null;
+    }
     if (loadedModel === null) {
-      const snapshot = storage.snapshot();
-      if (snapshot.modelPath === null) {
+      if (current.modelPath === null) {
         throw new SwapAIError(
           "service_unavailable",
           `Classifier "${config.name}" is trained but has no saved model`
         );
       }
       await runtime.ready();
-      loadedModel = await runtime.loadModel({
-        modelPath: snapshot.modelPath,
+      const restored = await runtime.loadModel({
+        modelPath: current.modelPath,
         resultConfig: config.result
       });
+      const afterLoad = refreshStoredState();
+      if (afterLoad.dataEpoch !== expectedEpoch || afterLoad.clearPending || !afterLoad.trained) {
+        await restored.close();
+        throw new SwapAIError(
+          "not_trained",
+          `Classifier "${config.name}" has been cleared`
+        );
+      }
+      loadedModel = restored;
+      loadedModelEpoch = expectedEpoch;
     }
     return loadedModel;
   }
   async function callReference(input, reference) {
     return validateResult(config.result, await reference(input));
   }
-  async function fallbackToReference(input, reference, candidateError, retestDue) {
+  async function fallbackToReference(input, reference, candidateError, retestDue, epoch) {
     if (reference === void 0) {
       throw toSwapAIError(
         candidateError,
@@ -1554,22 +2099,26 @@ function createClassifier(config, storage, runtime) {
       )
     );
     const referenceResult = await callReference(input, reference);
-    await persistExample(input, referenceResult);
-    if (retestDue) {
-      storage.recordLocalClassification();
-      const failures = storage.recordRetest(false);
-      if (failures >= config.retestRevertOn) await disableModel();
+    await persistExample(input, referenceResult, epoch);
+    if (retestDue && epoch === dataEpoch) {
+      if (storage.recordLocalClassification(epoch) === null) return referenceResult;
+      const failures = storage.recordRetest(false, epoch);
+      if (failures !== null && failures >= config.retestRevertOn) {
+        await disableModel(epoch);
+      }
     }
     return referenceResult;
   }
-  async function disableModel() {
-    storage.archiveAndReset();
+  async function disableModel(epoch) {
+    if (storage.archiveAndReset(epoch) === null) return;
+    refreshStoredState();
     trained = false;
     const previousModel = loadedModel;
     loadedModel = null;
+    loadedModelEpoch = null;
     if (previousModel !== null) await previousModel.close();
   }
-  async function classifyNow(input, reference) {
+  async function classifyNow(input, reference, epoch) {
     assertOpen2(closed);
     if (!trained) {
       if (reference === void 0) {
@@ -1579,7 +2128,7 @@ function createClassifier(config, storage, runtime) {
         );
       }
       const referenceResult2 = await callReference(input, reference);
-      await persistExample(input, referenceResult2);
+      await persistExample(input, referenceResult2, epoch);
       return referenceResult2;
     }
     const snapshot = storage.snapshot();
@@ -1588,37 +2137,190 @@ function createClassifier(config, storage, runtime) {
     try {
       candidateResult = validateResult(
         config.result,
-        await (await model()).classify(input)
+        await (await model(epoch)).classify(input)
       );
     } catch (error) {
-      return fallbackToReference(input, reference, error, retestDue);
+      return fallbackToReference(input, reference, error, retestDue, epoch);
+    }
+    const afterClassification = refreshStoredState();
+    if (afterClassification.dataEpoch !== epoch || afterClassification.clearPending) {
+      return fallbackToReference(
+        input,
+        reference,
+        new SwapAIError(
+          "not_trained",
+          `Classifier "${config.name}" was cleared during classification`
+        ),
+        retestDue,
+        epoch
+      );
     }
     if (!retestDue || reference === void 0) {
-      storage.recordLocalClassification();
+      if (epoch !== dataEpoch || storage.recordLocalClassification(epoch) === null) {
+        return fallbackToReference(
+          input,
+          reference,
+          new SwapAIError(
+            "not_trained",
+            `Classifier "${config.name}" was cleared before returning its result`
+          ),
+          retestDue,
+          epoch
+        );
+      }
       return candidateResult;
     }
     const referenceResult = await callReference(input, reference);
-    await persistExample(input, referenceResult);
-    storage.recordLocalClassification();
+    await persistExample(input, referenceResult, epoch);
+    if (epoch !== dataEpoch) return referenceResult;
+    if (storage.recordLocalClassification(epoch) === null) return referenceResult;
     const passed = resultError(config.result, referenceResult, candidateResult) <= config.acceptableError;
-    const failures = storage.recordRetest(passed);
-    if (!passed && failures >= config.retestRevertOn) {
-      await disableModel();
+    const failures = storage.recordRetest(passed, epoch);
+    if (!passed && failures !== null && failures >= config.retestRevertOn) {
+      await disableModel(epoch);
     }
     return referenceResult;
   }
+  async function performDurableClear(clearEpoch) {
+    trained = false;
+    const deletionFailures = [];
+    const throwDeletionFailures = () => {
+      if (deletionFailures.length === 1) throw deletionFailures[0];
+      if (deletionFailures.length > 1) {
+        throw new AggregateError(
+          deletionFailures,
+          `Could not completely clear classifier "${config.name}"`
+        );
+      }
+    };
+    const previousModel = loadedModel;
+    if (previousModel !== null) {
+      try {
+        await previousModel.close();
+        if (loadedModel === previousModel) {
+          loadedModel = null;
+          loadedModelEpoch = null;
+        }
+      } catch (error) {
+        deletionFailures.push(error);
+      }
+    }
+    const lockWaitStarted = Date.now();
+    let clearState = refreshStoredState();
+    while (clearState.clearPending && clearState.dataEpoch === clearEpoch && (clearState.trainingLeaseOwner !== null && clearState.trainingLeaseEpoch === null && clearState.trainingLeaseUntil !== null && clearState.trainingLeaseUntil > Date.now() || !storage.claimArtifactWriteLock())) {
+      if (Date.now() - lockWaitStarted >= ARTIFACT_LOCK_WAIT_MS) {
+        throw new SwapAIError(
+          "storage_failed",
+          `Timed out waiting to clear classifier "${config.name}" while training was still running`
+        );
+      }
+      await new Promise((resolve2) => setTimeout(resolve2, 10));
+      clearState = refreshStoredState();
+    }
+    if (!clearState.clearPending || clearState.dataEpoch !== clearEpoch) {
+      throwDeletionFailures();
+      return;
+    }
+    try {
+      try {
+        storage.eraseTrainingData(clearEpoch);
+      } catch (error) {
+        deletionFailures.push(error);
+      }
+      clearState = refreshStoredState();
+      if (clearState.clearPending && clearState.dataEpoch === clearEpoch && clearState.clearErased && clearState.clearArtifactGenerationMax !== null) {
+        try {
+          await runtime.clearClassifierArtifactsThroughGeneration(
+            config.name,
+            clearState.clearArtifactGenerationMax
+          );
+        } catch (error) {
+          deletionFailures.push(error);
+        }
+      }
+      throwDeletionFailures();
+      storage.finishClearTrainingData(clearEpoch);
+      const current = refreshStoredState();
+      if (!current.clearPending) {
+        clearFailure = null;
+        clearedEpoch = current.dataEpoch;
+      }
+    } finally {
+      storage.releaseArtifactWriteLock();
+    }
+  }
+  async function flushNow() {
+    const clearFailureBeforeFlush = clearFailure;
+    await startup;
+    while (true) {
+      const operations = operationQueue;
+      await operations;
+      const training = trainingQueue;
+      await training;
+      if (operations === operationQueue && training === trainingQueue) break;
+    }
+    if (clearFailure !== null && clearFailure !== clearFailureBeforeFlush) {
+      throw clearFailure;
+    }
+    const pending = refreshStoredState();
+    if (pending.clearPending || clearFailureBeforeFlush !== null) {
+      try {
+        await performDurableClear(pending.dataEpoch);
+        if (!refreshStoredState().clearPending) clearFailure = null;
+      } catch (error) {
+        rememberClearFailure(error);
+      }
+    }
+    if (refreshStoredState().clearPending) {
+      clearFailure ??= new SwapAIError(
+        "storage_failed",
+        `Could not completely clear classifier "${config.name}"`
+      );
+      throw clearFailure;
+    }
+    if (clearFailure !== null) throw clearFailure;
+    if (queuedFailure !== null) {
+      const failure = queuedFailure;
+      queuedFailure = null;
+      throw failure;
+    }
+  }
   const classifier = {
     isTrained() {
+      assertOpen2(closed || closing);
+      refreshStoredState();
       return trained;
     },
     logClassification(input, result) {
-      assertOpen2(closed);
+      assertOpen2(closed || closing);
       const validResult = validateResult(config.result, result);
-      void persistExample(input, validResult);
+      const epoch = refreshStoredState().dataEpoch;
+      void persistExample(input, validResult, epoch);
+    },
+    clearTrainingData() {
+      assertOpen2(closed || closing);
+      trained = false;
+      const clearEpoch = storage.beginClearTrainingData();
+      dataEpoch = clearEpoch;
+      const trainingBeforeClear = trainingQueue;
+      const classificationsBeforeClear = classificationQueue;
+      void enqueue(async () => {
+        await startup;
+        await trainingBeforeClear;
+        await classificationsBeforeClear;
+        try {
+          await performDurableClear(clearEpoch);
+        } catch (error) {
+          rememberClearFailure(error);
+        }
+      });
     },
     classify(input, reference) {
-      assertOpen2(closed);
-      const task = classificationQueue.then(() => classifyNow(input, reference));
+      assertOpen2(closed || closing);
+      const epoch = refreshStoredState().dataEpoch;
+      const task = classificationQueue.then(
+        () => classifyNow(input, reference, epoch)
+      );
       classificationQueue = task.then(
         () => void 0,
         () => void 0
@@ -1626,36 +2328,53 @@ function createClassifier(config, storage, runtime) {
       return task;
     },
     async flush() {
-      await startup;
-      while (true) {
-        const operations = operationQueue;
-        await operations;
-        const training = trainingQueue;
-        await training;
-        if (operations === operationQueue && training === trainingQueue) break;
-      }
-      if (queuedFailure !== null) {
-        const failure = queuedFailure;
-        queuedFailure = null;
-        throw failure;
-      }
+      assertOpen2(closed || closing);
+      await flushNow();
     },
-    async close() {
-      if (closed) return;
-      closed = true;
-      let failure;
-      try {
-        await classificationQueue;
-        await classifier.flush();
-      } catch (error) {
-        failure = error;
-      } finally {
-        if (loadedModel !== null) await loadedModel.close();
+    close() {
+      if (closed) return Promise.resolve();
+      if (closePromise !== null) return closePromise;
+      closing = true;
+      closePromise = (async () => {
+        try {
+          await classificationQueue;
+          await flushNow();
+        } catch (error) {
+          closing = false;
+          closePromise = null;
+          throw error;
+        }
+        const failures = [];
+        if (loadedModel !== null) {
+          try {
+            await loadedModel.close();
+          } catch (error) {
+            failures.push(error);
+          }
+        }
         loadedModel = null;
-        await runtime.close();
-        storage.close();
-      }
-      if (failure !== void 0) throw failure;
+        loadedModelEpoch = null;
+        try {
+          await runtime.close();
+        } catch (error) {
+          failures.push(error);
+        }
+        try {
+          storage.close();
+        } catch (error) {
+          failures.push(error);
+        }
+        closed = true;
+        closing = false;
+        if (failures.length === 1) throw failures[0];
+        if (failures.length > 1) {
+          throw new AggregateError(
+            failures,
+            `Could not completely close classifier "${config.name}"`
+          );
+        }
+      })();
+      return closePromise;
     }
   };
   return classifier;