@swapai/core 0.1.0 → 0.2.1

package/dist/index.js

@@ -46,6 +46,9 @@ function validateResultConfig(result) {
     if (result.max <= result.min) {
       invalidConfiguration("number result max must be greater than min");
     }
+    if (!Number.isFinite(result.max - result.min)) {
+      invalidConfiguration("number result range must be finite");
+    }
     return;
   }
   if (result.type === "string") {
@@ -122,30 +125,82 @@ function averageError(config, comparisons) {
 // src/runtime.ts
 import { createHash, randomUUID } from "crypto";
 import { spawn } from "child_process";
+import { readFileSync, statSync } from "fs";
 import {
   chmod,
   copyFile,
   mkdir,
   readFile,
   readdir,
+  rm,
   stat,
   writeFile
 } from "fs/promises";
 import { createInterface } from "readline";
-import { dirname, join } from "path";
+import { dirname, join, resolve, sep } from "path";
 import { fileURLToPath } from "url";
 
 // src/needle.ts
-function createNeedleTool(config) {
+var MAX_NEEDLE_NUMBER_LABELS = 64;
+function createNeedleNumberLabels(values, config, acceptableError = 0) {
+  if (typeof acceptableError !== "number" || !Number.isFinite(acceptableError) || acceptableError < 0 || acceptableError > 1) {
+    throw new TypeError("acceptableError must be between 0 and 1.");
+  }
+  const valid = values.map(
+    (value) => validateNeedleResult(value, config)
+  );
+  const unique = [...new Set(valid)].sort((left, right) => left - right);
+  const desiredLabelCount = acceptableError === 0 ? MAX_NEEDLE_NUMBER_LABELS : Math.min(
+    MAX_NEEDLE_NUMBER_LABELS,
+    Math.max(1, Math.floor(1 / (2 * acceptableError)) + 1)
+  );
+  if (unique.length <= desiredLabelCount) return { values: unique };
+  const width = (config.max - config.min) / desiredLabelCount;
+  const occupied = /* @__PURE__ */ new Set();
+  for (const value of unique) {
+    occupied.add(Math.min(
+      desiredLabelCount - 1,
+      Math.floor((value - config.min) / width)
+    ));
+  }
+  return {
+    values: [...occupied].sort((left, right) => left - right).map((index) => config.min + (index + 0.5) * width)
+  };
+}
+function numberLabel(index, count) {
+  if (count === 1) return "only_score";
+  if (count === 2) return index === 0 ? "lower_score" : "higher_score";
+  return `score_bucket_${index}`;
+}
+function encodeNumberResult(value, labels) {
+  if (labels.values.length === 0) {
+    throw new TypeError("Numeric result has no Needle label.");
+  }
+  let closestIndex = 0;
+  let closestDistance = Math.abs(value - labels.values[0]);
+  for (let index = 1; index < labels.values.length; index += 1) {
+    const distance = Math.abs(value - labels.values[index]);
+    if (distance < closestDistance) {
+      closestIndex = index;
+      closestDistance = distance;
+    }
+  }
+  return numberLabel(closestIndex, labels.values.length);
+}
+function createNeedleTool(config, numberLabels) {
   const description = "The classification result.";
   let result;
   switch (config.type) {
     case "number":
+      if (!numberLabels || numberLabels.values.length === 0) {
+        throw new TypeError("Numeric Needle tools require at least one result label.");
+      }
       result = {
-        type: "number",
-        minimum: config.min,
-        maximum: config.max,
-        description
+        type: "string",
+        enum: numberLabels.values.map(
+          (_, index) => numberLabel(index, numberLabels.values.length)
+        ),
+        description: `The numeric classification bucket from ${config.min} to ${config.max}.`
       };
       break;
     case "boolean":
@@ -166,12 +221,13 @@ function createNeedleTool(config) {
     }
   };
 }
-function createTrainingLine(input, value, config) {
+function createTrainingLine(input, value, config, numberLabels) {
   const result = validateNeedleResult(value, config);
+  const needleResult = config.type === "number" ? encodeNumberResult(result, numberLabels ?? { values: [] }) : result;
   return JSON.stringify({
     query: input,
-    tools: [createNeedleTool(config)],
-    answers: [{ name: "classify", arguments: { result } }]
+    tools: [createNeedleTool(config, numberLabels)],
+    answers: [{ name: "classify", arguments: { result: needleResult } }]
   });
 }
 function validateNeedleResult(value, config) {
@@ -195,11 +251,62 @@ function validateNeedleResult(value, config) {
       return value;
   }
 }
+function decodeNeedleResult(result, config, numberLabels) {
+  if (config.type === "number") {
+    if (typeof result !== "string") {
+      throw new TypeError(
+        `Needle returned a number outside ${config.min} to ${config.max}.`
+      );
+    }
+    const index = numberLabels?.values.findIndex(
+      (_, candidateIndex) => result === numberLabel(candidateIndex, numberLabels.values.length)
+    ) ?? -1;
+    const decoded = numberLabels?.values[index];
+    if (index < 0 || decoded === void 0) {
+      throw new TypeError(
+        `Needle returned a number outside ${config.min} to ${config.max}.`
+      );
+    }
+    return validateNeedleResult(decoded, config);
+  }
+  return validateNeedleResult(result, config);
+}
 
 // src/runtime.ts
 var NEEDLE_VERSION = "2.0.14";
+var NEEDLE_NUMBER_MODEL_VERSION = `${NEEDLE_VERSION}/number-buckets-v1`;
 var UV_VERSION = "0.11.4";
 var PYTHON_VERSION = "3.12";
+function needleModelVersion(resultConfig) {
+  return resultConfig.type === "number" ? NEEDLE_NUMBER_MODEL_VERSION : NEEDLE_VERSION;
+}
+var NeedleModelArtifactError = class extends SwapAIError {
+  constructor(message, cause) {
+    super(
+      "service_unavailable",
+      message,
+      cause === void 0 ? void 0 : { cause }
+    );
+    this.name = "NeedleModelArtifactError";
+  }
+};
+function hasNeedleModelArtifacts(options) {
+  try {
+    if (!statSync(options.modelPath).isFile()) return false;
+    if (options.resultConfig.type !== "number") return true;
+    const saved = JSON.parse(
+      readFileSync(`${options.modelPath}.numbers.json`, "utf8")
+    );
+    if (saved.format !== 1 || !Array.isArray(saved.values) || saved.values.length === 0) return false;
+    const savedValues = saved.values;
+    const labels = createNeedleNumberLabels(savedValues, options.resultConfig);
+    return labels.values.length === savedValues.length && labels.values.every(
+      (value, index) => value === savedValues[index]
+    );
+  } catch {
+    return false;
+  }
+}
 var NeedleRuntimeError = class extends SwapAIError {
   constructor(code, message, cause) {
     super(code, message, cause === void 0 ? void 0 : { cause });
@@ -224,8 +331,8 @@ var ManagedNeedleRuntime = class {
   #environment;
   #closed = false;
   constructor(options) {
-    this.#dataDirectory = options.dataDirectory;
-    this.#runtimeDirectory = join(options.dataDirectory, "runtime");
+    this.#dataDirectory = resolve(options.dataDirectory);
+    this.#runtimeDirectory = join(this.#dataDirectory, "runtime");
     this.#runCommand = options.dependencies?.runCommand ?? runCommand;
     this.#spawnProcess = options.dependencies?.spawnProcess ?? spawnProcess;
     this.#fetch = options.dependencies?.fetch ?? globalThis.fetch;
@@ -252,6 +359,12 @@ var ManagedNeedleRuntime = class {
         "Needle generation must be a non-negative integer."
       );
     }
+    if (!Number.isSafeInteger(options.expectedEpoch) || options.expectedEpoch < 0) {
+      throw new NeedleRuntimeError(
+        "service_unavailable",
+        "Needle data epoch must be a non-negative integer."
+      );
+    }
     if (options.examples.length === 0) {
       throw new NeedleRuntimeError(
         "service_unavailable",
@@ -278,49 +391,110 @@ var ManagedNeedleRuntime = class {
     const modelPath = join(candidateDirectory, "model.cact");
     await mkdir(checkpointDirectory, { recursive: true, mode: 448 });
     await mkdir(candidateDirectory, { recursive: true, mode: 448 });
+    const numberLabels = options.resultConfig.type === "number" ? createNeedleNumberLabels(
+      options.examples.map((example) => example.result),
+      options.resultConfig,
+      options.acceptableError
+    ) : void 0;
     const lines = options.examples.map(
-      (example) => createTrainingLine(example.input, example.result, options.resultConfig)
+      (example) => createTrainingLine(
+        example.input,
+        example.result,
+        options.resultConfig,
+        numberLabels
+      )
+    );
+    const lockPath = join(
+      this.#dataDirectory,
+      "locks",
+      `${createHash("sha256").update(options.classifierName).digest("hex")}.sqlite`
     );
-    await writeFile(trainingPath, `${lines.join("\n")}
-`, "utf8");
     try {
+      const trainArguments = [
+        this.#workerPath,
+        "train",
+        "--training-data",
+        trainingPath,
+        "--output",
+        modelPath,
+        "--checkpoint-dir",
+        checkpointDirectory,
+        "--epochs",
+        String(options.epochs ?? 10),
+        "--artifact-lock-database",
+        lockPath,
+        "--main-database",
+        join(this.#dataDirectory, "swapai.sqlite"),
+        "--classifier-name",
+        options.classifierName,
+        "--expected-epoch",
+        String(options.expectedEpoch)
+      ];
+      if (numberLabels) {
+        trainArguments.push("--numeric-labels-from-stdin");
+      }
       await this.#runCommand(
         this.#pythonPath,
-        [
-          this.#workerPath,
-          "train",
-          "--training-data",
-          trainingPath,
-          "--output",
-          modelPath,
-          "--checkpoint-dir",
-          checkpointDirectory,
-          "--epochs",
-          String(options.epochs ?? 10)
-        ],
+        trainArguments,
         {
           cwd: candidateDirectory,
           env: this.#environment,
-          timeoutMs: 6 * 60 * 60 * 1e3
+          timeoutMs: 6 * 60 * 60 * 1e3,
+          stdin: numberLabels ? `${JSON.stringify({ format: 1, values: numberLabels.values })}
+${lines.join("\n")}
+` : `${lines.join("\n")}
+`
         }
       );
       await stat(modelPath);
     } catch (error) {
       throw toRuntimeError("Needle training failed.", error);
     }
-    return { modelPath, needleVersion: NEEDLE_VERSION };
+    return {
+      modelPath,
+      needleVersion: needleModelVersion(options.resultConfig)
+    };
   }
   async loadModel(options) {
     await this.ready();
     try {
       await stat(options.modelPath);
     } catch (error) {
-      throw toRuntimeError(`Needle model does not exist: ${options.modelPath}`, error);
+      throw new NeedleModelArtifactError(
+        `Needle model does not exist: ${options.modelPath}`,
+        error
+      );
+    }
+    let numberLabels;
+    if (options.resultConfig.type === "number") {
+      try {
+        const saved = JSON.parse(
+          await readFile(`${options.modelPath}.numbers.json`, "utf8")
+        );
+        if (saved.format !== 1 || !Array.isArray(saved.values) || saved.values.length === 0) {
+          throw new TypeError("Numeric label map has an invalid format.");
+        }
+        const savedValues = saved.values;
+        numberLabels = createNeedleNumberLabels(
+          savedValues,
+          options.resultConfig
+        );
+        if (numberLabels.values.length !== savedValues.length || numberLabels.values.some(
+          (value, index) => value !== savedValues[index]
+        )) {
+          throw new TypeError("Numeric label map is not sorted and unique.");
+        }
+      } catch (error) {
+        throw new NeedleModelArtifactError(
+          "Could not load Needle numeric labels.",
+          error
+        );
+      }
     }
     const schemaPath = `${options.modelPath}.schema.json`;
     await writeFile(
       schemaPath,
-      JSON.stringify([createNeedleTool(options.resultConfig)]),
+      JSON.stringify([createNeedleTool(options.resultConfig, numberLabels)]),
       "utf8"
     );
     const process2 = this.#spawnProcess(
@@ -339,6 +513,7 @@ var ManagedNeedleRuntime = class {
     const model = new NeedleModelProcess(
       process2,
       options.resultConfig,
+      numberLabels,
       (error) => this.#onBackgroundError?.(error),
       () => this.#models.delete(model)
     );
@@ -352,6 +527,70 @@ var ManagedNeedleRuntime = class {
       throw toRuntimeError("Needle model could not start.", error);
     }
   }
+  async clearClassifierArtifacts(classifierName) {
+    await rm(
+      join(this.#dataDirectory, "classifiers", classifierKey(classifierName)),
+      { recursive: true, force: true }
+    );
+  }
+  async clearClassifierGenerationArtifacts(classifierName, generation) {
+    if (!Number.isSafeInteger(generation) || generation < 0) {
+      throw new NeedleRuntimeError(
+        "service_unavailable",
+        "Classifier generation must be a non-negative safe integer."
+      );
+    }
+    const classifierDirectory = resolve(
+      this.#dataDirectory,
+      "classifiers",
+      classifierKey(classifierName)
+    );
+    const generationDirectory = resolve(
+      classifierDirectory,
+      `generation-${generation}`
+    );
+    if (!generationDirectory.startsWith(`${classifierDirectory}${sep}`)) {
+      throw new NeedleRuntimeError(
+        "service_unavailable",
+        "Classifier generation path is outside its classifier directory."
+      );
+    }
+    await rm(generationDirectory, { recursive: true, force: true });
+  }
+  async clearClassifierArtifactsThroughGeneration(classifierName, maximumGeneration) {
+    if (!Number.isSafeInteger(maximumGeneration) || maximumGeneration < 0) {
+      throw new NeedleRuntimeError(
+        "service_unavailable",
+        "Maximum classifier generation must be a non-negative safe integer."
+      );
+    }
+    const classifierDirectory = join(
+      this.#dataDirectory,
+      "classifiers",
+      classifierKey(classifierName)
+    );
+    let entries;
+    try {
+      entries = await readdir(classifierDirectory, { withFileTypes: true });
+    } catch (error) {
+      if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
+        return;
+      }
+      throw error;
+    }
+    await Promise.all(
+      entries.map(async (entry) => {
+        const match = /^generation-(\d+)$/.exec(entry.name);
+        if (!entry.isDirectory() || match === null) return;
+        const generation = Number(match[1]);
+        if (generation > maximumGeneration) return;
+        await rm(join(classifierDirectory, entry.name), {
+          recursive: true,
+          force: true
+        });
+      })
+    );
+  }
   async close() {
     if (this.#closed) return;
     this.#closed = true;
@@ -515,6 +754,7 @@ var ManagedNeedleRuntime = class {
 var NeedleModelProcess = class {
   #process;
   #resultConfig;
+  #numberLabels;
   #onBackgroundError;
   #onClose;
   #pending = /* @__PURE__ */ new Map();
@@ -528,17 +768,18 @@ var NeedleModelProcess = class {
   #exited = false;
   #closing = false;
   #stderr = "";
-  constructor(process2, resultConfig, onBackgroundError, onClose) {
+  constructor(process2, resultConfig, numberLabels, onBackgroundError, onClose) {
     this.#process = process2;
     this.#resultConfig = resultConfig;
+    this.#numberLabels = numberLabels;
     this.#onBackgroundError = onBackgroundError;
     this.#onClose = onClose;
-    this.#readyPromise = new Promise((resolve, reject) => {
-      this.#resolveReady = resolve;
+    this.#readyPromise = new Promise((resolve2, reject) => {
+      this.#resolveReady = resolve2;
       this.#rejectReady = reject;
     });
-    this.#exitPromise = new Promise((resolve) => {
-      this.#resolveExit = resolve;
+    this.#exitPromise = new Promise((resolve2) => {
+      this.#resolveExit = resolve2;
     });
     this.#listen();
   }
@@ -556,8 +797,8 @@ var NeedleModelProcess = class {
       );
     }
     const id = ++this.#requestId;
-    const result = new Promise((resolve, reject) => {
-      this.#pending.set(id, { resolve, reject });
+    const result = new Promise((resolve2, reject) => {
+      this.#pending.set(id, { resolve: resolve2, reject });
       this.#process.stdin.write(
         `${JSON.stringify({ id, type: "classify", input })}
 `,
@@ -596,10 +837,7 @@ var NeedleModelProcess = class {
         await withTimeout(this.#exitPromise, 2e3, "Needle ignored SIGTERM.");
       } catch {
         this.#process.kill("SIGKILL");
-        await Promise.race([
-          this.#exitPromise,
-          new Promise((resolve) => setTimeout(resolve, 2e3))
-        ]);
+        await withTimeout(this.#exitPromise, 2e3, "Needle ignored SIGKILL.");
       }
     }
   }
@@ -684,7 +922,13 @@ var NeedleModelProcess = class {
       return;
     }
     try {
-      pending.resolve(validateNeedleResult(message.result, this.#resultConfig));
+      pending.resolve(
+        decodeNeedleResult(
+          message.result,
+          this.#resultConfig,
+          this.#numberLabels
+        )
+      );
     } catch (error) {
       pending.reject(
         new NeedleRuntimeError(
@@ -710,12 +954,16 @@ var spawnProcess = (command, args, options) => spawn(command, [...args], {
   env: options.env,
   stdio: ["pipe", "pipe", "pipe"]
 });
-var runCommand = (command, args, options) => new Promise((resolve, reject) => {
+var runCommand = (command, args, options) => new Promise((resolve2, reject) => {
   const child = spawn(command, [...args], {
     cwd: options?.cwd,
     env: options?.env,
-    stdio: ["ignore", "pipe", "pipe"]
+    stdio: [options?.stdin === void 0 ? "ignore" : "pipe", "pipe", "pipe"]
   });
+  if (options?.stdin !== void 0 && child.stdin !== null) {
+    child.stdin.on("error", () => void 0);
+    child.stdin.end(options.stdin, "utf8");
+  }
   let stdout = "";
   let stderr = "";
   let timedOut = false;
@@ -764,7 +1012,7 @@ var runCommand = (command, args, options) => new Promise((resolve, reject) => {
       return;
     }
     if (code === 0) {
-      resolve({ stdout, stderr });
+      resolve2({ stdout, stderr });
       return;
     }
     reject(
@@ -831,7 +1079,7 @@ async function findFile(directory, name) {
   return void 0;
 }
 function withTimeout(promise, milliseconds, message, onTimeout) {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     const timer = setTimeout(() => {
       onTimeout?.();
       reject(new NeedleRuntimeError("service_unavailable", message));
@@ -840,7 +1088,7 @@ function withTimeout(promise, milliseconds, message, onTimeout) {
     promise.then(
       (value) => {
         clearTimeout(timer);
-        resolve(value);
+        resolve2(value);
       },
       (error) => {
         clearTimeout(timer);
@@ -879,7 +1127,12 @@ var SCHEMA = `
     training_lease_owner TEXT,
     training_lease_until INTEGER,
     created_at INTEGER NOT NULL,
-    updated_at INTEGER NOT NULL
+    updated_at INTEGER NOT NULL,
+    data_epoch INTEGER NOT NULL DEFAULT 0,
+    clear_pending INTEGER NOT NULL DEFAULT 0,
+    clear_erased INTEGER NOT NULL DEFAULT 0,
+    clear_artifact_generation_max INTEGER,
+    training_lease_epoch INTEGER
   );
 
   CREATE TABLE IF NOT EXISTS generations (
@@ -911,6 +1164,43 @@ var SCHEMA = `
     ON examples(classifier_name, generation, id);
   CREATE INDEX IF NOT EXISTS examples_by_split
     ON examples(classifier_name, generation, split, id);
+
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_classifiers_insert
+    BEFORE INSERT ON classifiers
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_classifiers_update
+    BEFORE UPDATE ON classifiers
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_classifiers_delete
+    BEFORE DELETE ON classifiers
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_generations_insert
+    BEFORE INSERT ON generations
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_generations_update
+    BEFORE UPDATE ON generations
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_generations_delete
+    BEFORE DELETE ON generations
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_examples_insert
+    BEFORE INSERT ON examples
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_examples_update
+    BEFORE UPDATE ON examples
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
+  CREATE TRIGGER IF NOT EXISTS swapai_v2_examples_delete
+    BEFORE DELETE ON examples
+    WHEN swapai_writer_version() < 2
+    BEGIN SELECT RAISE(ABORT, 'SwapAI writer is too old'); END;
 `;
 function assignExampleSplit(name, input) {
   const hash = createHash2("sha256").update(name).update("\0").update(input).digest();
@@ -924,40 +1214,85 @@ function openStorage(options) {
   chmodSync(options.dataDirectory, 448);
   const databasePath = join2(options.dataDirectory, "swapai.sqlite");
   const database = new DatabaseSync(databasePath);
-  chmodSync(databasePath, 384);
-  database.exec("PRAGMA journal_mode = WAL");
-  database.exec("PRAGMA foreign_keys = ON");
-  database.exec("PRAGMA busy_timeout = 5000");
-  database.exec(SCHEMA);
-  const now = Date.now();
-  const configJson = stringifyJson(options.config, "classifier config");
-  const existing = database.prepare(`
-    SELECT config_json FROM classifiers WHERE name = ?
-  `).get(options.name);
-  if (existing !== void 0 && criticalConfigJson(existing.config_json) !== criticalConfigJson(configJson)) {
-    database.close();
-    throw new TypeError(
-      `Classifier "${options.name}" already exists with different result or behavior settings`
+  let artifactLockDatabase = null;
+  try {
+    database.function(
+      "swapai_writer_version",
+      { deterministic: true },
+      () => 2
+    );
+    chmodSync(databasePath, 384);
+    database.exec("PRAGMA journal_mode = WAL");
+    database.exec("PRAGMA foreign_keys = ON");
+    database.exec("PRAGMA secure_delete = ON");
+    database.exec("PRAGMA busy_timeout = 5000");
+    const secureDelete = requiredRow(
+      database.prepare("PRAGMA secure_delete").get()
+    );
+    if (secureDelete.secure_delete !== 1) {
+      throw new Error("SQLite secure deletion could not be enabled");
+    }
+    database.exec(SCHEMA);
+    migrateClassifierColumns(database);
+    const lockDirectory = join2(options.dataDirectory, "locks");
+    mkdirSync(lockDirectory, { recursive: true, mode: 448 });
+    chmodSync(lockDirectory, 448);
+    const lockPath = join2(
+      lockDirectory,
+      `${createHash2("sha256").update(options.name).digest("hex")}.sqlite`
+    );
+    artifactLockDatabase = new DatabaseSync(lockPath);
+    chmodSync(lockPath, 384);
+    artifactLockDatabase.exec("PRAGMA busy_timeout = 0");
+    artifactLockDatabase.exec(
+      "CREATE TABLE IF NOT EXISTS artifact_lock (id INTEGER PRIMARY KEY)"
     );
+    const now = Date.now();
+    const configJson = stringifyJson(options.config, "classifier config");
+    const existing = database.prepare(`
+      SELECT config_json FROM classifiers WHERE name = ?
+    `).get(options.name);
+    if (existing !== void 0 && criticalConfigJson(existing.config_json) !== criticalConfigJson(configJson)) {
+      throw new TypeError(
+        `Classifier "${options.name}" already exists with different result or behavior settings`
+      );
+    }
+    transaction(database, () => {
+      database.prepare(`
+        INSERT INTO classifiers (
+          name, config_json, max_training_set, created_at, updated_at
+        ) VALUES (?, ?, ?, ?, ?)
+        ON CONFLICT(name) DO UPDATE SET
+          config_json = excluded.config_json,
+          max_training_set = excluded.max_training_set,
+          updated_at = excluded.updated_at
+      `).run(options.name, configJson, options.maxTrainingSet, now, now);
+      database.prepare(`
+        INSERT OR IGNORE INTO generations (
+          classifier_name, generation, status, created_at
+        ) VALUES (?, 1, 'active', ?)
+      `).run(options.name, now);
+      trimExamples(
+        database,
+        options.name,
+        activeGeneration(database, options.name),
+        options.maxTrainingSet
+      );
+    });
+  } catch (error) {
+    try {
+      artifactLockDatabase?.close();
+    } catch {
+    }
+    try {
+      database.close();
+    } catch {
+    }
+    throw error;
   }
-  transaction(database, () => {
-    database.prepare(`
-      INSERT INTO classifiers (
-        name, config_json, max_training_set, created_at, updated_at
-      ) VALUES (?, ?, ?, ?, ?)
-      ON CONFLICT(name) DO UPDATE SET
-        config_json = excluded.config_json,
-        max_training_set = excluded.max_training_set,
-        updated_at = excluded.updated_at
-    `).run(options.name, configJson, options.maxTrainingSet, now, now);
-    database.prepare(`
-      INSERT OR IGNORE INTO generations (
-        classifier_name, generation, status, created_at
-      ) VALUES (?, 1, 'active', ?)
-    `).run(options.name, now);
-    trimExamples(database, options.name, activeGeneration(database, options.name), options.maxTrainingSet);
-  });
+  const initializedArtifactLockDatabase = artifactLockDatabase;
   let closed = false;
+  let artifactLockHeld = false;
   const storage = {
     databasePath,
     snapshot() {
@@ -978,6 +1313,13 @@ function openStorage(options) {
           g.trained,
           g.model_path,
           g.needle_version,
+          c.data_epoch,
+          c.clear_pending,
+          c.clear_erased,
+          c.clear_artifact_generation_max,
+          c.training_lease_owner,
+          c.training_lease_epoch,
+          c.training_lease_until,
           (
             SELECT COUNT(*)
             FROM examples e
@@ -1005,23 +1347,36 @@ function openStorage(options) {
         consecutiveRetestFailures: row2.consecutive_retest_failures,
         trained: row2.trained === 1,
         modelPath: row2.model_path,
-        needleVersion: row2.needle_version
+        needleVersion: row2.needle_version,
+        dataEpoch: row2.data_epoch,
+        clearPending: row2.clear_pending === 1,
+        clearErased: row2.clear_erased === 1,
+        clearArtifactGenerationMax: row2.clear_artifact_generation_max,
+        trainingLeaseOwner: row2.training_lease_owner,
+        trainingLeaseEpoch: row2.training_lease_epoch,
+        trainingLeaseUntil: row2.training_lease_until
       };
     },
-    addExample(input, result) {
+    addExample(input, result, expectedDataEpoch) {
       assertOpen(closed);
-      const generation = activeGeneration(database, options.name);
       const split = assignExampleSplit(options.name, input);
       const createdAt = Date.now();
       const resultJson = stringifyJson(result, "classification result");
       let id = 0;
+      let generation = 0;
+      let accepted = false;
       transaction(database, () => {
+        const state = classifierState(database, options.name);
+        const epoch = expectedDataEpoch ?? state.data_epoch;
+        if (state.data_epoch !== epoch || state.clear_pending === 1) return;
+        generation = state.active_generation;
         const insertion = database.prepare(`
           INSERT INTO examples (
             classifier_name, generation, input, result_json, split, created_at
           ) VALUES (?, ?, ?, ?, ?, ?)
         `).run(options.name, generation, input, resultJson, split, createdAt);
         id = Number(insertion.lastInsertRowid);
+        accepted = true;
         database.prepare(`
           UPDATE classifiers
           SET total_examples_logged = total_examples_logged + 1,
@@ -1031,7 +1386,7 @@ function openStorage(options) {
         `).run(createdAt, options.name);
         trimExamples(database, options.name, generation, options.maxTrainingSet);
       });
-      return { id, generation, input, result, split, createdAt };
+      return accepted ? { id, generation, input, result, split, createdAt } : null;
     },
     listExamples(split, generation) {
       assertOpen(closed);
@@ -1049,44 +1404,76 @@ function openStorage(options) {
           `).all(options.name, selectedGeneration, split);
       return rows.map((value) => mapExample(row(value)));
     },
-    markTrainingAttempted(exampleCount) {
+    listExamplesForTraining(split, generation, expectedDataEpoch) {
+      assertOpen(closed);
+      let examples = null;
+      transaction(database, () => {
+        const state = classifierState(database, options.name);
+        if (state.data_epoch !== expectedDataEpoch || state.clear_pending === 1 || state.active_generation !== generation) {
+          return;
+        }
+        examples = database.prepare(`
+          SELECT id, generation, input, result_json, split, created_at
+          FROM examples
+          WHERE classifier_name = ? AND generation = ? AND split = ?
+          ORDER BY id
+        `).all(options.name, generation, split).map(
+          (value) => mapExample(row(value))
+        );
+      });
+      return examples;
+    },
+    markTrainingAttempted(exampleCount, expectedDataEpoch) {
       assertOpen(closed);
       if (!Number.isSafeInteger(exampleCount) || exampleCount <= 0) {
         throw new TypeError("training example count must be a positive integer");
       }
-      database.prepare(`
+      const epoch = expectedDataEpoch ?? storage.snapshot().dataEpoch;
+      const result = database.prepare(`
         UPDATE classifiers
         SET new_examples_since_training = 0,
             training_attempts = training_attempts + 1,
             examples_used_for_training = ?,
             updated_at = ?
-        WHERE name = ?
-      `).run(exampleCount, Date.now(), options.name);
+        WHERE name = ? AND data_epoch = ? AND clear_pending = 0
+      `).run(exampleCount, Date.now(), options.name, epoch);
+      return result.changes === 1;
     },
-    promoteGeneration(model) {
+    promoteGeneration(model, expectedDataEpoch) {
       assertOpen(closed);
-      const generation = activeGeneration(database, options.name);
-      database.prepare(`
-        UPDATE generations
-        SET trained = 1, model_path = ?, needle_version = ?
-        WHERE classifier_name = ? AND generation = ?
-      `).run(model.modelPath, model.needleVersion, options.name, generation);
-      return generationByNumber(database, options.name, generation);
+      let generation = 0;
+      let promoted = false;
+      transaction(database, () => {
+        const state = classifierState(database, options.name);
+        const epoch = expectedDataEpoch ?? state.data_epoch;
+        if (state.data_epoch !== epoch || state.clear_pending === 1) return;
+        generation = state.active_generation;
+        database.prepare(`
+          UPDATE generations
+          SET trained = 1, model_path = ?, needle_version = ?
+          WHERE classifier_name = ? AND generation = ?
+        `).run(model.modelPath, model.needleVersion, options.name, generation);
+        promoted = true;
+      });
+      return promoted ? generationByNumber(database, options.name, generation) : null;
     },
-    recordLocalClassification() {
+    recordLocalClassification(expectedDataEpoch) {
       assertOpen(closed);
-      database.prepare(`
+      const epoch = expectedDataEpoch ?? storage.snapshot().dataEpoch;
+      const result = database.prepare(`
         UPDATE classifiers
         SET local_classifications_since_retest = local_classifications_since_retest + 1,
             total_local_classifications = total_local_classifications + 1,
             updated_at = ?
-        WHERE name = ?
-      `).run(Date.now(), options.name);
+        WHERE name = ? AND data_epoch = ? AND clear_pending = 0
+      `).run(Date.now(), options.name, epoch);
+      if (result.changes !== 1) return null;
       return storage.snapshot().localClassificationsSinceRetest;
     },
-    recordRetest(passed) {
+    recordRetest(passed, expectedDataEpoch) {
       assertOpen(closed);
-      database.prepare(`
+      const epoch = expectedDataEpoch ?? storage.snapshot().dataEpoch;
+      const result = database.prepare(`
         UPDATE classifiers
         SET local_classifications_since_retest = 0,
             total_retests = total_retests + 1,
@@ -1095,28 +1482,33 @@ function openStorage(options) {
               ELSE consecutive_retest_failures + 1
             END,
             updated_at = ?
-        WHERE name = ?
-      `).run(passed ? 1 : 0, Date.now(), options.name);
+        WHERE name = ? AND data_epoch = ? AND clear_pending = 0
+      `).run(passed ? 1 : 0, Date.now(), options.name, epoch);
+      if (result.changes !== 1) return null;
       return storage.snapshot().consecutiveRetestFailures;
     },
-    claimTrainingLease(owner, durationMs) {
+    claimTrainingLease(owner, durationMs, expectedDataEpoch) {
       assertOpen(closed);
       if (owner.trim() === "" || !Number.isSafeInteger(durationMs) || durationMs <= 0) {
         throw new TypeError("training lease needs an owner and positive duration");
       }
-      const now2 = Date.now();
+      const now = Date.now();
+      const epoch = expectedDataEpoch ?? storage.snapshot().dataEpoch;
       const result = database.prepare(`
         UPDATE classifiers
         SET training_lease_owner = ?,
             training_lease_until = ?,
+            training_lease_epoch = ?,
             updated_at = ?
         WHERE name = ?
+          AND data_epoch = ?
+          AND clear_pending = 0
           AND (
             training_lease_owner IS NULL
             OR training_lease_until <= ?
             OR training_lease_owner = ?
           )
-      `).run(owner, now2 + durationMs, now2, options.name, now2, owner);
+      `).run(owner, now + durationMs, epoch, now, options.name, epoch, now, owner);
       return result.changes === 1;
     },
     releaseTrainingLease(owner) {
@@ -1125,16 +1517,21 @@ function openStorage(options) {
         UPDATE classifiers
         SET training_lease_owner = NULL,
             training_lease_until = NULL,
+            training_lease_epoch = NULL,
             updated_at = ?
         WHERE name = ? AND training_lease_owner = ?
       `).run(Date.now(), options.name, owner);
     },
-    archiveAndReset() {
+    archiveAndReset(expectedDataEpoch, resetOptions) {
       assertOpen(closed);
       let nextGeneration = 0;
       const changedAt = Date.now();
+      let reset = false;
       transaction(database, () => {
-        const currentGeneration = activeGeneration(database, options.name);
+        const state = classifierState(database, options.name);
+        const epoch = expectedDataEpoch ?? state.data_epoch;
+        if (state.data_epoch !== epoch || state.clear_pending === 1) return;
+        const currentGeneration = state.active_generation;
         database.prepare(`
           UPDATE generations
           SET status = 'archived', archived_at = ?
@@ -1151,22 +1548,157 @@ function openStorage(options) {
             classifier_name, generation, status, created_at
           ) VALUES (?, ?, 'active', ?)
         `).run(options.name, nextGeneration, changedAt);
+        let retainedExampleCount = 0;
+        if (resetOptions?.retainExamples) {
+          retainedExampleCount = Number(database.prepare(`
+            UPDATE examples
+            SET generation = ?
+            WHERE classifier_name = ? AND generation = ?
+          `).run(nextGeneration, options.name, currentGeneration).changes);
+        }
         database.prepare(`
           UPDATE classifiers
           SET active_generation = ?,
-              new_examples_since_training = 0,
+              data_epoch = data_epoch + 1,
+              new_examples_since_training = ?,
               training_attempts = 0,
               examples_used_for_training = 0,
               training_lease_owner = NULL,
               training_lease_until = NULL,
+              training_lease_epoch = NULL,
               local_classifications_since_retest = 0,
               consecutive_retest_failures = 0,
               updated_at = ?
           WHERE name = ?
+        `).run(
+          nextGeneration,
+          retainedExampleCount,
+          changedAt,
+          options.name
+        );
+        reset = true;
+      });
+      return reset ? generationByNumber(database, options.name, nextGeneration) : null;
+    },
+    beginClearTrainingData() {
+      assertOpen(closed);
+      database.prepare(`
+        UPDATE classifiers
+        SET data_epoch = data_epoch + 1,
+            clear_pending = 1,
+            clear_erased = 0,
+            clear_artifact_generation_max = (
+              SELECT MAX(generation)
+              FROM generations
+              WHERE classifier_name = ?
+            ),
+            updated_at = ?
+        WHERE name = ?
+      `).run(options.name, Date.now(), options.name);
+      return classifierState(database, options.name).data_epoch;
+    },
+    eraseTrainingData(dataEpoch) {
+      assertOpen(closed);
+      let nextGeneration = 0;
+      const changedAt = Date.now();
+      let erased = false;
+      transaction(database, () => {
+        const state = classifierState(database, options.name);
+        if (state.data_epoch !== dataEpoch || state.clear_pending !== 1) return;
+        if (state.clear_erased === 1) return;
+        const maximum = requiredRow(database.prepare(`
+          SELECT MAX(generation) AS maximum
+          FROM generations
+          WHERE classifier_name = ?
+        `).get(options.name));
+        nextGeneration = maximum.maximum + 1;
+        database.prepare(`
+          DELETE FROM examples WHERE classifier_name = ?
+        `).run(options.name);
+        database.prepare(`
+          DELETE FROM generations WHERE classifier_name = ?
+        `).run(options.name);
+        database.prepare(`
+          INSERT INTO generations (
+            classifier_name, generation, status, created_at
+          ) VALUES (?, ?, 'active', ?)
+        `).run(options.name, nextGeneration, changedAt);
+        database.prepare(`
+          UPDATE classifiers
+          SET active_generation = ?,
+              total_examples_logged = 0,
+              new_examples_since_training = 0,
+              training_attempts = 0,
+              examples_used_for_training = 0,
+              local_classifications_since_retest = 0,
+              total_local_classifications = 0,
+              total_retests = 0,
+              consecutive_retest_failures = 0,
+              training_lease_owner = NULL,
+              training_lease_until = NULL,
+              training_lease_epoch = NULL,
+              clear_erased = 1,
+              updated_at = ?
+          WHERE name = ?
         `).run(nextGeneration, changedAt, options.name);
+        erased = true;
       });
+      if (!erased) return null;
+      truncateWriteAheadLog(database);
       return generationByNumber(database, options.name, nextGeneration);
     },
+    finishClearTrainingData(dataEpoch) {
+      assertOpen(closed);
+      const state = classifierState(database, options.name);
+      if (state.data_epoch !== dataEpoch || state.clear_pending !== 1 || state.clear_erased !== 1) {
+        return false;
+      }
+      truncateWriteAheadLog(database);
+      const result = database.prepare(`
+        UPDATE classifiers
+        SET clear_pending = 0, updated_at = ?
+        WHERE name = ?
+          AND data_epoch = ?
+          AND clear_pending = 1
+          AND clear_erased = 1
+      `).run(Date.now(), options.name, dataEpoch);
+      return result.changes === 1;
+    },
+    clearTrainingData() {
+      const dataEpoch = storage.beginClearTrainingData();
+      if (!storage.claimArtifactWriteLock()) {
+        throw new Error("Classifier artifacts are currently being written");
+      }
+      try {
+        const generation = storage.eraseTrainingData(dataEpoch);
+        if (generation === null || !storage.finishClearTrainingData(dataEpoch)) {
+          throw new Error("Training data clear was superseded");
+        }
+        return generation;
+      } finally {
+        storage.releaseArtifactWriteLock();
+      }
+    },
+    claimArtifactWriteLock() {
+      assertOpen(closed);
+      if (artifactLockHeld) return true;
+      try {
+        initializedArtifactLockDatabase.exec("BEGIN IMMEDIATE");
+        artifactLockHeld = true;
+        return true;
+      } catch (error) {
+        if (typeof error === "object" && error !== null && "code" in error && error.code === "ERR_SQLITE_ERROR" && "message" in error && typeof error.message === "string" && /locked|busy/i.test(error.message)) {
+          return false;
+        }
+        throw error;
+      }
+    },
+    releaseArtifactWriteLock() {
+      assertOpen(closed);
+      if (!artifactLockHeld) return;
+      initializedArtifactLockDatabase.exec("COMMIT");
+      artifactLockHeld = false;
+    },
     listGenerations() {
       assertOpen(closed);
       return database.prepare(`
@@ -1182,6 +1714,11 @@ function openStorage(options) {
       if (closed) {
         return;
       }
+      if (artifactLockHeld) {
+        initializedArtifactLockDatabase.exec("ROLLBACK");
+        artifactLockHeld = false;
+      }
+      initializedArtifactLockDatabase.close();
       closed = true;
       database.close();
     }
@@ -1194,6 +1731,69 @@ function activeGeneration(database, name) {
   `).get(name));
   return active.active_generation;
 }
+function truncateWriteAheadLog(database) {
+  database.exec("PRAGMA busy_timeout = 0");
+  let checkpoint;
+  try {
+    checkpoint = requiredRow(database.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get());
+  } finally {
+    database.exec("PRAGMA busy_timeout = 5000");
+  }
+  if (checkpoint.busy !== 0) {
+    throw new Error(
+      "SQLite WAL could not be truncated after clearing training data"
+    );
+  }
+}
+function classifierState(database, name) {
+  return requiredRow(database.prepare(`
+    SELECT
+      active_generation,
+      data_epoch,
+      clear_pending,
+      clear_erased,
+      clear_artifact_generation_max,
+      training_lease_owner,
+      training_lease_epoch,
+      training_lease_until
+    FROM classifiers
+    WHERE name = ?
+  `).get(name));
+}
+function migrateClassifierColumns(database) {
+  transaction(database, () => {
+    const columns = new Set(
+      database.prepare("PRAGMA table_info(classifiers)").all().map(
+        (value) => row(value).name
+      )
+    );
+    if (!columns.has("data_epoch")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN data_epoch INTEGER NOT NULL DEFAULT 0"
+      );
+    }
+    if (!columns.has("clear_pending")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN clear_pending INTEGER NOT NULL DEFAULT 0"
+      );
+    }
+    if (!columns.has("clear_erased")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN clear_erased INTEGER NOT NULL DEFAULT 0"
+      );
+    }
+    if (!columns.has("clear_artifact_generation_max")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN clear_artifact_generation_max INTEGER"
+      );
+    }
+    if (!columns.has("training_lease_epoch")) {
+      database.exec(
+        "ALTER TABLE classifiers ADD COLUMN training_lease_epoch INTEGER"
+      );
+    }
+  });
+}
 function generationByNumber(database, name, generation) {
   const stored = requiredRow(database.prepare(`
     SELECT
@@ -1325,6 +1925,7 @@ function requiredRow(value) {
 // src/classifier.ts
 var TRAINING_LEASE_DURATION_MS = 5 * 60 * 1e3;
 var TRAINING_LEASE_RENEWAL_MS = 60 * 1e3;
+var ARTIFACT_LOCK_WAIT_MS = 3e4;
 function init(inputConfig) {
   const config = normalizeConfig(inputConfig);
   const dataDirectory = config.dataDirectory ?? join3(process.cwd(), ".swapai");
@@ -1362,44 +1963,123 @@ function openClassifierStorage(config, dataDirectory) {
   }
 }
 function createClassifier(config, storage, runtime) {
+  const initialState = storage.snapshot();
   let closed = false;
-  let trained = storage.snapshot().trained;
+  let closing = false;
+  let closePromise = null;
+  let trained = initialState.trained && !initialState.clearPending;
   let loadedModel = null;
+  let loadedModelEpoch = null;
   let operationQueue = Promise.resolve();
   let trainingQueue = Promise.resolve();
   let trainingScheduled = false;
   const trainingLeaseOwner = `${process.pid}:${randomUUID2()}`;
   let classificationQueue = Promise.resolve();
   let queuedFailure = null;
-  const saved = storage.snapshot();
-  if (saved.trained && (saved.modelPath === null || saved.needleVersion !== NEEDLE_VERSION)) {
-    storage.archiveAndReset();
-    trained = false;
+  let clearFailure = null;
+  let dataEpoch = initialState.dataEpoch;
+  let clearedEpoch = initialState.clearPending ? initialState.dataEpoch - 1 : initialState.dataEpoch;
+  const saved = initialState;
+  if (!saved.clearPending && saved.trained && (saved.modelPath === null || saved.needleVersion !== needleModelVersion(config.result) || !hasNeedleModelArtifacts({
+    modelPath: saved.modelPath,
+    resultConfig: config.result
+  }))) {
+    if (storage.archiveAndReset(saved.dataEpoch, { retainExamples: true }) !== null) {
+      const reset = storage.snapshot();
+      dataEpoch = reset.dataEpoch;
+      clearedEpoch = reset.dataEpoch;
+      trained = false;
+    }
   }
   const startup = startRuntime();
   async function startRuntime() {
+    const startupEpoch = dataEpoch;
+    const beforeRuntime = storage.snapshot();
+    if (beforeRuntime.clearPending) {
+      trained = false;
+      dataEpoch = beforeRuntime.dataEpoch;
+      try {
+        await performDurableClear(beforeRuntime.dataEpoch);
+      } catch (error) {
+        rememberClearFailure(error);
+      }
+    }
     try {
       await runtime.ready();
-      const snapshot = storage.snapshot();
-      if (snapshot.trained && snapshot.modelPath !== null) {
-        loadedModel = await runtime.loadModel({
-          modelPath: snapshot.modelPath,
-          resultConfig: config.result
-        });
-      }
     } catch (error) {
-      const swapAIError = toSwapAIError(
+      rememberBackgroundFailure(toSwapAIError(
         error,
         "service_unavailable",
         "Needle could not start"
-      );
-      rememberBackgroundFailure(swapAIError);
+      ));
+      return;
     }
+    const snapshot = storage.snapshot();
+    if (snapshot.clearPending) {
+      trained = false;
+      dataEpoch = snapshot.dataEpoch;
+      return;
+    }
+    if (snapshot.trained && snapshot.modelPath !== null) {
+      try {
+        const restoredModel = await runtime.loadModel({
+          modelPath: snapshot.modelPath,
+          resultConfig: config.result
+        });
+        const current = storage.snapshot();
+        if (startupEpoch === current.dataEpoch && !current.clearPending && current.trained) {
+          loadedModel = restoredModel;
+          loadedModelEpoch = startupEpoch;
+          trained = true;
+        } else {
+          await restoredModel.close();
+        }
+      } catch (error) {
+        if (error instanceof NeedleModelArtifactError && storage.archiveAndReset(snapshot.dataEpoch, {
+          retainExamples: true
+        }) !== null) {
+          const reset = refreshStoredState();
+          dataEpoch = reset.dataEpoch;
+          clearedEpoch = reset.dataEpoch;
+          trained = false;
+        }
+        const loadError = toSwapAIError(
+          error,
+          "service_unavailable",
+          error instanceof NeedleModelArtifactError ? `Could not load classifier "${config.name}"; retraining from saved examples` : `Could not load classifier "${config.name}"`
+        );
+        if (error instanceof NeedleModelArtifactError) {
+          reportBackgroundError(config, loadError);
+        } else {
+          rememberBackgroundFailure(loadError);
+        }
+      }
+    }
+    scheduleTraining();
   }
   function rememberBackgroundFailure(error) {
     queuedFailure = error;
     reportBackgroundError(config, error);
   }
+  function rememberClearFailure(error) {
+    clearFailure = toSwapAIError(
+      error,
+      "storage_failed",
+      `Could not completely clear classifier "${config.name}"`
+    );
+    reportBackgroundError(config, clearFailure);
+  }
+  function refreshStoredState() {
+    const snapshot = storage.snapshot();
+    dataEpoch = snapshot.dataEpoch;
+    if (snapshot.clearPending) {
+      trained = false;
+    } else {
+      trained = snapshot.trained;
+      clearedEpoch = snapshot.dataEpoch;
+    }
+    return snapshot;
+  }
   function enqueue(operation) {
     const result = operationQueue.then(operation);
     operationQueue = result.catch((error) => {
@@ -1409,14 +2089,15 @@ function createClassifier(config, storage, runtime) {
     });
     return result;
   }
-  function persistExample(input, result) {
+  function persistExample(input, result, epoch = dataEpoch) {
+    if (epoch !== dataEpoch) return Promise.resolve();
     return enqueue(() => {
-      storage.addExample(input, result);
-      scheduleTraining();
+      if (epoch !== dataEpoch) return;
+      if (storage.addExample(input, result, epoch) !== null) scheduleTraining();
     });
   }
   function shouldTrain(snapshot = storage.snapshot()) {
-    return !snapshot.trained && snapshot.examplesUsedForTraining < config.maxTrainingSet && snapshot.newExamplesSinceTraining >= config.retrainOnCount;
+    return clearedEpoch === dataEpoch && snapshot.dataEpoch === dataEpoch && !snapshot.clearPending && !snapshot.trained && snapshot.examplesUsedForTraining < config.maxTrainingSet && snapshot.newExamplesSinceTraining >= config.retrainOnCount;
   }
   function scheduleTraining() {
     if (trainingScheduled || !shouldTrain()) return;
@@ -1435,8 +2116,13 @@ function createClassifier(config, storage, runtime) {
     });
   }
   async function trainWhenDue() {
+    const trainingEpoch = dataEpoch;
     if (!shouldTrain()) return false;
-    if (!storage.claimTrainingLease(trainingLeaseOwner, TRAINING_LEASE_DURATION_MS)) {
+    if (!storage.claimTrainingLease(
+      trainingLeaseOwner,
+      TRAINING_LEASE_DURATION_MS,
+      trainingEpoch
+    )) {
       return false;
     }
     let leaseHeld = true;
@@ -1445,7 +2131,8 @@ function createClassifier(config, storage, runtime) {
       try {
         leaseHeld = storage.claimTrainingLease(
           trainingLeaseOwner,
-          TRAINING_LEASE_DURATION_MS
+          TRAINING_LEASE_DURATION_MS,
+          trainingEpoch
         );
       } catch {
         leaseHeld = false;
@@ -1460,17 +2147,33 @@ function createClassifier(config, storage, runtime) {
     try {
       const snapshot = storage.snapshot();
       if (!shouldTrain(snapshot)) return false;
-      const trainingExamples = storage.listExamples("training");
-      const heldOutExamples = storage.listExamples("held_out");
-      storage.markTrainingAttempted(snapshot.activeExampleCount);
+      if (!storage.markTrainingAttempted(
+        snapshot.activeExampleCount,
+        trainingEpoch
+      )) {
+        return false;
+      }
+      const trainingExamples = storage.listExamplesForTraining(
+        "training",
+        snapshot.activeGeneration,
+        trainingEpoch
+      );
+      const heldOutExamples = storage.listExamplesForTraining(
+        "held_out",
+        snapshot.activeGeneration,
+        trainingEpoch
+      );
+      if (trainingExamples === null || heldOutExamples === null) return false;
       if (trainingExamples.length === 0 || heldOutExamples.length === 0) {
         return true;
       }
       const candidate = await runtime.train({
         classifierName: config.name,
         generation: snapshot.activeGeneration,
+        expectedEpoch: trainingEpoch,
         examples: trainingExamples.map(({ input, result }) => ({ input, result })),
-        resultConfig: config.result
+        resultConfig: config.result,
+        acceptableError: config.acceptableError
       });
       const candidateModel = await runtime.loadModel({
         modelPath: candidate.modelPath,
@@ -1492,14 +2195,43 @@ function createClassifier(config, storage, runtime) {
           return true;
         }
         if (!renewTrainingLease()) {
+          const current = refreshStoredState();
+          if (current.dataEpoch !== trainingEpoch || current.clearPending) {
+            await runtime.clearClassifierGenerationArtifacts(
+              config.name,
+              snapshot.activeGeneration
+            );
+            return false;
+          }
           throw new SwapAIError(
             "service_unavailable",
             `Classifier "${config.name}" lost its training lease`
           );
         }
-        storage.promoteGeneration(candidate);
+        if (trainingEpoch !== refreshStoredState().dataEpoch) {
+          await runtime.clearClassifierGenerationArtifacts(
+            config.name,
+            snapshot.activeGeneration
+          );
+          return false;
+        }
+        if (storage.promoteGeneration(candidate, trainingEpoch) === null) {
+          await runtime.clearClassifierGenerationArtifacts(
+            config.name,
+            snapshot.activeGeneration
+          );
+          return false;
+        }
         if (loadedModel !== null) await loadedModel.close();
+        if (trainingEpoch !== refreshStoredState().dataEpoch) {
+          await runtime.clearClassifierGenerationArtifacts(
+            config.name,
+            snapshot.activeGeneration
+          );
+          return false;
+        }
         loadedModel = candidateModel;
+        loadedModelEpoch = trainingEpoch;
         trained = true;
         return true;
       } finally {
@@ -1510,34 +2242,61 @@ function createClassifier(config, storage, runtime) {
       storage.releaseTrainingLease(trainingLeaseOwner);
     }
   }
-  async function model() {
+  async function model(expectedEpoch) {
     await startup;
-    if (!trained) {
+    const current = refreshStoredState();
+    if (!trained || current.clearPending || current.dataEpoch !== expectedEpoch) {
       throw new SwapAIError(
         "not_trained",
         `Classifier "${config.name}" has not passed its held-out test`
       );
     }
+    if (loadedModel !== null && loadedModelEpoch !== expectedEpoch) {
+      await loadedModel.close();
+      loadedModel = null;
+      loadedModelEpoch = null;
+    }
     if (loadedModel === null) {
-      const snapshot = storage.snapshot();
-      if (snapshot.modelPath === null) {
+      if (current.modelPath === null) {
         throw new SwapAIError(
           "service_unavailable",
           `Classifier "${config.name}" is trained but has no saved model`
         );
       }
       await runtime.ready();
-      loadedModel = await runtime.loadModel({
-        modelPath: snapshot.modelPath,
-        resultConfig: config.result
-      });
+      let restored;
+      try {
+        restored = await runtime.loadModel({
+          modelPath: current.modelPath,
+          resultConfig: config.result
+        });
+      } catch (error) {
+        if (error instanceof NeedleModelArtifactError && storage.archiveAndReset(expectedEpoch, { retainExamples: true }) !== null) {
+          const reset = refreshStoredState();
+          dataEpoch = reset.dataEpoch;
+          clearedEpoch = reset.dataEpoch;
+          trained = false;
+          scheduleTraining();
+        }
+        throw error;
+      }
+      const afterLoad = refreshStoredState();
+      if (afterLoad.dataEpoch !== expectedEpoch || afterLoad.clearPending || !afterLoad.trained) {
+        await restored.close();
+        throw new SwapAIError(
+          "not_trained",
+          `Classifier "${config.name}" has been cleared`
+        );
+      }
+      loadedModel = restored;
+      loadedModelEpoch = expectedEpoch;
     }
     return loadedModel;
   }
   async function callReference(input, reference) {
     return validateResult(config.result, await reference(input));
   }
-  async function fallbackToReference(input, reference, candidateError, retestDue) {
+  async function fallbackToReference(input, reference, candidateError, retestDue, epoch) {
     if (reference === void 0) {
       throw toSwapAIError(
         candidateError,
@@ -1554,22 +2313,26 @@ function createClassifier(config, storage, runtime) {
       )
     );
     const referenceResult = await callReference(input, reference);
-    await persistExample(input, referenceResult);
-    if (retestDue) {
-      storage.recordLocalClassification();
-      const failures = storage.recordRetest(false);
-      if (failures >= config.retestRevertOn) await disableModel();
+    await persistExample(input, referenceResult, epoch);
+    if (retestDue && epoch === dataEpoch) {
+      if (storage.recordLocalClassification(epoch) === null) return referenceResult;
+      const failures = storage.recordRetest(false, epoch);
+      if (failures !== null && failures >= config.retestRevertOn) {
+        await disableModel(epoch);
+      }
     }
     return referenceResult;
   }
-  async function disableModel() {
-    storage.archiveAndReset();
+  async function disableModel(epoch) {
+    if (storage.archiveAndReset(epoch) === null) return;
+    refreshStoredState();
     trained = false;
     const previousModel = loadedModel;
     loadedModel = null;
+    loadedModelEpoch = null;
     if (previousModel !== null) await previousModel.close();
   }
-  async function classifyNow(input, reference) {
+  async function classifyNow(input, reference, epoch) {
     assertOpen2(closed);
     if (!trained) {
       if (reference === void 0) {
@@ -1579,7 +2342,7 @@ function createClassifier(config, storage, runtime) {
         );
       }
       const referenceResult2 = await callReference(input, reference);
-      await persistExample(input, referenceResult2);
+      await persistExample(input, referenceResult2, epoch);
       return referenceResult2;
     }
     const snapshot = storage.snapshot();
@@ -1588,37 +2351,190 @@ function createClassifier(config, storage, runtime) {
     try {
       candidateResult = validateResult(
         config.result,
-        await (await model()).classify(input)
+        await (await model(epoch)).classify(input)
       );
     } catch (error) {
-      return fallbackToReference(input, reference, error, retestDue);
+      return fallbackToReference(input, reference, error, retestDue, epoch);
+    }
+    const afterClassification = refreshStoredState();
+    if (afterClassification.dataEpoch !== epoch || afterClassification.clearPending) {
+      return fallbackToReference(
+        input,
+        reference,
+        new SwapAIError(
+          "not_trained",
+          `Classifier "${config.name}" was cleared during classification`
+        ),
+        retestDue,
+        epoch
+      );
     }
     if (!retestDue || reference === void 0) {
-      storage.recordLocalClassification();
+      if (epoch !== dataEpoch || storage.recordLocalClassification(epoch) === null) {
+        return fallbackToReference(
+          input,
+          reference,
+          new SwapAIError(
+            "not_trained",
+            `Classifier "${config.name}" was cleared before returning its result`
+          ),
+          retestDue,
+          epoch
+        );
+      }
       return candidateResult;
     }
     const referenceResult = await callReference(input, reference);
-    await persistExample(input, referenceResult);
-    storage.recordLocalClassification();
+    await persistExample(input, referenceResult, epoch);
+    if (epoch !== dataEpoch) return referenceResult;
+    if (storage.recordLocalClassification(epoch) === null) return referenceResult;
     const passed = resultError(config.result, referenceResult, candidateResult) <= config.acceptableError;
-    const failures = storage.recordRetest(passed);
-    if (!passed && failures >= config.retestRevertOn) {
-      await disableModel();
+    const failures = storage.recordRetest(passed, epoch);
+    if (!passed && failures !== null && failures >= config.retestRevertOn) {
+      await disableModel(epoch);
     }
     return referenceResult;
   }
+  async function performDurableClear(clearEpoch) {
+    trained = false;
+    const deletionFailures = [];
+    const throwDeletionFailures = () => {
+      if (deletionFailures.length === 1) throw deletionFailures[0];
+      if (deletionFailures.length > 1) {
+        throw new AggregateError(
+          deletionFailures,
+          `Could not completely clear classifier "${config.name}"`
+        );
+      }
+    };
+    const previousModel = loadedModel;
+    if (previousModel !== null) {
+      try {
+        await previousModel.close();
+        if (loadedModel === previousModel) {
+          loadedModel = null;
+          loadedModelEpoch = null;
+        }
+      } catch (error) {
+        deletionFailures.push(error);
+      }
+    }
+    const lockWaitStarted = Date.now();
+    let clearState = refreshStoredState();
+    while (clearState.clearPending && clearState.dataEpoch === clearEpoch && (clearState.trainingLeaseOwner !== null && clearState.trainingLeaseEpoch === null && clearState.trainingLeaseUntil !== null && clearState.trainingLeaseUntil > Date.now() || !storage.claimArtifactWriteLock())) {
+      if (Date.now() - lockWaitStarted >= ARTIFACT_LOCK_WAIT_MS) {
+        throw new SwapAIError(
+          "storage_failed",
+          `Timed out waiting to clear classifier "${config.name}" while training was still running`
+        );
+      }
+      await new Promise((resolve2) => setTimeout(resolve2, 10));
+      clearState = refreshStoredState();
+    }
+    if (!clearState.clearPending || clearState.dataEpoch !== clearEpoch) {
+      throwDeletionFailures();
+      return;
+    }
+    try {
+      try {
+        storage.eraseTrainingData(clearEpoch);
+      } catch (error) {
+        deletionFailures.push(error);
+      }
+      clearState = refreshStoredState();
+      if (clearState.clearPending && clearState.dataEpoch === clearEpoch && clearState.clearErased && clearState.clearArtifactGenerationMax !== null) {
+        try {
+          await runtime.clearClassifierArtifactsThroughGeneration(
+            config.name,
+            clearState.clearArtifactGenerationMax
+          );
+        } catch (error) {
+          deletionFailures.push(error);
+        }
+      }
+      throwDeletionFailures();
+      storage.finishClearTrainingData(clearEpoch);
+      const current = refreshStoredState();
+      if (!current.clearPending) {
+        clearFailure = null;
+        clearedEpoch = current.dataEpoch;
+      }
+    } finally {
+      storage.releaseArtifactWriteLock();
+    }
+  }
+  async function flushNow() {
+    const clearFailureBeforeFlush = clearFailure;
+    await startup;
+    while (true) {
+      const operations = operationQueue;
+      await operations;
+      const training = trainingQueue;
+      await training;
+      if (operations === operationQueue && training === trainingQueue) break;
+    }
+    if (clearFailure !== null && clearFailure !== clearFailureBeforeFlush) {
+      throw clearFailure;
+    }
+    const pending = refreshStoredState();
+    if (pending.clearPending || clearFailureBeforeFlush !== null) {
+      try {
+        await performDurableClear(pending.dataEpoch);
+        if (!refreshStoredState().clearPending) clearFailure = null;
+      } catch (error) {
+        rememberClearFailure(error);
+      }
+    }
+    if (refreshStoredState().clearPending) {
+      clearFailure ??= new SwapAIError(
+        "storage_failed",
+        `Could not completely clear classifier "${config.name}"`
+      );
+      throw clearFailure;
+    }
+    if (clearFailure !== null) throw clearFailure;
+    if (queuedFailure !== null) {
+      const failure = queuedFailure;
+      queuedFailure = null;
+      throw failure;
+    }
+  }
   const classifier = {
     isTrained() {
+      assertOpen2(closed || closing);
+      refreshStoredState();
       return trained;
     },
     logClassification(input, result) {
-      assertOpen2(closed);
+      assertOpen2(closed || closing);
       const validResult = validateResult(config.result, result);
-      void persistExample(input, validResult);
+      const epoch = refreshStoredState().dataEpoch;
+      void persistExample(input, validResult, epoch);
+    },
+    clearTrainingData() {
+      assertOpen2(closed || closing);
+      trained = false;
+      const clearEpoch = storage.beginClearTrainingData();
+      dataEpoch = clearEpoch;
+      const trainingBeforeClear = trainingQueue;
+      const classificationsBeforeClear = classificationQueue;
+      void enqueue(async () => {
+        await startup;
+        await trainingBeforeClear;
+        await classificationsBeforeClear;
+        try {
+          await performDurableClear(clearEpoch);
+        } catch (error) {
+          rememberClearFailure(error);
+        }
+      });
     },
     classify(input, reference) {
-      assertOpen2(closed);
-      const task = classificationQueue.then(() => classifyNow(input, reference));
+      assertOpen2(closed || closing);
+      const epoch = refreshStoredState().dataEpoch;
+      const task = classificationQueue.then(
+        () => classifyNow(input, reference, epoch)
+      );
       classificationQueue = task.then(
         () => void 0,
         () => void 0
@@ -1626,36 +2542,53 @@ function createClassifier(config, storage, runtime) {
       return task;
     },
     async flush() {
-      await startup;
-      while (true) {
-        const operations = operationQueue;
-        await operations;
-        const training = trainingQueue;
-        await training;
-        if (operations === operationQueue && training === trainingQueue) break;
-      }
-      if (queuedFailure !== null) {
-        const failure = queuedFailure;
-        queuedFailure = null;
-        throw failure;
-      }
+      assertOpen2(closed || closing);
+      await flushNow();
     },
-    async close() {
-      if (closed) return;
-      closed = true;
-      let failure;
-      try {
-        await classificationQueue;
-        await classifier.flush();
-      } catch (error) {
-        failure = error;
-      } finally {
-        if (loadedModel !== null) await loadedModel.close();
+    close() {
+      if (closed) return Promise.resolve();
+      if (closePromise !== null) return closePromise;
+      closing = true;
+      closePromise = (async () => {
+        try {
+          await classificationQueue;
+          await flushNow();
+        } catch (error) {
+          closing = false;
+          closePromise = null;
+          throw error;
+        }
+        const failures = [];
+        if (loadedModel !== null) {
+          try {
+            await loadedModel.close();
+          } catch (error) {
+            failures.push(error);
+          }
+        }
         loadedModel = null;
-        await runtime.close();
-        storage.close();
-      }
-      if (failure !== void 0) throw failure;
+        loadedModelEpoch = null;
+        try {
+          await runtime.close();
+        } catch (error) {
+          failures.push(error);
+        }
+        try {
+          storage.close();
+        } catch (error) {
+          failures.push(error);
+        }
+        closed = true;
+        closing = false;
+        if (failures.length === 1) throw failures[0];
+        if (failures.length > 1) {
+          throw new AggregateError(
+            failures,
+            `Could not completely close classifier "${config.name}"`
+          );
+        }
+      })();
+      return closePromise;
     }
   };
   return classifier;