@littlebigbrain/create 0.4.0 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@littlebigbrain/create",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Create a project from a little big brain starter workflow template: npm create @littlebigbrain",
   "license": "Apache-2.0",
   "repository": {

package/templates/_shared/testing.ts

@@ -358,10 +358,23 @@ export interface MockFile {
  * files: `files.content` and `workflows.derivations.check` / `record`, the
  * shape of an `LbbClient`. `upload` puts the file's `Document` in `graph`
  * with its name, SHA-256 and media type, as the upload route does.
+ *
+ * Each upload has its `uploaded_at`, as the server's file record: an upload
+ * of a live file keeps it, an upload after `remove` gets a new one. A
+ * derivation belongs to the upload of its record. The check names the
+ * upload of a live file; a removed file has no upload and no derivation.
+ * A record that names another upload than the live one throws `LbbError`
+ * 409 `derivation_upload_changed`.
  */
 export class MockContent {
   readonly held = new Map<string, MockFile>();
   readonly recorded = new Map<string, Schemas["Derivation"]>();
+  /** The upload of each content's file, live or removed. */
+  readonly uploads = new Map<string, { uploadedAt: string; live: boolean }>();
+  /** The upload each recorded derivation belongs to. */
+  private readonly stamps = new Map<string, string | undefined>();
+  /** Every derivation record request, in order. */
+  readonly records: Schemas["DerivationRecordRequest"][] = [];
   /** Reads of a file's bytes. */
   reads = 0;
   /** Derivation checks. */
@@ -373,6 +386,11 @@ export class MockContent {
   upload(name: string, body: string | Uint8Array, mediaType: string): MockFile {
     const bytes = typeof body === "string" ? new TextEncoder().encode(body) : body;
     const sha256 = createHash("sha256").update(bytes).digest("hex");
+    if (!this.uploads.get(sha256)?.live) {
+      this.newUploads += 1;
+      const at = new Date(Date.UTC(2026, 9, 15, 9) + this.newUploads * 1_000);
+      this.uploads.set(sha256, { uploadedAt: at.toISOString(), live: true });
+    }
     const doc = this.graph.entity({ type: "Document", name, key: `file:${sha256}` });
     doc.name = name;
     Object.assign(doc.properties, {
@@ -386,6 +404,22 @@ export class MockContent {
     return file;
   }
 
+  /** Uploads that gave a content a new `uploaded_at`. */
+  private newUploads = 0;
+
+  /** Delete a file, as `DELETE /v1/files`: its bytes are no longer read; its `Document` stays here. */
+  remove(sha256: string): void {
+    const upload = this.uploads.get(sha256);
+    if (!upload?.live) throw new Error(`no live file ${sha256}`);
+    this.uploads.set(sha256, { ...upload, live: false });
+    this.held.delete(sha256);
+  }
+
+  /** The `uploaded_at` of a content's file. */
+  uploadedAt(sha256: string): string | undefined {
+    return this.uploads.get(sha256)?.uploadedAt;
+  }
+
   /** The byte range of each read, `null` for a whole file, in order. */
   readonly ranges: Array<{ start: number; end: number | null } | null> = [];
   /** The bytes all reads answered. */
@@ -428,14 +462,30 @@ export class MockContent {
     derivations: {
       check: async (key: Schemas["DerivationKey"]): Promise<Schemas["DerivationCheckResponse"]> => {
         this.checks += 1;
-        const derivation = this.recorded.get(derivationName(key));
-        return derivation ? { exists: true, derivation } : { exists: false };
+        const file = this.uploads.get(key.sha256);
+        const upload = file?.live ? file.uploadedAt : undefined;
+        const derivation =
+          file && !file.live ? undefined : this.current(derivationName(key), key.sha256);
+        return {
+          exists: Boolean(derivation),
+          ...(derivation ? { derivation } : {}),
+          ...(upload ? { content_uploaded_at: upload } : {}),
+        };
       },
       record: async (
         request: Schemas["DerivationRecordRequest"],
       ): Promise<Schemas["DerivationRecordResponse"]> => {
+        this.records.push(request);
+        const named = request.content_uploaded_at;
+        const upload = this.uploads.get(request.sha256);
+        if (named && !(upload?.live && upload.uploadedAt === named))
+          throw new LbbError(409, "{}", {
+            code: "derivation_upload_changed",
+            message: `the file was deleted since upload ${named}: nothing was recorded`,
+            param: "content_uploaded_at",
+          });
         const name = derivationName(request);
-        const kept = this.recorded.get(name);
+        const kept = this.current(name, request.sha256);
         if (kept) return { recorded: false, derivation: kept };
         const outputs = JSON.stringify(request.outputs ?? null);
         if (outputs.length > 16 * 1024)
@@ -453,10 +503,19 @@ export class MockContent {
           recorded_at: "2026-10-15T09:00:00Z",
         };
         this.recorded.set(name, derivation);
+        this.stamps.set(name, upload?.uploadedAt);
         return { recorded: true, derivation };
       },
     },
   };
+
+  /** The derivation `name` when it belongs to the current upload of its content. */
+  private current(name: string, sha256: string): Schemas["Derivation"] | undefined {
+    const derivation = this.recorded.get(name);
+    const upload = this.uploadedAt(sha256);
+    if (!derivation || (upload !== undefined && this.stamps.get(name) !== upload)) return undefined;
+    return derivation;
+  }
 }
 
 const derivationName = (key: Schemas["DerivationKey"]) =>

package/templates/rows/README.md

@@ -62,7 +62,14 @@ same offset. A cursor without `byte`, from an earlier version of the
 template, reads the file again from its start. The last turn
 records the derivation with what it wrote: the rows, the instances, the
 columns it mapped and left out, the instance the file represents, and the
-commits of its writes.
+commits of its writes. The record names the upload the check
+answered (`content_uploaded_at`). The cursor carries it from turn to turn.
+When the file was deleted while the run read it, the record writes nothing
+(`409 derivation_upload_changed`). The run ends and counts the file as
+skipped. The run of a new upload reads the file again. Say the file was
+deleted and not uploaded again. The next read of its bytes answers
+`404 file_not_found`. The run ends there too, and counts the file as
+skipped.
 
 The parser follows RFC 4180: fields in double quotes can hold the
 delimiter, line breaks and quotes (`""`). A row whose quoted field holds a

package/templates/rows/workflow.test.ts

@@ -526,6 +526,62 @@ test("a turn that continues starts at the byte offset where the last one stopped
   assert.equal(restarted.turn.keys[0], "check");
 });
 
+test("a file deleted and uploaded again while it is read records nothing; the new upload's run reads it", async () => {
+  const csv = customerList(10);
+  const settings = { ...EVERY_FILE, rowsPerTurn: 4 };
+  const { lbb, content, definition } = setup(settings, []);
+  const file = content.upload("customers-ten.csv", csv, "text/csv");
+  const first = content.uploadedAt(file.sha256);
+
+  // Turn 1 checks the file; the turns after it carry the upload the check named.
+  const one = await runTurn<RowsState>(definition, {}, contentMessage("rows", file));
+  type Continued = { cursor: { plan: { uploadedAt?: string } } };
+  const carried = one.result.continuation!.message as unknown as Continued;
+  assert.equal(carried.cursor.plan.uploadedAt, first);
+
+  // The file is deleted and uploaded again; the old run reads on to its end.
+  content.remove(file.sha256);
+  const again = content.upload("customers-ten.csv", csv, "text/csv");
+  assert.notEqual(content.uploadedAt(again.sha256), first);
+  const old = await runMessage<RowsState>(
+    definition,
+    one.result.continuation!.message,
+    one.result.state,
+  );
+  assert.deepEqual(old.state, { contents: 0, skipped: 1, rows: 10, instances: 10, represents: 0 });
+  assert.equal(content.records.at(-1)?.content_uploaded_at, first);
+  assert.equal(content.recorded.size, 0, "nothing is recorded for the deleted upload");
+
+  // The run of the new upload checks, reads the file and records its derivation.
+  const fresh = await runMessage<RowsState>(definition, contentMessage("rows", again));
+  assert.equal(fresh.state.contents, 1);
+  assert.equal(fresh.state.skipped, 0);
+  assert.equal(content.records.at(-1)?.content_uploaded_at, content.uploadedAt(again.sha256));
+  assert.equal(content.recorded.size, 1);
+  assertEveryRow(lbb.graph, again, csv);
+});
+
+test("a file deleted while it is read, and not uploaded again, ends the run as skipped", async () => {
+  const csv = customerList(10);
+  const settings = { ...EVERY_FILE, rowsPerTurn: 4 };
+  const { content, definition } = setup(settings, []);
+  const file = content.upload("customers-ten.csv", csv, "text/csv");
+  const one = await runTurn<RowsState>(definition, {}, contentMessage("rows", file));
+  assert.equal(one.result.state.rows, 4);
+
+  // The next read of the bytes answers 404: the run ends without an error.
+  content.remove(file.sha256);
+  const old = await runMessage<RowsState>(
+    definition,
+    one.result.continuation!.message,
+    one.result.state,
+  );
+  assert.equal(old.turns.length, 1, "the run ends at the turn that read nothing");
+  assert.deepEqual(old.state, { contents: 0, skipped: 1, rows: 4, instances: 4, represents: 0 });
+  assert.equal(content.records.length, 0, "no derivation is recorded");
+  assert.equal(content.recorded.size, 0);
+});
+
 test("a row longer than the longest row stops the read: a quote that never closes", async () => {
   const bytes = new TextEncoder().encode(`a,b\n1,"open${"x".repeat(500)}`);
   await assert.rejects(

package/templates/rows/workflow.ts

@@ -18,7 +18,11 @@
 //    header is row 1);
 // 4. with `represents`, writes `Document REPRESENTS <instance>` when the
 //    file's name names exactly one instance of that class;
-// 5. records the derivation.
+// 5. records the derivation, for the upload its check named. When the file
+//    was deleted (or deleted and uploaded again) since, nothing is recorded,
+//    and the run of the new upload reads the file again. A file deleted and
+//    not uploaded again answers 404 to the next read of its bytes: the run
+//    ends there and counts the file as skipped.
 //
 // A turn reads at most `rowsPerTurn` rows; the rest goes to a new turn
 // (`ctx.continue`) that starts where the last one stopped. The file is read
@@ -29,6 +33,7 @@ import { createHash } from "node:crypto";
 import {
   LbbError,
   entityRef,
+  isDerivationUploadChanged,
   triggeredWorkflow,
   type LbbActivities,
   type Schemas,
@@ -138,7 +143,7 @@ export interface ContentPorts {
 export interface RowsState {
   /** Files read to their end. */
   contents: number;
-  /** Files skipped: read before with these settings, another name, or no file. */
+  /** Files skipped: read before with these settings, another name, no file, or deleted while read. */
   skipped: number;
   rows: number;
   instances: number;
@@ -165,6 +170,8 @@ interface Plan {
   key: number | null;
   label: number | null;
   represents: string | null;
+  /** The upload of the file the check named: the record names it. */
+  uploadedAt?: string;
 }
 
 /** Where a file's next turn starts, and what its turns did so far. */
@@ -196,9 +203,11 @@ interface FileInfo {
   mediaType: string;
 }
 
-/** What the check of a file found: the file to read, or why it is skipped. */
+/** What the check of a file found: the file to read and its upload, or why it is skipped. */
 interface Checked {
   file?: FileInfo;
+  /** The `content_uploaded_at` of the check: absent without a file record. */
+  uploadedAt?: string;
   skip?: string;
 }
 
@@ -227,6 +236,10 @@ const SCALAR: Record<string, ValueType> = {
 /** The media type without parameters, in lower case. */
 const baseType = (mediaType: string) => mediaType.split(";")[0]!.trim().toLowerCase();
 
+/** Whether an error is the 404 of a file's bytes: the file was deleted and not uploaded again. */
+export const fileGone = (error: unknown) =>
+  error instanceof LbbError && error.status === 404 && error.code === "file_not_found";
+
 /** A file name without its last extension. */
 export const stem = (name: string) => name.replace(/\.[^./\\]*$/, "");
 
@@ -381,158 +394,178 @@ export function createWorkflow(
         delete plain.cursor;
         return ctx.continue(next, plain);
       };
-      const document = message.instances[0];
-      if (!document) return { state };
-      // A cursor without a byte offset (a message of an earlier version of
-      // this template) reads the file again from its start.
-      let cursor = typeof message.cursor?.byte === "number" ? message.cursor : undefined;
-      // The bytes the header read holds after the header, for the rows of
-      // the same turn: they are not read again.
-      let held: HeldBytes | undefined;
-
-      if (!cursor) {
-        const checked = await ctx.step("check", async (s): Promise<Checked> => {
-          s.progress("checking the file", { done: 0, total: 1, unit: "documents" });
-          const file = await readDocument(message, document);
-          if (!file) return { skip: "no file" };
-          const named = /\.(csv|tsv)$/i.test(file.name);
-          if (!MEDIA_TYPES.includes(file.mediaType) && !named) return { skip: "media type" };
-          if (match && !match.test(file.name)) return { skip: "name" };
-          const found = await ports.workflows.derivations.check(
-            derivationKey(file.sha256, settings),
-          );
-          return found.exists ? { skip: "derived" } : { file };
-        });
-        if (!checked.file) {
-          const next = { ...state, skipped: state.skipped + 1 };
-          return rest(next);
+      try {
+        const document = message.instances[0];
+        if (!document) return { state };
+        // A cursor without a byte offset (a message of an earlier version of
+        // this template) reads the file again from its start.
+        let cursor = typeof message.cursor?.byte === "number" ? message.cursor : undefined;
+        // The bytes the header read holds after the header, for the rows of
+        // the same turn: they are not read again.
+        let held: HeldBytes | undefined;
+
+        if (!cursor) {
+          const checked = await ctx.step("check", async (s): Promise<Checked> => {
+            s.progress("checking the file", { done: 0, total: 1, unit: "documents" });
+            const file = await readDocument(message, document);
+            if (!file) return { skip: "no file" };
+            const named = /\.(csv|tsv)$/i.test(file.name);
+            if (!MEDIA_TYPES.includes(file.mediaType) && !named) return { skip: "media type" };
+            if (match && !match.test(file.name)) return { skip: "name" };
+            const found = await ports.workflows.derivations.check(
+              derivationKey(file.sha256, settings),
+            );
+            if (found.exists) return { skip: "derived" };
+            const uploadedAt = found.content_uploaded_at ?? undefined;
+            return uploadedAt ? { file, uploadedAt } : { file };
+          });
+          if (!checked.file) {
+            const next = { ...state, skipped: state.skipped + 1 };
+            return rest(next);
+          }
+          const file = checked.file;
+          const planned = await ctx.step("plan", async (s) => {
+            s.progress("reading the header", { done: 0, total: 1, unit: "documents" });
+            const delimiter = delimiterOf(file.mediaType, file.name);
+            const header = await readRecords(rangeOf(file.sha256), delimiter, 0, 1, 1, windows);
+            held = header.held;
+            return plan(file, delimiter, header, checked.uploadedAt);
+          });
+          const first: number[] = [];
+          if (planned.plan.represents) {
+            const written = await lbb.writeOwned(ctx, "represents", (s) => {
+              s.progress("writing facts", { done: 0, total: 1, unit: "documents" });
+              const p = planned.plan;
+              return {
+                subjects: [entityRef(p.document, "Document", p.name)],
+                relations: [REPRESENTS],
+                triplets: [
+                  {
+                    source: entityRef(p.document, "Document", p.name),
+                    relation: REPRESENTS,
+                    target: entityRef(
+                      p.represents!,
+                      settings.represents!.class,
+                      planned.label ?? undefined,
+                    ),
+                    confidence: 1,
+                    evidence: { text: p.name, source_id: p.document },
+                  },
+                ],
+              };
+            });
+            if (written.commit_seq !== null) first.push(written.commit_seq);
+          }
+          cursor = {
+            byte: planned.at,
+            row: 2,
+            plan: planned.plan,
+            done: { rows: 0, instances: 0, keyless: 0, rejected: 0, commits: first },
+          };
         }
-        const file = checked.file;
-        const planned = await ctx.step("plan", async (s) => {
-          s.progress("reading the header", { done: 0, total: 1, unit: "documents" });
-          const delimiter = delimiterOf(file.mediaType, file.name);
-          const header = await readRecords(rangeOf(file.sha256), delimiter, 0, 1, 1, windows);
-          held = header.held;
-          return plan(file, delimiter, header);
+
+        // The rows of this turn, read from the byte offset where the last turn
+        // stopped, then their writes in parts. A turn run again after a crash
+        // reads its rows again from the same offset.
+        const from = cursor;
+        const filePlan = from.plan;
+        let rows: Promise<RecordsRead> | undefined;
+        const turnRows = () =>
+          (rows ??= readRecords(
+            rangeOf(filePlan.sha256),
+            filePlan.delimiter,
+            from.byte,
+            from.row,
+            settings.rowsPerTurn,
+            windows,
+            held,
+          ));
+        const windowRecords = async (): Promise<CsvRecord[]> => (await turnRows()).records;
+        const window = await ctx.step("rows", async (s) => {
+          s.progress("reading rows", { done: 0, unit: "rows" });
+          const read = await turnRows();
+          const taken = read.records;
+          s.progress("reading rows", { done: taken.length, unit: "rows" });
+          const counts = { instances: 0, keyless: 0, rejected: 0 };
+          for (const record of taken) {
+            const built = rowFacts(filePlan, record);
+            if (built === "blank") continue;
+            if (built === "keyless") counts.keyless += 1;
+            else {
+              counts.instances += 1;
+              counts.rejected += built.rejected;
+            }
+          }
+          return { count: taken.length, byte: read.next, row: read.row, end: read.end, ...counts };
         });
-        const first: number[] = [];
-        if (planned.plan.represents) {
-          const written = await lbb.writeOwned(ctx, "represents", (s) => {
-            s.progress("writing facts", { done: 0, total: 1, unit: "documents" });
-            const p = planned.plan;
-            return {
-              subjects: [entityRef(p.document, "Document", p.name)],
-              relations: [REPRESENTS],
-              triplets: [
-                {
-                  source: entityRef(p.document, "Document", p.name),
-                  relation: REPRESENTS,
-                  target: entityRef(
-                    p.represents!,
-                    settings.represents!.class,
-                    planned.label ?? undefined,
-                  ),
-                  confidence: 1,
-                  evidence: { text: p.name, source_id: p.document },
-                },
-              ],
-            };
+        const commits: number[] = [];
+        for (let n = 0; n * PART < window.count; n++) {
+          const written = await lbb.writeOwned(ctx, `write-${n}`, async (s) => {
+            s.progress("writing facts", { done: n * PART, total: window.count, unit: "rows" });
+            const part = (await windowRecords()).slice(n * PART, (n + 1) * PART);
+            return partWrite(filePlan, part);
           });
-          if (written.commit_seq !== null) first.push(written.commit_seq);
+          if (written.commit_seq !== null) commits.push(written.commit_seq);
         }
-        cursor = {
-          byte: planned.at,
-          row: 2,
-          plan: planned.plan,
-          done: { rows: 0, instances: 0, keyless: 0, rejected: 0, commits: first },
+        const done: Done = {
+          rows: from.done.rows + window.count,
+          instances: from.done.instances + window.instances,
+          keyless: from.done.keyless + window.keyless,
+          rejected: from.done.rejected + window.rejected,
+          commits: [...from.done.commits, ...commits].slice(0, 256),
         };
-      }
+        const counted = {
+          ...state,
+          rows: state.rows + window.count,
+          instances: state.instances + window.instances,
+        };
+        if (!window.end)
+          return ctx.continue(counted, {
+            ...message,
+            removed: [],
+            cursor: { byte: window.byte, row: window.row, plan: filePlan, done },
+          });
 
-      // The rows of this turn, read from the byte offset where the last turn
-      // stopped, then their writes in parts. A turn run again after a crash
-      // reads its rows again from the same offset.
-      const from = cursor;
-      const filePlan = from.plan;
-      let rows: Promise<RecordsRead> | undefined;
-      const turnRows = () =>
-        (rows ??= readRecords(
-          rangeOf(filePlan.sha256),
-          filePlan.delimiter,
-          from.byte,
-          from.row,
-          settings.rowsPerTurn,
-          windows,
-          held,
-        ));
-      const windowRecords = async (): Promise<CsvRecord[]> => (await turnRows()).records;
-      const window = await ctx.step("rows", async (s) => {
-        s.progress("reading rows", { done: 0, unit: "rows" });
-        const read = await turnRows();
-        const taken = read.records;
-        s.progress("reading rows", { done: taken.length, unit: "rows" });
-        const counts = { instances: 0, keyless: 0, rejected: 0 };
-        for (const record of taken) {
-          const built = rowFacts(filePlan, record);
-          if (built === "blank") continue;
-          if (built === "keyless") counts.keyless += 1;
-          else {
-            counts.instances += 1;
-            counts.rejected += built.rejected;
+        const recorded = await ctx.step("record", async (s) => {
+          s.progress("recording the derivation", { done: 0, total: 1, unit: "documents" });
+          try {
+            const answer = await ports.workflows.derivations.record({
+              ...derivationKey(filePlan.sha256, settings),
+              run_id: ctx.messageId,
+              outputs: {
+                rows: done.rows,
+                instances: done.instances,
+                rows_without_key: done.keyless,
+                values_left_out: done.rejected,
+                columns: filePlan.columns.slice(0, 100).map((c) => `${c.header} → ${c.property}`),
+                columns_left_out: filePlan.left,
+                represents: filePlan.represents,
+                commits: done.commits,
+              },
+              ...(filePlan.uploadedAt ? { content_uploaded_at: filePlan.uploadedAt } : {}),
+            });
+            return answer.recorded;
+          } catch (error) {
+            // The file was deleted while this run read it: nothing is
+            // recorded, and the run of a new upload reads it again.
+            if (isDerivationUploadChanged(error)) return "upload changed";
+            throw error;
           }
-        }
-        return { count: taken.length, byte: read.next, row: read.row, end: read.end, ...counts };
-      });
-      const commits: number[] = [];
-      for (let n = 0; n * PART < window.count; n++) {
-        const written = await lbb.writeOwned(ctx, `write-${n}`, async (s) => {
-          s.progress("writing facts", { done: n * PART, total: window.count, unit: "rows" });
-          const part = (await windowRecords()).slice(n * PART, (n + 1) * PART);
-          return partWrite(filePlan, part);
-        });
-        if (written.commit_seq !== null) commits.push(written.commit_seq);
-      }
-      const done: Done = {
-        rows: from.done.rows + window.count,
-        instances: from.done.instances + window.instances,
-        keyless: from.done.keyless + window.keyless,
-        rejected: from.done.rejected + window.rejected,
-        commits: [...from.done.commits, ...commits].slice(0, 256),
-      };
-      const counted = {
-        ...state,
-        rows: state.rows + window.count,
-        instances: state.instances + window.instances,
-      };
-      if (!window.end)
-        return ctx.continue(counted, {
-          ...message,
-          removed: [],
-          cursor: { byte: window.byte, row: window.row, plan: filePlan, done },
         });
-
-      await ctx.step("record", async (s) => {
-        s.progress("recording the derivation", { done: 0, total: 1, unit: "documents" });
-        const recorded = await ports.workflows.derivations.record({
-          ...derivationKey(filePlan.sha256, settings),
-          run_id: ctx.messageId,
-          outputs: {
-            rows: done.rows,
-            instances: done.instances,
-            rows_without_key: done.keyless,
-            values_left_out: done.rejected,
-            columns: filePlan.columns.slice(0, 100).map((c) => `${c.header} → ${c.property}`),
-            columns_left_out: filePlan.left,
-            represents: filePlan.represents,
-            commits: done.commits,
-          },
+        if (recorded === "upload changed")
+          return rest({ ...counted, skipped: counted.skipped + 1 });
+        return rest({
+          ...counted,
+          contents: counted.contents + 1,
+          represents: counted.represents + (filePlan.represents ? 1 : 0),
         });
-        return recorded.recorded;
-      });
-      return rest({
-        ...counted,
-        contents: counted.contents + 1,
-        represents: counted.represents + (filePlan.represents ? 1 : 0),
-      });
+      } catch (error) {
+        // The file was deleted while this run read it, and not uploaded
+        // again: its bytes answer 404. Nothing is recorded, and the file
+        // counts as skipped.
+        if (!fileGone(error)) throw error;
+        return rest({ ...state, skipped: state.skipped + 1 });
+      }
     },
   });
 
@@ -557,8 +590,8 @@ export function createWorkflow(
     };
   }
 
-  /** The plan of a file: its delimiter, the mapped columns, the key and label columns, and what it represents. */
-  async function plan(file: FileInfo, delimiter: string, read: RecordsRead) {
+  /** The plan of a file: its delimiter, the mapped columns, the key and label columns, what it represents, and its upload. */
+  async function plan(file: FileInfo, delimiter: string, read: RecordsRead, uploadedAt?: string) {
     const first = read.records[0];
     const header = first ? first.fields.map((h) => h.trim()) : [];
     const registry = header.length ? ((await lbb.ontology()).property_defs ?? []) : [];
@@ -580,6 +613,7 @@ export function createWorkflow(
         key: key >= 0 ? key : null,
         label,
         represents: found?.iri ?? null,
+        ...(uploadedAt ? { uploadedAt } : {}),
       } satisfies Plan,
     };
   }

package/templates/sections/README.md

@@ -54,7 +54,14 @@ The text before the first heading, when it is not blank, is the section at
 position 0, of level 0, titled with the file's name without its extension.
 A section's lines run from its heading to its last line that is not
 blank. The last turn records the derivation: the sections, their count per
-level, the instance the file represents, and the commits of the writes.
+level, the instance the file represents, and the commits of the writes. The record names the upload the check
+answered (`content_uploaded_at`). The cursor carries it from turn to turn.
+When the file was deleted while the run read it, the record writes nothing
+(`409 derivation_upload_changed`). The run ends and counts the file as
+skipped. The run of a new upload reads the file again. Say the file was
+deleted and not uploaded again. The next read of its bytes answers
+`404 file_not_found`. The run ends there too, and counts the file as
+skipped.
 
 ## Settings
 

package/templates/sections/workflow.test.ts

@@ -12,6 +12,7 @@ import {
   phases,
   readJson,
   runMessage,
+  runTurn,
   samplesDir,
   type MemoryGraph,
   type MockFile,
@@ -74,7 +75,7 @@ function setup(settings: Settings = WITH_GUIDES, guides = GUIDES.map((g) => g.na
   const definition = createWorkflow(lbb, content, settings);
   const run = (file: MockFile, state: unknown = {}) =>
     runMessage<SectionsState>(definition, contentMessage("sections", file), state);
-  return { lbb, content, run };
+  return { lbb, content, run, definition };
 }
 
 /** The written sections by position. */
@@ -207,6 +208,55 @@ test("a content with a derivation for these settings is skipped, under any name"
   assert.deepEqual(again.turns[0]!.keys, ["check"]);
 });
 
+test("a file deleted and uploaded again while it is read records nothing; the new upload's run reads it", async () => {
+  const { lbb, content, definition } = setup({ ...WITH_GUIDES, sectionsPerTurn: 3 });
+  const file = content.upload("support-handbook.md", HANDBOOK, "text/markdown");
+  const first = content.uploadedAt(file.sha256);
+
+  // Turn 1 checks the file; the turns after it carry the upload the check named.
+  const one = await runTurn<SectionsState>(definition, {}, contentMessage("sections", file));
+  type Continued = { cursor: { plan: { uploadedAt?: string } } };
+  const carried = one.result.continuation!.message as unknown as Continued;
+  assert.equal(carried.cursor.plan.uploadedAt, first);
+
+  // The file is deleted and uploaded again; the old run writes on to its end.
+  content.remove(file.sha256);
+  const again = content.upload("support-handbook.md", HANDBOOK, "text/markdown");
+  const old = await runMessage<SectionsState>(
+    definition,
+    one.result.continuation!.message,
+    one.result.state,
+  );
+  assert.deepEqual(old.state, { contents: 0, skipped: 1, sections: 8, represents: 0 });
+  assert.equal(content.records.at(-1)?.content_uploaded_at, first);
+  assert.equal(content.recorded.size, 0, "nothing is recorded for the deleted upload");
+
+  // The run of the new upload checks, writes the sections and records.
+  const fresh = await runMessage<SectionsState>(definition, contentMessage("sections", again));
+  assert.deepEqual(fresh.state, { contents: 1, skipped: 0, sections: 8, represents: 1 });
+  assert.equal(content.records.at(-1)?.content_uploaded_at, content.uploadedAt(again.sha256));
+  assert.equal(content.recorded.size, 1);
+  assert.equal(written(lbb.graph).length, EXPECTED.sections.length);
+});
+
+test("a file deleted while it is read, and not uploaded again, ends the run as skipped", async () => {
+  const { content, definition } = setup({ ...WITH_GUIDES, sectionsPerTurn: 3 });
+  const file = content.upload("support-handbook.md", HANDBOOK, "text/markdown");
+  const one = await runTurn<SectionsState>(definition, {}, contentMessage("sections", file));
+  assert.equal(one.result.state.sections, 3);
+
+  // The next read of the bytes answers 404: the run ends without an error.
+  content.remove(file.sha256);
+  const old = await runMessage<SectionsState>(
+    definition,
+    one.result.continuation!.message,
+    one.result.state,
+  );
+  assert.equal(old.turns.length, 1, "the run ends at the turn that read nothing");
+  assert.deepEqual(old.state, { contents: 0, skipped: 1, sections: 3, represents: 0 });
+  assert.equal(content.records.length, 0, "no derivation is recorded");
+});
+
 test("REPRESENTS needs exactly one instance whose label is the file's name", async () => {
   const counts: number[] = [];
   for (const guides of [

package/templates/sections/workflow.ts

@@ -19,13 +19,19 @@
 //    to 44);
 // 4. with `represents`, writes `Document REPRESENTS <instance>` when the
 //    file's name names exactly one instance of that class;
-// 5. records the derivation.
+// 5. records the derivation, for the upload its check named. When the file
+//    was deleted (or deleted and uploaded again) since, nothing is recorded,
+//    and the run of the new upload reads the file again. A file deleted and
+//    not uploaded again answers 404 to the next read of its bytes: the run
+//    ends there and counts the file as skipped.
 //
 // A turn writes at most `sectionsPerTurn` sections; the rest goes to a new
 // turn (`ctx.continue`).
 import { createHash } from "node:crypto";
 import {
+  LbbError,
   entityRef,
+  isDerivationUploadChanged,
   triggeredWorkflow,
   type LbbActivities,
   type Schemas,
@@ -121,7 +127,7 @@ export interface ContentPorts {
 export interface SectionsState {
   /** Files read to their end. */
   contents: number;
-  /** Files skipped: read before with these settings, another name or type, or no file. */
+  /** Files skipped: read before with these settings, another name or type, no file, or deleted while read. */
   skipped: number;
   sections: number;
   represents: number;
@@ -135,9 +141,11 @@ interface FileInfo {
   mediaType: string;
 }
 
-/** What the check of a file found: the file to read, or why it is skipped. */
+/** What the check of a file found: the file to read and its upload, or why it is skipped. */
 interface Checked {
   file?: FileInfo;
+  /** The `content_uploaded_at` of the check: absent without a file record. */
+  uploadedAt?: string;
   skip?: string;
 }
 
@@ -148,6 +156,8 @@ interface Plan {
   name: string;
   count: number;
   represents: string | null;
+  /** The upload of the file the check named: the record names it. */
+  uploadedAt?: string;
 }
 
 /** Where a file's next turn starts, and the commits of its writes so far. */
@@ -174,6 +184,10 @@ const LABEL = "http://www.w3.org/2000/01/rdf-schema#label";
 /** The media type without parameters, in lower case. */
 const baseType = (mediaType: string) => mediaType.split(";")[0]!.trim().toLowerCase();
 
+/** Whether an error is the 404 of a file's bytes: the file was deleted and not uploaded again. */
+export const fileGone = (error: unknown) =>
+  error instanceof LbbError && error.status === 404 && error.code === "file_not_found";
+
 /** A file name without its last extension. */
 export const stem = (name: string) => name.replace(/\.[^./\\]*$/, "");
 
@@ -244,108 +258,133 @@ export function createWorkflow(
         delete plain.cursor;
         return ctx.continue(next, plain);
       };
-      const document = message.instances[0];
-      if (!document) return { state };
-      let split: Promise<Section[]> | undefined;
-      const sectionsOf = (sha256: string, name: string) =>
-        (split ??= ports.files
-          .content(sha256)
-          .then((bytes) => markdownSections(new TextDecoder().decode(bytes), stem(name))));
-
-      let cursor = message.cursor;
-      if (!cursor) {
-        const checked = await ctx.step("check", async (s): Promise<Checked> => {
-          s.progress("checking the file", { done: 0, total: 1, unit: "documents" });
-          const file = await readDocument(message, document);
-          if (!file) return { skip: "no file" };
-          if (!isMarkdown(file.mediaType, file.name)) return { skip: "media type" };
-          if (match && !match.test(file.name)) return { skip: "name" };
-          const found = await ports.workflows.derivations.check(
-            derivationKey(file.sha256, settings),
-          );
-          return found.exists ? { skip: "derived" } : { file };
-        });
-        if (!checked.file) return rest({ ...state, skipped: state.skipped + 1 });
-        const file = checked.file;
-        const planned = await ctx.step("plan", async (s) => {
-          s.progress("reading sections", { done: 0, unit: "sections" });
-          const count = (await sectionsOf(file.sha256, file.name)).length;
-          s.progress("reading sections", { done: count, total: count, unit: "sections" });
-          const found = settings.represents ? await representsOf(file.name) : null;
-          const plan: Plan = { ...file, count, represents: found?.iri ?? null };
-          return { plan, label: found?.label ?? null };
-        });
-        const commits: number[] = [];
-        const p = planned.plan;
-        if (p.represents) {
-          const written = await lbb.writeOwned(ctx, "represents", (s) => {
-            s.progress("writing facts", { done: 0, total: 1, unit: "documents" });
-            return {
-              subjects: [entityRef(p.document, "Document", p.name)],
-              relations: [REPRESENTS],
-              triplets: [
-                {
-                  source: entityRef(p.document, "Document", p.name),
-                  relation: REPRESENTS,
-                  target: entityRef(
-                    p.represents!,
-                    settings.represents!.class,
-                    planned.label ?? undefined,
-                  ),
-                  confidence: 1,
-                  evidence: { text: p.name, source_id: p.document },
-                },
-              ],
+      try {
+        const document = message.instances[0];
+        if (!document) return { state };
+        let split: Promise<Section[]> | undefined;
+        const sectionsOf = (sha256: string, name: string) =>
+          (split ??= ports.files
+            .content(sha256)
+            .then((bytes) => markdownSections(new TextDecoder().decode(bytes), stem(name))));
+
+        let cursor = message.cursor;
+        if (!cursor) {
+          const checked = await ctx.step("check", async (s): Promise<Checked> => {
+            s.progress("checking the file", { done: 0, total: 1, unit: "documents" });
+            const file = await readDocument(message, document);
+            if (!file) return { skip: "no file" };
+            if (!isMarkdown(file.mediaType, file.name)) return { skip: "media type" };
+            if (match && !match.test(file.name)) return { skip: "name" };
+            const found = await ports.workflows.derivations.check(
+              derivationKey(file.sha256, settings),
+            );
+            if (found.exists) return { skip: "derived" };
+            const uploadedAt = found.content_uploaded_at ?? undefined;
+            return uploadedAt ? { file, uploadedAt } : { file };
+          });
+          if (!checked.file) return rest({ ...state, skipped: state.skipped + 1 });
+          const file = checked.file;
+          const planned = await ctx.step("plan", async (s) => {
+            s.progress("reading sections", { done: 0, unit: "sections" });
+            const count = (await sectionsOf(file.sha256, file.name)).length;
+            s.progress("reading sections", { done: count, total: count, unit: "sections" });
+            const found = settings.represents ? await representsOf(file.name) : null;
+            const plan: Plan = {
+              ...file,
+              count,
+              represents: found?.iri ?? null,
+              ...(checked.uploadedAt ? { uploadedAt: checked.uploadedAt } : {}),
             };
+            return { plan, label: found?.label ?? null };
+          });
+          const commits: number[] = [];
+          const p = planned.plan;
+          if (p.represents) {
+            const written = await lbb.writeOwned(ctx, "represents", (s) => {
+              s.progress("writing facts", { done: 0, total: 1, unit: "documents" });
+              return {
+                subjects: [entityRef(p.document, "Document", p.name)],
+                relations: [REPRESENTS],
+                triplets: [
+                  {
+                    source: entityRef(p.document, "Document", p.name),
+                    relation: REPRESENTS,
+                    target: entityRef(
+                      p.represents!,
+                      settings.represents!.class,
+                      planned.label ?? undefined,
+                    ),
+                    confidence: 1,
+                    evidence: { text: p.name, source_id: p.document },
+                  },
+                ],
+              };
+            });
+            if (written.commit_seq !== null) commits.push(written.commit_seq);
+          }
+          cursor = { next: 0, plan: p, commits };
+        }
+
+        // The sections of this turn, written in parts.
+        const { plan, next: from } = cursor;
+        const to = Math.min(plan.count, from + settings.sectionsPerTurn);
+        const commits = [...cursor.commits];
+        for (let n = 0; from + n * PART < to; n++) {
+          const written = await lbb.writeOwned(ctx, `write-${n}`, async (s) => {
+            s.progress("writing facts", { done: n * PART, total: to - from, unit: "sections" });
+            const all = await sectionsOf(plan.sha256, plan.name);
+            const start = from + n * PART;
+            return partWrite(plan, all, all.slice(start, Math.min(to, start + PART)));
           });
           if (written.commit_seq !== null) commits.push(written.commit_seq);
         }
-        cursor = { next: 0, plan: p, commits };
-      }
+        const counted = { ...state, sections: state.sections + (to - from) };
+        if (to < plan.count)
+          return ctx.continue(counted, {
+            ...message,
+            removed: [],
+            cursor: { next: to, plan, commits: commits.slice(0, 256) },
+          });
 
-      // The sections of this turn, written in parts.
-      const { plan, next: from } = cursor;
-      const to = Math.min(plan.count, from + settings.sectionsPerTurn);
-      const commits = [...cursor.commits];
-      for (let n = 0; from + n * PART < to; n++) {
-        const written = await lbb.writeOwned(ctx, `write-${n}`, async (s) => {
-          s.progress("writing facts", { done: n * PART, total: to - from, unit: "sections" });
-          const all = await sectionsOf(plan.sha256, plan.name);
-          const start = from + n * PART;
-          return partWrite(plan, all, all.slice(start, Math.min(to, start + PART)));
+        const recorded = await ctx.step("record", async (s) => {
+          s.progress("recording the derivation", { done: 0, total: 1, unit: "documents" });
+          const all = plan.count ? await sectionsOf(plan.sha256, plan.name) : [];
+          const levels: Record<string, number> = {};
+          for (const section of all) levels[section.level] = (levels[section.level] ?? 0) + 1;
+          try {
+            const answer = await ports.workflows.derivations.record({
+              ...derivationKey(plan.sha256, settings),
+              run_id: ctx.messageId,
+              outputs: {
+                sections: plan.count,
+                levels,
+                represents: plan.represents,
+                commits: commits.slice(0, 256),
+              },
+              ...(plan.uploadedAt ? { content_uploaded_at: plan.uploadedAt } : {}),
+            });
+            return answer.recorded;
+          } catch (error) {
+            // The file was deleted while this run read it: nothing is
+            // recorded, and the run of a new upload reads it again.
+            if (isDerivationUploadChanged(error)) return "upload changed";
+            throw error;
+          }
         });
-        if (written.commit_seq !== null) commits.push(written.commit_seq);
-      }
-      const counted = { ...state, sections: state.sections + (to - from) };
-      if (to < plan.count)
-        return ctx.continue(counted, {
-          ...message,
-          removed: [],
-          cursor: { next: to, plan, commits: commits.slice(0, 256) },
+        if (recorded === "upload changed")
+          return rest({ ...counted, skipped: counted.skipped + 1 });
+        return rest({
+          ...counted,
+          contents: counted.contents + 1,
+          represents: counted.represents + (plan.represents ? 1 : 0),
         });
-
-      await ctx.step("record", async (s) => {
-        s.progress("recording the derivation", { done: 0, total: 1, unit: "documents" });
-        const all = plan.count ? await sectionsOf(plan.sha256, plan.name) : [];
-        const levels: Record<string, number> = {};
-        for (const section of all) levels[section.level] = (levels[section.level] ?? 0) + 1;
-        const recorded = await ports.workflows.derivations.record({
-          ...derivationKey(plan.sha256, settings),
-          run_id: ctx.messageId,
-          outputs: {
-            sections: plan.count,
-            levels,
-            represents: plan.represents,
-            commits: commits.slice(0, 256),
-          },
-        });
-        return recorded.recorded;
-      });
-      return rest({
-        ...counted,
-        contents: counted.contents + 1,
-        represents: counted.represents + (plan.represents ? 1 : 0),
-      });
+      } catch (error) {
+        // The file was deleted while this run read it, and not uploaded
+        // again: its bytes answer 404. Nothing is recorded, and the file
+        // counts as skipped.
+        if (!fileGone(error)) throw error;
+        return rest({ ...state, skipped: state.skipped + 1 });
+      }
     },
   });
 

package/templates/versions.json

@@ -1,3 +1,3 @@
 {
-  "client": "0.27.0"
+  "client": "0.28.0"
 }