@dxos/pipeline-email 0.0.0

package/src/testing/email-extraction.bench.test.ts

@@ -0,0 +1,117 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Stream from 'effect/Stream';
+import { existsSync } from 'node:fs';
+import { readdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { describe, test } from 'vitest';
+
+import { Provider } from '@dxos/ai';
+import { OllamaAiServiceLayer } from '@dxos/ai/testing';
+import { EffectEx } from '@dxos/effect';
+import { log } from '@dxos/log';
+import { Pipeline } from '@dxos/pipeline';
+import { type DocumentFacts, extractFactsStage } from '@dxos/pipeline-rdf';
+import { captureSink, instrument, renderBenchmark, runBenchmark } from '@dxos/pipeline/testing';
+
+import { EMAIL_EXTRACT_OPTIONS, messageToDocument } from '../stages/facts';
+import { emailToMessage } from './email-fixtures';
+import { parquetSource } from './parquet';
+
+// Compares fact-extraction quality across local models over the head of the Enron dataset. Same
+// gating as email-pipeline.test.ts: needs the dataset under ROOT_DIR and a running Ollama, so CI and
+// un-provisioned checkouts skip it. This is the first concrete use of the `@dxos/pipeline/testing`
+// benchmark framework: an instrumented extraction stage feeds per-stage counters, and an effectful
+// evaluator derives the domain metrics (facts per message, extraction success rate).
+const DEFAULT_ROOT_DIR = fileURLToPath(new URL('../../data/enron-emails', import.meta.url));
+const ROOT_DIR = process.env.ROOT_DIR ?? DEFAULT_ROOT_DIR;
+const HAS_DATASET = existsSync(join(ROOT_DIR, 'data'));
+
+// Where the benchmark's serialized comparison is written (git-ignored, under the package's ./data).
+const BENCH_FILE = fileURLToPath(new URL('../../data/benchmark.json', import.meta.url));
+
+// Models to compare (Ollama DXNs).
+const BENCH_MODELS = [
+  'com.meta.model.llama-3-2-3b.instruct',
+  'com.alibaba.model.qwen-2-5-7b.instruct',
+  'com.google.model.gemma-4-12b.default',
+];
+
+// Emails drawn from the head of the dataset per model. Kept modest by default (each is one LLM call
+// per model); override via BENCH_COUNT for a heavier comparison.
+const BENCH_COUNT = Number(process.env.BENCH_COUNT ?? 10);
+
+describe.skipIf(!HAS_DATASET)('email extraction benchmark (ROOT_DIR + Ollama gated)', () => {
+  test(
+    'compares fact extraction across models',
+    async ({ expect }) => {
+      const dataDir = join(ROOT_DIR, 'data');
+      const files = (await readdir(dataDir))
+        .filter((name) => /^train-.*\.parquet$/.test(name))
+        .sort()
+        .map((name) => join(dataDir, name));
+      expect(files.length).toBeGreaterThan(0);
+
+      const variants = BENCH_MODELS.map((model) => ({ name: modelLabel(model), config: { model } }));
+
+      const result = await EffectEx.runPromise(
+        runBenchmark({
+          variants,
+          // Stream the first BENCH_COUNT emails through the instrumented extraction stage under the
+          // variant's model; the fallback in pipeline-rdf degrades a failed extraction to no facts, so
+          // a weak model yields low facts-per-message rather than aborting the run.
+          program: ({ model }) => {
+            const { sink, items } = captureSink<DocumentFacts>();
+            return parquetSource(files)
+              .pipe(
+                Stream.take(BENCH_COUNT),
+                Stream.map(emailToMessage),
+                Stream.map(messageToDocument),
+                instrument(
+                  'extract',
+                  extractFactsStage({ ...EMAIL_EXTRACT_OPTIONS, model, provider: Provider.ollama.id, strict: false }),
+                ),
+                Pipeline.run({ sink }),
+              )
+              .pipe(Effect.provide(OllamaAiServiceLayer), Effect.as(items));
+          },
+          // Domain metrics computed from the captured per-document facts.
+          evaluate: (_config, docs) => {
+            const facts = docs.reduce((total, entry) => total + entry.facts.length, 0);
+            const withFacts = docs.filter((entry) => entry.facts.length > 0).length;
+            return Effect.succeed({
+              'messages': docs.length,
+              facts,
+              'facts.per.msg': docs.length ? round(facts / docs.length) : 0,
+              'success.rate': docs.length ? round(withFacts / docs.length) : 0,
+            });
+          },
+        }),
+      );
+
+      const table = renderBenchmark(result);
+      log.info(`email extraction benchmark (${BENCH_COUNT} emails)\n${table}`);
+      await writeFile(BENCH_FILE, JSON.stringify(result, null, 2));
+
+      expect(result.variants.length).toBe(variants.length);
+      // Every variant ran the extraction stage over the whole sample (fewer only if the dataset is
+      // smaller). A model that produced zero facts still counts as a completed, comparable run.
+      for (const variant of result.variants) {
+        expect(variant.metrics['extract.in']).toBeGreaterThan(0);
+        expect(variant.metrics.messages).toBe(variant.metrics['extract.out']);
+      }
+    },
+    // Two LLM calls at most per message per model; budget generously so larger BENCH_COUNT runs and
+    // slower models don't trip the per-test timeout.
+    Math.max(5 * 60_000, BENCH_MODELS.length * BENCH_COUNT * 45_000),
+  );
+});
+
+// Short display label from a model DXN, e.g. `com.openai.model.gpt-oss-20b.default` -> `gpt-oss-20b`.
+const modelLabel = (model: string): string => model.split('.').at(-2) ?? model;
+
+const round = (value: number): number => Math.round(value * 100) / 100;

package/src/testing/email-fixtures.ts

@@ -0,0 +1,62 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { asyncBufferFromUrl, parquetMetadataAsync, parquetReadObjects } from 'hyparquet';
+
+import { type ContentBlock, Message } from '@dxos/types';
+
+import { type ParquetRow } from './parquet';
+
+// Maps rows from the Enron email dataset (see the dataset's `dataset_info` schema) onto ECHO Messages.
+
+/**
+ * First shard of the Enron email dataset (https://huggingface.co/datasets/corbt/enron-emails). The
+ * HuggingFace CDN serves it with range requests + permissive CORS, so a browser reads only the first
+ * N rows (a few row-group byte ranges) rather than the whole ~185 MB file.
+ */
+export const DEFAULT_ENRON_URL =
+  'https://huggingface.co/datasets/corbt/enron-emails/resolve/main/data/train-00000-of-00003.parquet';
+
+export type LoadEnronOptions = {
+  /** Number of messages to read from the start of the dataset. */
+  readonly count?: number;
+  /** Parquet URL to read from; defaults to {@link DEFAULT_ENRON_URL}. */
+  readonly url?: string;
+};
+
+/**
+ * Load the first `count` (default 100) Enron messages directly from the dataset parquet over HTTP.
+ * Browser-safe: uses hyparquet's URL async buffer (range requests), so only the byte ranges the first
+ * rows need are fetched. Rows are mapped to ECHO Messages via {@link emailToMessage}.
+ */
+export const loadEnronMessages = async ({ count = 100, url = DEFAULT_ENRON_URL }: LoadEnronOptions = {}): Promise<
+  Message.Message[]
+> => {
+  const file = await asyncBufferFromUrl({ url });
+  const metadata = await parquetMetadataAsync(file);
+  const rows = await parquetReadObjects({ file, metadata, rowStart: 0, rowEnd: count });
+  return rows.map((row) => emailToMessage(row as ParquetRow));
+};
+
+/** Coerce a parquet timestamp cell (a `Date`, or a string/number) to an ISO string. */
+export const asIso = (value: unknown): string =>
+  (value instanceof Date ? value : new Date(String(value))).toISOString();
+
+/** Map one email row (see the dataset's `dataset_info` schema) to a Message carrying the body as a text block. */
+export const emailToMessage = (row: ParquetRow): Message.Message => {
+  const block: ContentBlock.Text = { _tag: 'text', text: String(row.body ?? '') };
+  return Message.make({
+    created: asIso(row.date),
+    sender: { email: String(row.from ?? '') },
+    blocks: [block],
+    properties: {
+      messageId: row.message_id,
+      subject: row.subject,
+      to: row.to,
+      cc: row.cc,
+      bcc: row.bcc,
+      fileName: row.file_name,
+    },
+  });
+};

package/src/testing/email-pipeline.test.ts

@@ -0,0 +1,483 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as LanguageModel from '@effect/ai/LanguageModel';
+import * as Cause from 'effect/Cause';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as ManagedRuntime from 'effect/ManagedRuntime';
+import * as Schema from 'effect/Schema';
+import * as Stream from 'effect/Stream';
+import { existsSync } from 'node:fs';
+import { readdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterAll, beforeAll, describe, test } from 'vitest';
+
+import { AiService, Provider } from '@dxos/ai';
+import { OllamaAiServiceLayer } from '@dxos/ai/testing';
+import { type Database, Filter, Obj } from '@dxos/echo';
+import { EchoTestBuilder } from '@dxos/echo-client/testing';
+import { EffectEx } from '@dxos/effect';
+import { extractContact } from '@dxos/extractor-lib';
+import { log } from '@dxos/log';
+import { Pipeline, Stage } from '@dxos/pipeline';
+import { FactPipeline, FactStore } from '@dxos/pipeline-rdf';
+import { Metrics, captureSink, instrument, makeMetrics } from '@dxos/pipeline/testing';
+import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
+import { trim } from '@dxos/util';
+
+import { buildDigest, narrateDigest, renderDigest } from '../corpus/digest';
+import { commitmentLedger } from '../corpus/ledger';
+import { type Summarizer } from '../corpus/prompts';
+import { buildRollups } from '../corpus/rollups';
+import { clusterThreads, materializeTopics, summarizeTopics } from '../corpus/topics';
+import { buildEntityIndex, reconcileFactEntities } from '../internal/fact-index';
+import { buildThreads } from '../internal/threads';
+import { type FactIndexer, extractFactsStage } from '../stages/extract-facts';
+import { EMAIL_EXTRACT_OPTIONS, messageToDocument } from '../stages/facts';
+import { Thread, Topic } from '../types';
+import { emailToMessage } from './email-fixtures';
+import { parquetSource } from './parquet';
+
+// The email dataset (https://huggingface.co/datasets/corbt/enron-emails) lives under ROOT_DIR with
+// layout `${ROOT_DIR}/data/train-*.parquet`. ROOT_DIR defaults to the local checkout produced by
+// `moon run pipeline-email:setup` (packages/core/compute/pipeline-email/data/enron-emails); override
+// it via the ROOT_DIR env var. The suite is skipped unless that dataset is actually present — it also
+// needs a running Ollama (localhost:11434) and the native better-sqlite3 addon — so CI and
+// un-provisioned checkouts skip it.
+const DEFAULT_ROOT_DIR = fileURLToPath(new URL('../../data/enron-emails', import.meta.url));
+const ROOT_DIR = process.env.ROOT_DIR ?? DEFAULT_ROOT_DIR;
+
+// Where the run's serialized outputs are written (git-ignored, under the package's ./data).
+const RESULTS_FILE = fileURLToPath(new URL('../../data/results.json', import.meta.url));
+
+const TEST_ORGS = [
+  {
+    name: 'DXOS',
+    website: 'https://dxos.org',
+  },
+  {
+    name: 'Enron',
+    website: 'https://enron.com',
+  },
+];
+
+// Gate on the dataset shard directory existing locally.
+const HAS_DATASET = existsSync(join(ROOT_DIR, 'data'));
+
+// Local model served by Ollama (model DXN; its final NSID segment must be camelCase).
+const MODEL = process.env.OLLAMA_MODEL ?? 'com.openai.model.gpt-oss-20b.default';
+
+// Number of emails drawn from the head of the dataset for one run. Defaults to 50 so the gated run
+// spans multiple threads/senders and actually exercises the corpus-layer assertions (clustering,
+// rollups, digest); override via EMAIL_COUNT (e.g. EMAIL_COUNT=1) for fast local iteration.
+const EMAIL_COUNT = Number(process.env.EMAIL_COUNT ?? 50);
+
+const SUMMARIZE_PROMPT = trim`
+  Summarize the following email in one sentence, decide whether it is spam, and list up to five keywords.
+  Respond with ONLY a JSON object of the form {"summary": string, "isSpam": boolean, "keywords": string[]}.
+`;
+
+// Structured output the summarize stage asks the model for.
+const SummarySchema = Schema.Struct({
+  summary: Schema.String,
+  isSpam: Schema.Boolean,
+  keywords: Schema.Array(Schema.String),
+});
+
+type Summary = Schema.Schema.Type<typeof SummarySchema>;
+
+// Model output is untyped JSON, often wrapped in prose/reasoning by local models; extract the first
+// object and coerce leniently (field names and types vary between models) rather than strict schema
+// validation, which gpt-oss/Ollama frequently fails (`generateObject` → MalformedOutput).
+const parseSummary = (raw: string): Summary => {
+  const match = raw.match(/\{[\s\S]*\}/);
+  if (!match) {
+    return { summary: '', isSpam: false, keywords: [] };
+  }
+
+  try {
+    const parsed = JSON.parse(match[0]);
+    const keywords = parsed.keywords;
+    return {
+      summary: typeof parsed.summary === 'string' ? parsed.summary : '',
+      isSpam: parsed.isSpam === true || parsed.spam === true || parsed.is_spam === true,
+      keywords: Array.isArray(keywords)
+        ? keywords.map((keyword: unknown) => String(keyword))
+        : typeof keywords === 'string'
+          ? keywords
+              .split(',')
+              .map((keyword) => keyword.trim())
+              .filter(Boolean)
+          : [],
+    };
+  } catch {
+    return {
+      summary: '',
+      isSpam: false,
+      keywords: [],
+    };
+  }
+};
+
+// Running tallies produced by the stats stage; readable after the run for assertions.
+type Stats = {
+  readonly from: Map<string, number>;
+  readonly to: Map<string, number>;
+  total: number;
+  spam: number;
+};
+
+// Shared context threaded through every stage via Effect's Requirements channel. The service
+// closure and db are built once up-front and provided at the edge (`Effect.provide` below), so the
+// individual stage Effects carry `R = Ctx` but the fully-composed program is reduced back to
+// `R = never` before `EffectEx.runPromise` — the same "provide at the edge" idiom used elsewhere in
+// this repo (e.g. `Database.layer`). The heavy bits (Ollama runtime, better-sqlite3-backed db) live
+// here in the test harness, which is why the whole suite is env-gated.
+class Ctx extends Context.Tag('EmailPipelineCtx')<
+  Ctx,
+  {
+    readonly summarize: (text: string) => Promise<Summary>;
+    readonly db: Database.Database;
+    readonly stats: Stats;
+  }
+>() {}
+
+// Stage 1: LLM summarization. Appends a second text block carrying the summary and records spam /
+// keyword metadata on `Message.properties` (ContentBlock.Text has no metadata field). Produces a new
+// Message rather than mutating in place. The model call degrades gracefully to an empty summary when
+// Ollama is unreachable or a row fails (`tryPromise` captures the rejection so `orElse` can recover),
+// so the suite stays green whenever the dataset is checked out — real summaries when Ollama is up, an
+// empty-summary no-op otherwise.
+const summarizeStage: Stage.Stage<Message.Message, Message.Message, never, Ctx> = Stage.map('summarize', (message) =>
+  Effect.gen(function* () {
+    const { summarize } = yield* Ctx;
+    const text = Message.extractText(message);
+    const result = yield* Effect.tryPromise(() => summarize(text)).pipe(
+      Effect.orElse(() => Effect.succeed<Summary>({ summary: '', isSpam: false, keywords: [] })),
+    );
+    const summaryBlock: ContentBlock.Text = { _tag: 'text', text: result.summary };
+    return Message.make({
+      created: message.created,
+      sender: message.sender,
+      blocks: [...message.blocks, summaryBlock],
+      properties: {
+        ...message.properties,
+        spam: result.isSpam,
+        keywords: result.keywords,
+      },
+    });
+  }),
+);
+
+// Stage 2: run the shared `contactExtractor` (the `@dxos/extractor` abstraction) to build a Person
+// (+ Organization link by domain) from the sender, then persist the extractor's uncommitted output
+// (normally the dispatcher's role). The extractor's `extract` has R = never, so it composes into
+// the Requirements channel the same way any other stage dependency does. Passes the Message through
+// unchanged.
+const extractStage: Stage.Stage<Message.Message, Message.Message, never, Ctx> = Stage.map(
+  'extract-contact',
+  (message) =>
+    Effect.gen(function* () {
+      const { db } = yield* Ctx;
+      const result = yield* extractContact({ db, source: message });
+      for (const object of result.created) {
+        db.add(object);
+      }
+      return message;
+    }),
+);
+
+// Stage 3: pure-JS running tallies (no analysis libs — a Cloudflare-safe option would be a small
+// pure-JS stats package, but none is added here). Mutates the context-carried accumulator and passes
+// the Message through so it reaches the sink for collection.
+const statsStage: Stage.Stage<Message.Message, Message.Message, never, Ctx> = Stage.map('stats', (message) =>
+  Effect.gen(function* () {
+    const { stats } = yield* Ctx;
+    stats.total += 1;
+    const sender = message.sender.email;
+    if (sender) {
+      stats.from.set(sender, (stats.from.get(sender) ?? 0) + 1);
+    }
+    const recipients = message.properties?.to;
+    if (Array.isArray(recipients)) {
+      for (const recipient of recipients) {
+        const address = String(recipient);
+        stats.to.set(address, (stats.to.get(address) ?? 0) + 1);
+      }
+    }
+    if (message.properties?.spam) {
+      stats.spam += 1;
+    }
+    return message;
+  }),
+);
+
+// Bookend logging stage: emits one line describing the message as it enters/leaves the pipeline and
+// passes it through unchanged. (Visibility is controlled by LOG_FILTER, e.g. `LOG_FILTER=info`.)
+// Logs only the sender domain, not the full address, to avoid recording PII. Needs no shared context,
+// so it stays `R = never` even inside a pipeline that otherwise requires `Ctx`.
+const logStage = (label: string): Stage.Stage<Message.Message, Message.Message> =>
+  Stage.map(`log:${label}`, (message) =>
+    Effect.sync(() => {
+      const senderDomain = message.sender.email?.split('@')[1];
+      log.info(label, {
+        senderDomain,
+        subject: message.properties?.subject,
+        blocks: message.blocks.length,
+        spam: message.properties?.spam,
+      });
+      return message;
+    }),
+  );
+
+describe.skipIf(!HAS_DATASET)('Enron email pipeline (ROOT_DIR + Ollama gated)', () => {
+  // Model layer built ONCE so it is not rebuilt per message.
+  // `AiService.model` provides the `LanguageModel`, resolved through the local Ollama provider;
+  // `OllamaAiServiceLayer` provides the `AiService` it requires.
+  const modelLayer = AiService.model(MODEL, { provider: Provider.ollama.id }).pipe(Layer.provide(OllamaAiServiceLayer));
+  const runtime = ManagedRuntime.make(modelLayer.pipe(Layer.orDie));
+
+  // In-memory fact substrate for this run; shares the Ollama-backed AiService the extraction resolves
+  // its model through (pipeline-rdf's ExtractOptions carries the model + provider).
+  const factRuntime = ManagedRuntime.make(FactStore.layerMemory.pipe(Layer.provideMerge(OllamaAiServiceLayer)));
+
+  let builder: EchoTestBuilder;
+  let db: Database.Database;
+
+  beforeAll(async () => {
+    builder = await new EchoTestBuilder().open();
+    ({ db } = await builder.createDatabase({ types: [Organization.Organization, Person.Person, Thread, Topic] }));
+    // Seed a known Organization so domain-matching can link a sender's Person to it.
+    for (const org of TEST_ORGS) {
+      db.add(Obj.make(Organization.Organization, org));
+    }
+    await db.flush({ indexes: true });
+  });
+
+  afterAll(async () => {
+    await runtime.dispose();
+    await factRuntime.dispose();
+    await builder.close();
+  });
+
+  test(
+    'summarizes, extracts contacts, and tallies stats over the first emails',
+    async ({ expect }) => {
+      const dataDir = join(ROOT_DIR, 'data');
+      const files = (await readdir(dataDir))
+        .filter((name) => /^train-.*\.parquet$/.test(name))
+        .sort()
+        .map((name) => join(dataDir, name));
+      expect(files.length).toBeGreaterThan(0);
+
+      const stats: Stats = { from: new Map(), to: new Map(), total: 0, spam: 0 };
+      // `generateText` + lenient parse rather than `generateObject`: local reasoning models (gpt-oss
+      // via Ollama) return valid JSON that fails strict schema conformance (→ MalformedOutput). The
+      // cause is logged and degraded to an empty summary on failure, which also keeps the inner fiber
+      // from raising a FiberFailure.
+      const summarize = (text: string): Promise<Summary> =>
+        runtime.runPromise(
+          Effect.scoped(LanguageModel.generateText({ prompt: [SUMMARIZE_PROMPT, text].join('\n\n') })).pipe(
+            Effect.map((response) => parseSummary(response.text)),
+            Effect.catchAllCause((cause) =>
+              Effect.sync(() => {
+                log.warn('summarize failed; using empty summary', { model: MODEL, cause: Cause.pretty(cause) });
+                return { summary: '', isSpam: false, keywords: [] } satisfies Summary;
+              }),
+            ),
+          ),
+        );
+
+      const context: Context.Tag.Service<typeof Ctx> = { summarize, db, stats };
+
+      // Index each message into the fact substrate. The model + provider ride on ExtractOptions so
+      // pipeline-rdf resolves the Ollama model (its default is Anthropic); a failed extraction
+      // degrades to no facts in `extractFactsStage`, so the run stays green. The counter proves the
+      // stage actually invoked indexing (facts themselves are advisory — see the assertions below).
+      let indexedMessageCount = 0;
+      const indexFacts: FactIndexer = (message) => {
+        indexedMessageCount += 1;
+        return factRuntime.runPromise(
+          FactPipeline.run([messageToDocument(message)], {
+            ...EMAIL_EXTRACT_OPTIONS,
+            model: MODEL,
+            provider: Provider.ollama.id,
+            // Ollama reliably fails strict generateObject; skip it and salvage from generateText.
+            strict: false,
+          }),
+        );
+      };
+
+      // Per-stage timing: each LLM-bearing stage is instrumented so the run reports where the
+      // per-message latency goes (`extract-facts.ms` dominates — it makes up to two model calls per
+      // message, see extractChunk). One Metrics instance for the run, read back after it completes.
+      const metrics = makeMetrics();
+      const { sink, items } = captureSink<Message.Message>();
+      await EffectEx.runPromise(
+        Effect.provide(
+          parquetSource(files).pipe(
+            Stream.take(EMAIL_COUNT),
+            Stream.map(emailToMessage),
+            logStage('email.in'),
+            instrument('summarize', summarizeStage),
+            instrument('extract-facts', extractFactsStage(indexFacts)),
+            instrument('extract-contact', extractStage),
+            instrument('stats', statsStage),
+            logStage('email.out'),
+            Pipeline.run({ sink }),
+          ),
+          Layer.merge(Layer.succeed(Ctx, context), Layer.succeed(Metrics, metrics)),
+        ),
+      );
+
+      // Timing breakdown (ms is the summed per-stage latency; divide by the stage's `.out` count for
+      // the per-message average). Surfaced in the log and serialized with the run's results.
+      const timing = await EffectEx.runPromise(metrics.snapshot);
+      log.info('pipeline timing', timing);
+
+      // The pipeline processes up to EMAIL_COUNT messages (fewer only if the dataset is smaller).
+      expect(items.length).toBeGreaterThan(0);
+      expect(items.length).toBeLessThanOrEqual(EMAIL_COUNT);
+
+      for (const message of items) {
+        const textBlocks = message.blocks.filter((block) => block._tag === 'text');
+        // Original body block plus the appended summary block.
+        expect(textBlocks.length).toBeGreaterThanOrEqual(2);
+        expect(typeof message.properties?.spam).toBe('boolean');
+        expect(Array.isArray(message.properties?.keywords)).toBe(true);
+      }
+
+      await db.flush({ indexes: true });
+      const persons = await db.query(Filter.type(Person.Person)).run();
+      expect(persons.length).toBeGreaterThan(0);
+
+      expect(stats.total).toBe(items.length);
+      expect(stats.from.size).toBeGreaterThan(0);
+      expect(stats.to.size).toBeGreaterThan(0);
+
+      // Facts are advisory: gpt-oss's structured `generateObject` output is only intermittently
+      // schema-conforming (the summarize stage sidesteps this with lenient `generateText`), so the
+      // fact count is best-effort here — the deterministic proof of extraction lives in
+      // extract-stage.test.ts (mockAiService). Assert only that the substrate is queryable.
+      // The extract-facts stage ran for every captured message (proves the wiring executed, even
+      // though the facts it produced are advisory and may be empty under a flaky model).
+      expect(indexedMessageCount).toBe(items.length);
+      const facts = await factRuntime.runPromise(
+        Effect.gen(function* () {
+          const store = yield* FactStore;
+          return yield* store.query({});
+        }),
+      );
+      expect(Array.isArray(facts)).toBe(true);
+
+      // Reconcile advisory fact entities against the canonical ECHO Person/Organization objects the
+      // pipeline created (slice-1 §9.1). Facts are sparse/flaky under gpt-oss, so the resolved count
+      // is best-effort here — the reconciliation logic itself is proven deterministically in
+      // fact-index.test.ts.
+      const orgsForIndex = await db.query(Filter.type(Organization.Organization)).run();
+      const entityIndex = buildEntityIndex([...persons, ...orgsForIndex]);
+      const resolved = facts.flatMap((fact) => {
+        const refs = reconcileFactEntities(fact, entityIndex);
+        return refs.subject || refs.object ? [refs] : [];
+      });
+      expect(Array.isArray(resolved)).toBe(true);
+
+      // Threads are deterministic (grouped from the captured messages), so assert them strictly: at
+      // least one thread is materialized and every stored thread references its messages.
+      const threads = buildThreads(items, { ownerEmail: 'owner@dxos.org', now: new Date().toISOString() });
+      expect(threads.length).toBeGreaterThan(0);
+      for (const thread of threads) {
+        db.add(thread);
+      }
+      await db.flush({ indexes: true });
+      const storedThreads = await db.query(Filter.type(Thread)).run();
+      expect(storedThreads.length).toBe(threads.length);
+      expect(storedThreads.every((thread) => thread.messageIds.length > 0)).toBe(true);
+
+      // Corpus layer (phase 3). Clustering is deterministic, so assert it strictly: every thread
+      // lands in exactly one topic. LLM enrichment (topic summaries, digest narrative) is advisory
+      // and degradable — proven deterministically in topics/digest unit tests.
+      const narrate: Summarizer = (prompt) =>
+        runtime.runPromise(
+          Effect.scoped(LanguageModel.generateText({ prompt })).pipe(
+            Effect.map((response) => response.text),
+            Effect.catchAllCause(() => Effect.succeed('')),
+          ),
+        );
+      const drafts = await summarizeTopics(clusterThreads(threads), narrate);
+      expect(drafts.length).toBeGreaterThan(0);
+      expect(drafts.flatMap((draft) => [...draft.threadIds]).sort()).toEqual(
+        threads.map((thread) => thread.threadId).sort(),
+      );
+      const topics = materializeTopics(drafts);
+      for (const topic of topics) {
+        db.add(topic);
+      }
+      await db.flush({ indexes: true });
+      const storedTopics = await db.query(Filter.type(Topic)).run();
+      expect(storedTopics.length).toBe(topics.length);
+
+      // Commitment ledger over the advisory fact store: rows (if any) must be grounded in a fact.
+      const commitments = await factRuntime.runPromise(
+        Effect.gen(function* () {
+          const store = yield* FactStore;
+          return yield* commitmentLedger(store);
+        }),
+      );
+      expect(commitments.every((commitment) => commitment.factId.length > 0 && commitment.source.length > 0)).toBe(
+        true,
+      );
+
+      // Rollups are deterministic: one per distinct sender among the processed messages.
+      const rollups = buildRollups(items);
+      const senders = new Set(
+        items.flatMap((message) => (message.sender.email ? [message.sender.email.toLowerCase()] : [])),
+      );
+      expect(rollups.length).toBe(senders.size);
+
+      // Digest skeleton is deterministic; the narrative is advisory (may be empty when degraded).
+      const digest = await narrateDigest(
+        buildDigest({ threads, topics: drafts, commitments, rollups }, { now: new Date().toISOString() }),
+        narrate,
+      );
+      expect(digest.threadCount).toBe(threads.length);
+      expect(digest.topicCount).toBe(drafts.length);
+      expect(typeof digest.narrative).toBe('string');
+
+      // Serialize the run's outputs for inspection (git-ignored under ./data). `Obj.toJSON` is the
+      // canonical ECHO serialization (encodes refs as DXNs).
+      const organizations = await db.query(Filter.type(Organization.Organization)).run();
+      await writeFile(
+        RESULTS_FILE,
+        JSON.stringify(
+          {
+            messages: items.map((message) => Obj.toJSON(message)),
+            organizations: organizations.map((organization) => Obj.toJSON(organization)),
+            persons: persons.map((person) => Obj.toJSON(person)),
+            topics: storedTopics.map((topic) => Obj.toJSON(topic)),
+            commitments,
+            rollups,
+            digest: renderDigest(digest),
+            timing,
+            stats: {
+              total: stats.total,
+              spam: stats.spam,
+              from: Object.fromEntries(stats.from),
+              to: Object.fromEntries(stats.to),
+            },
+          },
+          null,
+          2,
+        ),
+      );
+    },
+    // LLM calls dominate and scale with EMAIL_COUNT: per message a summarize + an extraction (which
+    // may make a second lenient call), plus per-topic summaries and the digest narration. Budget
+    // ~45s/message over a 5-minute floor so larger demo runs (e.g. EMAIL_COUNT=50) don't time out.
+    Math.max(5 * 60_000, EMAIL_COUNT * 45_000),
+  );
+});

package/src/testing/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './email-fixtures';
+// NOTE: parquet.ts (node:fs-backed `parquetSource`) is intentionally NOT re-exported here — the
+// testing entry must stay browser-safe (the Pipeline story imports `loadEnronMessages`). Node tests
+// import `./parquet` directly.