@dxos/pipeline-email 0.0.0

package/src/testing/parquet-email.test.ts

@@ -0,0 +1,114 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Stream from 'effect/Stream';
+import { readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { describe, test } from 'vitest';
+
+import { EffectEx } from '@dxos/effect';
+import { Pipeline, Stage } from '@dxos/pipeline';
+import { captureSink } from '@dxos/pipeline/testing';
+import { Message } from '@dxos/types';
+
+import { emailToMessage } from './email-fixtures';
+import { type ParquetRow, parquetSource } from './parquet';
+
+// The email dataset (https://huggingface.co/datasets/corbt/enron-emails) is exposed via ROOT_DIR;
+// its layout is `${ROOT_DIR}/data/train-*.parquet`.
+const ROOT_DIR = process.env.ROOT_DIR;
+
+type SenderCount = { readonly sender: string; readonly count: number };
+
+describe('email parquet → Message', () => {
+  test('maps an email row to a Message with a text body block', ({ expect }) => {
+    const row: ParquetRow = {
+      message_id: '<abc@example.com>',
+      subject: 'Hello',
+      from: 'alice@example.com',
+      to: ['bob@example.com'],
+      cc: [],
+      bcc: [],
+      date: new Date('2020-01-02T03:04:05.000Z'),
+      body: 'Body text.',
+      file_name: 'inbox/1.',
+    };
+
+    const message = emailToMessage(row);
+    expect(message.sender.email).toBe('alice@example.com');
+    expect(message.created).toBe('2020-01-02T03:04:05.000Z');
+    expect(Message.extractText(message)).toBe('Body text.');
+    expect(message.properties?.subject).toBe('Hello');
+    expect(message.properties?.to).toEqual(['bob@example.com']);
+    expect(message.properties?.messageId).toBe('<abc@example.com>');
+  });
+
+  test('a pipeline stage counts emails per sender', async ({ expect }) => {
+    const rows: ParquetRow[] = [
+      { from: 'alice@example.com', body: 'one' },
+      { from: 'bob@example.com', body: 'two' },
+      { from: 'alice@example.com', body: 'three' },
+      { from: 'carol@example.com', body: 'four' },
+    ];
+
+    const { sink, items } = captureSink<SenderCount>();
+    await EffectEx.runPromise(Stream.fromIterable(rows).pipe(countBySenderStage<never>(), Pipeline.run({ sink })));
+
+    // The stage emits a running count per email; the final tally is the last count seen per sender.
+    const totals = new Map(items.map(({ sender, count }) => [sender, count]));
+    expect(totals.get('alice@example.com')).toBe(2);
+    expect(totals.get('bob@example.com')).toBe(1);
+    expect(totals.get('carol@example.com')).toBe(1);
+  });
+
+  // Runs only when ROOT_DIR points at the local dataset; skipped in CI so the suite stays green.
+  describe.skipIf(!ROOT_DIR)('local dataset', () => {
+    test('reads shards and converts the first rows to Messages', async ({ expect }) => {
+      // Unreachable when unset (guarded by describe.skipIf); narrows to string without a cast.
+      const rootDir = ROOT_DIR;
+      if (!rootDir) {
+        return;
+      }
+
+      const dataDir = join(rootDir, 'data');
+      const files = (await readdir(dataDir))
+        .filter((name) => /^train-.*\.parquet$/.test(name))
+        .sort()
+        .map((name) => join(dataDir, name));
+      expect(files.length).toBeGreaterThan(0);
+
+      const messages = await EffectEx.runPromise(
+        parquetSource(files).pipe(
+          Stream.take(5),
+          Stream.map(emailToMessage),
+          Stream.runCollect,
+          Effect.map((chunk) => [...chunk]),
+        ),
+      );
+
+      expect(messages).toHaveLength(5);
+      for (const message of messages) {
+        expect(typeof message.sender.email).toBe('string');
+        expect(message.blocks.some((block) => block._tag === 'text')).toBe(true);
+        expect(message.properties).toBeDefined();
+      }
+    });
+  });
+});
+
+// A pipeline stage that maintains a running count of emails per sender, emitting the updated count
+// for each email's sender as it flows through (a stateful scan; the counter map is bounded by the
+// number of distinct senders, not the stream length).
+const countBySenderStage =
+  <E = never>(): Stage.Stage<ParquetRow, SenderCount, E> =>
+  (input) =>
+    input.pipe(
+      Stream.mapAccum(new Map<string, number>(), (counts, row) => {
+        const sender = String(row.from ?? '');
+        const count = (counts.get(sender) ?? 0) + 1;
+        counts.set(sender, count);
+        return [counts, { sender, count }];
+      }),
+    );

package/src/testing/parquet.test.ts

@@ -0,0 +1,73 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Stream from 'effect/Stream';
+import { parquetWriteFile } from 'hyparquet-writer';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, test } from 'vitest';
+
+import { EffectEx } from '@dxos/effect';
+
+import { ParquetReadError, type ParquetRow, parquetSource } from './parquet';
+
+// git clone https://huggingface.co/datasets/corbt/enron-emails
+
+describe('parquetSource', () => {
+  let dir: string;
+  let fileA: string;
+  let fileB: string;
+
+  beforeAll(async () => {
+    dir = await mkdtemp(join(tmpdir(), 'dxos-pipeline-parquet-'));
+    fileA = join(dir, 'a.parquet');
+    fileB = join(dir, 'b.parquet');
+    // rowGroupSize forces multiple row groups in fileA so the lazy per-group path is exercised.
+    parquetWriteFile({
+      filename: fileA,
+      columnData: [
+        { name: 'id', data: [1, 2, 3, 4], type: 'INT32' },
+        { name: 'name', data: ['a', 'b', 'c', 'd'], type: 'STRING' },
+      ],
+      rowGroupSize: 2,
+    });
+    parquetWriteFile({
+      filename: fileB,
+      columnData: [
+        { name: 'id', data: [5], type: 'INT32' },
+        { name: 'name', data: ['e'], type: 'STRING' },
+      ],
+    });
+  });
+
+  afterAll(async () => {
+    await rm(dir, { recursive: true, force: true });
+  });
+
+  test('streams rows across files in order, row group by row group', async ({ expect }) => {
+    const rows = await collect([fileA, fileB]);
+    expect(rows.map((row) => row.id)).toEqual([1, 2, 3, 4, 5]);
+    expect(rows.map((row) => row.name)).toEqual(['a', 'b', 'c', 'd', 'e']);
+  });
+
+  test('fails with ParquetReadError for a missing file', async ({ expect }) => {
+    const result = await EffectEx.runPromise(
+      parquetSource([join(dir, 'missing.parquet')]).pipe(Stream.runCollect, Effect.either),
+    );
+    expect(result._tag).toBe('Left');
+    if (result._tag === 'Left') {
+      expect(result.left).toBeInstanceOf(ParquetReadError);
+    }
+  });
+});
+
+const collect = (files: readonly string[]): Promise<readonly ParquetRow[]> =>
+  EffectEx.runPromise(
+    parquetSource(files).pipe(
+      Stream.runCollect,
+      Effect.map((chunk) => [...chunk]),
+    ),
+  );

package/src/testing/parquet.ts

@@ -0,0 +1,82 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Data from 'effect/Data';
+import * as Effect from 'effect/Effect';
+import * as Stream from 'effect/Stream';
+import { type AsyncBuffer, parquetMetadataAsync, parquetReadObjects } from 'hyparquet';
+import { open } from 'node:fs/promises';
+
+/** A decoded parquet row, keyed by column name. */
+export type ParquetRow = Record<string, unknown>;
+
+/** Failure opening, parsing, or decoding a parquet file; carries the offending path. */
+export class ParquetReadError extends Data.TaggedError('ParquetReadError')<{
+  readonly file: string;
+  readonly cause: unknown;
+}> {}
+
+type RowGroupRange = { readonly rowStart: number; readonly rowEnd: number };
+
+/**
+ * A pipeline source over a set of parquet files. Emits decoded rows as a back-pressured stream, one
+ * row group at a time, so decoded-row memory stays bounded regardless of file size. Files are read
+ * in the given order; within a file, rows follow row-group order.
+ */
+export const parquetSource = (files: readonly string[]): Stream.Stream<ParquetRow, ParquetReadError> =>
+  Stream.fromIterable(files).pipe(Stream.flatMap(fileRows, { concurrency: 1 }));
+
+// Absolute [rowStart, rowEnd) ranges, one per row group; `num_rows` is a bigint in the metadata.
+const rowGroupRanges = (metadata: Awaited<ReturnType<typeof parquetMetadataAsync>>): RowGroupRange[] => {
+  const ranges: RowGroupRange[] = [];
+  let rowStart = 0;
+  for (const group of metadata.row_groups) {
+    const rowEnd = rowStart + Number(group.num_rows);
+    ranges.push({ rowStart, rowEnd });
+    rowStart = rowEnd;
+  }
+  return ranges;
+};
+
+// One file's rows, decoded a row group at a time. The fs handle is scoped to the stream (opened on
+// start, closed on completion/interruption via acquireRelease + unwrapScoped); the AsyncBuffer
+// slices byte ranges on demand so only the bytes a row group needs are read.
+const fileRows = (file: string): Stream.Stream<ParquetRow, ParquetReadError> =>
+  Stream.unwrapScoped(
+    Effect.gen(function* () {
+      const handle = yield* Effect.acquireRelease(
+        Effect.tryPromise({ try: () => open(file, 'r'), catch: (cause) => new ParquetReadError({ file, cause }) }),
+        // `tryPromise` + `ignore` so a `close()` rejection is a swallowed failure, not a scope-killing defect.
+        (handle) => Effect.tryPromise(() => handle.close()).pipe(Effect.ignore),
+      );
+      const { size } = yield* Effect.tryPromise({
+        try: () => handle.stat(),
+        catch: (cause) => new ParquetReadError({ file, cause }),
+      });
+      const asyncBuffer: AsyncBuffer = {
+        byteLength: size,
+        slice: async (start, end = size) => {
+          // Ranges hyparquet requests are always within [0, byteLength) (derived from the metadata),
+          // so the read fills the whole view; no short-read handling is needed.
+          const view = new Uint8Array(end - start);
+          await handle.read(view, 0, view.byteLength, start);
+          return view.buffer;
+        },
+      };
+      const metadata = yield* Effect.tryPromise({
+        try: () => parquetMetadataAsync(asyncBuffer),
+        catch: (cause) => new ParquetReadError({ file, cause }),
+      });
+      return Stream.fromIterable(rowGroupRanges(metadata)).pipe(
+        Stream.mapEffect((range) =>
+          Effect.tryPromise({
+            try: () =>
+              parquetReadObjects({ file: asyncBuffer, metadata, rowStart: range.rowStart, rowEnd: range.rowEnd }),
+            catch: (cause) => new ParquetReadError({ file, cause }),
+          }),
+        ),
+        Stream.flattenIterables,
+      );
+    }),
+  );

package/src/types/Thread.test.ts

@@ -0,0 +1,29 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { Obj } from '@dxos/echo';
+
+import { Thread } from './Thread';
+
+describe('Thread', () => {
+  test('constructs a Thread object with expected fields', ({ expect }) => {
+    const thread = Obj.make(Thread, {
+      threadId: 'deal terms',
+      subject: 'Deal terms',
+      summary: 'Negotiation over Q2 deal terms.',
+      state: 'awaiting-mine',
+      participants: ['a@x.com', 'b@x.com'],
+      messageIds: ['<m-1@x.com>', '<m-2@x.com>'],
+      openQuestions: ['What is the close date?'],
+      actionItems: ['Send revised terms'],
+    });
+
+    expect(Obj.instanceOf(Thread, thread)).toBe(true);
+    expect(thread.threadId).toBe('deal terms');
+    expect(thread.state).toBe('awaiting-mine');
+    expect(thread.participants).toHaveLength(2);
+  });
+});

package/src/types/Thread.ts

@@ -0,0 +1,26 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { DXN, Type } from '@dxos/echo';
+
+export const ThreadState = Schema.Literal('awaiting-mine', 'awaiting-theirs', 'resolved', 'stalled');
+export type ThreadState = Schema.Schema.Type<typeof ThreadState>;
+
+// A canonical (spec §4) conversation aggregate: messages sharing a derived threadId, with a rolling
+// summary, a coarse state, and consolidated open questions / action items. Authoritative in ECHO;
+// facts are advisory evidence, not the source of this object's state.
+export class Thread extends Type.makeObject<Thread>(DXN.make('org.dxos.type.emailThread', '0.1.0'))(
+  Schema.Struct({
+    threadId: Schema.String,
+    subject: Schema.String,
+    summary: Schema.String,
+    state: ThreadState,
+    participants: Schema.Array(Schema.String),
+    messageIds: Schema.Array(Schema.String),
+    openQuestions: Schema.Array(Schema.String),
+    actionItems: Schema.Array(Schema.String),
+  }),
+) {}

package/src/types/Topic.test.ts

@@ -0,0 +1,25 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { Obj } from '@dxos/echo';
+
+import { Topic } from './Topic';
+
+describe('Topic', () => {
+  test('constructs a Topic object with expected fields', ({ expect }) => {
+    const topic = Obj.make(Topic, {
+      label: 'Q2 reporting',
+      summary: 'Threads about the Q2 report and budget confirmation.',
+      threadIds: ['q2 report', 'budget numbers'],
+      participants: ['alice@enron.com', 'bob@enron.com'],
+      keywords: ['q2', 'report', 'budget'],
+    });
+
+    expect(Obj.instanceOf(Topic, topic)).toBe(true);
+    expect(topic.threadIds).toHaveLength(2);
+    expect(topic.label).toBe('Q2 reporting');
+  });
+});

package/src/types/Topic.ts

@@ -0,0 +1,20 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { DXN, Type } from '@dxos/echo';
+
+// A canonical (spec §4) meta-thread: a cluster of related conversations discovered at the corpus
+// layer, with an optional LLM label/summary. Authoritative in ECHO; string-keyed thread references
+// (like Thread.messageIds) until the entity layer firms up to DXN refs.
+export class Topic extends Type.makeObject<Topic>(DXN.make('org.dxos.type.emailTopic', '0.1.0'))(
+  Schema.Struct({
+    label: Schema.String,
+    summary: Schema.String,
+    threadIds: Schema.Array(Schema.String),
+    participants: Schema.Array(Schema.String),
+    keywords: Schema.Array(Schema.String),
+  }),
+) {}

package/src/types/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './Thread';
+export * from './Topic';