@dxos/pipeline-email 0.0.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/lib/index.mjs +994 -0
  2. package/dist/lib/index.mjs.map +1 -0
  3. package/dist/lib/testing.mjs +49 -0
  4. package/dist/lib/testing.mjs.map +1 -0
  5. package/dist/types/src/corpus/digest.d.ts +0 -4
  6. package/dist/types/src/corpus/digest.d.ts.map +1 -1
  7. package/dist/types/src/corpus/topics.d.ts +12 -4
  8. package/dist/types/src/corpus/topics.d.ts.map +1 -1
  9. package/dist/types/src/fact-pipeline.d.ts +28 -3
  10. package/dist/types/src/fact-pipeline.d.ts.map +1 -1
  11. package/dist/types/src/index.d.ts +4 -0
  12. package/dist/types/src/index.d.ts.map +1 -1
  13. package/dist/types/src/internal/threads.d.ts +5 -1
  14. package/dist/types/src/internal/threads.d.ts.map +1 -1
  15. package/dist/types/src/model-policy.d.ts +17 -0
  16. package/dist/types/src/model-policy.d.ts.map +1 -0
  17. package/dist/types/src/model-policy.test.d.ts +2 -0
  18. package/dist/types/src/model-policy.test.d.ts.map +1 -0
  19. package/dist/types/src/stages/EmailStage.d.ts +149 -0
  20. package/dist/types/src/stages/EmailStage.d.ts.map +1 -0
  21. package/dist/types/src/stages/extract-facts-commit.d.ts +1 -1
  22. package/dist/types/src/stages/extract-facts-commit.d.ts.map +1 -1
  23. package/dist/types/src/stages/index.d.ts +2 -0
  24. package/dist/types/src/stages/index.d.ts.map +1 -1
  25. package/dist/types/src/stages/tag.d.ts +38 -0
  26. package/dist/types/src/stages/tag.d.ts.map +1 -0
  27. package/dist/types/src/stages/tag.test.d.ts +2 -0
  28. package/dist/types/src/stages/tag.test.d.ts.map +1 -0
  29. package/dist/types/src/topics-pipeline.d.ts +55 -0
  30. package/dist/types/src/topics-pipeline.d.ts.map +1 -0
  31. package/dist/types/src/topics-pipeline.test.d.ts +2 -0
  32. package/dist/types/src/topics-pipeline.test.d.ts.map +1 -0
  33. package/dist/types/src/types/index.d.ts +0 -1
  34. package/dist/types/src/types/index.d.ts.map +1 -1
  35. package/dist/types/tsconfig.tsbuildinfo +1 -1
  36. package/package.json +22 -17
  37. package/src/corpus/digest.test.ts +1 -1
  38. package/src/corpus/digest.ts +6 -6
  39. package/src/corpus/topics.test.ts +72 -18
  40. package/src/corpus/topics.ts +61 -15
  41. package/src/fact-pipeline.ts +114 -4
  42. package/src/index.ts +4 -0
  43. package/src/internal/threads.test.ts +12 -0
  44. package/src/internal/threads.ts +11 -3
  45. package/src/model-policy.test.ts +21 -0
  46. package/src/model-policy.ts +45 -0
  47. package/src/stages/EmailStage.ts +398 -0
  48. package/src/stages/extract-facts-commit.ts +16 -7
  49. package/src/stages/index.ts +3 -0
  50. package/src/stages/tag.test.ts +98 -0
  51. package/src/stages/tag.ts +150 -0
  52. package/src/testing/email-pipeline.test.ts +9 -6
  53. package/src/topics-pipeline.test.ts +95 -0
  54. package/src/topics-pipeline.ts +106 -0
  55. package/src/types/index.ts +0 -1
  56. package/dist/lib/neutral/index.mjs +0 -650
  57. package/dist/lib/neutral/index.mjs.map +0 -7
  58. package/dist/lib/neutral/meta.json +0 -1
  59. package/dist/lib/neutral/testing/index.mjs +0 -48
  60. package/dist/lib/neutral/testing/index.mjs.map +0 -7
  61. package/dist/types/src/types/Topic.d.ts +0 -12
  62. package/dist/types/src/types/Topic.d.ts.map +0 -1
  63. package/dist/types/src/types/Topic.test.d.ts +0 -2
  64. package/dist/types/src/types/Topic.test.d.ts.map +0 -1
  65. package/src/types/Topic.test.ts +0 -25
  66. package/src/types/Topic.ts +0 -20
@@ -0,0 +1,45 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ // Declarative model routing for the email pipeline stages (productized from the stories-brain research
6
+ // harness): one place that maps each LLM-bearing stage to the model it runs on, instead of hard-coding
7
+ // a model DXN at every call site. Product tiers only (Anthropic); the research ladder's open-weight
8
+ // (ollama) variants stay in the harness. Each layer overrides the previous: default ← per-run policy.
9
+
10
+ /** Anthropic model DXNs available to product pipelines. */
11
+ export const MODEL_HAIKU = 'com.anthropic.model.claude-haiku-4-5.default';
12
+ export const MODEL_SONNET = 'com.anthropic.model.claude-sonnet-4-6.default';
13
+ export const MODEL_OPUS = 'com.anthropic.model.claude-opus-4-8.default';
14
+
15
+ /** The LLM-bearing email pipeline stages. Deterministic stages (threading, dedup, stats) never appear. */
16
+ export type StageId =
17
+ | 'classify-sender'
18
+ | 'tag'
19
+ | 'summarize-message'
20
+ | 'summarize-thread'
21
+ | 'summarize-topic'
22
+ | 'extract-facts'
23
+ | 'draft';
24
+
25
+ /** A partial policy: stage → a model DXN. */
26
+ export type ModelPolicy = Partial<Record<StageId, string>>;
27
+
28
+ /**
29
+ * Default stage → model routing, seeded from the model-ladder findings (REPORT §4/§5): labeling and
30
+ * summaries want a premier model, drafting tolerates a cheaper one. Product defaults to haiku across
31
+ * the board (the cheapest premier tier); callers override per run via a `ModelPolicy`.
32
+ */
33
+ export const DEFAULT_MODEL_POLICY: Record<StageId, string> = {
34
+ 'classify-sender': MODEL_HAIKU,
35
+ 'tag': MODEL_HAIKU,
36
+ 'summarize-message': MODEL_HAIKU,
37
+ 'summarize-thread': MODEL_HAIKU,
38
+ 'summarize-topic': MODEL_HAIKU,
39
+ 'extract-facts': MODEL_HAIKU,
40
+ 'draft': MODEL_HAIKU,
41
+ };
42
+
43
+ /** Resolves the model DXN for a stage: per-run `policy` wins over `DEFAULT_MODEL_POLICY`. */
44
+ export const resolveModel = (stage: StageId, policy?: ModelPolicy): string =>
45
+ policy?.[stage] ?? DEFAULT_MODEL_POLICY[stage];
@@ -0,0 +1,398 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ // @import-as-namespace
6
+
7
+ import * as Effect from 'effect/Effect';
8
+ import * as Stream from 'effect/Stream';
9
+
10
+ import { Blob, Database, Feed, Filter, Obj, Ref } from '@dxos/echo';
11
+ import { type IdentityIndex, overlayIdentityIndex } from '@dxos/extractor';
12
+ import { buildContactFromActor, getIdentityIndex, identitySpecs } from '@dxos/extractor-lib';
13
+ import { Cursor } from '@dxos/link';
14
+ import { log } from '@dxos/log';
15
+ import { normalizeText } from '@dxos/markdown';
16
+ import { Stage } from '@dxos/pipeline';
17
+ import { Tagging, TagIndex } from '@dxos/schema';
18
+ import { DraftMessage, Message, Person } from '@dxos/types';
19
+
20
+ // TODO(burdon): Factor out.
21
+
22
+ /**
23
+ * Reusable, generic email-processing pipeline stages: {@link Change} → {@link Change}, composing in any
24
+ * order, each simply recording what it found (`message.attachments`, `contact`, …) rather than writing
25
+ * directly. {@link toCommitUnit} is the terminal stage that turns those into a generic
26
+ * `Cursor.CommitUnit`, and must run last. Provider-specific decode/map stages construct the initial
27
+ * `Change` (an `Insert`, tagged from the start) and stay in each sync op.
28
+ */
29
+
30
+ /** An item carrying a body string that may contain HTML. */
31
+ export type Bodied = { readonly body: string };
32
+
33
+ /**
34
+ * Normalizes an item's `body` from (possibly) HTML into markdown via `normalizeText` (turndown when
35
+ * HTML, passthrough for plaintext). Requires nothing from the Requirements channel. Written as a
36
+ * generic pipeable so the item type infers from the upstream stream (no type argument needed).
37
+ */
38
+ export const htmlToMarkdown = <In extends Bodied, E, R>(self: Stream.Stream<In, E, R>): Stream.Stream<In, E, R> =>
39
+ Stage.map('html-to-markdown', (item: In) => Effect.sync(() => ({ ...item, body: normalizeText(item.body) })))(self);
40
+
41
+ /**
42
+ * A new message to append, tagged `insert` from the moment a provider decodes it. The upstream dedup
43
+ * stage (`Cursor.dedupStage`) guarantees a message reaching this variant was never previously committed,
44
+ * so it's a one-time insert, never a resync/overwrite. The generic email stages act only on this variant
45
+ * of {@link Change}.
46
+ */
47
+ export type Insert = {
48
+ readonly _tag: 'insert';
49
+ readonly message: Message.Message;
50
+ readonly foreignId: string;
51
+ readonly key: number;
52
+ /** Provider label/folder tag URIs to apply on commit; omit for providers/stages that don't tag. */
53
+ readonly tagUris?: readonly string[];
54
+ /** Attachments fetched by a provider-specific stage upstream of {@link processAttachments}. */
55
+ readonly attachments?: readonly Attachment[];
56
+ /** Contact resolved by {@link extractContacts}, added to the database by {@link toCommitUnit}. */
57
+ readonly contact?: Person.Person;
58
+ /** Local drafts this synced message supersedes ({@link reconcileDrafts}), removed by {@link toCommitUnit}. */
59
+ readonly supersededDrafts?: readonly Message.Message[];
60
+ };
61
+
62
+ /**
63
+ * A label change on an already-committed message, resolved to its feed message's `entityId`.
64
+ * {@link toCommitUnit} turns it into an objectless commit unit that applies the tag adds/removes — no
65
+ * feed append, so the body stages ({@link processAttachments}, {@link extractContacts},
66
+ * {@link reconcileDrafts}) pass it through untouched.
67
+ */
68
+ export type Retag = {
69
+ readonly _tag: 'retag';
70
+ readonly foreignId: string;
71
+ readonly entityId: Obj.ID;
72
+ readonly addTagIds: readonly string[];
73
+ readonly removeTagIds: readonly string[];
74
+ };
75
+
76
+ /**
77
+ * A remote deletion of an already-committed message. Wired but STUBBED — {@link toCommitUnit} emits an
78
+ * objectless unit with no effect (see the TODO there) pending safe in-sync feed object deletion.
79
+ */
80
+ export type Delete = {
81
+ readonly _tag: 'delete';
82
+ readonly foreignId: string;
83
+ readonly entityId: Obj.ID;
84
+ };
85
+
86
+ /**
87
+ * The discriminated item flowing through the shared email tail. `insert` is a new message; `retag`/
88
+ * `delete` act on existing feed messages by `entityId`. All converge on the single {@link toCommitUnit}
89
+ * → `Cursor.commit`.
90
+ */
91
+ export type Change = Insert | Retag | Delete;
92
+
93
+ /** A decoded email attachment, ready for {@link processAttachments} to turn into a Blob. */
94
+ export type Attachment = {
95
+ readonly name?: string;
96
+ readonly mimeType?: string;
97
+ readonly size: number;
98
+ readonly bytes: Uint8Array;
99
+ /** The attachment's `Content-ID`, if any — matches a `cid:` reference in the message's HTML body. */
100
+ readonly contentId?: string;
101
+ };
102
+
103
+ /**
104
+ * Builds a Person (+ Organization link by domain) from the message sender and records it on the
105
+ * item as {@link Insert.contact} — the stage writes nothing itself; {@link toCommitUnit} defers
106
+ * the actual `db.add` to commit.
107
+ *
108
+ * Dedup is the space's shared identity index (see `getIdentityIndex`), refreshed once at the start
109
+ * of the run: it covers contacts committed before the run, contacts built earlier in it, and — since
110
+ * the index is shared per database — contacts a concurrently-syncing mailbox is building right now.
111
+ *
112
+ * Not every sender earns a record: {@link senderSignals} feeds the allow-list in
113
+ * `shouldExtractContact`, so bulk and automated mail no longer fills the space with contacts nobody
114
+ * will look at. NOTE: no provider records an outbound signal yet, so in practice the allow-list is
115
+ * "the domain matches a known Organization" — stricter than the policy intends. Extracting the
116
+ * recipients of sent mail is what closes that gap.
117
+ */
118
+ export const extractContacts = (): Stage.Stage<Change, Change, never, Database.Service> => {
119
+ // Run-scoped overlay over the space's shared index: reads see everything committed (including by a
120
+ // concurrent sync), writes stay local, so a run that dies mid-way leaves nothing claiming a contact
121
+ // it never committed.
122
+ let index: IdentityIndex | undefined;
123
+ return Stage.map('extract-contacts', (change: Change) =>
124
+ Effect.gen(function* () {
125
+ // Only new messages carry a body/sender; retag/delete pass through untouched.
126
+ if (change._tag !== 'insert') {
127
+ return change;
128
+ }
129
+ const { db } = yield* Database.Service;
130
+ if (!index) {
131
+ // Re-seed once per run so the shared index picks up whatever was committed since it was built.
132
+ index = overlayIdentityIndex(identitySpecs, yield* getIdentityIndex(db, { refresh: true }));
133
+ }
134
+ const sender = change.message.sender;
135
+ const contact = sender
136
+ ? yield* buildContactFromActor(sender, db, { signals: senderSignals(change.message), index })
137
+ : undefined;
138
+ return contact ? { ...change, contact } : change;
139
+ }),
140
+ );
141
+ };
142
+
143
+ /**
144
+ * The sender signals the shared extraction gate reads, taken from the fields provider mappers
145
+ * already record on `properties`. `Precedence`/`Auto-Submitted` are folded into `bulk` by the mapper
146
+ * where available.
147
+ */
148
+ const senderSignals = (message: Message.Message) => {
149
+ const properties = message.properties ?? {};
150
+ return {
151
+ noReply: properties.noReply === true,
152
+ listUnsubscribe: typeof properties.listUnsubscribe === 'string' ? properties.listUnsubscribe : undefined,
153
+ bulk: properties.bulk === true,
154
+ };
155
+ };
156
+
157
+ /**
158
+ * Turns each of the item's `attachments` into a Blob object (via the database's configured storage
159
+ * backend — edge in Composer) and adds a {@link Message.Attachment} pointing at it to the message.
160
+ * {@link toCommitUnit} finds these blobs again via each attachment's ref (inlined, since the
161
+ * blob isn't attached to a database yet) to defer their feed append to commit. A no-op when the item
162
+ * has no attachments.
163
+ *
164
+ * One bad or oversized attachment must not fail the whole message: each `Blob.fromBytes` is caught
165
+ * and logged individually, so a rejected attachment is simply dropped from the message.
166
+ */
167
+ export const processAttachments = (): Stage.Stage<Change, Change, never, Database.Service> =>
168
+ Stage.map('process-attachments', (change: Change) =>
169
+ Effect.gen(function* () {
170
+ // Only new messages carry attachments; retag/delete pass through untouched.
171
+ if (change._tag !== 'insert') {
172
+ return change;
173
+ }
174
+ const mapped = change;
175
+ if (!mapped.attachments?.length) {
176
+ return mapped;
177
+ }
178
+
179
+ const attachments: Message.Attachment[] = [];
180
+ for (const attachment of mapped.attachments) {
181
+ const blob: Blob.Blob | undefined = yield* Blob.fromBytes(attachment.bytes, {
182
+ type: attachment.mimeType,
183
+ }).pipe(
184
+ Effect.catchAll((error) => {
185
+ log.catch(error, { foreignId: mapped.foreignId, name: attachment.name, size: attachment.size });
186
+ return Effect.succeed(undefined);
187
+ }),
188
+ );
189
+ if (!blob) {
190
+ continue;
191
+ }
192
+ attachments.push({ name: attachment.name, ref: Ref.make(blob), contentId: attachment.contentId });
193
+ }
194
+ if (attachments.length === 0) {
195
+ return mapped;
196
+ }
197
+
198
+ // The message is not yet appended to the feed, so mutating it here is safe — mirrors how the
199
+ // mappers assemble `blocks` before the message is ever persisted.
200
+ Obj.update(mapped.message, (message) => {
201
+ message.attachments = [...(message.attachments ?? []), ...attachments];
202
+ });
203
+
204
+ return mapped;
205
+ }),
206
+ );
207
+
208
+ /**
209
+ * Queries the parent container's already-sent drafts once per sync run and pools them by the provider
210
+ * message id captured at send time (`properties.sentMessageId`) — the key {@link reconcileDrafts}
211
+ * matches each incoming message's `foreignId` against. Doing this once (rather than per item) keeps
212
+ * reconciliation off the database as messages stream through. Unsent drafts (no `sentMessageId`) are
213
+ * never pooled and thus never eligible for removal.
214
+ *
215
+ * `parent` is the local root object drafts are scoped to (e.g. a Mailbox) — only its URI is read, so
216
+ * any object works.
217
+ */
218
+ export const queryDraftPool = Effect.fn('queryDraftPool')(function* (parent: Obj.Unknown) {
219
+ const parentUri = Obj.getURI(parent);
220
+ const drafts = (yield* Database.query(Filter.type(Message.Message, { properties: { mailbox: parentUri } }))
221
+ .run).filter((candidate) => DraftMessage.belongsTo(candidate, parentUri) && candidate.properties?.sentMessageId);
222
+
223
+ const pool = new Map<string, Message.Message[]>();
224
+ for (const draft of drafts) {
225
+ const sentMessageId = draft.properties?.sentMessageId;
226
+ if (!sentMessageId) {
227
+ continue;
228
+ }
229
+ const existing = pool.get(sentMessageId);
230
+ if (existing) {
231
+ existing.push(draft);
232
+ } else {
233
+ pool.set(sentMessageId, [draft]);
234
+ }
235
+ }
236
+ return pool;
237
+ });
238
+
239
+ /**
240
+ * Records the local drafts each incoming message supersedes — matched by the provider message id the
241
+ * draft captured at send time (`properties.sentMessageId`), which equals the synced message's
242
+ * `foreignId` — from a pool built once per run and keyed by that id. Writes nothing itself;
243
+ * {@link toCommitUnit} defers the `db.remove` to commit, so the canonical message is appended
244
+ * to the feed before its now-redundant drafts are deleted. A stage factory: pass the run's draft
245
+ * pool once.
246
+ *
247
+ * The pool is queried a single time by the caller (per sync operation), so this stage never touches
248
+ * the database or feed per item.
249
+ */
250
+ export const reconcileDrafts = (
251
+ draftPool: ReadonlyMap<string, readonly Message.Message[]>,
252
+ ): Stage.Stage<Change, Change, never, never> =>
253
+ Stage.map('reconcile-drafts', (change: Change) =>
254
+ Effect.sync(() => {
255
+ // Only new messages can supersede a draft; retag/delete pass through untouched.
256
+ if (change._tag !== 'insert') {
257
+ return change;
258
+ }
259
+ const drafts = draftPool.get(change.foreignId);
260
+ return drafts?.length ? { ...change, supersededDrafts: drafts } : change;
261
+ }),
262
+ );
263
+
264
+ /**
265
+ * Terminal stage converting a run's {@link Change} into the generic `Cursor.CommitUnit` the commit sink
266
+ * consumes. For an `insert`: `message.attachments` (populated by {@link processAttachments}) becomes a
267
+ * feed-append commit effect for the referenced blobs, `mapped.contact` (from {@link extractContacts})
268
+ * becomes a `db.add` commit effect, and `mapped.supersededDrafts` (from {@link reconcileDrafts})
269
+ * becomes a `db.remove` commit effect. Every other email stage is Change → Change and composes in
270
+ * any order; this is the one stage that must run last.
271
+ *
272
+ * Tagging is email-specific (`Cursor.CommitUnit` carries no tag data), so it's applied here rather
273
+ * than by the generic commit sink: `tagIndex` is optional (omit for providers/tests that don't tag),
274
+ * and every tagged unit attaches the *same* `applyTags` closure (keyed by `mapped.tagUris` recorded in
275
+ * a run-scoped map, read back by object identity) so the commit's identity-based batching applies
276
+ * every tag pair for the page in one `Tagging.setBatch` call rather than one per message.
277
+ */
278
+ export const toCommitUnit = (
279
+ options: { readonly tagIndex?: TagIndex.TagIndex } = {},
280
+ ): Stage.Stage<Change, Cursor.CommitUnit, never, Cursor.Service> => {
281
+ const { tagIndex } = options;
282
+ const tagUrisByObject = new WeakMap<Obj.Any, readonly string[]>();
283
+ const applyTags: Cursor.CommitEffect = Effect.fn('email.commit.tags')(function* (
284
+ units: readonly Cursor.CommitUnit[],
285
+ ) {
286
+ if (!tagIndex) {
287
+ return;
288
+ }
289
+ const tagEntries: { object: Obj.Any; tagId: string }[] = [];
290
+ for (const unit of units) {
291
+ if (!unit.object) {
292
+ continue;
293
+ }
294
+ for (const uri of tagUrisByObject.get(unit.object) ?? []) {
295
+ tagEntries.push({ object: unit.object, tagId: uri });
296
+ }
297
+ }
298
+ if (tagEntries.length > 0) {
299
+ Tagging.setBatch(tagEntries, { index: tagIndex });
300
+ }
301
+ });
302
+
303
+ // Retag adds/removes recorded per foreign id, applied in one shared effect (identity-batched by the
304
+ // commit) so a page of label changes resolves to a single pass over the tag index.
305
+ const retagByForeignId = new Map<string, Pick<Retag, 'entityId' | 'addTagIds' | 'removeTagIds'>>();
306
+ const applyRetags: Cursor.CommitEffect = Effect.fn('email.commit.retags')(function* (
307
+ units: readonly Cursor.CommitUnit[],
308
+ ) {
309
+ if (!tagIndex) {
310
+ return;
311
+ }
312
+ const accessor = TagIndex.bind(tagIndex);
313
+ for (const unit of units) {
314
+ const retag = retagByForeignId.get(unit.foreignId);
315
+ if (!retag) {
316
+ continue;
317
+ }
318
+ for (const tagId of retag.addTagIds) {
319
+ accessor.setTag(tagId, retag.entityId);
320
+ }
321
+ for (const tagId of retag.removeTagIds) {
322
+ accessor.unsetTag(tagId, retag.entityId);
323
+ }
324
+ }
325
+ });
326
+
327
+ return Stage.map('to-commit-unit', (change: Change) =>
328
+ Effect.gen(function* () {
329
+ // Label change on an existing message: an objectless unit (no feed append, key 0) whose effect
330
+ // retags by entityId.
331
+ if (change._tag === 'retag') {
332
+ retagByForeignId.set(change.foreignId, {
333
+ entityId: change.entityId,
334
+ addTagIds: change.addTagIds,
335
+ removeTagIds: change.removeTagIds,
336
+ });
337
+ return {
338
+ foreignId: change.foreignId,
339
+ key: 0,
340
+ commitEffects: tagIndex ? [applyRetags] : undefined,
341
+ } satisfies Cursor.CommitUnit;
342
+ }
343
+
344
+ // Remote deletion: wired but STUBBED — the branch resolves an objectless unit that does nothing.
345
+ // TODO(wittjosiah): Remove the message from the feed once in-sync feed object deletion is safe
346
+ // (cursor/dedup-set interaction across the non-transactional queue↔space-db seam).
347
+ if (change._tag === 'delete') {
348
+ return { foreignId: change.foreignId, key: 0 } satisfies Cursor.CommitUnit;
349
+ }
350
+
351
+ const mapped = change;
352
+ const { feed } = yield* Cursor.Service;
353
+ const commitEffects: Cursor.CommitEffect[] = [];
354
+
355
+ // Each attachment's ref is inlined (its blob isn't attached to a database yet), so `.target`
356
+ // resolves it synchronously without an Insert.attachmentBlobs field to carry it separately.
357
+ const blobs = (mapped.message.attachments ?? [])
358
+ .map((attachment) => attachment.ref.target)
359
+ .filter((target): target is Blob.Blob => Obj.instanceOf(Blob.Blob, target));
360
+ if (feed && blobs.length > 0) {
361
+ commitEffects.push(() => Feed.append(feed, blobs));
362
+ }
363
+
364
+ if (mapped.contact) {
365
+ const contact = mapped.contact;
366
+ commitEffects.push(
367
+ Effect.fn('email.commit.addContact')(function* () {
368
+ yield* Database.add(contact);
369
+ }),
370
+ );
371
+ }
372
+
373
+ if (mapped.supersededDrafts?.length) {
374
+ const drafts = mapped.supersededDrafts;
375
+ commitEffects.push(
376
+ Effect.fn('email.commit.removeDrafts')(function* () {
377
+ const { db } = yield* Database.Service;
378
+ for (const draft of drafts) {
379
+ db.remove(draft);
380
+ }
381
+ }),
382
+ );
383
+ }
384
+
385
+ if (tagIndex && mapped.tagUris?.length) {
386
+ tagUrisByObject.set(mapped.message, mapped.tagUris);
387
+ commitEffects.push(applyTags);
388
+ }
389
+
390
+ return {
391
+ object: mapped.message,
392
+ foreignId: mapped.foreignId,
393
+ key: mapped.key,
394
+ commitEffects: commitEffects.length > 0 ? commitEffects : undefined,
395
+ };
396
+ }),
397
+ );
398
+ };
@@ -29,13 +29,22 @@ export type FactUnit = {
29
29
  // instead of persisting them in-stage, so the sink can write a page atomically with the cursor
30
30
  // advance. Extraction degrades to no facts on failure (advisory layer) but logs first, matching
31
31
  // `extractFactsStage`'s degradation, so the loss stays observable.
32
- export const extractFactsUnitStage = (extract: FactExtractor): Stage.Stage<Message.Message, FactUnit> =>
33
- Stage.map('extract-facts-unit', (message) =>
34
- Effect.tryPromise(() => extract(message)).pipe(
35
- Effect.tapError((error) => Effect.logWarning('extract-facts-unit failed; degrading to no facts', error)),
36
- Effect.orElse(() => Effect.succeed<RDF.Fact[]>([])),
37
- Effect.map((facts): FactUnit => ({ facts, foreignId: messageSource(message), key: keyOf(message) })),
38
- ),
32
+ // `concurrency` bounds in-flight extractions; >1 runs the (independent, network-bound) per-message
33
+ // LLM calls in parallel. Facts are order-independent — the sink commits by max cursor key — so the
34
+ // reordering `mapEffect` may introduce is harmless here.
35
+ export const extractFactsUnitStage = (
36
+ extract: FactExtractor,
37
+ concurrency = 1,
38
+ ): Stage.Stage<Message.Message, FactUnit> =>
39
+ Stage.map(
40
+ 'extract-facts-unit',
41
+ (message) =>
42
+ Effect.tryPromise(() => extract(message)).pipe(
43
+ Effect.tapError((error) => Effect.logWarning('extract-facts-unit failed; degrading to no facts', error)),
44
+ Effect.orElse(() => Effect.succeed<RDF.Fact[]>([])),
45
+ Effect.map((facts): FactUnit => ({ facts, foreignId: messageSource(message), key: keyOf(message) })),
46
+ ),
47
+ { concurrency },
39
48
  );
40
49
 
41
50
  // NOTE(workaround): `message.created` is the incremental cursor key because ECHO's native feed
@@ -9,3 +9,6 @@ export * from './extract-facts-commit';
9
9
  export * from './facts';
10
10
  export * from './stats';
11
11
  export * from './summarize';
12
+ export * from './tag';
13
+
14
+ export * as EmailStage from './EmailStage';
@@ -0,0 +1,98 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import { describe, test } from 'vitest';
6
+
7
+ import { applyBulkTag, classifyBulk, parseTagResult } from './tag';
8
+
9
+ describe('parseTagResult', () => {
10
+ test('parses tags (lowercased) and spam flag', ({ expect }) => {
11
+ const result = parseTagResult('{"tags":["Invoice","Finance"],"spam":false}');
12
+ expect(result.tags).toEqual(['invoice', 'finance']);
13
+ expect(result.spam).toBe(false);
14
+ expect(result.bulk).toBe(false);
15
+ });
16
+
17
+ test('flags bulk when the model emits the tag', ({ expect }) => {
18
+ const result = parseTagResult('{"tags":["receipt","bulk"],"spam":false}');
19
+ expect(result.bulk).toBe(true);
20
+ });
21
+
22
+ test('extracts JSON wrapped in prose', ({ expect }) => {
23
+ const result = parseTagResult('Here you go:\n{"tags":["personal"],"spam":false}\nHope that helps.');
24
+ expect(result.tags).toEqual(['personal']);
25
+ });
26
+
27
+ test('infers spam from a spam tag and dedups it', ({ expect }) => {
28
+ const result = parseTagResult('{"tags":["spam","marketing"]}');
29
+ expect(result.spam).toBe(true);
30
+ expect(result.tags.filter((tag) => tag === 'spam')).toHaveLength(1);
31
+ });
32
+
33
+ test('adds the spam tag when the flag is set but the tag is missing', ({ expect }) => {
34
+ const result = parseTagResult('{"tags":["promo"],"spam":true}');
35
+ expect(result.spam).toBe(true);
36
+ expect(result.tags).toContain('spam');
37
+ });
38
+
39
+ test('degrades to empty on unparseable output', ({ expect }) => {
40
+ const result = parseTagResult('no json here');
41
+ expect(result.tags).toEqual([]);
42
+ expect(result.spam).toBe(false);
43
+ expect(result.bulk).toBe(false);
44
+ });
45
+ });
46
+
47
+ describe('classifyBulk', () => {
48
+ test('receipts and login/security notices are bulk', ({ expect }) => {
49
+ expect(classifyBulk({ subject: 'Receipt from Anthropic PBC' })).toBe('bulk');
50
+ expect(classifyBulk({ subject: 'Your receipt from HeyGen Inc' })).toBe('bulk');
51
+ expect(classifyBulk({ subject: 'Linear login' })).toBe('bulk');
52
+ expect(classifyBulk({ subject: 'New sign-in to your account' })).toBe('bulk');
53
+ expect(classifyBulk({ subject: 'Your order has shipped' })).toBe('bulk');
54
+ });
55
+
56
+ test('invoices and payment requests are action-required, never bulk', ({ expect }) => {
57
+ expect(classifyBulk({ subject: 'Invoice #4021 — payment due' })).toBe('action');
58
+ expect(classifyBulk({ subject: 'Payment reminder', senderEmail: 'billing@acme.com' })).toBe('action');
59
+ expect(classifyBulk({ subject: 'Your invoice receipt' })).toBe('action');
60
+ });
61
+
62
+ test('a person with no automated signal is unknown (left to the model / person gate)', ({ expect }) => {
63
+ expect(classifyBulk({ subject: 'Barbera Corporate — Dmytro', senderEmail: 'dmytro@barbera.com' })).toBe('unknown');
64
+ });
65
+
66
+ test('a no-reply / notifications local part is bulk when the subject is silent', ({ expect }) => {
67
+ expect(classifyBulk({ subject: 'Update', senderEmail: 'no-reply@service.io' })).toBe('bulk');
68
+ expect(classifyBulk({ subject: 'Update', senderEmail: 'notifications@service.io' })).toBe('bulk');
69
+ });
70
+
71
+ test('an unsubscribe affordance (header or body link) deterministically marks bulk', ({ expect }) => {
72
+ // List-Unsubscribe header (real synced mail).
73
+ expect(classifyBulk({ subject: 'Interview with Eric Ries', listUnsubscribe: '<https://x.io/unsub>' })).toBe('bulk');
74
+ // Unsubscribe link in the body (e.g. the fixture, no header).
75
+ expect(classifyBulk({ subject: 'Interview with Eric Ries', bodyText: 'Great read.\n\nUnsubscribe here.' })).toBe(
76
+ 'bulk',
77
+ );
78
+ // Outranks an action-looking subject — transactional invoices don't carry an unsubscribe link.
79
+ expect(classifyBulk({ subject: 'Invoice #12 payment due', bodyText: 'Click to unsubscribe.' })).toBe('bulk');
80
+ // No unsubscribe → the normal person path.
81
+ expect(classifyBulk({ subject: 'Interview with Eric Ries', bodyText: 'Looking forward to it.' })).toBe('unknown');
82
+ });
83
+ });
84
+
85
+ describe('applyBulkTag', () => {
86
+ test('adds bulk for no-action mail (deduped)', ({ expect }) => {
87
+ expect(applyBulkTag(['receipt'], 'bulk')).toEqual(['receipt', 'bulk']);
88
+ expect(applyBulkTag(['receipt', 'bulk'], 'bulk')).toEqual(['receipt', 'bulk']);
89
+ });
90
+
91
+ test('strips bulk the model wrongly added to action-required mail', ({ expect }) => {
92
+ expect(applyBulkTag(['invoice', 'bulk'], 'action')).toEqual(['invoice']);
93
+ });
94
+
95
+ test('leaves tags untouched when unknown', ({ expect }) => {
96
+ expect(applyBulkTag(['personal'], 'unknown')).toEqual(['personal']);
97
+ });
98
+ });