@dxos/pipeline-email 0.0.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/index.mjs +994 -0
- package/dist/lib/index.mjs.map +1 -0
- package/dist/lib/testing.mjs +49 -0
- package/dist/lib/testing.mjs.map +1 -0
- package/dist/types/src/corpus/digest.d.ts +0 -4
- package/dist/types/src/corpus/digest.d.ts.map +1 -1
- package/dist/types/src/corpus/topics.d.ts +12 -4
- package/dist/types/src/corpus/topics.d.ts.map +1 -1
- package/dist/types/src/fact-pipeline.d.ts +28 -3
- package/dist/types/src/fact-pipeline.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +4 -0
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/internal/threads.d.ts +5 -1
- package/dist/types/src/internal/threads.d.ts.map +1 -1
- package/dist/types/src/model-policy.d.ts +17 -0
- package/dist/types/src/model-policy.d.ts.map +1 -0
- package/dist/types/src/model-policy.test.d.ts +2 -0
- package/dist/types/src/model-policy.test.d.ts.map +1 -0
- package/dist/types/src/stages/EmailStage.d.ts +149 -0
- package/dist/types/src/stages/EmailStage.d.ts.map +1 -0
- package/dist/types/src/stages/extract-facts-commit.d.ts +1 -1
- package/dist/types/src/stages/extract-facts-commit.d.ts.map +1 -1
- package/dist/types/src/stages/index.d.ts +2 -0
- package/dist/types/src/stages/index.d.ts.map +1 -1
- package/dist/types/src/stages/tag.d.ts +38 -0
- package/dist/types/src/stages/tag.d.ts.map +1 -0
- package/dist/types/src/stages/tag.test.d.ts +2 -0
- package/dist/types/src/stages/tag.test.d.ts.map +1 -0
- package/dist/types/src/topics-pipeline.d.ts +55 -0
- package/dist/types/src/topics-pipeline.d.ts.map +1 -0
- package/dist/types/src/topics-pipeline.test.d.ts +2 -0
- package/dist/types/src/topics-pipeline.test.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +0 -1
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +22 -17
- package/src/corpus/digest.test.ts +1 -1
- package/src/corpus/digest.ts +6 -6
- package/src/corpus/topics.test.ts +72 -18
- package/src/corpus/topics.ts +61 -15
- package/src/fact-pipeline.ts +114 -4
- package/src/index.ts +4 -0
- package/src/internal/threads.test.ts +12 -0
- package/src/internal/threads.ts +11 -3
- package/src/model-policy.test.ts +21 -0
- package/src/model-policy.ts +45 -0
- package/src/stages/EmailStage.ts +398 -0
- package/src/stages/extract-facts-commit.ts +16 -7
- package/src/stages/index.ts +3 -0
- package/src/stages/tag.test.ts +98 -0
- package/src/stages/tag.ts +150 -0
- package/src/testing/email-pipeline.test.ts +9 -6
- package/src/topics-pipeline.test.ts +95 -0
- package/src/topics-pipeline.ts +106 -0
- package/src/types/index.ts +0 -1
- package/dist/lib/neutral/index.mjs +0 -650
- package/dist/lib/neutral/index.mjs.map +0 -7
- package/dist/lib/neutral/meta.json +0 -1
- package/dist/lib/neutral/testing/index.mjs +0 -48
- package/dist/lib/neutral/testing/index.mjs.map +0 -7
- package/dist/types/src/types/Topic.d.ts +0 -12
- package/dist/types/src/types/Topic.d.ts.map +0 -1
- package/dist/types/src/types/Topic.test.d.ts +0 -2
- package/dist/types/src/types/Topic.test.d.ts.map +0 -1
- package/src/types/Topic.test.ts +0 -25
- package/src/types/Topic.ts +0 -20
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// Declarative model routing for the email pipeline stages (productized from the stories-brain research
|
|
6
|
+
// harness): one place that maps each LLM-bearing stage to the model it runs on, instead of hard-coding
|
|
7
|
+
// a model DXN at every call site. Product tiers only (Anthropic); the research ladder's open-weight
|
|
8
|
+
// (ollama) variants stay in the harness. Each layer overrides the previous: default ← per-run policy.
|
|
9
|
+
|
|
10
|
+
/** Anthropic model DXNs available to product pipelines. */
|
|
11
|
+
export const MODEL_HAIKU = 'com.anthropic.model.claude-haiku-4-5.default';
|
|
12
|
+
export const MODEL_SONNET = 'com.anthropic.model.claude-sonnet-4-6.default';
|
|
13
|
+
export const MODEL_OPUS = 'com.anthropic.model.claude-opus-4-8.default';
|
|
14
|
+
|
|
15
|
+
/** The LLM-bearing email pipeline stages. Deterministic stages (threading, dedup, stats) never appear. */
|
|
16
|
+
export type StageId =
|
|
17
|
+
| 'classify-sender'
|
|
18
|
+
| 'tag'
|
|
19
|
+
| 'summarize-message'
|
|
20
|
+
| 'summarize-thread'
|
|
21
|
+
| 'summarize-topic'
|
|
22
|
+
| 'extract-facts'
|
|
23
|
+
| 'draft';
|
|
24
|
+
|
|
25
|
+
/** A partial policy: stage → a model DXN. */
|
|
26
|
+
export type ModelPolicy = Partial<Record<StageId, string>>;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Default stage → model routing, seeded from the model-ladder findings (REPORT §4/§5): labeling and
|
|
30
|
+
* summaries want a premier model, drafting tolerates a cheaper one. Product defaults to haiku across
|
|
31
|
+
* the board (the cheapest premier tier); callers override per run via a `ModelPolicy`.
|
|
32
|
+
*/
|
|
33
|
+
export const DEFAULT_MODEL_POLICY: Record<StageId, string> = {
|
|
34
|
+
'classify-sender': MODEL_HAIKU,
|
|
35
|
+
'tag': MODEL_HAIKU,
|
|
36
|
+
'summarize-message': MODEL_HAIKU,
|
|
37
|
+
'summarize-thread': MODEL_HAIKU,
|
|
38
|
+
'summarize-topic': MODEL_HAIKU,
|
|
39
|
+
'extract-facts': MODEL_HAIKU,
|
|
40
|
+
'draft': MODEL_HAIKU,
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/** Resolves the model DXN for a stage: per-run `policy` wins over `DEFAULT_MODEL_POLICY`. */
|
|
44
|
+
export const resolveModel = (stage: StageId, policy?: ModelPolicy): string =>
|
|
45
|
+
policy?.[stage] ?? DEFAULT_MODEL_POLICY[stage];
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// @import-as-namespace
|
|
6
|
+
|
|
7
|
+
import * as Effect from 'effect/Effect';
|
|
8
|
+
import * as Stream from 'effect/Stream';
|
|
9
|
+
|
|
10
|
+
import { Blob, Database, Feed, Filter, Obj, Ref } from '@dxos/echo';
|
|
11
|
+
import { type IdentityIndex, overlayIdentityIndex } from '@dxos/extractor';
|
|
12
|
+
import { buildContactFromActor, getIdentityIndex, identitySpecs } from '@dxos/extractor-lib';
|
|
13
|
+
import { Cursor } from '@dxos/link';
|
|
14
|
+
import { log } from '@dxos/log';
|
|
15
|
+
import { normalizeText } from '@dxos/markdown';
|
|
16
|
+
import { Stage } from '@dxos/pipeline';
|
|
17
|
+
import { Tagging, TagIndex } from '@dxos/schema';
|
|
18
|
+
import { DraftMessage, Message, Person } from '@dxos/types';
|
|
19
|
+
|
|
20
|
+
// TODO(burdon): Factor out.
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Reusable, generic email-processing pipeline stages: {@link Change} → {@link Change}, composing in any
|
|
24
|
+
* order, each simply recording what it found (`message.attachments`, `contact`, …) rather than writing
|
|
25
|
+
* directly. {@link toCommitUnit} is the terminal stage that turns those into a generic
|
|
26
|
+
* `Cursor.CommitUnit`, and must run last. Provider-specific decode/map stages construct the initial
|
|
27
|
+
* `Change` (an `Insert`, tagged from the start) and stay in each sync op.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/** An item carrying a body string that may contain HTML. */
|
|
31
|
+
export type Bodied = { readonly body: string };
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Normalizes an item's `body` from (possibly) HTML into markdown via `normalizeText` (turndown when
|
|
35
|
+
* HTML, passthrough for plaintext). Requires nothing from the Requirements channel. Written as a
|
|
36
|
+
* generic pipeable so the item type infers from the upstream stream (no type argument needed).
|
|
37
|
+
*/
|
|
38
|
+
export const htmlToMarkdown = <In extends Bodied, E, R>(self: Stream.Stream<In, E, R>): Stream.Stream<In, E, R> =>
|
|
39
|
+
Stage.map('html-to-markdown', (item: In) => Effect.sync(() => ({ ...item, body: normalizeText(item.body) })))(self);
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* A new message to append, tagged `insert` from the moment a provider decodes it. The upstream dedup
|
|
43
|
+
* stage (`Cursor.dedupStage`) guarantees a message reaching this variant was never previously committed,
|
|
44
|
+
* so it's a one-time insert, never a resync/overwrite. The generic email stages act only on this variant
|
|
45
|
+
* of {@link Change}.
|
|
46
|
+
*/
|
|
47
|
+
export type Insert = {
|
|
48
|
+
readonly _tag: 'insert';
|
|
49
|
+
readonly message: Message.Message;
|
|
50
|
+
readonly foreignId: string;
|
|
51
|
+
readonly key: number;
|
|
52
|
+
/** Provider label/folder tag URIs to apply on commit; omit for providers/stages that don't tag. */
|
|
53
|
+
readonly tagUris?: readonly string[];
|
|
54
|
+
/** Attachments fetched by a provider-specific stage upstream of {@link processAttachments}. */
|
|
55
|
+
readonly attachments?: readonly Attachment[];
|
|
56
|
+
/** Contact resolved by {@link extractContacts}, added to the database by {@link toCommitUnit}. */
|
|
57
|
+
readonly contact?: Person.Person;
|
|
58
|
+
/** Local drafts this synced message supersedes ({@link reconcileDrafts}), removed by {@link toCommitUnit}. */
|
|
59
|
+
readonly supersededDrafts?: readonly Message.Message[];
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* A label change on an already-committed message, resolved to its feed message's `entityId`.
|
|
64
|
+
* {@link toCommitUnit} turns it into an objectless commit unit that applies the tag adds/removes — no
|
|
65
|
+
* feed append, so the body stages ({@link processAttachments}, {@link extractContacts},
|
|
66
|
+
* {@link reconcileDrafts}) pass it through untouched.
|
|
67
|
+
*/
|
|
68
|
+
export type Retag = {
|
|
69
|
+
readonly _tag: 'retag';
|
|
70
|
+
readonly foreignId: string;
|
|
71
|
+
readonly entityId: Obj.ID;
|
|
72
|
+
readonly addTagIds: readonly string[];
|
|
73
|
+
readonly removeTagIds: readonly string[];
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* A remote deletion of an already-committed message. Wired but STUBBED — {@link toCommitUnit} emits an
|
|
78
|
+
* objectless unit with no effect (see the TODO there) pending safe in-sync feed object deletion.
|
|
79
|
+
*/
|
|
80
|
+
export type Delete = {
|
|
81
|
+
readonly _tag: 'delete';
|
|
82
|
+
readonly foreignId: string;
|
|
83
|
+
readonly entityId: Obj.ID;
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The discriminated item flowing through the shared email tail. `insert` is a new message; `retag`/
|
|
88
|
+
* `delete` act on existing feed messages by `entityId`. All converge on the single {@link toCommitUnit}
|
|
89
|
+
* → `Cursor.commit`.
|
|
90
|
+
*/
|
|
91
|
+
export type Change = Insert | Retag | Delete;
|
|
92
|
+
|
|
93
|
+
/** A decoded email attachment, ready for {@link processAttachments} to turn into a Blob. */
|
|
94
|
+
export type Attachment = {
|
|
95
|
+
readonly name?: string;
|
|
96
|
+
readonly mimeType?: string;
|
|
97
|
+
readonly size: number;
|
|
98
|
+
readonly bytes: Uint8Array;
|
|
99
|
+
/** The attachment's `Content-ID`, if any — matches a `cid:` reference in the message's HTML body. */
|
|
100
|
+
readonly contentId?: string;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Builds a Person (+ Organization link by domain) from the message sender and records it on the
|
|
105
|
+
* item as {@link Insert.contact} — the stage writes nothing itself; {@link toCommitUnit} defers
|
|
106
|
+
* the actual `db.add` to commit.
|
|
107
|
+
*
|
|
108
|
+
* Dedup is the space's shared identity index (see `getIdentityIndex`), refreshed once at the start
|
|
109
|
+
* of the run: it covers contacts committed before the run, contacts built earlier in it, and — since
|
|
110
|
+
* the index is shared per database — contacts a concurrently-syncing mailbox is building right now.
|
|
111
|
+
*
|
|
112
|
+
* Not every sender earns a record: {@link senderSignals} feeds the allow-list in
|
|
113
|
+
* `shouldExtractContact`, so bulk and automated mail no longer fills the space with contacts nobody
|
|
114
|
+
* will look at. NOTE: no provider records an outbound signal yet, so in practice the allow-list is
|
|
115
|
+
* "the domain matches a known Organization" — stricter than the policy intends. Extracting the
|
|
116
|
+
* recipients of sent mail is what closes that gap.
|
|
117
|
+
*/
|
|
118
|
+
export const extractContacts = (): Stage.Stage<Change, Change, never, Database.Service> => {
|
|
119
|
+
// Run-scoped overlay over the space's shared index: reads see everything committed (including by a
|
|
120
|
+
// concurrent sync), writes stay local, so a run that dies mid-way leaves nothing claiming a contact
|
|
121
|
+
// it never committed.
|
|
122
|
+
let index: IdentityIndex | undefined;
|
|
123
|
+
return Stage.map('extract-contacts', (change: Change) =>
|
|
124
|
+
Effect.gen(function* () {
|
|
125
|
+
// Only new messages carry a body/sender; retag/delete pass through untouched.
|
|
126
|
+
if (change._tag !== 'insert') {
|
|
127
|
+
return change;
|
|
128
|
+
}
|
|
129
|
+
const { db } = yield* Database.Service;
|
|
130
|
+
if (!index) {
|
|
131
|
+
// Re-seed once per run so the shared index picks up whatever was committed since it was built.
|
|
132
|
+
index = overlayIdentityIndex(identitySpecs, yield* getIdentityIndex(db, { refresh: true }));
|
|
133
|
+
}
|
|
134
|
+
const sender = change.message.sender;
|
|
135
|
+
const contact = sender
|
|
136
|
+
? yield* buildContactFromActor(sender, db, { signals: senderSignals(change.message), index })
|
|
137
|
+
: undefined;
|
|
138
|
+
return contact ? { ...change, contact } : change;
|
|
139
|
+
}),
|
|
140
|
+
);
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* The sender signals the shared extraction gate reads, taken from the fields provider mappers
|
|
145
|
+
* already record on `properties`. `Precedence`/`Auto-Submitted` are folded into `bulk` by the mapper
|
|
146
|
+
* where available.
|
|
147
|
+
*/
|
|
148
|
+
const senderSignals = (message: Message.Message) => {
|
|
149
|
+
const properties = message.properties ?? {};
|
|
150
|
+
return {
|
|
151
|
+
noReply: properties.noReply === true,
|
|
152
|
+
listUnsubscribe: typeof properties.listUnsubscribe === 'string' ? properties.listUnsubscribe : undefined,
|
|
153
|
+
bulk: properties.bulk === true,
|
|
154
|
+
};
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Turns each of the item's `attachments` into a Blob object (via the database's configured storage
|
|
159
|
+
* backend — edge in Composer) and adds a {@link Message.Attachment} pointing at it to the message.
|
|
160
|
+
* {@link toCommitUnit} finds these blobs again via each attachment's ref (inlined, since the
|
|
161
|
+
* blob isn't attached to a database yet) to defer their feed append to commit. A no-op when the item
|
|
162
|
+
* has no attachments.
|
|
163
|
+
*
|
|
164
|
+
* One bad or oversized attachment must not fail the whole message: each `Blob.fromBytes` is caught
|
|
165
|
+
* and logged individually, so a rejected attachment is simply dropped from the message.
|
|
166
|
+
*/
|
|
167
|
+
export const processAttachments = (): Stage.Stage<Change, Change, never, Database.Service> =>
|
|
168
|
+
Stage.map('process-attachments', (change: Change) =>
|
|
169
|
+
Effect.gen(function* () {
|
|
170
|
+
// Only new messages carry attachments; retag/delete pass through untouched.
|
|
171
|
+
if (change._tag !== 'insert') {
|
|
172
|
+
return change;
|
|
173
|
+
}
|
|
174
|
+
const mapped = change;
|
|
175
|
+
if (!mapped.attachments?.length) {
|
|
176
|
+
return mapped;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const attachments: Message.Attachment[] = [];
|
|
180
|
+
for (const attachment of mapped.attachments) {
|
|
181
|
+
const blob: Blob.Blob | undefined = yield* Blob.fromBytes(attachment.bytes, {
|
|
182
|
+
type: attachment.mimeType,
|
|
183
|
+
}).pipe(
|
|
184
|
+
Effect.catchAll((error) => {
|
|
185
|
+
log.catch(error, { foreignId: mapped.foreignId, name: attachment.name, size: attachment.size });
|
|
186
|
+
return Effect.succeed(undefined);
|
|
187
|
+
}),
|
|
188
|
+
);
|
|
189
|
+
if (!blob) {
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
attachments.push({ name: attachment.name, ref: Ref.make(blob), contentId: attachment.contentId });
|
|
193
|
+
}
|
|
194
|
+
if (attachments.length === 0) {
|
|
195
|
+
return mapped;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// The message is not yet appended to the feed, so mutating it here is safe — mirrors how the
|
|
199
|
+
// mappers assemble `blocks` before the message is ever persisted.
|
|
200
|
+
Obj.update(mapped.message, (message) => {
|
|
201
|
+
message.attachments = [...(message.attachments ?? []), ...attachments];
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
return mapped;
|
|
205
|
+
}),
|
|
206
|
+
);
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Queries the parent container's already-sent drafts once per sync run and pools them by the provider
|
|
210
|
+
* message id captured at send time (`properties.sentMessageId`) — the key {@link reconcileDrafts}
|
|
211
|
+
* matches each incoming message's `foreignId` against. Doing this once (rather than per item) keeps
|
|
212
|
+
* reconciliation off the database as messages stream through. Unsent drafts (no `sentMessageId`) are
|
|
213
|
+
* never pooled and thus never eligible for removal.
|
|
214
|
+
*
|
|
215
|
+
* `parent` is the local root object drafts are scoped to (e.g. a Mailbox) — only its URI is read, so
|
|
216
|
+
* any object works.
|
|
217
|
+
*/
|
|
218
|
+
export const queryDraftPool = Effect.fn('queryDraftPool')(function* (parent: Obj.Unknown) {
|
|
219
|
+
const parentUri = Obj.getURI(parent);
|
|
220
|
+
const drafts = (yield* Database.query(Filter.type(Message.Message, { properties: { mailbox: parentUri } }))
|
|
221
|
+
.run).filter((candidate) => DraftMessage.belongsTo(candidate, parentUri) && candidate.properties?.sentMessageId);
|
|
222
|
+
|
|
223
|
+
const pool = new Map<string, Message.Message[]>();
|
|
224
|
+
for (const draft of drafts) {
|
|
225
|
+
const sentMessageId = draft.properties?.sentMessageId;
|
|
226
|
+
if (!sentMessageId) {
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const existing = pool.get(sentMessageId);
|
|
230
|
+
if (existing) {
|
|
231
|
+
existing.push(draft);
|
|
232
|
+
} else {
|
|
233
|
+
pool.set(sentMessageId, [draft]);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return pool;
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Records the local drafts each incoming message supersedes — matched by the provider message id the
|
|
241
|
+
* draft captured at send time (`properties.sentMessageId`), which equals the synced message's
|
|
242
|
+
* `foreignId` — from a pool built once per run and keyed by that id. Writes nothing itself;
|
|
243
|
+
* {@link toCommitUnit} defers the `db.remove` to commit, so the canonical message is appended
|
|
244
|
+
* to the feed before its now-redundant drafts are deleted. A stage factory: pass the run's draft
|
|
245
|
+
* pool once.
|
|
246
|
+
*
|
|
247
|
+
* The pool is queried a single time by the caller (per sync operation), so this stage never touches
|
|
248
|
+
* the database or feed per item.
|
|
249
|
+
*/
|
|
250
|
+
export const reconcileDrafts = (
|
|
251
|
+
draftPool: ReadonlyMap<string, readonly Message.Message[]>,
|
|
252
|
+
): Stage.Stage<Change, Change, never, never> =>
|
|
253
|
+
Stage.map('reconcile-drafts', (change: Change) =>
|
|
254
|
+
Effect.sync(() => {
|
|
255
|
+
// Only new messages can supersede a draft; retag/delete pass through untouched.
|
|
256
|
+
if (change._tag !== 'insert') {
|
|
257
|
+
return change;
|
|
258
|
+
}
|
|
259
|
+
const drafts = draftPool.get(change.foreignId);
|
|
260
|
+
return drafts?.length ? { ...change, supersededDrafts: drafts } : change;
|
|
261
|
+
}),
|
|
262
|
+
);
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Terminal stage converting a run's {@link Change} into the generic `Cursor.CommitUnit` the commit sink
|
|
266
|
+
* consumes. For an `insert`: `message.attachments` (populated by {@link processAttachments}) becomes a
|
|
267
|
+
* feed-append commit effect for the referenced blobs, `mapped.contact` (from {@link extractContacts})
|
|
268
|
+
* becomes a `db.add` commit effect, and `mapped.supersededDrafts` (from {@link reconcileDrafts})
|
|
269
|
+
* becomes a `db.remove` commit effect. Every other email stage is Change → Change and composes in
|
|
270
|
+
* any order; this is the one stage that must run last.
|
|
271
|
+
*
|
|
272
|
+
* Tagging is email-specific (`Cursor.CommitUnit` carries no tag data), so it's applied here rather
|
|
273
|
+
* than by the generic commit sink: `tagIndex` is optional (omit for providers/tests that don't tag),
|
|
274
|
+
* and every tagged unit attaches the *same* `applyTags` closure (keyed by `mapped.tagUris` recorded in
|
|
275
|
+
* a run-scoped map, read back by object identity) so the commit's identity-based batching applies
|
|
276
|
+
* every tag pair for the page in one `Tagging.setBatch` call rather than one per message.
|
|
277
|
+
*/
|
|
278
|
+
export const toCommitUnit = (
|
|
279
|
+
options: { readonly tagIndex?: TagIndex.TagIndex } = {},
|
|
280
|
+
): Stage.Stage<Change, Cursor.CommitUnit, never, Cursor.Service> => {
|
|
281
|
+
const { tagIndex } = options;
|
|
282
|
+
const tagUrisByObject = new WeakMap<Obj.Any, readonly string[]>();
|
|
283
|
+
const applyTags: Cursor.CommitEffect = Effect.fn('email.commit.tags')(function* (
|
|
284
|
+
units: readonly Cursor.CommitUnit[],
|
|
285
|
+
) {
|
|
286
|
+
if (!tagIndex) {
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
const tagEntries: { object: Obj.Any; tagId: string }[] = [];
|
|
290
|
+
for (const unit of units) {
|
|
291
|
+
if (!unit.object) {
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
for (const uri of tagUrisByObject.get(unit.object) ?? []) {
|
|
295
|
+
tagEntries.push({ object: unit.object, tagId: uri });
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
if (tagEntries.length > 0) {
|
|
299
|
+
Tagging.setBatch(tagEntries, { index: tagIndex });
|
|
300
|
+
}
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
// Retag adds/removes recorded per foreign id, applied in one shared effect (identity-batched by the
|
|
304
|
+
// commit) so a page of label changes resolves to a single pass over the tag index.
|
|
305
|
+
const retagByForeignId = new Map<string, Pick<Retag, 'entityId' | 'addTagIds' | 'removeTagIds'>>();
|
|
306
|
+
const applyRetags: Cursor.CommitEffect = Effect.fn('email.commit.retags')(function* (
|
|
307
|
+
units: readonly Cursor.CommitUnit[],
|
|
308
|
+
) {
|
|
309
|
+
if (!tagIndex) {
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
const accessor = TagIndex.bind(tagIndex);
|
|
313
|
+
for (const unit of units) {
|
|
314
|
+
const retag = retagByForeignId.get(unit.foreignId);
|
|
315
|
+
if (!retag) {
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
for (const tagId of retag.addTagIds) {
|
|
319
|
+
accessor.setTag(tagId, retag.entityId);
|
|
320
|
+
}
|
|
321
|
+
for (const tagId of retag.removeTagIds) {
|
|
322
|
+
accessor.unsetTag(tagId, retag.entityId);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
return Stage.map('to-commit-unit', (change: Change) =>
|
|
328
|
+
Effect.gen(function* () {
|
|
329
|
+
// Label change on an existing message: an objectless unit (no feed append, key 0) whose effect
|
|
330
|
+
// retags by entityId.
|
|
331
|
+
if (change._tag === 'retag') {
|
|
332
|
+
retagByForeignId.set(change.foreignId, {
|
|
333
|
+
entityId: change.entityId,
|
|
334
|
+
addTagIds: change.addTagIds,
|
|
335
|
+
removeTagIds: change.removeTagIds,
|
|
336
|
+
});
|
|
337
|
+
return {
|
|
338
|
+
foreignId: change.foreignId,
|
|
339
|
+
key: 0,
|
|
340
|
+
commitEffects: tagIndex ? [applyRetags] : undefined,
|
|
341
|
+
} satisfies Cursor.CommitUnit;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// Remote deletion: wired but STUBBED — the branch resolves an objectless unit that does nothing.
|
|
345
|
+
// TODO(wittjosiah): Remove the message from the feed once in-sync feed object deletion is safe
|
|
346
|
+
// (cursor/dedup-set interaction across the non-transactional queue↔space-db seam).
|
|
347
|
+
if (change._tag === 'delete') {
|
|
348
|
+
return { foreignId: change.foreignId, key: 0 } satisfies Cursor.CommitUnit;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
const mapped = change;
|
|
352
|
+
const { feed } = yield* Cursor.Service;
|
|
353
|
+
const commitEffects: Cursor.CommitEffect[] = [];
|
|
354
|
+
|
|
355
|
+
// Each attachment's ref is inlined (its blob isn't attached to a database yet), so `.target`
|
|
356
|
+
// resolves it synchronously without an Insert.attachmentBlobs field to carry it separately.
|
|
357
|
+
const blobs = (mapped.message.attachments ?? [])
|
|
358
|
+
.map((attachment) => attachment.ref.target)
|
|
359
|
+
.filter((target): target is Blob.Blob => Obj.instanceOf(Blob.Blob, target));
|
|
360
|
+
if (feed && blobs.length > 0) {
|
|
361
|
+
commitEffects.push(() => Feed.append(feed, blobs));
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
if (mapped.contact) {
|
|
365
|
+
const contact = mapped.contact;
|
|
366
|
+
commitEffects.push(
|
|
367
|
+
Effect.fn('email.commit.addContact')(function* () {
|
|
368
|
+
yield* Database.add(contact);
|
|
369
|
+
}),
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
if (mapped.supersededDrafts?.length) {
|
|
374
|
+
const drafts = mapped.supersededDrafts;
|
|
375
|
+
commitEffects.push(
|
|
376
|
+
Effect.fn('email.commit.removeDrafts')(function* () {
|
|
377
|
+
const { db } = yield* Database.Service;
|
|
378
|
+
for (const draft of drafts) {
|
|
379
|
+
db.remove(draft);
|
|
380
|
+
}
|
|
381
|
+
}),
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
if (tagIndex && mapped.tagUris?.length) {
|
|
386
|
+
tagUrisByObject.set(mapped.message, mapped.tagUris);
|
|
387
|
+
commitEffects.push(applyTags);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
return {
|
|
391
|
+
object: mapped.message,
|
|
392
|
+
foreignId: mapped.foreignId,
|
|
393
|
+
key: mapped.key,
|
|
394
|
+
commitEffects: commitEffects.length > 0 ? commitEffects : undefined,
|
|
395
|
+
};
|
|
396
|
+
}),
|
|
397
|
+
);
|
|
398
|
+
};
|
|
@@ -29,13 +29,22 @@ export type FactUnit = {
|
|
|
29
29
|
// instead of persisting them in-stage, so the sink can write a page atomically with the cursor
|
|
30
30
|
// advance. Extraction degrades to no facts on failure (advisory layer) but logs first, matching
|
|
31
31
|
// `extractFactsStage`'s degradation, so the loss stays observable.
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
32
|
+
// `concurrency` bounds in-flight extractions; >1 runs the (independent, network-bound) per-message
|
|
33
|
+
// LLM calls in parallel. Facts are order-independent — the sink commits by max cursor key — so the
|
|
34
|
+
// reordering `mapEffect` may introduce is harmless here.
|
|
35
|
+
export const extractFactsUnitStage = (
|
|
36
|
+
extract: FactExtractor,
|
|
37
|
+
concurrency = 1,
|
|
38
|
+
): Stage.Stage<Message.Message, FactUnit> =>
|
|
39
|
+
Stage.map(
|
|
40
|
+
'extract-facts-unit',
|
|
41
|
+
(message) =>
|
|
42
|
+
Effect.tryPromise(() => extract(message)).pipe(
|
|
43
|
+
Effect.tapError((error) => Effect.logWarning('extract-facts-unit failed; degrading to no facts', error)),
|
|
44
|
+
Effect.orElse(() => Effect.succeed<RDF.Fact[]>([])),
|
|
45
|
+
Effect.map((facts): FactUnit => ({ facts, foreignId: messageSource(message), key: keyOf(message) })),
|
|
46
|
+
),
|
|
47
|
+
{ concurrency },
|
|
39
48
|
);
|
|
40
49
|
|
|
41
50
|
// NOTE(workaround): `message.created` is the incremental cursor key because ECHO's native feed
|
package/src/stages/index.ts
CHANGED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { describe, test } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import { applyBulkTag, classifyBulk, parseTagResult } from './tag';
|
|
8
|
+
|
|
9
|
+
describe('parseTagResult', () => {
|
|
10
|
+
test('parses tags (lowercased) and spam flag', ({ expect }) => {
|
|
11
|
+
const result = parseTagResult('{"tags":["Invoice","Finance"],"spam":false}');
|
|
12
|
+
expect(result.tags).toEqual(['invoice', 'finance']);
|
|
13
|
+
expect(result.spam).toBe(false);
|
|
14
|
+
expect(result.bulk).toBe(false);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test('flags bulk when the model emits the tag', ({ expect }) => {
|
|
18
|
+
const result = parseTagResult('{"tags":["receipt","bulk"],"spam":false}');
|
|
19
|
+
expect(result.bulk).toBe(true);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test('extracts JSON wrapped in prose', ({ expect }) => {
|
|
23
|
+
const result = parseTagResult('Here you go:\n{"tags":["personal"],"spam":false}\nHope that helps.');
|
|
24
|
+
expect(result.tags).toEqual(['personal']);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test('infers spam from a spam tag and dedups it', ({ expect }) => {
|
|
28
|
+
const result = parseTagResult('{"tags":["spam","marketing"]}');
|
|
29
|
+
expect(result.spam).toBe(true);
|
|
30
|
+
expect(result.tags.filter((tag) => tag === 'spam')).toHaveLength(1);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test('adds the spam tag when the flag is set but the tag is missing', ({ expect }) => {
|
|
34
|
+
const result = parseTagResult('{"tags":["promo"],"spam":true}');
|
|
35
|
+
expect(result.spam).toBe(true);
|
|
36
|
+
expect(result.tags).toContain('spam');
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test('degrades to empty on unparseable output', ({ expect }) => {
|
|
40
|
+
const result = parseTagResult('no json here');
|
|
41
|
+
expect(result.tags).toEqual([]);
|
|
42
|
+
expect(result.spam).toBe(false);
|
|
43
|
+
expect(result.bulk).toBe(false);
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
describe('classifyBulk', () => {
|
|
48
|
+
test('receipts and login/security notices are bulk', ({ expect }) => {
|
|
49
|
+
expect(classifyBulk({ subject: 'Receipt from Anthropic PBC' })).toBe('bulk');
|
|
50
|
+
expect(classifyBulk({ subject: 'Your receipt from HeyGen Inc' })).toBe('bulk');
|
|
51
|
+
expect(classifyBulk({ subject: 'Linear login' })).toBe('bulk');
|
|
52
|
+
expect(classifyBulk({ subject: 'New sign-in to your account' })).toBe('bulk');
|
|
53
|
+
expect(classifyBulk({ subject: 'Your order has shipped' })).toBe('bulk');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test('invoices and payment requests are action-required, never bulk', ({ expect }) => {
|
|
57
|
+
expect(classifyBulk({ subject: 'Invoice #4021 — payment due' })).toBe('action');
|
|
58
|
+
expect(classifyBulk({ subject: 'Payment reminder', senderEmail: 'billing@acme.com' })).toBe('action');
|
|
59
|
+
expect(classifyBulk({ subject: 'Your invoice receipt' })).toBe('action');
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('a person with no automated signal is unknown (left to the model / person gate)', ({ expect }) => {
|
|
63
|
+
expect(classifyBulk({ subject: 'Barbera Corporate — Dmytro', senderEmail: 'dmytro@barbera.com' })).toBe('unknown');
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test('a no-reply / notifications local part is bulk when the subject is silent', ({ expect }) => {
|
|
67
|
+
expect(classifyBulk({ subject: 'Update', senderEmail: 'no-reply@service.io' })).toBe('bulk');
|
|
68
|
+
expect(classifyBulk({ subject: 'Update', senderEmail: 'notifications@service.io' })).toBe('bulk');
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test('an unsubscribe affordance (header or body link) deterministically marks bulk', ({ expect }) => {
|
|
72
|
+
// List-Unsubscribe header (real synced mail).
|
|
73
|
+
expect(classifyBulk({ subject: 'Interview with Eric Ries', listUnsubscribe: '<https://x.io/unsub>' })).toBe('bulk');
|
|
74
|
+
// Unsubscribe link in the body (e.g. the fixture, no header).
|
|
75
|
+
expect(classifyBulk({ subject: 'Interview with Eric Ries', bodyText: 'Great read.\n\nUnsubscribe here.' })).toBe(
|
|
76
|
+
'bulk',
|
|
77
|
+
);
|
|
78
|
+
// Outranks an action-looking subject — transactional invoices don't carry an unsubscribe link.
|
|
79
|
+
expect(classifyBulk({ subject: 'Invoice #12 payment due', bodyText: 'Click to unsubscribe.' })).toBe('bulk');
|
|
80
|
+
// No unsubscribe → the normal person path.
|
|
81
|
+
expect(classifyBulk({ subject: 'Interview with Eric Ries', bodyText: 'Looking forward to it.' })).toBe('unknown');
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
describe('applyBulkTag', () => {
|
|
86
|
+
test('adds bulk for no-action mail (deduped)', ({ expect }) => {
|
|
87
|
+
expect(applyBulkTag(['receipt'], 'bulk')).toEqual(['receipt', 'bulk']);
|
|
88
|
+
expect(applyBulkTag(['receipt', 'bulk'], 'bulk')).toEqual(['receipt', 'bulk']);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test('strips bulk the model wrongly added to action-required mail', ({ expect }) => {
|
|
92
|
+
expect(applyBulkTag(['invoice', 'bulk'], 'action')).toEqual(['invoice']);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test('leaves tags untouched when unknown', ({ expect }) => {
|
|
96
|
+
expect(applyBulkTag(['personal'], 'unknown')).toEqual(['personal']);
|
|
97
|
+
});
|
|
98
|
+
});
|