@dxos/pipeline-email 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +105 -0
- package/README.md +15 -0
- package/dist/lib/neutral/index.mjs +650 -0
- package/dist/lib/neutral/index.mjs.map +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/testing/index.mjs +48 -0
- package/dist/lib/neutral/testing/index.mjs.map +7 -0
- package/dist/types/src/corpus/digest.d.ts +50 -0
- package/dist/types/src/corpus/digest.d.ts.map +1 -0
- package/dist/types/src/corpus/digest.test.d.ts +2 -0
- package/dist/types/src/corpus/digest.test.d.ts.map +1 -0
- package/dist/types/src/corpus/index.d.ts +6 -0
- package/dist/types/src/corpus/index.d.ts.map +1 -0
- package/dist/types/src/corpus/ledger.d.ts +33 -0
- package/dist/types/src/corpus/ledger.d.ts.map +1 -0
- package/dist/types/src/corpus/ledger.test.d.ts +2 -0
- package/dist/types/src/corpus/ledger.test.d.ts.map +1 -0
- package/dist/types/src/corpus/prompts.d.ts +21 -0
- package/dist/types/src/corpus/prompts.d.ts.map +1 -0
- package/dist/types/src/corpus/prompts.test.d.ts +2 -0
- package/dist/types/src/corpus/prompts.test.d.ts.map +1 -0
- package/dist/types/src/corpus/rollups.d.ts +21 -0
- package/dist/types/src/corpus/rollups.d.ts.map +1 -0
- package/dist/types/src/corpus/rollups.test.d.ts +2 -0
- package/dist/types/src/corpus/rollups.test.d.ts.map +1 -0
- package/dist/types/src/corpus/topics.d.ts +46 -0
- package/dist/types/src/corpus/topics.d.ts.map +1 -0
- package/dist/types/src/corpus/topics.test.d.ts +2 -0
- package/dist/types/src/corpus/topics.test.d.ts.map +1 -0
- package/dist/types/src/fact-pipeline.d.ts +26 -0
- package/dist/types/src/fact-pipeline.d.ts.map +1 -0
- package/dist/types/src/fact-pipeline.test.d.ts +2 -0
- package/dist/types/src/fact-pipeline.test.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +6 -0
- package/dist/types/src/index.d.ts.map +1 -0
- package/dist/types/src/internal/fact-index.d.ts +8 -0
- package/dist/types/src/internal/fact-index.d.ts.map +1 -0
- package/dist/types/src/internal/fact-index.test.d.ts +2 -0
- package/dist/types/src/internal/fact-index.test.d.ts.map +1 -0
- package/dist/types/src/internal/threading.d.ts +4 -0
- package/dist/types/src/internal/threading.d.ts.map +1 -0
- package/dist/types/src/internal/threading.test.d.ts +2 -0
- package/dist/types/src/internal/threading.test.d.ts.map +1 -0
- package/dist/types/src/internal/threads.d.ts +9 -0
- package/dist/types/src/internal/threads.d.ts.map +1 -0
- package/dist/types/src/internal/threads.test.d.ts +2 -0
- package/dist/types/src/internal/threads.test.d.ts.map +1 -0
- package/dist/types/src/pipeline.d.ts +36 -0
- package/dist/types/src/pipeline.d.ts.map +1 -0
- package/dist/types/src/pipeline.test.d.ts +2 -0
- package/dist/types/src/pipeline.test.d.ts.map +1 -0
- package/dist/types/src/stages/context.d.ts +40 -0
- package/dist/types/src/stages/context.d.ts.map +1 -0
- package/dist/types/src/stages/extract-contacts.d.ts +6 -0
- package/dist/types/src/stages/extract-contacts.d.ts.map +1 -0
- package/dist/types/src/stages/extract-facts-commit.d.ts +13 -0
- package/dist/types/src/stages/extract-facts-commit.d.ts.map +1 -0
- package/dist/types/src/stages/extract-facts-commit.test.d.ts +2 -0
- package/dist/types/src/stages/extract-facts-commit.test.d.ts.map +1 -0
- package/dist/types/src/stages/extract-facts.d.ts +6 -0
- package/dist/types/src/stages/extract-facts.d.ts.map +1 -0
- package/dist/types/src/stages/extract-facts.test.d.ts +2 -0
- package/dist/types/src/stages/extract-facts.test.d.ts.map +1 -0
- package/dist/types/src/stages/facts.d.ts +7 -0
- package/dist/types/src/stages/facts.d.ts.map +1 -0
- package/dist/types/src/stages/facts.test.d.ts +2 -0
- package/dist/types/src/stages/facts.test.d.ts.map +1 -0
- package/dist/types/src/stages/index.d.ts +8 -0
- package/dist/types/src/stages/index.d.ts.map +1 -0
- package/dist/types/src/stages/stats.d.ts +6 -0
- package/dist/types/src/stages/stats.d.ts.map +1 -0
- package/dist/types/src/stages/stats.test.d.ts +2 -0
- package/dist/types/src/stages/stats.test.d.ts.map +1 -0
- package/dist/types/src/stages/summarize.d.ts +7 -0
- package/dist/types/src/stages/summarize.d.ts.map +1 -0
- package/dist/types/src/testing/email-extraction.bench.test.d.ts +2 -0
- package/dist/types/src/testing/email-extraction.bench.test.d.ts.map +1 -0
- package/dist/types/src/testing/email-fixtures.d.ts +25 -0
- package/dist/types/src/testing/email-fixtures.d.ts.map +1 -0
- package/dist/types/src/testing/email-pipeline.test.d.ts +2 -0
- package/dist/types/src/testing/email-pipeline.test.d.ts.map +1 -0
- package/dist/types/src/testing/index.d.ts +2 -0
- package/dist/types/src/testing/index.d.ts.map +1 -0
- package/dist/types/src/testing/parquet-email.test.d.ts +2 -0
- package/dist/types/src/testing/parquet-email.test.d.ts.map +1 -0
- package/dist/types/src/testing/parquet.d.ts +20 -0
- package/dist/types/src/testing/parquet.d.ts.map +1 -0
- package/dist/types/src/testing/parquet.test.d.ts +2 -0
- package/dist/types/src/testing/parquet.test.d.ts.map +1 -0
- package/dist/types/src/types/Thread.d.ts +18 -0
- package/dist/types/src/types/Thread.d.ts.map +1 -0
- package/dist/types/src/types/Thread.test.d.ts +2 -0
- package/dist/types/src/types/Thread.test.d.ts.map +1 -0
- package/dist/types/src/types/Topic.d.ts +12 -0
- package/dist/types/src/types/Topic.d.ts.map +1 -0
- package/dist/types/src/types/Topic.test.d.ts +2 -0
- package/dist/types/src/types/Topic.test.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +3 -0
- package/dist/types/src/types/index.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -0
- package/package.json +58 -0
- package/src/corpus/digest.test.ts +78 -0
- package/src/corpus/digest.ts +92 -0
- package/src/corpus/index.ts +9 -0
- package/src/corpus/ledger.test.ts +85 -0
- package/src/corpus/ledger.ts +73 -0
- package/src/corpus/prompts.test.ts +24 -0
- package/src/corpus/prompts.ts +43 -0
- package/src/corpus/rollups.test.ts +50 -0
- package/src/corpus/rollups.ts +65 -0
- package/src/corpus/topics.test.ts +90 -0
- package/src/corpus/topics.ts +211 -0
- package/src/fact-pipeline.test.ts +63 -0
- package/src/fact-pipeline.ts +51 -0
- package/src/index.ts +9 -0
- package/src/internal/fact-index.test.ts +54 -0
- package/src/internal/fact-index.ts +43 -0
- package/src/internal/threading.test.ts +28 -0
- package/src/internal/threading.ts +27 -0
- package/src/internal/threads.test.ts +76 -0
- package/src/internal/threads.ts +73 -0
- package/src/pipeline.test.ts +74 -0
- package/src/pipeline.ts +76 -0
- package/src/stages/context.ts +32 -0
- package/src/stages/extract-contacts.ts +25 -0
- package/src/stages/extract-facts-commit.test.ts +75 -0
- package/src/stages/extract-facts-commit.ts +44 -0
- package/src/stages/extract-facts.test.ts +74 -0
- package/src/stages/extract-facts.ts +26 -0
- package/src/stages/facts.test.ts +40 -0
- package/src/stages/facts.ts +37 -0
- package/src/stages/index.ts +11 -0
- package/src/stages/stats.test.ts +38 -0
- package/src/stages/stats.ts +35 -0
- package/src/stages/summarize.ts +76 -0
- package/src/testing/email-extraction.bench.test.ts +117 -0
- package/src/testing/email-fixtures.ts +62 -0
- package/src/testing/email-pipeline.test.ts +483 -0
- package/src/testing/index.ts +8 -0
- package/src/testing/parquet-email.test.ts +114 -0
- package/src/testing/parquet.test.ts +73 -0
- package/src/testing/parquet.ts +82 -0
- package/src/types/Thread.test.ts +29 -0
- package/src/types/Thread.ts +26 -0
- package/src/types/Topic.test.ts +25 -0
- package/src/types/Topic.ts +20 -0
- package/src/types/index.ts +6 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { Obj } from '@dxos/echo';
|
|
6
|
+
|
|
7
|
+
import { type Thread, Topic } from '../types';
|
|
8
|
+
import { DEFAULT_EMAIL_PROMPTS, type EmailPrompts, type Summarizer, mergePrompts } from './prompts';
|
|
9
|
+
|
|
10
|
+
// Subject tokens that carry no topical signal; excluded from signatures and keywords.
|
|
11
|
+
const DEFAULT_STOPWORDS: readonly string[] = [
|
|
12
|
+
'the',
|
|
13
|
+
'and',
|
|
14
|
+
'for',
|
|
15
|
+
'with',
|
|
16
|
+
'from',
|
|
17
|
+
'about',
|
|
18
|
+
'this',
|
|
19
|
+
'that',
|
|
20
|
+
'your',
|
|
21
|
+
'our',
|
|
22
|
+
'are',
|
|
23
|
+
'was',
|
|
24
|
+
'will',
|
|
25
|
+
'has',
|
|
26
|
+
'have',
|
|
27
|
+
'not',
|
|
28
|
+
'you',
|
|
29
|
+
'all',
|
|
30
|
+
'any',
|
|
31
|
+
'can',
|
|
32
|
+
'per',
|
|
33
|
+
'new',
|
|
34
|
+
];
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Tunable knobs for the deterministic shared-signal clustering (spec §10.3: agglomeration over
|
|
38
|
+
* participant/keyword overlap; embedding/LLM grouping deferred). All values have defaults so
|
|
39
|
+
* callers can adjust the algorithm without forking it.
|
|
40
|
+
*/
|
|
41
|
+
export type TopicOptions = {
|
|
42
|
+
/** Minimum Jaccard similarity between subject-token signatures that joins a topic. */
|
|
43
|
+
readonly similarityThreshold?: number;
|
|
44
|
+
/** Shared-participant count that joins a topic regardless of token overlap. */
|
|
45
|
+
readonly minSharedParticipants?: number;
|
|
46
|
+
/** Subject tokens shorter than this are ignored. */
|
|
47
|
+
readonly minTokenLength?: number;
|
|
48
|
+
/** Tokens excluded from signatures and keywords. */
|
|
49
|
+
readonly stopwords?: readonly string[];
|
|
50
|
+
/** Keywords retained per topic (most frequent first). */
|
|
51
|
+
readonly maxKeywords?: number;
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
export const DEFAULT_TOPIC_OPTIONS: Required<TopicOptions> = {
|
|
55
|
+
similarityThreshold: 0.25,
|
|
56
|
+
minSharedParticipants: 2,
|
|
57
|
+
minTokenLength: 3,
|
|
58
|
+
stopwords: DEFAULT_STOPWORDS,
|
|
59
|
+
maxKeywords: 5,
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
/** A topic before ECHO materialization: plain values so LLM enrichment can rewrite them freely. */
|
|
63
|
+
export type TopicDraft = {
|
|
64
|
+
readonly label: string;
|
|
65
|
+
readonly summary: string;
|
|
66
|
+
readonly threadIds: readonly string[];
|
|
67
|
+
readonly participants: readonly string[];
|
|
68
|
+
readonly keywords: readonly string[];
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
type Signature = {
|
|
72
|
+
readonly thread: Thread;
|
|
73
|
+
readonly tokens: Set<string>;
|
|
74
|
+
readonly participants: Set<string>;
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const tokenize = (subject: string, options: Required<TopicOptions>): Set<string> =>
|
|
78
|
+
new Set(
|
|
79
|
+
subject
|
|
80
|
+
.toLowerCase()
|
|
81
|
+
.split(/[^a-z0-9]+/)
|
|
82
|
+
.filter((token) => token.length >= options.minTokenLength && !options.stopwords.includes(token)),
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
const jaccard = (left: Set<string>, right: Set<string>): number => {
|
|
86
|
+
if (left.size === 0 || right.size === 0) {
|
|
87
|
+
return 0;
|
|
88
|
+
}
|
|
89
|
+
let shared = 0;
|
|
90
|
+
for (const token of left) {
|
|
91
|
+
if (right.has(token)) {
|
|
92
|
+
shared += 1;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return shared / (left.size + right.size - shared);
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const sharedCount = (left: Set<string>, right: Set<string>): number => {
|
|
99
|
+
let shared = 0;
|
|
100
|
+
for (const value of left) {
|
|
101
|
+
if (right.has(value)) {
|
|
102
|
+
shared += 1;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return shared;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Greedy agglomerative clustering of threads into topic drafts. Deterministic (Compute mode): a
|
|
110
|
+
* thread joins the first cluster whose aggregate signature overlaps enough — by subject-token
|
|
111
|
+
* Jaccard similarity or by shared participants — else starts its own. Bounded by O(threads ×
|
|
112
|
+
* clusters); no LLM involved, so results are stable and unit-testable.
|
|
113
|
+
*/
|
|
114
|
+
export const clusterThreads = (threads: readonly Thread[], options?: TopicOptions): TopicDraft[] => {
|
|
115
|
+
const opts = { ...DEFAULT_TOPIC_OPTIONS, ...options };
|
|
116
|
+
const clusters: { members: Signature[]; tokens: Set<string>; participants: Set<string> }[] = [];
|
|
117
|
+
|
|
118
|
+
for (const thread of threads) {
|
|
119
|
+
const signature: Signature = {
|
|
120
|
+
thread,
|
|
121
|
+
tokens: tokenize(thread.subject, opts),
|
|
122
|
+
participants: new Set(thread.participants),
|
|
123
|
+
};
|
|
124
|
+
const home = clusters.find(
|
|
125
|
+
(cluster) =>
|
|
126
|
+
jaccard(cluster.tokens, signature.tokens) >= opts.similarityThreshold ||
|
|
127
|
+
sharedCount(cluster.participants, signature.participants) >= opts.minSharedParticipants,
|
|
128
|
+
);
|
|
129
|
+
if (home) {
|
|
130
|
+
home.members.push(signature);
|
|
131
|
+
for (const token of signature.tokens) {
|
|
132
|
+
home.tokens.add(token);
|
|
133
|
+
}
|
|
134
|
+
for (const participant of signature.participants) {
|
|
135
|
+
home.participants.add(participant);
|
|
136
|
+
}
|
|
137
|
+
} else {
|
|
138
|
+
clusters.push({
|
|
139
|
+
members: [signature],
|
|
140
|
+
tokens: new Set(signature.tokens),
|
|
141
|
+
participants: new Set(signature.participants),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return clusters.map((cluster) => {
|
|
147
|
+
// Rank keywords by how many member threads mention them, then alphabetically for determinism.
|
|
148
|
+
const counts = new Map<string, number>();
|
|
149
|
+
for (const member of cluster.members) {
|
|
150
|
+
for (const token of member.tokens) {
|
|
151
|
+
counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
const keywords = [...counts.entries()]
|
|
155
|
+
.sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
|
|
156
|
+
.slice(0, opts.maxKeywords)
|
|
157
|
+
.map(([token]) => token);
|
|
158
|
+
|
|
159
|
+
const summaries = cluster.members.flatMap((member) =>
|
|
160
|
+
member.thread.summary.length > 0 ? [member.thread.summary] : [],
|
|
161
|
+
);
|
|
162
|
+
|
|
163
|
+
// Label from top keywords; a keyword-less cluster (e.g. blank subjects) falls back through the
|
|
164
|
+
// first thread's subject to its threadId so the label is never empty.
|
|
165
|
+
const first = cluster.members[0].thread;
|
|
166
|
+
return {
|
|
167
|
+
label: keywords.length > 0 ? keywords.slice(0, 3).join(' ') : first.subject || first.threadId,
|
|
168
|
+
summary: summaries.join('\n'),
|
|
169
|
+
threadIds: cluster.members.map((member) => member.thread.threadId),
|
|
170
|
+
participants: [...cluster.participants].sort(),
|
|
171
|
+
keywords,
|
|
172
|
+
};
|
|
173
|
+
});
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Optionally replace each draft's deterministic summary with an LLM one, using the editable
|
|
178
|
+
* `topicSummary` prompt. Degrades per-topic: a failed call keeps that draft's deterministic summary
|
|
179
|
+
* (advisory enrichment must not fail the corpus pass).
|
|
180
|
+
*/
|
|
181
|
+
export const summarizeTopics = async (
|
|
182
|
+
drafts: readonly TopicDraft[],
|
|
183
|
+
summarize: Summarizer,
|
|
184
|
+
options?: { readonly prompts?: Partial<EmailPrompts> },
|
|
185
|
+
): Promise<TopicDraft[]> => {
|
|
186
|
+
const prompts = options?.prompts ? mergePrompts(options.prompts) : DEFAULT_EMAIL_PROMPTS;
|
|
187
|
+
return Promise.all(
|
|
188
|
+
drafts.map(async (draft) => {
|
|
189
|
+
try {
|
|
190
|
+
const summary = await summarize(
|
|
191
|
+
[prompts.topicSummary, `Topic: ${draft.label}`, `Threads:\n${draft.summary}`].join('\n\n'),
|
|
192
|
+
);
|
|
193
|
+
return summary.trim().length > 0 ? { ...draft, summary: summary.trim() } : draft;
|
|
194
|
+
} catch {
|
|
195
|
+
return draft;
|
|
196
|
+
}
|
|
197
|
+
}),
|
|
198
|
+
);
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
/** Materialize drafts as canonical Topic ECHO objects (done last; drafts stay freely mutable). */
|
|
202
|
+
export const materializeTopics = (drafts: readonly TopicDraft[]): Topic[] =>
|
|
203
|
+
drafts.map((draft) =>
|
|
204
|
+
Obj.make(Topic, {
|
|
205
|
+
label: draft.label,
|
|
206
|
+
summary: draft.summary,
|
|
207
|
+
threadIds: [...draft.threadIds],
|
|
208
|
+
participants: [...draft.participants],
|
|
209
|
+
keywords: [...draft.keywords],
|
|
210
|
+
}),
|
|
211
|
+
);
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { afterEach, beforeEach, describe, test } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import { EchoTestBuilder } from '@dxos/echo-client/testing';
|
|
8
|
+
import { EffectEx } from '@dxos/effect';
|
|
9
|
+
import { Message, Organization, Person } from '@dxos/types';
|
|
10
|
+
|
|
11
|
+
import { EmailFactPipeline } from './fact-pipeline';
|
|
12
|
+
import { type FactIndexer } from './stages';
|
|
13
|
+
import { Thread } from './types';
|
|
14
|
+
|
|
15
|
+
describe('email fact pipeline (facts-only assembly)', () => {
|
|
16
|
+
let builder: EchoTestBuilder;
|
|
17
|
+
beforeEach(async () => {
|
|
18
|
+
builder = await new EchoTestBuilder().open();
|
|
19
|
+
});
|
|
20
|
+
afterEach(async () => {
|
|
21
|
+
await builder.close();
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test('EmailFactPipeline.run: stats → extract-facts', async ({ expect }) => {
|
|
25
|
+
const { db } = await builder.createDatabase({
|
|
26
|
+
types: [Person.Person, Organization.Organization, Thread],
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
const messages = [
|
|
30
|
+
makeMessage('<m-1@enron.com>', 'alice@enron.com', 'Q2 report', 'I will send the Q2 report by Friday.'),
|
|
31
|
+
makeMessage('<m-2@enron.com>', 'bob@enron.com', 'Q2 report', 'Thanks, looking forward to it.'),
|
|
32
|
+
];
|
|
33
|
+
|
|
34
|
+
const indexed: Message.Message[] = [];
|
|
35
|
+
const indexFacts: FactIndexer = async (message) => {
|
|
36
|
+
indexed.push(message);
|
|
37
|
+
return [];
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const result = await EffectEx.runPromise(EmailFactPipeline.run(messages, { db, indexFacts }));
|
|
41
|
+
|
|
42
|
+
// All messages flowed through the stream and were collected in order.
|
|
43
|
+
expect(result.messages).toHaveLength(2);
|
|
44
|
+
// stats tallied each message that flowed through.
|
|
45
|
+
expect(result.stats.total).toBe(2);
|
|
46
|
+
expect(result.stats.from.get('alice@enron.com')).toBe(1);
|
|
47
|
+
expect(result.stats.from.get('bob@enron.com')).toBe(1);
|
|
48
|
+
// extract-facts invoked the indexer once per message.
|
|
49
|
+
expect(indexed).toHaveLength(2);
|
|
50
|
+
expect(indexed.map((message) => message.properties?.messageId)).toEqual(['<m-1@enron.com>', '<m-2@enron.com>']);
|
|
51
|
+
// Facts-only assembly omits summarize/extract-contacts/threads: `result` has no such keys
|
|
52
|
+
// (enforced at compile time by `EmailFactPipelineResult`, which declares only `messages`/`stats`).
|
|
53
|
+
expect(Object.keys(result).sort()).toEqual(['messages', 'stats']);
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
const makeMessage = (messageId: string, email: string, subject: string, text: string): Message.Message =>
|
|
58
|
+
Message.make({
|
|
59
|
+
created: '2001-05-14T10:00:00.000Z',
|
|
60
|
+
sender: { email },
|
|
61
|
+
blocks: [{ _tag: 'text', text }],
|
|
62
|
+
properties: { messageId, subject },
|
|
63
|
+
});
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
import * as Stream from 'effect/Stream';
|
|
7
|
+
|
|
8
|
+
import { type Database } from '@dxos/echo';
|
|
9
|
+
import { Pipeline } from '@dxos/pipeline';
|
|
10
|
+
import { Message } from '@dxos/types';
|
|
11
|
+
|
|
12
|
+
import { EmailPipelineCtx, type FactIndexer, type Stats, emptyStats, extractFactsStage, statsStage } from './stages';
|
|
13
|
+
|
|
14
|
+
export type EmailFactPipelineOptions = {
|
|
15
|
+
/** ECHO space database, required by `EmailPipelineCtx` even though this assembly never writes to it. */
|
|
16
|
+
readonly db: Database.Database;
|
|
17
|
+
/** Persists one message's facts to the fact substrate (a closure over a `FactStore`). */
|
|
18
|
+
readonly indexFacts: FactIndexer;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export type EmailFactPipelineResult = {
|
|
22
|
+
readonly messages: Message.Message[];
|
|
23
|
+
readonly stats: Stats;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export const EmailFactPipeline = {
|
|
27
|
+
/**
|
|
28
|
+
* Facts-only assembly of the email stages over a batch of messages: stats → extract-facts. Omits
|
|
29
|
+
* summarize, extract-contacts, and {@link buildThreads} from {@link EmailPipeline}, so it needs no
|
|
30
|
+
* `AiService` and never writes to `db`. The cursored `EnrichMailbox` operation (in plugin-inbox)
|
|
31
|
+
* covers the same fact-extraction concern incrementally, via a sibling facts-returning stage
|
|
32
|
+
* (`extractFactsUnitStage`) that commits pages atomically with a cursor advance instead of indexing
|
|
33
|
+
* in-stage; this batch variant is for one-shot runs over an already-collected set of messages.
|
|
34
|
+
*/
|
|
35
|
+
run: (
|
|
36
|
+
messages: readonly Message.Message[],
|
|
37
|
+
options: EmailFactPipelineOptions,
|
|
38
|
+
): Effect.Effect<EmailFactPipelineResult> =>
|
|
39
|
+
Effect.gen(function* () {
|
|
40
|
+
const stats = emptyStats();
|
|
41
|
+
const collected: Message.Message[] = [];
|
|
42
|
+
yield* Stream.fromIterable(messages).pipe(
|
|
43
|
+
statsStage,
|
|
44
|
+
extractFactsStage(options.indexFacts),
|
|
45
|
+
Pipeline.run({ sink: (message) => Effect.sync(() => collected.push(message)) }),
|
|
46
|
+
Effect.provideService(EmailPipelineCtx, { db: options.db, stats, summaries: [] }),
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
return { messages: collected, stats };
|
|
50
|
+
}),
|
|
51
|
+
};
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { describe, test } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import { Obj } from '@dxos/echo';
|
|
8
|
+
import { RDF, normalizeEntityId } from '@dxos/pipeline-rdf';
|
|
9
|
+
import { Organization, Person } from '@dxos/types';
|
|
10
|
+
|
|
11
|
+
import { buildEntityIndex, reconcileFactEntities } from './fact-index';
|
|
12
|
+
|
|
13
|
+
const fact = (subjectLabel: string, objectLabel: string): RDF.Fact => ({
|
|
14
|
+
id: 'f1',
|
|
15
|
+
assertion: {
|
|
16
|
+
subject: { entity: normalizeEntityId(subjectLabel), label: subjectLabel },
|
|
17
|
+
predicate: 'works at',
|
|
18
|
+
object: { entity: normalizeEntityId(objectLabel), label: objectLabel },
|
|
19
|
+
},
|
|
20
|
+
factuality: { value: 'CT+', polarity: '+' },
|
|
21
|
+
attribution: { source: 'dxn:queue:m1', generatedAtTime: '2001-05-14T10:00:00.000Z' },
|
|
22
|
+
recordedAt: '2001-05-14T10:00:00.000Z',
|
|
23
|
+
extractor: { id: 'default', model: 'test', version: '1' },
|
|
24
|
+
sourceHash: 'h1',
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
describe('entity reconciliation', () => {
|
|
28
|
+
test('buildEntityIndex maps normalized names/emails to object DXNs', ({ expect }) => {
|
|
29
|
+
const alice = Obj.make(Person.Person, { fullName: 'Alice Smith', emails: [{ value: 'alice@enron.com' }] });
|
|
30
|
+
const enron = Obj.make(Organization.Organization, { name: 'Enron' });
|
|
31
|
+
|
|
32
|
+
const index = buildEntityIndex([alice, enron]);
|
|
33
|
+
expect(index.get(normalizeEntityId('Alice Smith'))).toBe(Obj.getURI(alice));
|
|
34
|
+
expect(index.get(normalizeEntityId('alice@enron.com'))).toBe(Obj.getURI(alice));
|
|
35
|
+
expect(index.get(normalizeEntityId('Enron'))).toBe(Obj.getURI(enron));
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test('reconcileFactEntities resolves subject/object to DXNs when known', ({ expect }) => {
|
|
39
|
+
const alice = Obj.make(Person.Person, { fullName: 'Alice Smith', emails: [{ value: 'alice@enron.com' }] });
|
|
40
|
+
const enron = Obj.make(Organization.Organization, { name: 'Enron' });
|
|
41
|
+
const index = buildEntityIndex([alice, enron]);
|
|
42
|
+
|
|
43
|
+
const resolved = reconcileFactEntities(fact('Alice Smith', 'Enron'), index);
|
|
44
|
+
expect(resolved.subject).toBe(Obj.getURI(alice));
|
|
45
|
+
expect(resolved.object).toBe(Obj.getURI(enron));
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test('reconcileFactEntities leaves unknown entities unresolved', ({ expect }) => {
|
|
49
|
+
const index = buildEntityIndex([]);
|
|
50
|
+
const resolved = reconcileFactEntities(fact('Nobody', 'Nowhere'), index);
|
|
51
|
+
expect(resolved.subject).toBeUndefined();
|
|
52
|
+
expect(resolved.object).toBeUndefined();
|
|
53
|
+
});
|
|
54
|
+
});
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { Obj } from '@dxos/echo';
|
|
6
|
+
import { RDF, normalizeEntityId } from '@dxos/pipeline-rdf';
|
|
7
|
+
import { Organization, Person } from '@dxos/types';
|
|
8
|
+
|
|
9
|
+
// Surface forms (name + emails) under which a Person may be referenced in extracted facts.
|
|
10
|
+
const personKeys = (person: Person.Person): string[] =>
|
|
11
|
+
[person.fullName, ...(person.emails ?? []).map((email) => email.value)].filter(
|
|
12
|
+
(value): value is string => typeof value === 'string' && value.length > 0,
|
|
13
|
+
);
|
|
14
|
+
|
|
15
|
+
const orgKeys = (org: Organization.Organization): string[] =>
|
|
16
|
+
[org.name].filter((value): value is string => typeof value === 'string' && value.length > 0);
|
|
17
|
+
|
|
18
|
+
// Reconciliation table: normalized entity id (as produced by the extractor) → canonical ECHO URI.
|
|
19
|
+
// ECHO objects remain the source of truth (spec §4); this table lets advisory facts point at them.
|
|
20
|
+
export const buildEntityIndex = (
|
|
21
|
+
objects: readonly (Person.Person | Organization.Organization)[],
|
|
22
|
+
): Map<string, string> => {
|
|
23
|
+
const index = new Map<string, string>();
|
|
24
|
+
for (const object of objects) {
|
|
25
|
+
const uri = Obj.getURI(object);
|
|
26
|
+
const keys = Obj.instanceOf(Person.Person, object) ? personKeys(object) : orgKeys(object);
|
|
27
|
+
for (const key of keys) {
|
|
28
|
+
index.set(normalizeEntityId(key), uri);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return index;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
// Resolve a fact's subject/object entity ids to canonical URIs where the index knows them; unknown
|
|
35
|
+
// referents are simply omitted (an advisory fact about an unresolved entity is still a valid fact).
|
|
36
|
+
export const reconcileFactEntities = (
|
|
37
|
+
fact: RDF.Fact,
|
|
38
|
+
index: Map<string, string>,
|
|
39
|
+
): { subject?: string; object?: string } => {
|
|
40
|
+
const subject = 'entity' in fact.assertion.subject ? index.get(fact.assertion.subject.entity) : undefined;
|
|
41
|
+
const object = 'entity' in fact.assertion.object ? index.get(fact.assertion.object.entity) : undefined;
|
|
42
|
+
return { ...(subject ? { subject } : {}), ...(object ? { object } : {}) };
|
|
43
|
+
};
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { describe, test } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import { Message } from '@dxos/types';
|
|
8
|
+
|
|
9
|
+
import { deriveThreadId, normalizeSubject } from './threading';
|
|
10
|
+
|
|
11
|
+
describe('threading', () => {
|
|
12
|
+
test('normalizeSubject strips reply/forward prefixes and normalizes', ({ expect }) => {
|
|
13
|
+
expect(normalizeSubject('Re: Q2 Report')).toBe('q2 report');
|
|
14
|
+
expect(normalizeSubject('FW: Fwd: Re: Q2 Report ')).toBe('q2 report');
|
|
15
|
+
expect(normalizeSubject('Q2 Report')).toBe('q2 report');
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test('deriveThreadId groups replies onto one thread', ({ expect }) => {
|
|
19
|
+
const original = Message.make({ sender: { email: 'a@x.com' }, properties: { subject: 'Deal terms' } });
|
|
20
|
+
const reply = Message.make({ sender: { email: 'b@x.com' }, properties: { subject: 'RE: Deal terms' } });
|
|
21
|
+
expect(deriveThreadId(original)).toBe(deriveThreadId(reply));
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test('deriveThreadId falls back for missing subject', ({ expect }) => {
|
|
25
|
+
const message = Message.make({ sender: { email: 'a@x.com' } });
|
|
26
|
+
expect(deriveThreadId(message)).toBe('no-subject');
|
|
27
|
+
});
|
|
28
|
+
});
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { Message } from '@dxos/types';
|
|
6
|
+
|
|
7
|
+
// Leading reply/forward markers, stripped repeatedly ("Fw: Re:" → "").
|
|
8
|
+
const PREFIX = /^\s*(re|fwd?|fw)\s*:\s*/i;
|
|
9
|
+
|
|
10
|
+
// Canonical subject for threading: drop reply/forward prefixes, collapse internal whitespace,
|
|
11
|
+
// lowercase. Two messages with the same normalized subject belong to the same thread.
|
|
12
|
+
export const normalizeSubject = (subject: string): string => {
|
|
13
|
+
let value = subject;
|
|
14
|
+
while (PREFIX.test(value)) {
|
|
15
|
+
value = value.replace(PREFIX, '');
|
|
16
|
+
}
|
|
17
|
+
return value.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
// Derive a thread id from the subject. The Enron dataset carries no References/In-Reply-To headers,
|
|
21
|
+
// so the normalized subject is the available grouping signal; blank subjects share a `no-subject`
|
|
22
|
+
// bucket rather than each forming a singleton thread.
|
|
23
|
+
export const deriveThreadId = (message: Message.Message): string => {
|
|
24
|
+
const subject = message.properties?.subject;
|
|
25
|
+
const normalized = typeof subject === 'string' ? normalizeSubject(subject) : '';
|
|
26
|
+
return normalized.length > 0 ? normalized : 'no-subject';
|
|
27
|
+
};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { describe, test } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import { Message } from '@dxos/types';
|
|
8
|
+
|
|
9
|
+
import { buildThreads } from './threads';
|
|
10
|
+
|
|
11
|
+
const msg = (subject: string, from: string, created: string, summary?: string) =>
|
|
12
|
+
Message.make({
|
|
13
|
+
created,
|
|
14
|
+
sender: { email: from },
|
|
15
|
+
blocks: [{ _tag: 'text', text: 'body' }],
|
|
16
|
+
properties: { subject, messageId: `<${from}:${created}>`, ...(summary ? { summary } : {}) },
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
const OWNER = 'me@enron.com';
|
|
20
|
+
|
|
21
|
+
describe('buildThreads', () => {
|
|
22
|
+
test('groups messages by normalized subject', ({ expect }) => {
|
|
23
|
+
const threads = buildThreads(
|
|
24
|
+
[
|
|
25
|
+
msg('Deal terms', 'a@x.com', '2001-05-01T10:00:00.000Z'),
|
|
26
|
+
msg('RE: Deal terms', OWNER, '2001-05-01T11:00:00.000Z'),
|
|
27
|
+
msg('Lunch?', 'b@x.com', '2001-05-01T12:00:00.000Z'),
|
|
28
|
+
],
|
|
29
|
+
{ ownerEmail: OWNER, now: '2001-05-02T00:00:00.000Z' },
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
expect(threads).toHaveLength(2);
|
|
33
|
+
const deal = threads.find((thread) => thread.threadId === 'deal terms');
|
|
34
|
+
expect(deal?.messageIds).toHaveLength(2);
|
|
35
|
+
expect([...(deal?.participants ?? [])].sort()).toEqual(['a@x.com', 'me@enron.com']);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test('state is awaiting-theirs when owner sent last', ({ expect }) => {
|
|
39
|
+
const [thread] = buildThreads(
|
|
40
|
+
[
|
|
41
|
+
msg('Deal terms', 'a@x.com', '2001-05-01T10:00:00.000Z'),
|
|
42
|
+
msg('RE: Deal terms', OWNER, '2001-05-01T11:00:00.000Z'),
|
|
43
|
+
],
|
|
44
|
+
{ ownerEmail: OWNER, now: '2001-05-02T00:00:00.000Z' },
|
|
45
|
+
);
|
|
46
|
+
expect(thread.state).toBe('awaiting-theirs');
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('state is awaiting-mine when other party sent last', ({ expect }) => {
|
|
50
|
+
const [thread] = buildThreads([msg('Deal terms', 'a@x.com', '2001-05-01T10:00:00.000Z')], {
|
|
51
|
+
ownerEmail: OWNER,
|
|
52
|
+
now: '2001-05-02T00:00:00.000Z',
|
|
53
|
+
});
|
|
54
|
+
expect(thread.state).toBe('awaiting-mine');
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('state is stalled when awaiting past the stale period', ({ expect }) => {
|
|
58
|
+
const [thread] = buildThreads([msg('Deal terms', 'a@x.com', '2001-05-01T10:00:00.000Z')], {
|
|
59
|
+
ownerEmail: OWNER,
|
|
60
|
+
now: '2001-06-01T00:00:00.000Z',
|
|
61
|
+
});
|
|
62
|
+
expect(thread.state).toBe('stalled');
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test('summary concatenates per-message summaries', ({ expect }) => {
|
|
66
|
+
const [thread] = buildThreads(
|
|
67
|
+
[
|
|
68
|
+
msg('Deal terms', 'a@x.com', '2001-05-01T10:00:00.000Z', 'Opening offer.'),
|
|
69
|
+
msg('RE: Deal terms', OWNER, '2001-05-01T11:00:00.000Z', 'Countered.'),
|
|
70
|
+
],
|
|
71
|
+
{ ownerEmail: OWNER, now: '2001-05-02T00:00:00.000Z' },
|
|
72
|
+
);
|
|
73
|
+
expect(thread.summary).toContain('Opening offer.');
|
|
74
|
+
expect(thread.summary).toContain('Countered.');
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { Obj } from '@dxos/echo';
|
|
6
|
+
import { Message } from '@dxos/types';
|
|
7
|
+
|
|
8
|
+
import { messageSource } from '../stages/facts';
|
|
9
|
+
import { Thread, type ThreadState } from '../types';
|
|
10
|
+
import { deriveThreadId } from './threading';
|
|
11
|
+
|
|
12
|
+
const DEFAULT_STALE_PERIOD_MS = 14 * 24 * 60 * 60 * 1000;
|
|
13
|
+
|
|
14
|
+
export type BuildThreadsOptions = {
|
|
15
|
+
readonly ownerEmail: string;
|
|
16
|
+
readonly now: string;
|
|
17
|
+
readonly stalePeriodMs?: number;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
// Coarse thread state from who spoke last and how long ago. `resolved` needs a signal we do not yet
|
|
21
|
+
// extract, so it is not inferred in this slice.
|
|
22
|
+
const computeState = (lastFromOwner: boolean, lastCreated: string, options: BuildThreadsOptions): ThreadState => {
|
|
23
|
+
const idleMs = new Date(options.now).getTime() - new Date(lastCreated).getTime();
|
|
24
|
+
const stale = idleMs > (options.stalePeriodMs ?? DEFAULT_STALE_PERIOD_MS);
|
|
25
|
+
if (stale) {
|
|
26
|
+
return 'stalled';
|
|
27
|
+
}
|
|
28
|
+
return lastFromOwner ? 'awaiting-theirs' : 'awaiting-mine';
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
// Group messages into canonical Thread objects. Messages are bucketed by derived threadId; within a
|
|
32
|
+
// bucket they are ordered by `created` to find the last speaker.
|
|
33
|
+
export const buildThreads = (messages: readonly Message.Message[], options: BuildThreadsOptions): Thread[] => {
|
|
34
|
+
const buckets = new Map<string, Message.Message[]>();
|
|
35
|
+
for (const message of messages) {
|
|
36
|
+
const threadId = deriveThreadId(message);
|
|
37
|
+
let bucket = buckets.get(threadId);
|
|
38
|
+
if (!bucket) {
|
|
39
|
+
bucket = [];
|
|
40
|
+
buckets.set(threadId, bucket);
|
|
41
|
+
}
|
|
42
|
+
bucket.push(message);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const threads: Thread[] = [];
|
|
46
|
+
for (const [threadId, bucket] of buckets) {
|
|
47
|
+
const ordered = [...bucket].sort((a, b) => a.created.localeCompare(b.created));
|
|
48
|
+
const last = ordered[ordered.length - 1];
|
|
49
|
+
// Email addresses are case-insensitive, so compare and dedup on a lowercased form; otherwise
|
|
50
|
+
// mixed-case senders split into distinct participants and can misclassify awaiting-mine/theirs.
|
|
51
|
+
const owner = options.ownerEmail.toLowerCase();
|
|
52
|
+
const lastFromOwner = last.sender.email?.toLowerCase() === owner;
|
|
53
|
+
const participants = [...new Set(ordered.flatMap((m) => (m.sender.email ? [m.sender.email.toLowerCase()] : [])))];
|
|
54
|
+
const summaries = ordered.flatMap((m) =>
|
|
55
|
+
typeof m.properties?.summary === 'string' && m.properties.summary.length > 0 ? [m.properties.summary] : [],
|
|
56
|
+
);
|
|
57
|
+
const subject = typeof ordered[0].properties?.subject === 'string' ? ordered[0].properties.subject : threadId;
|
|
58
|
+
|
|
59
|
+
threads.push(
|
|
60
|
+
Obj.make(Thread, {
|
|
61
|
+
threadId,
|
|
62
|
+
subject,
|
|
63
|
+
summary: summaries.length > 0 ? summaries.join('\n') : subject,
|
|
64
|
+
state: computeState(lastFromOwner, last.created, options),
|
|
65
|
+
participants,
|
|
66
|
+
messageIds: ordered.map(messageSource),
|
|
67
|
+
openQuestions: [],
|
|
68
|
+
actionItems: [],
|
|
69
|
+
}),
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
return threads;
|
|
73
|
+
};
|