@dxos/pipeline-email 0.0.0 → 0.11.1

package/dist/lib/index.mjs

@@ -0,0 +1,994 @@
+import { Project } from "@dxos/compute";
+import { trim } from "@dxos/util";
+import * as Effect from "effect/Effect";
+import * as Chunk from "effect/Chunk";
+import * as Stream from "effect/Stream";
+import { Blob, DXN, Database, Feed, Filter, Obj, Ref, Type } from "@dxos/echo";
+import { Cursor } from "@dxos/link";
+import { log } from "@dxos/log";
+import { Pipeline, Stage } from "@dxos/pipeline";
+import { FactStore } from "@dxos/pipeline-rdf";
+import { DraftMessage, Message } from "@dxos/types";
+import * as Context from "effect/Context";
+import { buildContactFromActor, extractContact, getIdentityIndex, identitySpecs } from "@dxos/extractor-lib";
+import * as LanguageModel from "@effect/ai/LanguageModel";
+import * as Layer from "effect/Layer";
+import { AiService } from "@dxos/ai";
+import { overlayIdentityIndex } from "@dxos/extractor";
+import { normalizeText } from "@dxos/markdown";
+import { TagIndex, Tagging } from "@dxos/schema";
+import * as Schema from "effect/Schema";
+//#region \0rolldown/runtime.js
+var __defProp = Object.defineProperty;
+var __exportAll = (all, no_symbols) => {
+	let target = {};
+	for (var name in all) __defProp(target, name, {
+		get: all[name],
+		enumerable: true
+	});
+	if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
+	return target;
+};
+//#endregion
+//#region src/corpus/prompts.ts
+var DEFAULT_EMAIL_PROMPTS = {
+	topicSummary: trim`
+    Summarize the following email conversation topic in 1-2 sentences.
+    Focus on what the threads have in common and any outcome or open question.
+    Respond with ONLY the summary text.
+  `,
+	digest: trim`
+    Write a short briefing (3-5 sentences) from the following inbox digest data.
+    Lead with what needs attention (stalled threads, due commitments), then notable topics.
+    Respond with ONLY the briefing text.
+  `
+};
+/** Overlay caller overrides onto the defaults; absent slots keep their default. */
+var mergePrompts = (overrides) => ({
+	...DEFAULT_EMAIL_PROMPTS,
+	...overrides
+});
+var DEFAULT_TOPIC_OPTIONS = {
+	similarityThreshold: .25,
+	minSharedParticipants: 2,
+	minTokenLength: 3,
+	stopwords: [
+		"the",
+		"and",
+		"for",
+		"with",
+		"from",
+		"about",
+		"this",
+		"that",
+		"your",
+		"our",
+		"are",
+		"was",
+		"will",
+		"has",
+		"have",
+		"not",
+		"you",
+		"all",
+		"any",
+		"can",
+		"per",
+		"new"
+	],
+	maxKeywords: 5,
+	dropIdTokens: true
+};
+var isIdToken = (token) => {
+	if (/^\d+$/.test(token)) return true;
+	const digits = (token.match(/\d/g) ?? []).length;
+	if (digits === 0) return false;
+	if (/^[0-9a-f]+$/.test(token) && token.length >= 6) return true;
+	return token.length >= 5 && digits / token.length >= .3;
+};
+var tokenize = (subject, options) => new Set(subject.toLowerCase().split(/[^a-z0-9]+/).filter((token) => token.length >= options.minTokenLength && !options.stopwords.includes(token) && !(options.dropIdTokens && isIdToken(token))));
+var jaccard = (left, right) => {
+	if (left.size === 0 || right.size === 0) return 0;
+	let shared = 0;
+	for (const token of left) if (right.has(token)) shared += 1;
+	return shared / (left.size + right.size - shared);
+};
+var sharedCount = (left, right) => {
+	let shared = 0;
+	for (const value of left) if (right.has(value)) shared += 1;
+	return shared;
+};
+/**
+* Greedy agglomerative clustering of threads into topic drafts. Deterministic (Compute mode): a
+* thread joins the first cluster whose aggregate signature overlaps enough — by subject-token
+* Jaccard similarity or by shared participants — else starts its own. Bounded by O(threads ×
+* clusters); no LLM involved, so results are stable and unit-testable.
+*/
+var clusterThreads = (threads, options) => {
+	const opts = {
+		...DEFAULT_TOPIC_OPTIONS,
+		...options
+	};
+	const clusters = [];
+	for (const thread of threads) {
+		const signature = {
+			thread,
+			tokens: tokenize(thread.subject, opts),
+			participants: new Set(thread.participants)
+		};
+		const home = clusters.find((cluster) => jaccard(cluster.tokens, signature.tokens) >= opts.similarityThreshold || sharedCount(cluster.participants, signature.participants) >= opts.minSharedParticipants);
+		if (home) {
+			home.members.push(signature);
+			for (const token of signature.tokens) home.tokens.add(token);
+			for (const participant of signature.participants) home.participants.add(participant);
+		} else clusters.push({
+			members: [signature],
+			tokens: new Set(signature.tokens),
+			participants: new Set(signature.participants)
+		});
+	}
+	return clusters.map((cluster) => {
+		const counts = /* @__PURE__ */ new Map();
+		for (const member of cluster.members) for (const token of member.tokens) counts.set(token, (counts.get(token) ?? 0) + 1);
+		const keywords = [...counts.entries()].sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0])).slice(0, opts.maxKeywords).map(([token]) => token);
+		const summaries = cluster.members.flatMap((member) => member.thread.summary.length > 0 ? [member.thread.summary] : []);
+		const questions = [...new Set(cluster.members.flatMap((member) => member.thread.openQuestions ?? []))];
+		const tasks = [...new Set(cluster.members.flatMap((member) => member.thread.actionItems ?? []))];
+		const first = cluster.members[0].thread;
+		return {
+			name: keywords.length > 0 ? keywords.slice(0, 3).join(" ") : first.subject || first.threadId,
+			summary: summaries.join("\n"),
+			threadIds: cluster.members.map((member) => member.thread.threadId),
+			participants: [...cluster.participants].sort(),
+			keywords,
+			questions,
+			tasks
+		};
+	});
+};
+/**
+* Optionally replace each draft's deterministic summary with an LLM one, using the editable
+* `topicSummary` prompt. Degrades per-topic: a failed call keeps that draft's deterministic summary
+* (advisory enrichment must not fail the corpus pass).
+*/
+var summarizeTopics = async (drafts, summarize, options) => {
+	const prompts = options?.prompts ? mergePrompts(options.prompts) : DEFAULT_EMAIL_PROMPTS;
+	return Promise.all(drafts.map(async (draft) => {
+		try {
+			const summary = await summarize([prompts.topicSummary, `Topic: ${draft.name}`].join("\n\n"));
+			return summary.trim().length > 0 ? {
+				...draft,
+				summary: summary.trim()
+			} : draft;
+		} catch {
+			return draft;
+		}
+	}));
+};
+/** Materialize drafts as canonical Project ECHO objects (done last; drafts stay freely mutable). */
+var materializeTopics = (drafts) => drafts.map((draft) => Project.make({ name: draft.name }));
+//#endregion
+//#region src/corpus/ledger.ts
+/**
+* Predicates treated as commitments (spec §3③ commitment ledger). Editable: extraction phrasing
+* varies by corpus and extraction rules, so callers tune this list rather than the query logic.
+*/
+var DEFAULT_COMMITMENT_PREDICATES = [
+	"owes",
+	"will send",
+	"will provide",
+	"must",
+	"due"
+];
+var termLabel = (term) => "entity" in term ? term.label ?? term.entity : term.literal;
+var toCommitment = (fact) => ({
+	who: termLabel(fact.assertion.subject),
+	what: termLabel(fact.assertion.object),
+	...fact.assertion.validTo ? { dueBy: fact.assertion.validTo } : {},
+	...fact.factuality.confidence !== void 0 ? { confidence: fact.factuality.confidence } : {},
+	factId: fact.id,
+	source: fact.attribution.source
+});
+/**
+* Query the fact store for outstanding commitments — the spec's cross-thread "owe / owed" ledger.
+* Uses the structured query path (one `query({ predicate })` per commitment predicate, compiled to
+* SPARQL on the sqlite backend), which works on both store backends; rows are advisory evidence
+* (§4 canonicality) and each carries its fact id + source message for grounding.
+*/
+var commitmentLedger = (store, options) => Effect.forEach(options?.predicates ?? DEFAULT_COMMITMENT_PREDICATES, (predicate) => store.query({ predicate })).pipe(Effect.map((results) => {
+	const seen = /* @__PURE__ */ new Set();
+	const commitments = [];
+	for (const fact of results.flat()) if (!seen.has(fact.id)) {
+		seen.add(fact.id);
+		commitments.push(toCommitment(fact));
+	}
+	return commitments.sort((a, b) => a.dueBy && b.dueBy ? a.dueBy.localeCompare(b.dueBy) : a.dueBy ? -1 : b.dueBy ? 1 : 0);
+}));
+//#endregion
+//#region src/internal/threading.ts
+var PREFIX = /^\s*(re|fwd?|fw)\s*:\s*/i;
+var normalizeSubject = (subject) => {
+	let value = subject;
+	while (PREFIX.test(value)) value = value.replace(PREFIX, "");
+	return value.replace(/\s+/g, " ").trim().toLowerCase();
+};
+var deriveThreadId = (message) => {
+	const subject = message.properties?.subject;
+	const normalized = typeof subject === "string" ? normalizeSubject(subject) : "";
+	return normalized.length > 0 ? normalized : "no-subject";
+};
+//#endregion
+//#region src/corpus/rollups.ts
+var MS_PER_DAY = 1440 * 60 * 1e3;
+/** Aggregate messages into per-sender rollups, most active sender first. Deterministic (Compute mode). */
+var buildRollups = (messages) => {
+	const bySender = /* @__PURE__ */ new Map();
+	for (const message of messages) {
+		const email = message.sender.email?.toLowerCase();
+		if (!email) continue;
+		let entry = bySender.get(email);
+		if (!entry) {
+			entry = {
+				created: [],
+				threadIds: []
+			};
+			bySender.set(email, entry);
+		}
+		entry.created.push(message.created);
+		const threadId = deriveThreadId(message);
+		if (!entry.threadIds.includes(threadId)) entry.threadIds.push(threadId);
+	}
+	const rollups = [];
+	for (const [email, entry] of bySender) {
+		const ordered = [...entry.created].sort();
+		const first = ordered[0];
+		const last = ordered[ordered.length - 1];
+		const spanMs = new Date(last).getTime() - new Date(first).getTime();
+		rollups.push({
+			email,
+			messageCount: ordered.length,
+			firstContact: first,
+			lastContact: last,
+			...ordered.length >= 2 ? { cadenceDays: spanMs / (ordered.length - 1) / MS_PER_DAY } : {},
+			threadIds: entry.threadIds
+		});
+	}
+	return rollups.sort((a, b) => b.messageCount - a.messageCount || a.email.localeCompare(b.email));
+};
+//#endregion
+//#region src/corpus/digest.ts
+/** Assemble the deterministic digest skeleton from the corpus outputs (Compute mode; no LLM). */
+var buildDigest = (input, options) => ({
+	generatedAt: options.now,
+	threadCount: input.threads.length,
+	topicCount: input.topics.length,
+	stalledThreads: input.threads.filter((thread) => thread.state === "stalled").map((thread) => thread.threadId),
+	awaitingMine: input.threads.filter((thread) => thread.state === "awaiting-mine").map((thread) => thread.threadId),
+	dueCommitments: input.commitments.filter((commitment) => commitment.dueBy !== void 0).sort((a, b) => (a.dueBy ?? "").localeCompare(b.dueBy ?? "")),
+	narrative: ""
+});
+/** Render the skeleton as plain text — the prompt payload for narration, and a fallback briefing. */
+var renderDigest = (digest) => [
+	`Inbox digest (${digest.generatedAt})`,
+	`Threads: ${digest.threadCount}  Topics: ${digest.topicCount}`,
+	`Awaiting my reply: ${digest.awaitingMine.join(", ") || "none"}`,
+	`Stalled: ${digest.stalledThreads.join(", ") || "none"}`,
+	`Due commitments: ${digest.dueCommitments.map((commitment) => `${commitment.who} → ${commitment.what} (${commitment.dueBy})`).join("; ") || "none"}`
+].join("\n");
+/**
+* Optionally add an LLM briefing over the skeleton, using the editable `digest` prompt. Degrades to
+* the un-narrated digest on failure — the skeleton is always available.
+*/
+var narrateDigest = async (digest, summarize, options) => {
+	const prompts = options?.prompts ? mergePrompts(options.prompts) : DEFAULT_EMAIL_PROMPTS;
+	try {
+		const narrative = await summarize([prompts.digest, renderDigest(digest)].join("\n\n"));
+		return {
+			...digest,
+			narrative: narrative.trim()
+		};
+	} catch {
+		return digest;
+	}
+};
+//#endregion
+//#region src/stages/context.ts
+var emptyStats = () => ({
+	from: /* @__PURE__ */ new Map(),
+	to: /* @__PURE__ */ new Map(),
+	total: 0,
+	spam: 0
+});
+/**
+* Shared context threaded through the email stages via Effect's Requirements channel. `db` is the
+* ECHO space database (browser-safe); `stats` and `summaries` are mutable accumulators read after
+* the run. Callers provide it once at the pipeline edge (`Effect.provide`).
+*/
+var EmailPipelineCtx = class extends Context.Tag("@dxos/pipeline-email/EmailPipelineCtx")() {};
+//#endregion
+//#region src/stages/extract-contacts.ts
+/** Extract a Person (+ Organization) from the sender and persist to the ECHO space; pass message through. */
+var extractContactsStage = Stage.map("extract-contacts", (message) => Effect.gen(function* () {
+	const { db } = yield* EmailPipelineCtx;
+	const result = yield* extractContact({
+		db,
+		source: message
+	});
+	for (const object of result.created) db.add(object);
+	return message;
+}));
+//#endregion
+//#region src/stages/extract-facts.ts
+var extractFactsStage = (indexFacts) => Stage.map("extract-facts", (message) => Effect.tryPromise(() => indexFacts(message)).pipe(Effect.tapError((error) => Effect.logWarning("extract-facts failed; degrading to no facts", error)), Effect.orElse(() => Effect.succeed([])), Effect.as(message)));
+//#endregion
+//#region src/stages/facts.ts
+var EMAIL_EXTRACTION_RULES = [
+	"Treat a request or promise as a commitment: \"please send X by Friday\" => subject (the owner), predicate \"owes\"/\"will send\", object X, validTo the deadline.",
+	"A deadline or due date becomes validTo on the relevant assertion (ISO date when known).",
+	"Attribute first-person statements (\"I will…\", \"we can…\") to the message sender as subject."
+];
+var EMAIL_EXTRACT_OPTIONS = { rules: EMAIL_EXTRACTION_RULES };
+var messageSource = (message) => {
+	const messageId = message.properties?.messageId;
+	if (typeof messageId === "string" && messageId.length > 0) return messageId;
+	return `${message.sender.email ?? "unknown"}:${message.created}`;
+};
+var messageToDocument = (message) => ({
+	text: Message.extractText(message),
+	source: messageSource(message),
+	...message.sender.email ? { author: message.sender.email } : {},
+	date: message.created
+});
+//#endregion
+//#region src/stages/extract-facts-commit.ts
+var extractFactsUnitStage = (extract, concurrency = 1) => Stage.map("extract-facts-unit", (message) => Effect.tryPromise(() => extract(message)).pipe(Effect.tapError((error) => Effect.logWarning("extract-facts-unit failed; degrading to no facts", error)), Effect.orElse(() => Effect.succeed([])), Effect.map((facts) => ({
+	facts,
+	foreignId: messageSource(message),
+	key: keyOf(message)
+}))), { concurrency });
+var keyOf = (message) => Date.parse(message.created);
+//#endregion
+//#region src/stages/stats.ts
+/** Pure-JS running tallies (senders, recipients, spam); pass message through. */
+var statsStage = Stage.map("stats", (message) => Effect.gen(function* () {
+	const { stats } = yield* EmailPipelineCtx;
+	stats.total += 1;
+	const sender = message.sender.email;
+	if (sender) stats.from.set(sender, (stats.from.get(sender) ?? 0) + 1);
+	const recipients = message.properties?.to;
+	if (Array.isArray(recipients)) for (const recipient of recipients) {
+		const address = String(recipient);
+		stats.to.set(address, (stats.to.get(address) ?? 0) + 1);
+	}
+	if (message.properties?.spam) stats.spam += 1;
+	return message;
+}));
+//#endregion
+//#region src/stages/summarize.ts
+var SUMMARIZE_MODEL = "com.anthropic.model.claude-haiku-4-5.default";
+var SUMMARIZE_PROMPT = trim`
+  Summarize the following email in one sentence, decide whether it is spam, and list up to five keywords.
+  Respond with ONLY a JSON object of the form {"summary": string, "isSpam": boolean, "keywords": string[]}.
+`;
+var parseSummary = (raw) => {
+	const match = raw.match(/\{[\s\S]*\}/);
+	if (!match) return {
+		summary: "",
+		isSpam: false,
+		keywords: []
+	};
+	try {
+		const parsed = JSON.parse(match[0]);
+		const keywords = parsed.keywords;
+		return {
+			summary: typeof parsed.summary === "string" ? parsed.summary : "",
+			isSpam: parsed.isSpam === true || parsed.spam === true || parsed.is_spam === true,
+			keywords: Array.isArray(keywords) ? keywords.map((keyword) => String(keyword)) : []
+		};
+	} catch {
+		return {
+			summary: "",
+			isSpam: false,
+			keywords: []
+		};
+	}
+};
+/** Summarize via AiService/LanguageModel; append a summary block and record spam/keywords + result. */
+var summarizeStage = Stage.map("summarize", (message) => Effect.gen(function* () {
+	const ctx = yield* EmailPipelineCtx;
+	const text = Message.extractText(message);
+	const summary = parseSummary(yield* LanguageModel.generateText({ prompt: `${SUMMARIZE_PROMPT}\n\n${text}` }).pipe(Effect.provide(AiService.model(SUMMARIZE_MODEL).pipe(Layer.orDie)), Effect.timeout("30 seconds"), Effect.map((response) => response.text), Effect.orElse(() => Effect.succeed(""))));
+	const messageId = String(message.properties?.messageId ?? message.id);
+	ctx.summaries.push({
+		messageId,
+		summary
+	});
+	const summaryBlock = {
+		_tag: "text",
+		text: summary.summary
+	};
+	return Message.make({
+		id: message.id,
+		created: message.created,
+		sender: message.sender,
+		blocks: [...message.blocks, summaryBlock],
+		properties: {
+			...message.properties,
+			spam: summary.isSpam,
+			keywords: summary.keywords
+		}
+	});
+}));
+//#endregion
+//#region src/model-policy.ts
+/** Anthropic model DXNs available to product pipelines. */
+var MODEL_HAIKU = "com.anthropic.model.claude-haiku-4-5.default";
+var MODEL_SONNET = "com.anthropic.model.claude-sonnet-4-6.default";
+var MODEL_OPUS = "com.anthropic.model.claude-opus-4-8.default";
+/**
+* Default stage → model routing, seeded from the model-ladder findings (REPORT §4/§5): labeling and
+* summaries want a premier model, drafting tolerates a cheaper one. Product defaults to haiku across
+* the board (the cheapest premier tier); callers override per run via a `ModelPolicy`.
+*/
+var DEFAULT_MODEL_POLICY = {
+	"classify-sender": MODEL_HAIKU,
+	"tag": MODEL_HAIKU,
+	"summarize-message": MODEL_HAIKU,
+	"summarize-thread": MODEL_HAIKU,
+	"summarize-topic": MODEL_HAIKU,
+	"extract-facts": MODEL_HAIKU,
+	"draft": MODEL_HAIKU
+};
+/** Resolves the model DXN for a stage: per-run `policy` wins over `DEFAULT_MODEL_POLICY`. */
+var resolveModel = (stage, policy) => policy?.[stage] ?? DEFAULT_MODEL_POLICY[stage];
+//#endregion
+//#region src/stages/tag.ts
+var TAG_PROMPT = trim`
+  Classify the following email. Assign between 1 and 5 short, lowercase, single-word topic tags
+  (e.g. "invoice", "newsletter", "security", "personal"). Decide whether the email is spam or a
+  cold marketing blast; if so include the tag "spam".
+  Decide whether the email is "bulk" — automated, no-action mail such as receipts, login or
+  security notices, order/shipping confirmations, or newsletters. If so include the tag "bulk".
+  Do NOT tag invoices, payment requests, or anything that needs a reply as "bulk".
+  Respond with ONLY a JSON object of the form {"tags": string[], "spam": boolean}.
+`;
+var BULK_SUBJECT_RE = /\b(receipts?|your (order|purchase|receipt)|order (confirmation|placed|shipped|#)|(has )?shipped|out for delivery|tracking (number|info)|sign[-\s]?in|log[-\s]?in|new login|verify( your)?|verification code|confirm your (email|address)|password reset|welcome to|newsletter|digest|weekly (update|digest)|unsubscribe|subscription (confirmed|renewed))\b/i;
+var ACTION_SUBJECT_RE = /\b(invoices?|payment (due|request|reminder|required)|amount due|balance due|past due|overdue|outstanding balance|please pay|remittance|bill(ing)? (due|statement)|pay now)\b/i;
+var BULK_LOCALPART_RE = /^(no-?reply|do-?not-?reply|receipts?|notifications?|notify|alerts?|updates?|news(letter)?|digest|mailer|postmaster)([._+-]|$)/i;
+var UNSUBSCRIBE_BODY_RE = /\bunsubscribe\b/i;
+/**
+* Deterministic bulk / no-action signal. Returns `'bulk'` for mailing-list / automated mail —
+* **any unsubscribe affordance** (a `List-Unsubscribe` header or an unsubscribe link in the body) is
+* the top-priority give-away (RFC 2369), followed by receipts / logins / order confirmations /
+* newsletters and no-reply role senders. Returns `'action'` when the mail requires a response
+* (invoice / payment request) and has no unsubscribe affordance; `'unknown'` otherwise. Pure.
+*/
+var classifyBulk = (input) => {
+	if ((input.listUnsubscribe ?? "").length > 0 || UNSUBSCRIBE_BODY_RE.test(input.bodyText ?? "")) return "bulk";
+	const subject = input.subject ?? "";
+	if (ACTION_SUBJECT_RE.test(subject)) return "action";
+	if (BULK_SUBJECT_RE.test(subject)) return "bulk";
+	const localPart = (input.senderEmail ?? "").split("@")[0] ?? "";
+	return BULK_LOCALPART_RE.test(localPart) ? "bulk" : "unknown";
+};
+/** Folds the deterministic bulk classification into a tag set: adds `bulk` for no-action mail, strips it for action-required mail. Pure. */
+var applyBulkTag = (tags, classification) => {
+	if (classification === "action") return tags.filter((tag) => tag !== "bulk");
+	if (classification === "bulk" && !tags.includes("bulk")) return [...tags, "bulk"];
+	return [...tags];
+};
+/** Parses the tag response leniently (models wrap JSON in prose); infers/dedups the spam tag. Pure. */
+var parseTagResult = (raw) => {
+	const match = raw.match(/\{[\s\S]*\}/);
+	if (!match) return {
+		tags: [],
+		spam: false,
+		bulk: false
+	};
+	try {
+		const parsed = JSON.parse(match[0]);
+		const tags = Array.isArray(parsed.tags) ? parsed.tags.map((tag) => String(tag).toLowerCase()).filter((tag) => tag.length > 0) : [];
+		const spam = parsed.spam === true || tags.includes("spam");
+		return {
+			tags: spam && !tags.includes("spam") ? [...tags, "spam"] : tags,
+			spam,
+			bulk: tags.includes("bulk")
+		};
+	} catch {
+		return {
+			tags: [],
+			spam: false,
+			bulk: false
+		};
+	}
+};
+/**
+* Tags a single message (topic tags + spam) via the AI service. The model comes from the `tag` stage
+* of the model policy (overridable per run). Degrades to no tags on timeout/error so one bad message
+* never fails the pipeline.
+*/
+var tagMessage = (message, options = {}) => Effect.gen(function* () {
+	const subject = String(message.properties?.subject ?? "");
+	const body = Message.extractText(message);
+	const parsed = parseTagResult(yield* LanguageModel.generateText({ prompt: `${TAG_PROMPT}\n\nSubject: ${subject}\n\n${body}` }).pipe(Effect.provide(AiService.model(resolveModel("tag", options.policy)).pipe(Layer.orDie)), Effect.timeout("30 seconds"), Effect.map((response) => response.text), Effect.orElse(() => Effect.succeed(""))));
+	const listUnsubscribe = message.properties?.listUnsubscribe;
+	const classification = classifyBulk({
+		subject,
+		senderEmail: message.sender?.email,
+		listUnsubscribe: typeof listUnsubscribe === "string" ? listUnsubscribe : void 0,
+		bodyText: body
+	});
+	const tags = applyBulkTag(parsed.tags, classification);
+	return {
+		tags,
+		spam: parsed.spam,
+		bulk: tags.includes("bulk")
+	};
+});
+//#endregion
+//#region src/stages/EmailStage.ts
+var EmailStage_exports = /* @__PURE__ */ __exportAll({
+	extractContacts: () => extractContacts,
+	htmlToMarkdown: () => htmlToMarkdown,
+	processAttachments: () => processAttachments,
+	queryDraftPool: () => queryDraftPool,
+	reconcileDrafts: () => reconcileDrafts,
+	toCommitUnit: () => toCommitUnit
+});
+var __dxlog_file$1 = "/__w/dxos/dxos/packages/core/compute/pipeline-email/src/stages/EmailStage.ts";
+/**
+* Normalizes an item's `body` from (possibly) HTML into markdown via `normalizeText` (turndown when
+* HTML, passthrough for plaintext). Requires nothing from the Requirements channel. Written as a
+* generic pipeable so the item type infers from the upstream stream (no type argument needed).
+*/
+var htmlToMarkdown = (self) => Stage.map("html-to-markdown", (item) => Effect.sync(() => ({
+	...item,
+	body: normalizeText(item.body)
+})))(self);
+/**
+* Builds a Person (+ Organization link by domain) from the message sender and records it on the
+* item as {@link Insert.contact} — the stage writes nothing itself; {@link toCommitUnit} defers
+* the actual `db.add` to commit.
+*
+* Dedup is the space's shared identity index (see `getIdentityIndex`), refreshed once at the start
+* of the run: it covers contacts committed before the run, contacts built earlier in it, and — since
+* the index is shared per database — contacts a concurrently-syncing mailbox is building right now.
+*
+* Not every sender earns a record: {@link senderSignals} feeds the allow-list in
+* `shouldExtractContact`, so bulk and automated mail no longer fills the space with contacts nobody
+* will look at. NOTE: no provider records an outbound signal yet, so in practice the allow-list is
+* "the domain matches a known Organization" — stricter than the policy intends. Extracting the
+* recipients of sent mail is what closes that gap.
+*/
+var extractContacts = () => {
+	let index;
+	return Stage.map("extract-contacts", (change) => Effect.gen(function* () {
+		if (change._tag !== "insert") return change;
+		const { db } = yield* Database.Service;
+		if (!index) index = overlayIdentityIndex(identitySpecs, yield* getIdentityIndex(db, { refresh: true }));
+		const sender = change.message.sender;
+		const contact = sender ? yield* buildContactFromActor(sender, db, {
+			signals: senderSignals(change.message),
+			index
+		}) : void 0;
+		return contact ? {
+			...change,
+			contact
+		} : change;
+	}));
+};
+/**
+* The sender signals the shared extraction gate reads, taken from the fields provider mappers
+* already record on `properties`. `Precedence`/`Auto-Submitted` are folded into `bulk` by the mapper
+* where available.
+*/
+var senderSignals = (message) => {
+	const properties = message.properties ?? {};
+	return {
+		noReply: properties.noReply === true,
+		listUnsubscribe: typeof properties.listUnsubscribe === "string" ? properties.listUnsubscribe : void 0,
+		bulk: properties.bulk === true
+	};
+};
+/**
+* Turns each of the item's `attachments` into a Blob object (via the database's configured storage
+* backend — edge in Composer) and adds a {@link Message.Attachment} pointing at it to the message.
+* {@link toCommitUnit} finds these blobs again via each attachment's ref (inlined, since the
+* blob isn't attached to a database yet) to defer their feed append to commit. A no-op when the item
+* has no attachments.
+*
+* One bad or oversized attachment must not fail the whole message: each `Blob.fromBytes` is caught
+* and logged individually, so a rejected attachment is simply dropped from the message.
+*/
+var processAttachments = () => Stage.map("process-attachments", (change) => Effect.gen(function* () {
+	if (change._tag !== "insert") return change;
+	const mapped = change;
+	if (!mapped.attachments?.length) return mapped;
+	const attachments = [];
+	for (const attachment of mapped.attachments) {
+		const blob = yield* Blob.fromBytes(attachment.bytes, { type: attachment.mimeType }).pipe(Effect.catchAll((error) => {
+			log.catch(error, {
+				foreignId: mapped.foreignId,
+				name: attachment.name,
+				size: attachment.size
+			}, {
+				"~LogMeta": "~LogMeta",
+				F: __dxlog_file$1,
+				L: 185,
+				S: this
+			});
+			return Effect.succeed(void 0);
+		}));
+		if (!blob) continue;
+		attachments.push({
+			name: attachment.name,
+			ref: Ref.make(blob),
+			contentId: attachment.contentId
+		});
+	}
+	if (attachments.length === 0) return mapped;
+	Obj.update(mapped.message, (message) => {
+		message.attachments = [...message.attachments ?? [], ...attachments];
+	});
+	return mapped;
+}));
+/**
+* Queries the parent container's already-sent drafts once per sync run and pools them by the provider
+* message id captured at send time (`properties.sentMessageId`) — the key {@link reconcileDrafts}
+* matches each incoming message's `foreignId` against. Doing this once (rather than per item) keeps
+* reconciliation off the database as messages stream through. Unsent drafts (no `sentMessageId`) are
+* never pooled and thus never eligible for removal.
+*
+* `parent` is the local root object drafts are scoped to (e.g. a Mailbox) — only its URI is read, so
+* any object works.
+*/
+var queryDraftPool = Effect.fn("queryDraftPool")(function* (parent) {
+	const parentUri = Obj.getURI(parent);
+	const drafts = (yield* Database.query(Filter.type(Message.Message, { properties: { mailbox: parentUri } })).run).filter((candidate) => DraftMessage.belongsTo(candidate, parentUri) && candidate.properties?.sentMessageId);
+	const pool = /* @__PURE__ */ new Map();
+	for (const draft of drafts) {
+		const sentMessageId = draft.properties?.sentMessageId;
+		if (!sentMessageId) continue;
+		const existing = pool.get(sentMessageId);
+		if (existing) existing.push(draft);
+		else pool.set(sentMessageId, [draft]);
+	}
+	return pool;
+});
+/**
+* Records the local drafts each incoming message supersedes — matched by the provider message id the
+* draft captured at send time (`properties.sentMessageId`), which equals the synced message's
+* `foreignId` — from a pool built once per run and keyed by that id. Writes nothing itself;
+* {@link toCommitUnit} defers the `db.remove` to commit, so the canonical message is appended
+* to the feed before its now-redundant drafts are deleted. A stage factory: pass the run's draft
+* pool once.
+*
+* The pool is queried a single time by the caller (per sync operation), so this stage never touches
+* the database or feed per item.
+*/
+var reconcileDrafts = (draftPool) => Stage.map("reconcile-drafts", (change) => Effect.sync(() => {
+	if (change._tag !== "insert") return change;
+	const drafts = draftPool.get(change.foreignId);
+	return drafts?.length ? {
+		...change,
+		supersededDrafts: drafts
+	} : change;
+}));
+/**
+* Terminal stage converting a run's {@link Change} into the generic `Cursor.CommitUnit` the commit sink
+* consumes. For an `insert`: `message.attachments` (populated by {@link processAttachments}) becomes a
+* feed-append commit effect for the referenced blobs, `mapped.contact` (from {@link extractContacts})
+* becomes a `db.add` commit effect, and `mapped.supersededDrafts` (from {@link reconcileDrafts})
+* becomes a `db.remove` commit effect. Every other email stage is Change → Change and composes in
+* any order; this is the one stage that must run last.
+*
+* Tagging is email-specific (`Cursor.CommitUnit` carries no tag data), so it's applied here rather
+* than by the generic commit sink: `tagIndex` is optional (omit for providers/tests that don't tag),
+* and every tagged unit attaches the *same* `applyTags` closure (keyed by `mapped.tagUris` recorded in
+* a run-scoped map, read back by object identity) so the commit's identity-based batching applies
+* every tag pair for the page in one `Tagging.setBatch` call rather than one per message.
+*/
+var toCommitUnit = (options = {}) => {
+	const { tagIndex } = options;
+	const tagUrisByObject = /* @__PURE__ */ new WeakMap();
+	const applyTags = Effect.fn("email.commit.tags")(function* (units) {
+		if (!tagIndex) return;
+		const tagEntries = [];
+		for (const unit of units) {
+			if (!unit.object) continue;
+			for (const uri of tagUrisByObject.get(unit.object) ?? []) tagEntries.push({
+				object: unit.object,
+				tagId: uri
+			});
+		}
+		if (tagEntries.length > 0) Tagging.setBatch(tagEntries, { index: tagIndex });
+	});
+	const retagByForeignId = /* @__PURE__ */ new Map();
+	const applyRetags = Effect.fn("email.commit.retags")(function* (units) {
+		if (!tagIndex) return;
+		const accessor = TagIndex.bind(tagIndex);
+		for (const unit of units) {
+			const retag = retagByForeignId.get(unit.foreignId);
+			if (!retag) continue;
+			for (const tagId of retag.addTagIds) accessor.setTag(tagId, retag.entityId);
+			for (const tagId of retag.removeTagIds) accessor.unsetTag(tagId, retag.entityId);
+		}
+	});
+	return Stage.map("to-commit-unit", (change) => Effect.gen(function* () {
+		if (change._tag === "retag") {
+			retagByForeignId.set(change.foreignId, {
+				entityId: change.entityId,
+				addTagIds: change.addTagIds,
+				removeTagIds: change.removeTagIds
+			});
+			return {
+				foreignId: change.foreignId,
+				key: 0,
+				commitEffects: tagIndex ? [applyRetags] : void 0
+			};
+		}
+		if (change._tag === "delete") return {
+			foreignId: change.foreignId,
+			key: 0
+		};
+		const mapped = change;
+		const { feed } = yield* Cursor.Service;
+		const commitEffects = [];
+		const blobs = (mapped.message.attachments ?? []).map((attachment) => attachment.ref.target).filter((target) => Obj.instanceOf(Blob.Blob, target));
+		if (feed && blobs.length > 0) commitEffects.push(() => Feed.append(feed, blobs));
+		if (mapped.contact) {
+			const contact = mapped.contact;
+			commitEffects.push(Effect.fn("email.commit.addContact")(function* () {
+				yield* Database.add(contact);
+			}));
+		}
+		if (mapped.supersededDrafts?.length) {
+			const drafts = mapped.supersededDrafts;
+			commitEffects.push(Effect.fn("email.commit.removeDrafts")(function* () {
+				const { db } = yield* Database.Service;
+				for (const draft of drafts) db.remove(draft);
+			}));
+		}
+		if (tagIndex && mapped.tagUris?.length) {
+			tagUrisByObject.set(mapped.message, mapped.tagUris);
+			commitEffects.push(applyTags);
+		}
+		return {
+			object: mapped.message,
+			foreignId: mapped.foreignId,
+			key: mapped.key,
+			commitEffects: commitEffects.length > 0 ? commitEffects : void 0
+		};
+	}));
+};
+//#endregion
+//#region src/fact-pipeline.ts
+var __dxlog_file = "/__w/dxos/dxos/packages/core/compute/pipeline-email/src/fact-pipeline.ts";
+var EmailFactPipeline = { 
+/**
+* Facts-only assembly of the email stages over a batch of messages: stats → extract-facts. Omits
+* summarize, extract-contacts, and {@link buildThreads} from {@link EmailPipeline}, so it needs no
+* `AiService` and never writes to `db`. The cursored `AnalyzeMailbox` operation (in plugin-inbox)
+* covers the same fact-extraction concern incrementally, via a sibling facts-returning stage
+* (`extractFactsUnitStage`) that commits pages atomically with a cursor advance instead of indexing
+* in-stage; this batch variant is for one-shot runs over an already-collected set of messages.
+*/
+run: (messages, options) => Effect.gen(function* () {
+	const stats = emptyStats();
+	const collected = [];
+	yield* Stream.fromIterable(messages).pipe(statsStage, extractFactsStage(options.indexFacts), Pipeline.run({ sink: (message) => Effect.sync(() => collected.push(message)) }), Effect.provideService(EmailPipelineCtx, {
+		db: options.db,
+		stats,
+		summaries: []
+	}));
+	return {
+		messages: collected,
+		stats
+	};
+}) };
+/**
+* Runs the cursored fact pipeline over a feed: dedup → extract facts → commit each page to the
+* {@link FactStore} while advancing `cursor`. Extraction is injected (`extract`) so the pipeline is
+* unit-testable with a deterministic stub — no `AiService` required. Feed-generic (the mailbox
+* `AnalyzeMailbox` operation is a thin wrapper that resolves a Mailbox to its feed and its persisted
+* `Cursor`).
+*
+* Dedup is by `cursor`'s high-water key (a coarse `key < cursorKey` skip) plus the set of message
+* sources already in the store (the precise idempotency backstop) — the store doesn't share `cursor`'s
+* persistence lifetime (it's still in-memory per space), so the source-dedup backstop still matters
+* even though the cursor itself is now durable.
+*/
+var runFactPipeline = (options) => Effect.gen(function* () {
+	const { feed, cursor, extract, pageSize, concurrency = 1 } = options;
+	const store = yield* FactStore;
+	const priorFacts = yield* store.query({}).pipe(Effect.orElseSucceed(() => []));
+	const indexedSources = new Set(priorFacts.map((fact) => fact.attribution.source));
+	let cursorKey = Cursor.parseKey(cursor.max);
+	let processed = 0;
+	let facts = 0;
+	const messages = yield* Feed.query(feed, Filter.type(Message.Message)).run;
+	log.info("analyze: pipeline start", {
+		messages: messages.length,
+		cursorKey,
+		indexed: indexedSources.size,
+		pageSize
+	}, {
+		"~LogMeta": "~LogMeta",
+		F: __dxlog_file,
+		L: 106,
+		S: this
+	});
+	yield* Stream.fromIterable(messages).pipe(Stage.map("facts-dedup", (message) => Effect.sync(() => Date.parse(message.created) < cursorKey || indexedSources.has(messageSource(message)) ? void 0 : message)), extractFactsUnitStage(extract, concurrency), Stage.map("facts-log", (unit) => Effect.sync(() => {
+		log.info("analyze: extracted unit", {
+			foreignId: unit.foreignId,
+			key: unit.key,
+			facts: unit.facts.length
+		}, {
+			"~LogMeta": "~LogMeta",
+			F: __dxlog_file,
+			L: 122,
+			S: this
+		});
+		return unit;
+	})), Stream.grouped(pageSize), Pipeline.run({ sink: (page) => Effect.gen(function* () {
+		const units = Chunk.toReadonlyArray(page);
+		if (units.length === 0) return;
+		const pageFacts = units.flatMap((unit) => unit.facts);
+		if (pageFacts.length > 0) yield* store.putFacts(pageFacts).pipe(Effect.orDie);
+		processed += units.length;
+		facts += pageFacts.length;
+		cursorKey = Math.max(cursorKey, ...units.map((unit) => unit.key));
+		Cursor.advance(cursor, Cursor.formatKey(cursorKey));
+		log.info("analyze: committed page", {
+			page: units.length,
+			pageFacts: pageFacts.length,
+			processed,
+			facts,
+			cursorKey
+		}, {
+			"~LogMeta": "~LogMeta",
+			F: __dxlog_file,
+			L: 148,
+			S: this
+		});
+	}) }));
+	log.info("analyze: pipeline done", {
+		processed,
+		facts
+	}, {
+		"~LogMeta": "~LogMeta",
+		F: __dxlog_file,
+		L: 159,
+		S: this
+	});
+	return {
+		processed,
+		facts
+	};
+});
+//#endregion
+//#region src/types/Thread.ts
+var ThreadState = Schema.Literal("awaiting-mine", "awaiting-theirs", "resolved", "stalled");
+var Thread = class extends Type.makeObject(DXN.make("org.dxos.type.emailThread", "0.1.0"))(Schema.Struct({
+	threadId: Schema.String,
+	subject: Schema.String,
+	summary: Schema.String,
+	state: ThreadState,
+	participants: Schema.Array(Schema.String),
+	messageIds: Schema.Array(Schema.String),
+	openQuestions: Schema.Array(Schema.String),
+	actionItems: Schema.Array(Schema.String)
+})) {};
+//#endregion
+//#region src/internal/threads.ts
+var DEFAULT_STALE_PERIOD_MS = 336 * 60 * 60 * 1e3;
+/** Lowercased set of the owner's addresses (handles the single-string and multi-alias forms). */
+var ownerAddressSet = (ownerEmail) => new Set((Array.isArray(ownerEmail) ? ownerEmail : [ownerEmail]).map((email) => email.toLowerCase()).filter(Boolean));
+var computeState = (lastFromOwner, lastCreated, options) => {
+	if (new Date(options.now).getTime() - new Date(lastCreated).getTime() > (options.stalePeriodMs ?? DEFAULT_STALE_PERIOD_MS)) return "stalled";
+	return lastFromOwner ? "awaiting-theirs" : "awaiting-mine";
+};
+var buildThreads = (messages, options) => {
+	const owners = ownerAddressSet(options.ownerEmail);
+	const buckets = /* @__PURE__ */ new Map();
+	for (const message of messages) {
+		const threadId = deriveThreadId(message);
+		let bucket = buckets.get(threadId);
+		if (!bucket) {
+			bucket = [];
+			buckets.set(threadId, bucket);
+		}
+		bucket.push(message);
+	}
+	const threads = [];
+	for (const [threadId, bucket] of buckets) {
+		const ordered = [...bucket].sort((a, b) => a.created.localeCompare(b.created));
+		const last = ordered[ordered.length - 1];
+		const lastFromOwner = owners.has(last.sender.email?.toLowerCase() ?? "");
+		const participants = [...new Set(ordered.flatMap((m) => m.sender.email ? [m.sender.email.toLowerCase()] : []))];
+		const summaries = ordered.flatMap((m) => typeof m.properties?.summary === "string" && m.properties.summary.length > 0 ? [m.properties.summary] : []);
+		const subject = typeof ordered[0].properties?.subject === "string" ? ordered[0].properties.subject : threadId;
+		threads.push(Obj.make(Thread, {
+			threadId,
+			subject,
+			summary: summaries.length > 0 ? summaries.join("\n") : subject,
+			state: computeState(lastFromOwner, last.created, options),
+			participants,
+			messageIds: ordered.map(messageSource),
+			openQuestions: [],
+			actionItems: []
+		}));
+	}
+	return threads;
+};
+//#endregion
+//#region src/pipeline.ts
+var EmailPipeline = { 
+/**
+* Default assembly of the email stages over a batch of messages: summarize → extract-contacts →
+* stats → extract-facts stream through with back pressure, then {@link buildThreads} groups the
+* collected messages into canonical threads. Contacts and threads are persisted to `db`; the fact
+* substrate is populated via `indexFacts`. Requires an `AiService` (summarize); the shared
+* {@link EmailPipelineCtx} is constructed and provided internally.
+*/
+run: (messages, options) => Effect.gen(function* () {
+	const stats = emptyStats();
+	const summaries = [];
+	const collected = [];
+	yield* Stream.fromIterable(messages).pipe(summarizeStage, extractContactsStage, statsStage, extractFactsStage(options.indexFacts), Pipeline.run({ sink: (message) => Effect.sync(() => collected.push(message)) }), Effect.provideService(EmailPipelineCtx, {
+		db: options.db,
+		stats,
+		summaries
+	}));
+	const threads = buildThreads(collected, {
+		ownerEmail: options.ownerEmail,
+		now: options.now
+	});
+	for (const thread of threads) options.db.add(thread);
+	return {
+		messages: collected,
+		stats,
+		summaries,
+		threads
+	};
+}) };
+//#endregion
+//#region src/topics-pipeline.ts
+var messageIdOf = (message) => String(message.properties?.messageId ?? message.id);
+/**
+* Runs the topics pipeline: tag (bounded/resumable) → `buildThreads` → `clusterThreads` →
+* `summarizeTopics`. Returns per-message tags and the summarized topic drafts for the caller to
+* materialize or suggest. Tagging failures degrade per message (the injected `tag` should not
+* reject); clustering is deterministic.
+*/
+var runTopicsPipeline = async (input, deps) => {
+	const { messages, ownerEmail, now, limit, skipMessage, skipTopic, keepTopic, topicOptions, onProgress, signal } = input;
+	const pending = messages.filter((message) => !skipMessage?.(messageIdOf(message)));
+	const toTag = limit !== void 0 ? pending.slice(0, limit) : pending;
+	const messageTags = [];
+	for (let index = 0; index < toTag.length && !signal?.aborted; index++) {
+		const message = toTag[index];
+		const result = await deps.tag(message);
+		messageTags.push({
+			messageId: messageIdOf(message),
+			tags: result.tags,
+			spam: result.spam
+		});
+		onProgress?.("tag", index + 1, toTag.length);
+	}
+	if (signal?.aborted) return {
+		messageTags,
+		topicDrafts: []
+	};
+	return {
+		messageTags,
+		topicDrafts: clusterThreads(buildThreads(messages, {
+			ownerEmail,
+			now
+		}), topicOptions)
+	};
+};
+//#endregion
+export { DEFAULT_COMMITMENT_PREDICATES, DEFAULT_EMAIL_PROMPTS, DEFAULT_MODEL_POLICY, DEFAULT_TOPIC_OPTIONS, EMAIL_EXTRACTION_RULES, EMAIL_EXTRACT_OPTIONS, EmailFactPipeline, EmailPipeline, EmailPipelineCtx, EmailStage_exports as EmailStage, MODEL_HAIKU, MODEL_OPUS, MODEL_SONNET, Thread, ThreadState, applyBulkTag, buildDigest, buildRollups, buildThreads, classifyBulk, clusterThreads, commitmentLedger, deriveThreadId, emptyStats, extractContactsStage, extractFactsStage, extractFactsUnitStage, materializeTopics, mergePrompts, messageSource, messageToDocument, narrateDigest, normalizeSubject, parseTagResult, renderDigest, resolveModel, runFactPipeline, runTopicsPipeline, statsStage, summarizeStage, summarizeTopics, tagMessage };
+
+//# sourceMappingURL=index.mjs.map