@dxos/pipeline-email 0.0.0

package/dist/lib/neutral/index.mjs

@@ -0,0 +1,650 @@
+// src/corpus/topics.ts
+import { Obj } from "@dxos/echo";
+
+// src/types/Thread.ts
+import * as Schema from "effect/Schema";
+import { DXN, Type } from "@dxos/echo";
+var ThreadState = Schema.Literal("awaiting-mine", "awaiting-theirs", "resolved", "stalled");
+var Thread = class extends Type.makeObject(DXN.make("org.dxos.type.emailThread", "0.1.0"))(Schema.Struct({
+  threadId: Schema.String,
+  subject: Schema.String,
+  summary: Schema.String,
+  state: ThreadState,
+  participants: Schema.Array(Schema.String),
+  messageIds: Schema.Array(Schema.String),
+  openQuestions: Schema.Array(Schema.String),
+  actionItems: Schema.Array(Schema.String)
+})) {
+};
+
+// src/types/Topic.ts
+import * as Schema2 from "effect/Schema";
+import { DXN as DXN2, Type as Type2 } from "@dxos/echo";
+var Topic = class extends Type2.makeObject(DXN2.make("org.dxos.type.emailTopic", "0.1.0"))(Schema2.Struct({
+  label: Schema2.String,
+  summary: Schema2.String,
+  threadIds: Schema2.Array(Schema2.String),
+  participants: Schema2.Array(Schema2.String),
+  keywords: Schema2.Array(Schema2.String)
+})) {
+};
+
+// src/corpus/prompts.ts
+import { trim } from "@dxos/util";
+var DEFAULT_EMAIL_PROMPTS = {
+  topicSummary: trim`
+    Summarize the following email conversation topic in 1-2 sentences.
+    Focus on what the threads have in common and any outcome or open question.
+    Respond with ONLY the summary text.
+  `,
+  digest: trim`
+    Write a short briefing (3-5 sentences) from the following inbox digest data.
+    Lead with what needs attention (stalled threads, due commitments), then notable topics.
+    Respond with ONLY the briefing text.
+  `
+};
+var mergePrompts = (overrides) => ({
+  ...DEFAULT_EMAIL_PROMPTS,
+  ...overrides
+});
+
+// src/corpus/topics.ts
+var DEFAULT_STOPWORDS = [
+  "the",
+  "and",
+  "for",
+  "with",
+  "from",
+  "about",
+  "this",
+  "that",
+  "your",
+  "our",
+  "are",
+  "was",
+  "will",
+  "has",
+  "have",
+  "not",
+  "you",
+  "all",
+  "any",
+  "can",
+  "per",
+  "new"
+];
+var DEFAULT_TOPIC_OPTIONS = {
+  similarityThreshold: 0.25,
+  minSharedParticipants: 2,
+  minTokenLength: 3,
+  stopwords: DEFAULT_STOPWORDS,
+  maxKeywords: 5
+};
+var tokenize = (subject, options) => new Set(subject.toLowerCase().split(/[^a-z0-9]+/).filter((token) => token.length >= options.minTokenLength && !options.stopwords.includes(token)));
+var jaccard = (left, right) => {
+  if (left.size === 0 || right.size === 0) {
+    return 0;
+  }
+  let shared = 0;
+  for (const token of left) {
+    if (right.has(token)) {
+      shared += 1;
+    }
+  }
+  return shared / (left.size + right.size - shared);
+};
+var sharedCount = (left, right) => {
+  let shared = 0;
+  for (const value of left) {
+    if (right.has(value)) {
+      shared += 1;
+    }
+  }
+  return shared;
+};
+var clusterThreads = (threads, options) => {
+  const opts = {
+    ...DEFAULT_TOPIC_OPTIONS,
+    ...options
+  };
+  const clusters = [];
+  for (const thread of threads) {
+    const signature = {
+      thread,
+      tokens: tokenize(thread.subject, opts),
+      participants: new Set(thread.participants)
+    };
+    const home = clusters.find((cluster) => jaccard(cluster.tokens, signature.tokens) >= opts.similarityThreshold || sharedCount(cluster.participants, signature.participants) >= opts.minSharedParticipants);
+    if (home) {
+      home.members.push(signature);
+      for (const token of signature.tokens) {
+        home.tokens.add(token);
+      }
+      for (const participant of signature.participants) {
+        home.participants.add(participant);
+      }
+    } else {
+      clusters.push({
+        members: [
+          signature
+        ],
+        tokens: new Set(signature.tokens),
+        participants: new Set(signature.participants)
+      });
+    }
+  }
+  return clusters.map((cluster) => {
+    const counts = /* @__PURE__ */ new Map();
+    for (const member of cluster.members) {
+      for (const token of member.tokens) {
+        counts.set(token, (counts.get(token) ?? 0) + 1);
+      }
+    }
+    const keywords = [
+      ...counts.entries()
+    ].sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0])).slice(0, opts.maxKeywords).map(([token]) => token);
+    const summaries = cluster.members.flatMap((member) => member.thread.summary.length > 0 ? [
+      member.thread.summary
+    ] : []);
+    const first = cluster.members[0].thread;
+    return {
+      label: keywords.length > 0 ? keywords.slice(0, 3).join(" ") : first.subject || first.threadId,
+      summary: summaries.join("\n"),
+      threadIds: cluster.members.map((member) => member.thread.threadId),
+      participants: [
+        ...cluster.participants
+      ].sort(),
+      keywords
+    };
+  });
+};
+var summarizeTopics = async (drafts, summarize, options) => {
+  const prompts = options?.prompts ? mergePrompts(options.prompts) : DEFAULT_EMAIL_PROMPTS;
+  return Promise.all(drafts.map(async (draft) => {
+    try {
+      const summary = await summarize([
+        prompts.topicSummary,
+        `Topic: ${draft.label}`,
+        `Threads:
+${draft.summary}`
+      ].join("\n\n"));
+      return summary.trim().length > 0 ? {
+        ...draft,
+        summary: summary.trim()
+      } : draft;
+    } catch {
+      return draft;
+    }
+  }));
+};
+var materializeTopics = (drafts) => drafts.map((draft) => Obj.make(Topic, {
+  label: draft.label,
+  summary: draft.summary,
+  threadIds: [
+    ...draft.threadIds
+  ],
+  participants: [
+    ...draft.participants
+  ],
+  keywords: [
+    ...draft.keywords
+  ]
+}));
+
+// src/corpus/ledger.ts
+import * as Effect from "effect/Effect";
+var DEFAULT_COMMITMENT_PREDICATES = [
+  "owes",
+  "will send",
+  "will provide",
+  "must",
+  "due"
+];
+var termLabel = (term) => "entity" in term ? term.label ?? term.entity : term.literal;
+var toCommitment = (fact) => ({
+  who: termLabel(fact.assertion.subject),
+  what: termLabel(fact.assertion.object),
+  ...fact.assertion.validTo ? {
+    dueBy: fact.assertion.validTo
+  } : {},
+  ...fact.factuality.confidence !== void 0 ? {
+    confidence: fact.factuality.confidence
+  } : {},
+  factId: fact.id,
+  source: fact.attribution.source
+});
+var commitmentLedger = (store, options) => Effect.forEach(options?.predicates ?? DEFAULT_COMMITMENT_PREDICATES, (predicate) => store.query({
+  predicate
+})).pipe(Effect.map((results) => {
+  const seen = /* @__PURE__ */ new Set();
+  const commitments = [];
+  for (const fact of results.flat()) {
+    if (!seen.has(fact.id)) {
+      seen.add(fact.id);
+      commitments.push(toCommitment(fact));
+    }
+  }
+  return commitments.sort((a, b) => a.dueBy && b.dueBy ? a.dueBy.localeCompare(b.dueBy) : a.dueBy ? -1 : b.dueBy ? 1 : 0);
+}));
+
+// src/internal/threading.ts
+var PREFIX = /^\s*(re|fwd?|fw)\s*:\s*/i;
+var normalizeSubject = (subject) => {
+  let value = subject;
+  while (PREFIX.test(value)) {
+    value = value.replace(PREFIX, "");
+  }
+  return value.replace(/\s+/g, " ").trim().toLowerCase();
+};
+var deriveThreadId = (message) => {
+  const subject = message.properties?.subject;
+  const normalized = typeof subject === "string" ? normalizeSubject(subject) : "";
+  return normalized.length > 0 ? normalized : "no-subject";
+};
+
+// src/corpus/rollups.ts
+var MS_PER_DAY = 24 * 60 * 60 * 1e3;
+var buildRollups = (messages) => {
+  const bySender = /* @__PURE__ */ new Map();
+  for (const message of messages) {
+    const email = message.sender.email?.toLowerCase();
+    if (!email) {
+      continue;
+    }
+    let entry = bySender.get(email);
+    if (!entry) {
+      entry = {
+        created: [],
+        threadIds: []
+      };
+      bySender.set(email, entry);
+    }
+    entry.created.push(message.created);
+    const threadId = deriveThreadId(message);
+    if (!entry.threadIds.includes(threadId)) {
+      entry.threadIds.push(threadId);
+    }
+  }
+  const rollups = [];
+  for (const [email, entry] of bySender) {
+    const ordered = [
+      ...entry.created
+    ].sort();
+    const first = ordered[0];
+    const last = ordered[ordered.length - 1];
+    const spanMs = new Date(last).getTime() - new Date(first).getTime();
+    rollups.push({
+      email,
+      messageCount: ordered.length,
+      firstContact: first,
+      lastContact: last,
+      ...ordered.length >= 2 ? {
+        cadenceDays: spanMs / (ordered.length - 1) / MS_PER_DAY
+      } : {},
+      threadIds: entry.threadIds
+    });
+  }
+  return rollups.sort((a, b) => b.messageCount - a.messageCount || a.email.localeCompare(b.email));
+};
+
+// src/corpus/digest.ts
+var buildDigest = (input, options) => ({
+  generatedAt: options.now,
+  threadCount: input.threads.length,
+  topicCount: input.topics.length,
+  stalledThreads: input.threads.filter((thread) => thread.state === "stalled").map((thread) => thread.threadId),
+  awaitingMine: input.threads.filter((thread) => thread.state === "awaiting-mine").map((thread) => thread.threadId),
+  dueCommitments: input.commitments.filter((commitment) => commitment.dueBy !== void 0).sort((a, b) => (a.dueBy ?? "").localeCompare(b.dueBy ?? "")),
+  topTopics: [
+    ...input.topics
+  ].sort((a, b) => b.threadIds.length - a.threadIds.length || a.label.localeCompare(b.label)).slice(0, options.maxTopTopics ?? 5).map((topic) => ({
+    label: topic.label,
+    threadCount: topic.threadIds.length
+  })),
+  narrative: ""
+});
+var renderDigest = (digest) => [
+  `Inbox digest (${digest.generatedAt})`,
+  `Threads: ${digest.threadCount}  Topics: ${digest.topicCount}`,
+  `Awaiting my reply: ${digest.awaitingMine.join(", ") || "none"}`,
+  `Stalled: ${digest.stalledThreads.join(", ") || "none"}`,
+  `Due commitments: ${digest.dueCommitments.map((commitment) => `${commitment.who} \u2192 ${commitment.what} (${commitment.dueBy})`).join("; ") || "none"}`,
+  `Top topics: ${digest.topTopics.map((topic) => `${topic.label} (${topic.threadCount})`).join("; ") || "none"}`
+].join("\n");
+var narrateDigest = async (digest, summarize, options) => {
+  const prompts = options?.prompts ? mergePrompts(options.prompts) : DEFAULT_EMAIL_PROMPTS;
+  try {
+    const narrative = await summarize([
+      prompts.digest,
+      renderDigest(digest)
+    ].join("\n\n"));
+    return {
+      ...digest,
+      narrative: narrative.trim()
+    };
+  } catch {
+    return digest;
+  }
+};
+
+// src/fact-pipeline.ts
+import * as Effect7 from "effect/Effect";
+import * as Stream from "effect/Stream";
+import { Pipeline } from "@dxos/pipeline";
+
+// src/stages/context.ts
+import * as Context from "effect/Context";
+var emptyStats = () => ({
+  from: /* @__PURE__ */ new Map(),
+  to: /* @__PURE__ */ new Map(),
+  total: 0,
+  spam: 0
+});
+var EmailPipelineCtx = class extends Context.Tag("@dxos/pipeline-email/EmailPipelineCtx")() {
+};
+
+// src/stages/extract-contacts.ts
+import * as Effect2 from "effect/Effect";
+import { extractContact } from "@dxos/extractor-lib";
+import { Stage } from "@dxos/pipeline";
+var extractContactsStage = Stage.map("extract-contacts", (message) => Effect2.gen(function* () {
+  const { db } = yield* EmailPipelineCtx;
+  const result = yield* extractContact({
+    db,
+    source: message
+  });
+  for (const object of result.created) {
+    db.add(object);
+  }
+  return message;
+}));
+
+// src/stages/extract-facts.ts
+import * as Effect3 from "effect/Effect";
+import { Stage as Stage2 } from "@dxos/pipeline";
+var extractFactsStage = (indexFacts) => Stage2.map("extract-facts", (message) => Effect3.tryPromise(() => indexFacts(message)).pipe(Effect3.tapError((error) => Effect3.logWarning("extract-facts failed; degrading to no facts", error)), Effect3.orElse(() => Effect3.succeed([])), Effect3.as(message)));
+
+// src/stages/extract-facts-commit.ts
+import * as Effect4 from "effect/Effect";
+import { Stage as Stage3 } from "@dxos/pipeline";
+
+// src/stages/facts.ts
+import { Message } from "@dxos/types";
+var EMAIL_EXTRACTION_RULES = [
+  'Treat a request or promise as a commitment: "please send X by Friday" => subject (the owner), predicate "owes"/"will send", object X, validTo the deadline.',
+  "A deadline or due date becomes validTo on the relevant assertion (ISO date when known).",
+  'Attribute first-person statements ("I will\u2026", "we can\u2026") to the message sender as subject.'
+];
+var EMAIL_EXTRACT_OPTIONS = {
+  rules: EMAIL_EXTRACTION_RULES
+};
+var messageSource = (message) => {
+  const messageId = message.properties?.messageId;
+  if (typeof messageId === "string" && messageId.length > 0) {
+    return messageId;
+  }
+  return `${message.sender.email ?? "unknown"}:${message.created}`;
+};
+var messageToDocument = (message) => ({
+  text: Message.extractText(message),
+  source: messageSource(message),
+  ...message.sender.email ? {
+    author: message.sender.email
+  } : {},
+  date: message.created
+});
+
+// src/stages/extract-facts-commit.ts
+var extractFactsUnitStage = (extract) => Stage3.map("extract-facts-unit", (message) => Effect4.tryPromise(() => extract(message)).pipe(Effect4.tapError((error) => Effect4.logWarning("extract-facts-unit failed; degrading to no facts", error)), Effect4.orElse(() => Effect4.succeed([])), Effect4.map((facts) => ({
+  facts,
+  foreignId: messageSource(message),
+  key: keyOf(message)
+}))));
+var keyOf = (message) => Date.parse(message.created);
+
+// src/stages/stats.ts
+import * as Effect5 from "effect/Effect";
+import { Stage as Stage4 } from "@dxos/pipeline";
+var statsStage = Stage4.map("stats", (message) => Effect5.gen(function* () {
+  const { stats } = yield* EmailPipelineCtx;
+  stats.total += 1;
+  const sender = message.sender.email;
+  if (sender) {
+    stats.from.set(sender, (stats.from.get(sender) ?? 0) + 1);
+  }
+  const recipients = message.properties?.to;
+  if (Array.isArray(recipients)) {
+    for (const recipient of recipients) {
+      const address = String(recipient);
+      stats.to.set(address, (stats.to.get(address) ?? 0) + 1);
+    }
+  }
+  if (message.properties?.spam) {
+    stats.spam += 1;
+  }
+  return message;
+}));
+
+// src/stages/summarize.ts
+import * as LanguageModel from "@effect/ai/LanguageModel";
+import * as Effect6 from "effect/Effect";
+import * as Layer from "effect/Layer";
+import { AiService } from "@dxos/ai";
+import { Stage as Stage5 } from "@dxos/pipeline";
+import { Message as Message2 } from "@dxos/types";
+import { trim as trim2 } from "@dxos/util";
+var SUMMARIZE_MODEL = "com.anthropic.model.claude-haiku-4-5.default";
+var SUMMARIZE_PROMPT = trim2`
+  Summarize the following email in one sentence, decide whether it is spam, and list up to five keywords.
+  Respond with ONLY a JSON object of the form {"summary": string, "isSpam": boolean, "keywords": string[]}.
+`;
+var parseSummary = (raw) => {
+  const match = raw.match(/\{[\s\S]*\}/);
+  if (!match) {
+    return {
+      summary: "",
+      isSpam: false,
+      keywords: []
+    };
+  }
+  try {
+    const parsed = JSON.parse(match[0]);
+    const keywords = parsed.keywords;
+    return {
+      summary: typeof parsed.summary === "string" ? parsed.summary : "",
+      isSpam: parsed.isSpam === true || parsed.spam === true || parsed.is_spam === true,
+      keywords: Array.isArray(keywords) ? keywords.map((keyword) => String(keyword)) : []
+    };
+  } catch {
+    return {
+      summary: "",
+      isSpam: false,
+      keywords: []
+    };
+  }
+};
+var summarizeStage = Stage5.map("summarize", (message) => Effect6.gen(function* () {
+  const ctx = yield* EmailPipelineCtx;
+  const text = Message2.extractText(message);
+  const raw = yield* LanguageModel.generateText({
+    prompt: `${SUMMARIZE_PROMPT}
+
+${text}`
+  }).pipe(Effect6.provide(AiService.model(SUMMARIZE_MODEL).pipe(Layer.orDie)), Effect6.timeout("30 seconds"), Effect6.map((response) => response.text), Effect6.orElse(() => Effect6.succeed("")));
+  const summary = parseSummary(raw);
+  const messageId = String(message.properties?.messageId ?? message.id);
+  ctx.summaries.push({
+    messageId,
+    summary
+  });
+  const summaryBlock = {
+    _tag: "text",
+    text: summary.summary
+  };
+  return Message2.make({
+    id: message.id,
+    created: message.created,
+    sender: message.sender,
+    blocks: [
+      ...message.blocks,
+      summaryBlock
+    ],
+    properties: {
+      ...message.properties,
+      spam: summary.isSpam,
+      keywords: summary.keywords
+    }
+  });
+}));
+
+// src/fact-pipeline.ts
+var EmailFactPipeline = {
+  /**
+  * Facts-only assembly of the email stages over a batch of messages: stats → extract-facts. Omits
+  * summarize, extract-contacts, and {@link buildThreads} from {@link EmailPipeline}, so it needs no
+  * `AiService` and never writes to `db`. The cursored `EnrichMailbox` operation (in plugin-inbox)
+  * covers the same fact-extraction concern incrementally, via a sibling facts-returning stage
+  * (`extractFactsUnitStage`) that commits pages atomically with a cursor advance instead of indexing
+  * in-stage; this batch variant is for one-shot runs over an already-collected set of messages.
+  */
+  run: (messages, options) => Effect7.gen(function* () {
+    const stats = emptyStats();
+    const collected = [];
+    yield* Stream.fromIterable(messages).pipe(statsStage, extractFactsStage(options.indexFacts), Pipeline.run({
+      sink: (message) => Effect7.sync(() => collected.push(message))
+    }), Effect7.provideService(EmailPipelineCtx, {
+      db: options.db,
+      stats,
+      summaries: []
+    }));
+    return {
+      messages: collected,
+      stats
+    };
+  })
+};
+
+// src/pipeline.ts
+import * as Effect8 from "effect/Effect";
+import * as Stream2 from "effect/Stream";
+import { Pipeline as Pipeline2 } from "@dxos/pipeline";
+
+// src/internal/threads.ts
+import { Obj as Obj2 } from "@dxos/echo";
+var DEFAULT_STALE_PERIOD_MS = 14 * 24 * 60 * 60 * 1e3;
+var computeState = (lastFromOwner, lastCreated, options) => {
+  const idleMs = new Date(options.now).getTime() - new Date(lastCreated).getTime();
+  const stale = idleMs > (options.stalePeriodMs ?? DEFAULT_STALE_PERIOD_MS);
+  if (stale) {
+    return "stalled";
+  }
+  return lastFromOwner ? "awaiting-theirs" : "awaiting-mine";
+};
+var buildThreads = (messages, options) => {
+  const buckets = /* @__PURE__ */ new Map();
+  for (const message of messages) {
+    const threadId = deriveThreadId(message);
+    let bucket = buckets.get(threadId);
+    if (!bucket) {
+      bucket = [];
+      buckets.set(threadId, bucket);
+    }
+    bucket.push(message);
+  }
+  const threads = [];
+  for (const [threadId, bucket] of buckets) {
+    const ordered = [
+      ...bucket
+    ].sort((a, b) => a.created.localeCompare(b.created));
+    const last = ordered[ordered.length - 1];
+    const owner = options.ownerEmail.toLowerCase();
+    const lastFromOwner = last.sender.email?.toLowerCase() === owner;
+    const participants = [
+      ...new Set(ordered.flatMap((m) => m.sender.email ? [
+        m.sender.email.toLowerCase()
+      ] : []))
+    ];
+    const summaries = ordered.flatMap((m) => typeof m.properties?.summary === "string" && m.properties.summary.length > 0 ? [
+      m.properties.summary
+    ] : []);
+    const subject = typeof ordered[0].properties?.subject === "string" ? ordered[0].properties.subject : threadId;
+    threads.push(Obj2.make(Thread, {
+      threadId,
+      subject,
+      summary: summaries.length > 0 ? summaries.join("\n") : subject,
+      state: computeState(lastFromOwner, last.created, options),
+      participants,
+      messageIds: ordered.map(messageSource),
+      openQuestions: [],
+      actionItems: []
+    }));
+  }
+  return threads;
+};
+
+// src/pipeline.ts
+var EmailPipeline = {
+  /**
+  * Default assembly of the email stages over a batch of messages: summarize → extract-contacts →
+  * stats → extract-facts stream through with back pressure, then {@link buildThreads} groups the
+  * collected messages into canonical threads. Contacts and threads are persisted to `db`; the fact
+  * substrate is populated via `indexFacts`. Requires an `AiService` (summarize); the shared
+  * {@link EmailPipelineCtx} is constructed and provided internally.
+  */
+  run: (messages, options) => Effect8.gen(function* () {
+    const stats = emptyStats();
+    const summaries = [];
+    const collected = [];
+    yield* Stream2.fromIterable(messages).pipe(summarizeStage, extractContactsStage, statsStage, extractFactsStage(options.indexFacts), Pipeline2.run({
+      sink: (message) => Effect8.sync(() => collected.push(message))
+    }), Effect8.provideService(EmailPipelineCtx, {
+      db: options.db,
+      stats,
+      summaries
+    }));
+    const threads = buildThreads(collected, {
+      ownerEmail: options.ownerEmail,
+      now: options.now
+    });
+    for (const thread of threads) {
+      options.db.add(thread);
+    }
+    return {
+      messages: collected,
+      stats,
+      summaries,
+      threads
+    };
+  })
+};
+export {
+  DEFAULT_COMMITMENT_PREDICATES,
+  DEFAULT_EMAIL_PROMPTS,
+  DEFAULT_TOPIC_OPTIONS,
+  EMAIL_EXTRACTION_RULES,
+  EMAIL_EXTRACT_OPTIONS,
+  EmailFactPipeline,
+  EmailPipeline,
+  EmailPipelineCtx,
+  Thread,
+  ThreadState,
+  Topic,
+  buildDigest,
+  buildRollups,
+  clusterThreads,
+  commitmentLedger,
+  emptyStats,
+  extractContactsStage,
+  extractFactsStage,
+  extractFactsUnitStage,
+  materializeTopics,
+  mergePrompts,
+  messageSource,
+  messageToDocument,
+  narrateDigest,
+  renderDigest,
+  statsStage,
+  summarizeStage,
+  summarizeTopics
+};
+//# sourceMappingURL=index.mjs.map