ei-tui 1.3.4 → 1.4.0

package/src/integrations/slack/reader.ts

@@ -0,0 +1,416 @@
+import type { SlackChannelState, SlackSettings } from "./types.js";
+
+export class SlackRateLimitError extends Error {
+  constructor(method: string) {
+    super(`Slack rate limited on ${method} (429) — will retry next cycle`);
+    this.name = "SlackRateLimitError";
+  }
+}
+
+// =============================================================================
+// Slack API types
+// =============================================================================
+
+export interface SlackMessage {
+  type: string;
+  subtype?: string;
+  hidden?: boolean;
+  user?: string;
+  bot_id?: string;
+  username?: string;
+  text: string;
+  ts: string;
+  thread_ts?: string;
+  reply_count?: number;
+  latest_reply?: string;
+  parent_user_id?: string;
+  client_msg_id?: string;
+  edited?: { user: string; ts: string };
+}
+
+export interface SlackUser {
+  id: string;
+  name: string;
+  real_name: string;
+  profile: {
+    display_name: string;
+    real_name: string;
+  };
+  is_bot: boolean;
+  deleted: boolean;
+}
+
+export interface SlackChannel {
+  id: string;
+  name: string;
+  is_channel: boolean;
+  is_group: boolean;
+  is_im: boolean;
+  is_mpim: boolean;
+  is_private: boolean;
+  is_archived: boolean;
+  is_member: boolean;
+  num_members?: number;
+}
+
+export type ChannelTier = "dm" | "private" | "public" | "broadcast" | "skip";
+
+// =============================================================================
+// Resolved message — Slack message with mentions substituted
+// =============================================================================
+
+export interface ResolvedMessage {
+  ts: string;
+  thread_ts?: string;
+  userId: string;
+  displayName: string;
+  text: string;
+  isThreadParent: boolean;
+  latestReply?: string;
+  replyCount?: number;
+}
+
+// =============================================================================
+// SlackReader
+// =============================================================================
+
+export class SlackReader {
+  private token: string;
+  private userCache: Map<string, string> = new Map();   // userId → displayName
+  private channelCache: Map<string, string> = new Map(); // channelId → name
+
+  constructor(token: string) {
+    this.token = token;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Core API fetch
+  // ---------------------------------------------------------------------------
+
+  private async slackFetch(method: string, params: Record<string, string | number | boolean>): Promise<Record<string, unknown>> {
+    const url = new URL(`https://slack.com/api/${method}`);
+    for (const [k, v] of Object.entries(params)) {
+      url.searchParams.set(k, String(v));
+    }
+    const resp = await fetch(url.toString(), {
+      headers: { Authorization: `Bearer ${this.token}` },
+    });
+    if (resp.status === 429) throw new SlackRateLimitError(method);
+    if (!resp.ok) throw new Error(`Slack API ${method} failed: ${resp.status}`);
+    const data = await resp.json() as Record<string, unknown>;
+    if (!data.ok) throw new Error(`Slack API ${method} error: ${data.error ?? "unknown"}`);
+    return data;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Channel list
+  // ---------------------------------------------------------------------------
+
+  async listChannels(): Promise<SlackChannel[]> {
+    const allChannels: SlackChannel[] = [];
+    let cursor = "";
+    do {
+      const params: Record<string, string | number | boolean> = {
+        types: "public_channel,private_channel,im,mpim",
+        limit: 200,
+        exclude_archived: true,
+      };
+      if (cursor) params.cursor = cursor;
+      const data = await this.slackFetch("users.conversations", params);
+      allChannels.push(...(data.channels as SlackChannel[]));
+      cursor = (data.response_metadata as Record<string, string>)?.next_cursor ?? "";
+    } while (cursor);
+    return allChannels;
+  }
+
+  classifyChannel(ch: SlackChannel, settings: SlackSettings): ChannelTier {
+    const override = settings.channel_overrides?.[ch.id];
+    if (override) return override === "skip" ? "skip" : override;
+    if (ch.is_im || ch.is_mpim) return "dm";
+    if (ch.is_private) return "private";
+    const broadcastThreshold = settings.broadcast_threshold ?? 100;
+    if ((ch.num_members ?? 0) >= broadcastThreshold) return "broadcast";
+    return "public";
+  }
+
+  backfillDaysForTier(tier: ChannelTier, settings: SlackSettings): number {
+    const defaults = { dm: 90, private: 90, public: 30 };
+    if (tier === "dm" || tier === "private") return settings.backfill_days?.dm ?? defaults[tier];
+    if (tier === "public") return settings.backfill_days?.public ?? defaults.public;
+    return 0;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Channel name resolution (lazy cache)
+  // ---------------------------------------------------------------------------
+
+  async resolveChannelName(channelId: string): Promise<string> {
+    const cached = this.channelCache.get(channelId);
+    if (cached) return cached;
+    try {
+      const data = await this.slackFetch("conversations.info", { channel: channelId });
+      const name = ((data.channel as Record<string, unknown>).name as string) ?? channelId;
+      this.channelCache.set(channelId, name);
+      return name;
+    } catch {
+      this.channelCache.set(channelId, channelId);
+      return channelId;
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+  // User display name resolution (lazy cache)
+  // ---------------------------------------------------------------------------
+
+  async resolveUserName(userId: string): Promise<string> {
+    const cached = this.userCache.get(userId);
+    if (cached) return cached;
+    try {
+      const data = await this.slackFetch("users.info", { user: userId });
+      const user = data.user as SlackUser;
+      const name = user.profile.display_name || user.profile.real_name || user.name || userId;
+      this.userCache.set(userId, name);
+      return name;
+    } catch {
+      this.userCache.set(userId, userId);
+      return userId;
+    }
+  }
+
+  seedUserCache(userId: string, displayName: string): void {
+    this.userCache.set(userId, displayName);
+  }
+
+  seedChannelCache(channelId: string, name: string): void {
+    this.channelCache.set(channelId, name);
+  }
+
+  // ---------------------------------------------------------------------------
+  // Mention resolution in message text
+  // ---------------------------------------------------------------------------
+
+  async resolveMentions(text: string): Promise<string> {
+    // Collect all unique IDs needing resolution before substituting
+    const userIds = new Set<string>();
+    const channelIds = new Set<string>();
+
+    for (const [, id] of text.matchAll(/<@([A-Z0-9]+)(?:\|[^>]*)?>/g)) userIds.add(id);
+    for (const [, id] of text.matchAll(/<#([A-Z0-9]+)(?:\|[^>]*)?>/g)) channelIds.add(id);
+
+    // Resolve unknowns in parallel
+    await Promise.all([
+      ...[...userIds].filter(id => !this.userCache.has(id)).map(id => this.resolveUserName(id)),
+      ...[...channelIds].filter(id => !this.channelCache.has(id)).map(id => this.resolveChannelName(id)),
+    ]);
+
+    return text
+      .replace(/<@([A-Z0-9]+)(?:\|[^>]*)?>/g, (_, id) => `@${this.userCache.get(id) ?? id}`)
+      .replace(/<#([A-Z0-9]+)(?:\|[^>]*)?>/g, (_, id) => `#${this.channelCache.get(id) ?? id}(${id})`)
+      .replace(/<https?:\/\/[^|>]+\|([^>]+)>/g, "$1")
+      .replace(/<https?:\/\/([^>]+)>/g, "$1")
+      .replace(/<!channel>/g, "@channel")
+      .replace(/<!here>/g, "@here")
+      .replace(/<!everyone>/g, "@everyone");
+  }
+
+  // ---------------------------------------------------------------------------
+  // Message filtering
+  // ---------------------------------------------------------------------------
+
+  private isUserMessage(msg: SlackMessage): boolean {
+    if (msg.hidden) return false;
+    if (!msg.subtype) return !!msg.user;
+    // Allow only these subtypes
+    return msg.subtype === "me_message";
+  }
+
+  // ---------------------------------------------------------------------------
+  // Probe for the next message after a timestamp
+  //
+  // Returns the ts of the first message strictly after sinceTs, or null if
+  // the channel has no more messages. Used to skip silent periods instantly
+  // instead of advancing 24h at a time through months of inactivity.
+  // ---------------------------------------------------------------------------
+
+  async probeNextMessageTs(channelId: string, sinceTs: string): Promise<string | null> {
+    const data = await this.slackFetch("conversations.history", {
+      channel: channelId,
+      oldest: sinceTs,
+      inclusive: false,
+      limit: 1,
+    });
+    const messages = data.messages as SlackMessage[];
+    const first = messages.find(m => this.isUserMessage(m));
+    return first?.ts ?? null;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Spine messages for a channel window
+  //
+  // Returns messages between start and end (exclusive of thread replies).
+  // Thread parents ARE included — they form the spine with reply metadata.
+  // ---------------------------------------------------------------------------
+
+  async spineMessagesBetween(
+    channelId: string,
+    startTs: string,
+    endTs: string,
+  ): Promise<ResolvedMessage[]> {
+    const raw: SlackMessage[] = [];
+    let cursor = "";
+    do {
+      const params: Record<string, string | number | boolean> = {
+        channel: channelId,
+        oldest: startTs,
+        latest: endTs,
+        limit: 200,
+        inclusive: false,
+      };
+      if (cursor) params.cursor = cursor;
+      const data = await this.slackFetch("conversations.history", params);
+      raw.push(...(data.messages as SlackMessage[]));
+      cursor = (data.response_metadata as Record<string, string>)?.next_cursor ?? "";
+    } while (cursor);
+
+    const userMessage = raw.filter(m => this.isUserMessage(m));
+
+    // Resolve all mentions in parallel per message
+    return Promise.all(userMessage.map(async m => {
+      const userId = m.user ?? m.bot_id ?? "unknown";
+      const [displayName, resolvedText] = await Promise.all([
+        this.resolveUserName(userId),
+        this.resolveMentions(m.text),
+      ]);
+      return {
+        ts: m.ts,
+        thread_ts: m.thread_ts,
+        userId,
+        displayName,
+        text: resolvedText,
+        isThreadParent: !!m.thread_ts && m.thread_ts === m.ts && (m.reply_count ?? 0) > 0,
+        latestReply: m.latest_reply,
+        replyCount: m.reply_count,
+      };
+    }));
+  }
+
+  // ---------------------------------------------------------------------------
+  // Known threads with new replies since lastRun
+  //
+  // For each threadTs in the channel state's threads map whose lastSeenReply
+  // is older than lastRun, fetch replies since lastRun.
+  // Returns only threads that actually have new replies.
+  // ---------------------------------------------------------------------------
+
+  async threadsWithUpdatesSince(
+    channelId: string,
+    threadTsMap: Record<string, string>,
+    lastRunTs: string,
+  ): Promise<Array<{ threadTs: string; newReplies: ResolvedMessage[]; allReplies: ResolvedMessage[] }>> {
+    const results: Array<{ threadTs: string; newReplies: ResolvedMessage[]; allReplies: ResolvedMessage[] }> = [];
+
+    await Promise.all(
+      Object.entries(threadTsMap).map(async ([threadTs, lastSeenReply]) => {
+        if (lastSeenReply >= lastRunTs) return;
+        const { newReplies, allReplies } = await this.fetchThread(channelId, threadTs, lastSeenReply);
+        if (newReplies.length > 0) {
+          results.push({ threadTs, newReplies, allReplies });
+        }
+      })
+    );
+
+    return results;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Fetch a full thread
+  //
+  // Returns:
+  //   allReplies — every reply (for context_messages)
+  //   newReplies — only replies after sinceTs (for messages_analyze)
+  // ---------------------------------------------------------------------------
+
+  async fetchThread(
+    channelId: string,
+    threadTs: string,
+    sinceTs: string,
+  ): Promise<{ allReplies: ResolvedMessage[]; newReplies: ResolvedMessage[] }> {
+    const raw: SlackMessage[] = [];
+    let cursor = "";
+    do {
+      const params: Record<string, string | number | boolean> = {
+        channel: channelId,
+        ts: threadTs,
+        limit: 200,
+        inclusive: false,
+      };
+      if (cursor) params.cursor = cursor;
+      const data = await this.slackFetch("conversations.replies", params);
+      raw.push(...(data.messages as SlackMessage[]));
+      cursor = (data.response_metadata as Record<string, string>)?.next_cursor ?? "";
+    } while (cursor);
+
+    // First message is always the parent — skip it, include only replies
+    const replies = raw.filter((m, i) => i > 0 && this.isUserMessage(m));
+
+    const resolved = await Promise.all(replies.map(async m => {
+      const userId = m.user ?? m.bot_id ?? "unknown";
+      const [displayName, resolvedText] = await Promise.all([
+        this.resolveUserName(userId),
+        this.resolveMentions(m.text),
+      ]);
+      return {
+        ts: m.ts,
+        thread_ts: m.thread_ts,
+        userId,
+        displayName,
+        text: resolvedText,
+        isThreadParent: false,
+        latestReply: undefined,
+        replyCount: undefined,
+      };
+    }));
+
+    const allReplies = resolved;
+    const newReplies = resolved.filter(m => m.ts > sinceTs);
+
+    return { allReplies, newReplies };
+  }
+
+  // ---------------------------------------------------------------------------
+  // Select the next channel to process
+  //
+  // Returns the channel with the oldest extraction_point that still has
+  // content to process (extraction_point < now).
+  // ---------------------------------------------------------------------------
+
+  selectCandidateChannel(
+    channels: SlackChannel[],
+    channelStates: Record<string, SlackChannelState>,
+    settings: SlackSettings,
+    now: string,
+  ): { channel: SlackChannel; state: SlackChannelState; tier: ChannelTier } | null {
+    const nowMs = new Date(now).getTime();
+
+    let oldest: { channel: SlackChannel; state: SlackChannelState; tier: ChannelTier; pointMs: number } | null = null;
+
+    for (const ch of channels) {
+      const tier = this.classifyChannel(ch, settings);
+      if (tier === "broadcast" || tier === "skip") continue;
+
+      const state = channelStates[ch.id] ?? {};
+      const backfillDays = this.backfillDaysForTier(tier, settings);
+      const defaultPoint = new Date(nowMs - backfillDays * 86400_000).toISOString();
+      const pointIso = state.extraction_point ?? defaultPoint;
+      const pointMs = new Date(pointIso).getTime();
+
+      if (pointMs >= nowMs) continue;
+
+      if (!oldest || pointMs < oldest.pointMs) {
+        oldest = { channel: ch, state: { ...state, extraction_point: pointIso }, tier, pointMs };
+      }
+    }
+
+    return oldest ? { channel: oldest.channel, state: oldest.state, tier: oldest.tier } : null;
+  }
+}

package/src/integrations/slack/types.ts

@@ -0,0 +1,30 @@
+export interface SlackAuth {
+  type: "pkce" | "xoxp";
+  token: string;
+  refresh_token?: string;
+  workspace_id?: string;
+  workspace_name?: string;
+}
+
+export interface SlackChannelState {
+  extraction_point?: string;         // ISO — how far we've advanced in the timeline (spine cursor)
+  last_run?: string;                 // ISO — when we last checked for updates (necro reply detection)
+  name?: string;                     // cached display name
+  threads?: Record<string, string>;  // threadTs → latest reply ts seen (reply cursor per thread)
+}
+
+export interface SlackSettings {
+  integration?: boolean;
+  polling_interval_ms?: number;
+  extraction_model?: string;
+  last_sync?: string;
+  auth?: SlackAuth;
+  backfill_days?: {
+    dm: number;
+    private: number;
+    public: number;
+  };
+  broadcast_threshold?: number;
+  channel_overrides?: Record<string, "dm" | "private" | "public" | "skip">;
+  channels?: Record<string, SlackChannelState>;
+}

package/src/prompts/human/person-scan.ts

@@ -1,6 +1,19 @@
-import type { PersonScanPromptData, ParticipantContext, PromptOutput } from "./types.js";
+import type { PersonScanPromptData, ParticipantContext, ExcludedParticipant, PromptOutput } from "./types.js";
 import { formatMessagesAsPlaceholders } from "../message-utils.js";
 
+function excludedParticipantsSection(excluded: ExcludedParticipant[] | undefined): string {
+  if (!excluded || excluded.length === 0) return "";
+  const lines = [
+    "## Known Participants — Do Not Flag",
+    "The following people are already identified and will be processed separately.",
+    "Do NOT include them in your output. They may appear in messages by name — that is expected.",
+    "",
+    ...excluded.map(p => `- ${p.name}(${p.id})`),
+    "",
+  ];
+  return lines.join("\n");
+}
+
 function participantContextSection(ctx: ParticipantContext | undefined): string {
   if (!ctx) return "";
   const lines: string[] = ["# Participant Context", "The following may help you understand who is in this conversation.", ""];
@@ -37,7 +50,8 @@ You are scanning a conversation to quickly identify PEOPLE in the HUMAN USER's l
 
 Detect and flag. Do NOT analyze deeply — that happens later.
 
-${participantContextSection(data.participant_context)}## What to Capture
+${participantContextSection(data.participant_context)}${excludedParticipantsSection(data.excluded_participants)}
+## What to Capture
 
 Flag a PERSON when they were meaningfully discussed — not just mentioned in passing.
 

package/src/prompts/human/types.ts

@@ -30,9 +30,15 @@ export interface TopicScanPromptData extends BaseScanPromptData {
   technical_context?: boolean;
 }
 
+export interface ExcludedParticipant {
+  name: string;
+  id: string;
+}
+
 export interface PersonScanPromptData extends BaseScanPromptData {
   participant_context?: ParticipantContext;
   known_identifier_types?: string[];
+  excluded_participants?: ExcludedParticipant[];
 }
 
 export interface FactFindPromptData {

package/src/prompts/response/sections.ts

@@ -407,7 +407,7 @@ Rooms are shared multi-persona conversations — a space where the Human and mul
 
 ## Learning About the Human
 As the human chats, the system learns about them:
-- **Facts**: Objective information (job, location, family members)
+- **Facts**: User demographics only (name, age, job title, location, family structure, physical traits) — not interests or opinions
 - **Topics**: Interests and how they feel about them
 - **People**: Relationships in their life
 - **Quotes**: Memorable things said in conversation (human selects these with ${viewQuotesAction})

package/src/prompts/synthesis/index.ts

@@ -14,7 +14,7 @@ Your goal is to produce a well-structured markdown document that a human could s
 
 Everything below is complete as provided — do not use tools to re-fetch records already present here. Only use tools to fill genuine gaps not covered by the data below.
 
-- **Facts**: Ground-truth statements.
+- **Facts**: User demographics only (name, age, job title, location, family structure, physical traits) — not interests or opinions.
 - **Topics**: Areas of interest, work, or concern with descriptions.
 - **People**: Individuals with relationship context.
 - **Quotes**: Verbatim things said, with a \`message_id\`. Use \`fetch_message\` with the \`message_id\` if you want the surrounding conversation for additional context.${hasEntityMap ? `

package/src/templates/slack.ts

@@ -0,0 +1,17 @@
+export const SLACK_PERSONA_DEFINITION = {
+  entity: "system" as const,
+  aliases: ["Slack", "slack"],
+  short_description: "Your Slack workspace — conversations, threads, and the people you work with, indexed for memory.",
+  long_description: `Slack is the quiet archivist of your working life. It reads your channels and threads, extracts what matters — who said what, what topics are alive, what decisions were made — and feeds that into Ei's memory so your personas know what's been going on without you having to explain it.
+
+It doesn't chat. It doesn't have opinions. It's infrastructure with good taste about what's worth remembering.`,
+  model: undefined,
+  group_primary: "Integrations",
+  groups_visible: [] as string[],
+  traits: [],
+  topics: [],
+  heartbeat_delay_ms: 0,
+  is_archived: false,
+  is_paused: false,
+  is_static: true,
+};

package/tui/README.md

@@ -41,6 +41,32 @@ Sessions are processed oldest-first, one per queue cycle. On first run Ei works
 
 OpenCode also supports reading Ei's extracted knowledge back out via the [CLI tool](../src/cli/README.md), giving it persistent memory across sessions.
 
+## Slack Integration
+
+Slack is different from the coding tool integrations — it reads human conversations rather than coding sessions, and requires OAuth instead of local file access.
+
+**Setup:**
+
+```
+/auth slack
+```
+
+This opens a browser, walks you through OAuth, and stores your token. Then enable in `/settings`:
+
+```yaml
+slack:
+  integration: true
+  extraction_model: default  # optional override
+```
+
+Ei will index channels and DMs you're a member of, extracting topics, people, and context. Everything is processed locally — nothing is sent to the developer.
+
+**Notes:**
+- Ei uses a [published Slack app](https://github.com/Flare576/ei) — your workspace admin may need to approve it
+- Non-Marketplace apps are subject to Slack's rate limits on `conversations.history` (1 req/min). Backfill is gradual; steady-state is fast.
+- Your workspace admin may need to approve the [Ei Slack app](https://slack.com/oauth/v2/authorize?client_id=11080256060354.11080294064034&scope=&user_scope=channels:history,channels:read,groups:history,groups:read,im:history,im:read,mpim:history,mpim:read,users:read,users:read.email) — that link goes directly to the install flow
+- The Slack app is read-only — Ei never posts, reacts, or takes any action in your workspace
+
 # Installation
 
 ```bash
@@ -146,6 +172,7 @@ Rooms have three modes, set at creation time:
 | `/settings` | `/set` | Edit your global settings in `$EDITOR` |
 | `/setsync <user> <pass>` | `/ss` | Set sync credentials (triggers restart) |
 | `/tools` | | Manage tool providers — enable/disable tools per persona |
+| `/auth <service>` | | Authenticate with an external service via OAuth. Supported: `spotify`, `slack` |
 
 ### Editor
 

package/tui/src/commands/auth.ts

@@ -1,17 +1,18 @@
 import type { Command } from "./registry.js";
 import { runSpotifyAuth } from "./spotify-auth.js";
+import { runSlackAuth } from "./slack-auth.js";
 
 export const authCommand: Command = {
   name: "auth",
   aliases: [],
   description: "Authenticate with a service (e.g. /auth spotify)",
-  usage: "/auth <service>  — supported: spotify",
+  usage: "/auth <service>  — supported: spotify, slack",
 
   async execute(args, ctx) {
     const service = args[0]?.toLowerCase();
 
     if (!service) {
-      ctx.showNotification("Usage: /auth <service>  (supported: spotify)", "error");
+      ctx.showNotification("Usage: /auth <service>  (supported: spotify, slack)", "error");
       return;
     }
 
@@ -19,8 +20,11 @@ export const authCommand: Command = {
       case "spotify":
         await runSpotifyAuth(ctx);
         break;
+      case "slack":
+        await runSlackAuth(ctx);
+        break;
       default:
-        ctx.showNotification(`Unknown service: ${service}. Supported: spotify`, "error");
+        ctx.showNotification(`Unknown service: ${service}. Supported: spotify, slack`, "error");
     }
   },
 };