ei-tui 1.3.4 → 1.4.0

package/README.md

@@ -185,6 +185,28 @@ Ei splits the document into segments, runs them through the extraction pipeline,
 
 Both surfaces show you which documents have been imported and let you remove their extracted knowledge (web: Delete button in the Documents tab; TUI: `/unsource <source_tag>`).
 
+## Slack Integration
+
+Ei can index your Slack workspace — channels, DMs, and threads — and extract the same topics, people, and context it pulls from your conversations. Your personas end up knowing what's been going on at work without you having to explain it.
+
+**TUI** (requires setup):
+```bash
+/auth slack
+```
+
+Opens a browser OAuth flow. Once authenticated, enable in `/settings`:
+
+```yaml
+slack:
+  integration: true
+```
+
+**Web**: Open **☰ menu** → **My Data** → **External** tab → **Connect Slack**.
+
+Ei is read-only — it never posts, reacts, or takes any action in your workspace. All processing happens locally. Your workspace admin may need to approve the [Ei Slack app](https://slack.com/oauth/v2/authorize?client_id=11080256060354.11080294064034&scope=&user_scope=channels:history,channels:read,groups:history,groups:read,im:history,im:read,mpim:history,mpim:read,users:read,users:read.email) before you can connect.
+
+> **Note on backfill speed**: Slack limits non-Marketplace apps to 1 request/minute on message history. Backfill through a large workspace is gradual — steady-state (indexing new messages) is fast once you're caught up.
+
 ## Knowledge Share
 
 Sometimes you want to take what Ei knows and turn it into something you can hand to another human. A new teammate joining a project. A briefing doc before a meeting. A brain dump before a vacation.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "1.3.4",
+  "version": "1.4.0",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/cli/mcp.ts

@@ -134,7 +134,7 @@ export function createMcpServer(): McpServer {
     "ei_find_memory",
     {
       description:
-        "Search Ei's persistent knowledge base — facts, topics, people, and quotes learned across ALL conversations over time. Use when you need context about the user, their life, relationships, or interests that may not be visible in the current exchange. Returns results grouped by type. Use `recent: true` to retrieve what's been discussed recently.",
+        "Search Ei's persistent knowledge base — facts, topics, people, and quotes learned across ALL conversations over time. Use when you need context about the user, their life, relationships, or interests that may not be visible in the current exchange. Returns results grouped by type. Use `recent: true` to retrieve what's been discussed recently. TYPE GUIDANCE: 'facts' are ONLY user demographics — name, age, job title, location, family structure, physical traits. For interests, opinions, hobbies, or anything the human cares about, use 'topics'. For named individuals, use 'people'. For verbatim things said, use 'quotes'.",
       inputSchema: {
         query: z
           .string()
@@ -145,7 +145,7 @@ export function createMcpServer(): McpServer {
         types: z
           .array(z.enum(["facts", "topics", "people", "quotes"]))
           .optional()
-          .describe("Limit search to specific memory types (default: all types)"),
+          .describe("Limit search to specific memory types (default: all types). Use 'facts' ONLY for user demographics (name, age, job, location, family). Use 'topics' for interests, opinions, and anything the human cares about."),
         limit: z
           .number()
           .optional()

package/src/core/orchestrators/human-extraction.ts

@@ -61,6 +61,7 @@ export interface ExtractionContext {
   extraction_flag?: "f" | "t" | "p" | "e";
   roomId?: string;
   sources?: string[];
+  excluded_participants?: import("../../prompts/human/types.js").ExcludedParticipant[];
 }
 
 export interface ExtractionOptions {
@@ -224,6 +225,7 @@ export function queuePersonScan(context: ExtractionContext, state: StateManager,
       messages_analyze: chunk.messages_analyze,
       participant_context: buildParticipantContext(context.personaId, state),
       known_identifier_types: userIdentifierTypesForScan,
+      excluded_participants: context.excluded_participants,
     });
 
     state.queue_enqueue({

package/src/core/processor.ts

@@ -169,6 +169,8 @@ export class Processor {
   private claudeCodeImportInProgress = false;
   private lastCursorSync = 0;
   private cursorImportInProgress = false;
+  private lastSlackSync = 0;
+  private slackImportInProgress = false;
   private pendingConflict: StateConflictData | null = null;
   private storage: Storage | null = null;
   private importAbortController = new AbortController();
@@ -1224,6 +1226,10 @@ export class Processor {
       console.log(`[Processor ${this.instanceId}] Clearing claudeCodeImportInProgress flag`);
       this.claudeCodeImportInProgress = false;
     }
+    if (this.slackImportInProgress) {
+      console.log(`[Processor ${this.instanceId}] Clearing slackImportInProgress flag`);
+      this.slackImportInProgress = false;
+    }
     await this.stateManager.flush();
     console.log(`[Processor ${this.instanceId}] pause() complete (main loop stopped, state flushed)`);
   }
@@ -1474,6 +1480,15 @@ const toolNextSteps = new Set([
       await this.checkAndSyncPersonaHistory(human);
     }
 
+    if (
+      this.isTUI &&
+      human.settings?.slack?.integration &&
+      human.settings?.slack?.auth?.token &&
+      this.stateManager.queue_length() === 0
+    ) {
+      await this.checkAndSyncSlack(human, now);
+    }
+
     if (human.settings?.ceremony && shouldStartCeremony(human.settings.ceremony, this.stateManager)) {
       if (human.settings?.sync && remoteSync.isConfigured()) {
         const state = this.stateManager.getStorageState();
@@ -1714,6 +1729,52 @@ const toolNextSteps = new Set([
       });
   }
 
+  private async checkAndSyncSlack(human: HumanEntity, now: number): Promise<void> {
+    if (this.slackImportInProgress) return;
+
+    const slack = human.settings?.slack;
+    const pollingInterval = slack?.polling_interval_ms ?? 60_000;
+    const lastSync = slack?.last_sync ? new Date(slack.last_sync).getTime() : 0;
+
+    if (now - lastSync < pollingInterval && this.lastSlackSync > 0) return;
+
+    this.lastSlackSync = now;
+    this.stateManager.setHuman({
+      ...this.stateManager.getHuman(),
+      settings: {
+        ...this.stateManager.getHuman().settings,
+        slack: { ...slack, last_sync: new Date(now).toISOString() },
+      },
+    });
+
+    this.slackImportInProgress = true;
+    import("../integrations/slack/importer.js")
+      .then(({ importSlackChannel }) =>
+        importSlackChannel({
+          stateManager: this.stateManager,
+          interface: this.interface,
+          signal: this.importAbortController.signal,
+        })
+      )
+      .then((result) => {
+        if (result.channelProcessed) {
+          console.log(
+            `[Processor] Slack sync: #${result.channelProcessed} — ` +
+            `${result.messagesImported} messages, ${result.threadsProcessed} threads, ` +
+            `${result.scansQueued} scans queued`
+          );
+        }
+      })
+      .catch((err) => {
+        const msg = err instanceof Error ? err.message : JSON.stringify(err);
+        const stack = err instanceof Error ? err.stack : undefined;
+        console.warn(`[Processor] Slack sync failed: ${msg}${stack ? `\n${stack}` : ''}`);
+      })
+      .finally(() => {
+        this.slackImportInProgress = false;
+      });
+  }
+
   private personaHistoryImportInProgress = false;
 
   private async checkAndSyncPersonaHistory(_human: HumanEntity): Promise<void> {

package/src/core/prompt-context-builder.ts

@@ -272,15 +272,24 @@ export async function buildResponsePromptData(
   const filteredHuman = await filterHumanDataByVisibility(human, persona, queries);
   const visiblePersonas = getVisiblePersonas(sm, persona);
 
+  const contextWindowMs = persona.context_window_ms ?? human.settings?.default_context_window_ms ?? 28800000;
+  const contextBoundaryMs = persona.context_boundary ? new Date(persona.context_boundary).getTime() : 0;
+  const windowStartMs = Date.now() - contextWindowMs;
+
   const alwaysMessages = sm.messages_getAlways(persona.id);
-  const temporalAnchors = alwaysMessages.map(m => ({
-    id: m.id,
-    role: m.role === "human" ? "human" as const : "system" as const,
-    content: m.content,
-    silence_reason: m.silence_reason,
-    timestamp: m.timestamp,
-    _synthesis: m._synthesis,
-  }));
+  const temporalAnchors = alwaysMessages
+    .filter(m => {
+      const msgMs = new Date(m.timestamp).getTime();
+      return msgMs < windowStartMs || (contextBoundaryMs > 0 && msgMs < contextBoundaryMs);
+    })
+    .map(m => ({
+      id: m.id,
+      role: m.role === "human" ? "human" as const : "system" as const,
+      content: m.content,
+      silence_reason: m.silence_reason,
+      timestamp: m.timestamp,
+      _synthesis: m._synthesis,
+    }));
 
   return {
     persona: {

package/src/core/tools/builtin/pkce.ts

@@ -1,18 +1,16 @@
 /**
- * PKCE helpers — shared by Web (SpotifyAuthButton) and TUI (spotify-auth command).
+ * PKCE helpers — shared by Web and TUI auth flows (Spotify, Slack, etc.).
  *
  * Uses the Web Crypto API (available in both browser and Bun/Node >= 19).
  * All functions are synchronous except generateChallenge which needs crypto.subtle.
  */
 
-/** Generate a random code verifier (128 chars, URL-safe base64). */
 export function generateVerifier(): string {
   const array = new Uint8Array(96); // 96 bytes → 128 chars base64url
   crypto.getRandomValues(array);
   return base64url(array);
 }
 
-/** Derive the PKCE code challenge (SHA-256 of verifier, base64url). */
 export async function generateChallenge(verifier: string): Promise<string> {
   const encoder = new TextEncoder();
   const data = encoder.encode(verifier);
@@ -20,14 +18,22 @@ export async function generateChallenge(verifier: string): Promise<string> {
   return base64url(new Uint8Array(digest));
 }
 
-/** Exchange an authorization code for tokens (used by both Web and TUI). */
 export async function exchangeCode(params: {
   code: string;
   verifier: string;
   redirectUri: string;
   clientId: string;
-}): Promise<{ access_token: string; refresh_token: string; expires_in: number }> {
-  const { code, verifier, redirectUri, clientId } = params;
+  tokenEndpoint?: string;
+  tokenResponsePath?: string[];
+}): Promise<{ access_token: string; refresh_token: string; expires_in: number; _raw: Record<string, unknown> }> {
+  const {
+    code,
+    verifier,
+    redirectUri,
+    clientId,
+    tokenEndpoint = "https://accounts.spotify.com/api/token",
+    tokenResponsePath,
+  } = params;
 
   const body = new URLSearchParams({
     grant_type: "authorization_code",
@@ -37,7 +43,7 @@ export async function exchangeCode(params: {
     code_verifier: verifier,
   });
 
-  const response = await fetch("https://accounts.spotify.com/api/token", {
+  const response = await fetch(tokenEndpoint, {
     method: "POST",
     headers: { "Content-Type": "application/x-www-form-urlencoded" },
     body: body.toString(),
@@ -45,47 +51,58 @@ export async function exchangeCode(params: {
 
   if (!response.ok) {
     const text = await response.text();
-    throw new Error(`Spotify token exchange failed (${response.status}): ${text}`);
+    throw new Error(`Token exchange failed (${response.status}): ${text}`);
   }
 
-  return response.json() as Promise<{
-    access_token: string;
-    refresh_token: string;
-    expires_in: number;
-  }>;
+  const json = await response.json() as Record<string, unknown>;
+
+  const payload = tokenResponsePath
+    ? tokenResponsePath.reduce<Record<string, unknown>>((obj, key) => {
+        const next = obj[key];
+        return (next && typeof next === "object" ? next : obj) as Record<string, unknown>;
+      }, json)
+    : json;
+
+  return { ...(payload as { access_token: string; refresh_token: string; expires_in: number }), _raw: json };
 }
 
-/** Build the Spotify authorization URL. */
 export function buildAuthUrl(params: {
   clientId: string;
   redirectUri: string;
   scopes: string[];
   challenge: string;
   state?: string;
+  userScopes?: string[];
+  authEndpoint?: string;
 }): string {
-  const { clientId, redirectUri, scopes, challenge, state } = params;
-  const url = new URL("https://accounts.spotify.com/authorize");
+  const {
+    clientId,
+    redirectUri,
+    scopes,
+    challenge,
+    state,
+    userScopes,
+    authEndpoint = "https://accounts.spotify.com/authorize",
+  } = params;
+
+  const url = new URL(authEndpoint);
   url.searchParams.set("client_id", clientId);
   url.searchParams.set("response_type", "code");
   url.searchParams.set("redirect_uri", redirectUri);
-  url.searchParams.set("scope", scopes.join(" "));
+  if (scopes.length > 0) url.searchParams.set("scope", scopes.join(" "));
+  if (userScopes && userScopes.length > 0) url.searchParams.set("user_scope", userScopes.join(" "));
   url.searchParams.set("code_challenge", challenge);
   url.searchParams.set("code_challenge_method", "S256");
   if (state) url.searchParams.set("state", state);
   return url.toString();
 }
 
-// ---------------------------------------------------------------------------
-// Internal helpers
 // ---------------------------------------------------------------------------
 
 function base64url(buffer: Uint8Array): string {
-  // btoa works in browser; Buffer works in Node/Bun
-  let binary: string;
   if (typeof btoa === "function") {
-    binary = String.fromCharCode(...buffer);
+    const binary = String.fromCharCode(...buffer);
     return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
   }
-  // Node/Bun fallback
   return Buffer.from(buffer).toString("base64url");
 }

package/src/core/tools/builtin/slack-auth.ts

@@ -0,0 +1,117 @@
+/**
+ * Slack token refresh — shared helper for the Slack integration.
+ *
+ * Caches the current access token in module scope so multiple API calls
+ * within the same session don't each trigger a refresh round-trip.
+ *
+ * The refresh token is read from human.settings.slack.auth at call time,
+ * so it always reflects the latest stored value.
+ *
+ * Auth flow uses PKCE (no client_secret required — public client).
+ * Slack issues rotating refresh tokens for localhost/desktop redirects;
+ * callers must persist the new refresh token via onTokenRotated.
+ */
+
+export const SLACK_CLIENT_ID = "11080256060354.11080294064034";
+
+export const SLACK_USER_SCOPES = [
+  "channels:history",
+  "channels:read",
+  "groups:history",
+  "groups:read",
+  "im:history",
+  "im:read",
+  "mpim:history",
+  "mpim:read",
+  "users:read",
+  "users:read.email",
+];
+
+// TUI redirect URI — Slack requires HTTPS for distributed apps, so we relay
+// through ei.flare576.com/callback/slack/tui which does a 302 to localhost.
+export const SLACK_TUI_REDIRECT_URI = "https://ei.flare576.com/callback/slack/tui";
+export const SLACK_TUI_PORT = 4243;
+
+// Web redirect URI — must match slack_manifest.yaml
+export const SLACK_WEB_REDIRECT_URI = "https://ei.flare576.com/callback/slack";
+
+interface CachedToken {
+  token: string;
+  expires_at: number; // Date.now() ms
+}
+
+let cachedToken: CachedToken | null = null;
+
+/**
+ * Get a valid Slack access token, refreshing if needed.
+ * @param refreshToken - The stored refresh token from human.settings.slack.auth
+ * @param onTokenRotated - Called with the new refresh token when Slack rotates it
+ */
+export async function getSlackAccessToken(
+  refreshToken: string,
+  onTokenRotated?: (newRefreshToken: string) => void
+): Promise<string> {
+  // Return cached token if still valid (60s buffer)
+  if (cachedToken && Date.now() < cachedToken.expires_at - 60_000) {
+    return cachedToken.token;
+  }
+
+  const body = new URLSearchParams({
+    grant_type: "refresh_token",
+    refresh_token: refreshToken,
+    client_id: SLACK_CLIENT_ID,
+  });
+
+  const response = await fetch("https://slack.com/api/oauth.v2.access", {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: body.toString(),
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Slack token refresh failed (${response.status}): ${text}`);
+  }
+
+  const data = (await response.json()) as {
+    ok: boolean;
+    error?: string;
+    authed_user?: {
+      access_token: string;
+      refresh_token?: string;
+      expires_in?: number;
+    };
+  };
+
+  if (!data.ok || !data.authed_user?.access_token) {
+    throw new Error(`Slack token refresh failed: ${data.error ?? "unknown error"}`);
+  }
+
+  const expiresIn = data.authed_user.expires_in ?? 43200; // 12h default
+  cachedToken = {
+    token: data.authed_user.access_token,
+    expires_at: Date.now() + expiresIn * 1000,
+  };
+
+  // Slack rotates the refresh token — persist the new one if provided
+  if (data.authed_user.refresh_token && data.authed_user.refresh_token !== refreshToken) {
+    onTokenRotated?.(data.authed_user.refresh_token);
+  }
+
+  return cachedToken.token;
+}
+
+/** Clear the cached token (call if refresh token changes). */
+export function clearSlackTokenCache(): void {
+  cachedToken = null;
+}
+
+/** Return a structured "not authenticated" error. */
+export function slackNotAuthenticatedError(): string {
+  return JSON.stringify({
+    error: "not_authenticated",
+    integration: "slack",
+    message:
+      "Slack is not connected. In the TUI: /auth slack. In the web app: My Data → External → Connect Slack.",
+  });
+}

package/src/core/tools/index.ts

@@ -29,7 +29,7 @@ export const SYSTEM_TOOLS: ToolDefinition[] = [
     provider_id: "ei",
     name: "find_memory",
     display_name: "Find Memory",
-    description: "Semantic search of your personal memory — facts, topics, people, and quotes learned across ALL conversations over time, not just this one. Use when the human references something from the past, mentions a person, or asks about a topic you might have learned about. People and topic results include a sentiment field (e.g. '72% positive', 'neutral', '45% slightly negative') indicating how the human generally feels about that person or subject. Supports optional filters: types (array of 'facts', 'topics', 'people', 'quotes'), limit (1-20, default 10), recent (true = sort by recency), persona (filter to what a specific persona has learned — use display name).",
+    description: "Semantic search of your personal memory — facts, topics, people, and quotes learned across ALL conversations over time, not just this one. Use when the human references something from the past, mentions a person, or asks about a topic you might have learned about. People and topic results include a sentiment field (e.g. '72% positive', 'neutral', '45% slightly negative') indicating how the human generally feels about that person or subject. Supports optional filters: types (array of 'facts', 'topics', 'people', 'quotes'), limit (1-20, default 10), recent (true = sort by recency), persona (filter to what a specific persona has learned — use display name). TYPE GUIDANCE: 'facts' are ONLY user demographics — name, age, job title, location, family structure, physical traits. For interests, opinions, hobbies, or anything the human cares about, use 'topics'. For named individuals, use 'people'. For verbatim things said, use 'quotes'.",
     input_schema: {
       type: "object",
       properties: {

package/src/core/types/entities.ts

@@ -132,6 +132,7 @@ export interface HumanSettings {
   active_theme?: string;
   custom_themes?: ThemeDefinition[];
   personaHistory?: import("../../integrations/persona-history/types.js").PersonaHistorySettings;
+  slack?: import("../../integrations/slack/types.js").SlackSettings;
 }
 
 export interface HumanEntity {

package/src/core/utils/message-id.ts

@@ -112,3 +112,7 @@ export function qualifyCursorMessage(machine: string, sessionId: string, nativeI
 export function qualifyDocumentMessage(slug: string, uuid: string): string {
   return `import:document:${slug}:${uuid}`
 }
+
+export function qualifySlackMessage(workspaceId: string, channelId: string, ts: string): string {
+  return `slack:${workspaceId}:${channelId}:${ts}`
+}