@memoire-ai/collector 0.1.0

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@memoire-ai/collector",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@memoire-ai/sdk": "0.1.0",
+    "@memoire-ai/shared": "0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "postgres": "^3.4.5",
+    "typescript": "^5.7.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "test": "node --test dist/*.test.js",
+    "test:integration": "node --test dist/*.integration.js"
+  }
+}

package/src/capture.integration.ts

@@ -0,0 +1,98 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import postgres, { type Sql } from 'postgres';
+import { Collector } from './capture.js';
+
+const integrationEnv = {
+  apiUrl: process.env.MEMOIRE_INTEGRATION_API_URL,
+  apiKey: process.env.MEMOIRE_INTEGRATION_API_KEY,
+  orgId: process.env.MEMOIRE_INTEGRATION_ORG_ID,
+  projectId: process.env.MEMOIRE_INTEGRATION_PROJECT_ID,
+  userId: process.env.MEMOIRE_INTEGRATION_USER_ID,
+  databaseUrl: process.env.MEMOIRE_INTEGRATION_DATABASE_URL,
+};
+
+const missingEnv = Object.entries(integrationEnv)
+  .filter(([, value]) => !value)
+  .map(([key]) => key);
+
+test(
+  'collector forwards captured events to the API and persists them',
+  { skip: missingEnv.length > 0 ? `missing env: ${missingEnv.join(', ')}` : false },
+  async () => {
+    const collector = new Collector({
+      apiUrl: integrationEnv.apiUrl!,
+      apiKey: integrationEnv.apiKey!,
+      orgId: integrationEnv.orgId!,
+      projectId: integrationEnv.projectId!,
+      userId: integrationEnv.userId!,
+      client: 'cursor',
+      repoId: 'memoire/integration',
+    });
+    const sql = postgres(integrationEnv.databaseUrl!);
+    const content = `collector integration ${Date.now()}`;
+
+    try {
+      await collector.observation(content, {
+        branchName: 'integration/collector',
+        filesModified: ['src/auth/config.ts'],
+        concepts: ['integration-test'],
+      });
+
+      const row = await waitForEvent(sql, {
+        content,
+        projectId: integrationEnv.projectId!,
+      });
+
+      assert.ok(row, 'expected collector event to be persisted');
+      assert.equal(row.event_type, 'observation');
+      assert.equal(row.client, 'cursor');
+      assert.equal(row.project_id, integrationEnv.projectId);
+      assert.match(row.files_modified, /src\/auth\/config\.ts/);
+
+      await sql`delete from events where id = ${row.id}`;
+    } finally {
+      collector.destroy();
+      await sql.end({ timeout: 5 });
+    }
+  }
+);
+
+async function waitForEvent(
+  sql: Sql,
+  input: { content: string; projectId: string }
+): Promise<
+  | {
+      id: string;
+      project_id: string;
+      event_type: string;
+      client: string;
+      files_modified: string;
+    }
+  | undefined
+> {
+  for (let attempt = 0; attempt < 20; attempt++) {
+    const rows = await sql<{
+      id: string;
+      project_id: string;
+      event_type: string;
+      client: string;
+      files_modified: string;
+    }[]>`
+      select id, project_id, event_type, client, files_modified::text as files_modified
+      from events
+      where project_id = ${input.projectId}
+        and content = ${input.content}
+      order by inserted_at desc
+      limit 1
+    `;
+
+    if (rows[0]) {
+      return rows[0];
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+
+  return undefined;
+}

package/src/capture.ts

@@ -0,0 +1,352 @@
+import { MemoireClient, OfflineQueue, type MemoireClientOptions } from '@memoire-ai/sdk';
+import type { MemoireEvent, MemoireClient as ClientType } from '@memoire-ai/shared';
+import { nanoid } from './utils.js';
+import { writeCursorRules, type CursorRulesContext } from './cursor-rules.js';
+import { stripPrivateTags } from './strip-private.js';
+import { ContentDedup } from './dedup.js';
+import { ClientSummarizer, type ClientSummarizerConfig } from './client-summarizer.js';
+
+export interface CollectorConfig {
+  apiUrl: string;
+  apiKey: string;
+  orgId: string;
+  projectId: string;
+  userId: string;
+  client: ClientType;
+  repoId?: string;
+  clientId?: string;
+  /** Workspace root path — if set, auto-writes .cursor/rules/ context */
+  workspacePath?: string;
+  /** Project name for cursor rules */
+  projectName?: string;
+  /** Dedup window in ms (default 30000). Set to 0 to disable. */
+  dedupWindowMs?: number;
+  /** Edge/native summarization config. When set, events are pre-summarized
+   *  before reaching the API, saving server-side summarization cost. */
+  summarizer?: ClientSummarizerConfig;
+}
+
+/** Structured session summary matching claude-mem's summary schema */
+export interface SessionSummary {
+  /** What the user requested */
+  request: string;
+  /** What was explored / investigated */
+  investigated?: string[];
+  /** Key learnings and discoveries */
+  learned?: string[];
+  /** What was delivered / completed */
+  completed?: string[];
+  /** Suggested follow-up work */
+  next_steps?: string[];
+  /** Additional notes */
+  notes?: string;
+}
+
+/**
+ * Collector captures events from IDE environments and forwards them
+ * to the Memoire API. If the API is unavailable, events are queued
+ * locally and retried.
+ *
+ * Follows the claude-mem hook lifecycle pattern:
+ * SessionStart → PostToolUse → SessionEnd
+ *
+ * Edge processing:
+ * - Privacy: <private> tags are stripped before storage
+ * - Dedup: Content-hash deduplication within a 30s sliding window
+ * - Structured summaries: Session end accepts typed summary objects
+ */
+export class Collector {
+  private memoireClient: MemoireClient;
+  private queue: OfflineQueue;
+  private config: CollectorConfig;
+  private sessionId: string;
+  private dedup: ContentDedup;
+  private summarizer: ClientSummarizer | null;
+
+  constructor(config: CollectorConfig) {
+    this.config = config;
+    this.sessionId = nanoid();
+    this.memoireClient = new MemoireClient({
+      apiUrl: config.apiUrl,
+      apiKey: config.apiKey,
+      clientId: config.clientId,
+    });
+    this.queue = new OfflineQueue(this.memoireClient);
+    this.dedup = new ContentDedup(config.dedupWindowMs ?? 30_000);
+    this.summarizer = config.summarizer ? new ClientSummarizer(config.summarizer) : null;
+  }
+
+  /** Start a new session */
+  async sessionStart(): Promise<void> {
+    this.sessionId = nanoid();
+    await this.emit({
+      event_type: 'observation',
+      content: `Session started on ${this.config.client}`,
+    });
+  }
+
+  /** Record an observation from a tool use */
+  async observation(content: string, opts?: {
+    filesRead?: string[];
+    filesModified?: string[];
+    concepts?: string[];
+    branchName?: string;
+  }): Promise<void> {
+    await this.emit({
+      event_type: 'observation',
+      content,
+      files_read: opts?.filesRead,
+      files_modified: opts?.filesModified,
+      concepts: opts?.concepts,
+      branch_name: opts?.branchName,
+    });
+  }
+
+  /** Record a user prompt */
+  async prompt(content: string, opts?: {
+    filesRead?: string[];
+    filesModified?: string[];
+    concepts?: string[];
+    branchName?: string;
+  }): Promise<void> {
+    await this.emit({
+      event_type: 'prompt',
+      content,
+      files_read: opts?.filesRead,
+      files_modified: opts?.filesModified,
+      concepts: opts?.concepts,
+      branch_name: opts?.branchName,
+    });
+  }
+
+  /** Record a decision */
+  async decision(content: string, opts?: {
+    filesModified?: string[];
+    concepts?: string[];
+    branchName?: string;
+  }): Promise<void> {
+    await this.emit({
+      event_type: 'decision',
+      content,
+      files_modified: opts?.filesModified,
+      concepts: opts?.concepts,
+      branch_name: opts?.branchName,
+    });
+  }
+
+  /** Record a failed attempt */
+  async attempt(content: string, opts?: {
+    filesModified?: string[];
+    concepts?: string[];
+    branchName?: string;
+  }): Promise<void> {
+    await this.emit({
+      event_type: 'attempt',
+      content,
+      files_modified: opts?.filesModified,
+      concepts: opts?.concepts,
+      branch_name: opts?.branchName,
+    });
+  }
+
+  /** Record a convention or standard */
+  async convention(content: string, opts?: {
+    filesModified?: string[];
+    concepts?: string[];
+    branchName?: string;
+  }): Promise<void> {
+    await this.emit({
+      event_type: 'convention',
+      content,
+      files_modified: opts?.filesModified,
+      concepts: opts?.concepts,
+      branch_name: opts?.branchName,
+    });
+  }
+
+  /** Record a branch lifecycle event */
+  async branchEvent(content: string, opts?: {
+    filesModified?: string[];
+    concepts?: string[];
+    branchName?: string;
+  }): Promise<void> {
+    await this.emit({
+      event_type: 'branch_event',
+      content,
+      files_modified: opts?.filesModified,
+      concepts: opts?.concepts,
+      branch_name: opts?.branchName,
+    });
+  }
+
+  /** End a session with a structured or plain-text summary */
+  async sessionEnd(summary: string | SessionSummary): Promise<void> {
+    const content = typeof summary === 'string'
+      ? summary
+      : formatStructuredSummary(summary);
+
+    await this.emit({
+      event_type: 'session_summary',
+      content,
+    });
+    // Flush any remaining events
+    await this.queue.flush();
+
+    // Auto-write Cursor rules context if workspace path is configured
+    if (this.config.workspacePath && this.config.client === 'cursor') {
+      try {
+        await this.updateCursorRules(content);
+      } catch {
+        // Non-fatal: cursor rules update is best-effort
+      }
+    }
+  }
+
+  /**
+   * Update .cursor/rules/memoire-context.mdc with latest project context.
+   * Called automatically on session end for Cursor clients.
+   */
+  private async updateCursorRules(sessionSummary: string): Promise<void> {
+    if (!this.config.workspacePath) return;
+
+    // Try to fetch profile from API
+    let profile: CursorRulesContext['profile'];
+
+    try {
+      const profileResp = await this.memoireClient.projectProfile({
+        org_id: this.config.orgId,
+        project_id: this.config.projectId,
+        viewer_user_id: this.config.userId,
+      });
+      if (profileResp?.project_profile) {
+        const pp = profileResp.project_profile;
+        profile = {
+          architecture: pp.architecture?.join(', '),
+          stack: pp.key_files?.slice(0, 5),
+          conventions: pp.conventions?.slice(0, 10),
+          currentFocus: pp.current_focus?.join(', '),
+        };
+      }
+    } catch {
+      // API unavailable — write rules with just the session summary
+    }
+
+    writeCursorRules(this.config.workspacePath, {
+      projectName: this.config.projectName ?? 'Project',
+      profile,
+      recentSummary: sessionSummary,
+      apiUrl: this.config.apiUrl,
+    });
+  }
+
+  /** Emit a raw event — applies privacy stripping, dedup, and optional client-side summarization */
+  private async emit(partial: Partial<MemoireEvent> & { event_type: string; content: string }): Promise<void> {
+    // Edge processing: strip <private> tags from content AND all metadata fields
+    const cleanContent = stripPrivateTags(partial.content);
+
+    // Skip empty content after stripping
+    if (!cleanContent) return;
+
+    const cleanFilesRead = stripPrivateArray(partial.files_read);
+    const cleanFilesModified = stripPrivateArray(partial.files_modified);
+    const cleanConcepts = stripPrivateArray(partial.concepts);
+    const cleanBranch = partial.branch_name ? stripPrivateTags(partial.branch_name) : partial.branch_name;
+
+    // Edge processing: deduplicate within sliding window (keyed on event_type + content)
+    if (this.dedup.isDuplicate(`${partial.event_type}\0${cleanContent}`)) return;
+
+    const event: MemoireEvent = {
+      org_id: this.config.orgId,
+      project_id: this.config.projectId,
+      repo_id: this.config.repoId,
+      user_id: this.config.userId,
+      session_id: this.sessionId,
+      client: this.config.client,
+      created_at: new Date().toISOString(),
+      idempotency_key: nanoid(),
+      private: false,
+      ...partial,
+      event_type: partial.event_type,
+      content: cleanContent,
+      files_read: cleanFilesRead,
+      files_modified: cleanFilesModified,
+      concepts: cleanConcepts,
+      branch_name: cleanBranch,
+    };
+
+    // Edge processing: client-side AI summarization (non-blocking on failure)
+    if (this.summarizer) {
+      try {
+        const result = await this.summarizer.summarize(
+          cleanContent,
+          partial.event_type,
+          {
+            filesRead: cleanFilesRead,
+            filesModified: cleanFilesModified,
+            concepts: cleanConcepts,
+            branchName: cleanBranch,
+          },
+        );
+        if (result) {
+          event.summary = result.summary;
+          event.summary_details = result.summary_details;
+          // Merge AI-extracted concepts with existing ones
+          if (result.concepts.length > 0) {
+            const merged = new Set([...(event.concepts ?? []), ...result.concepts]);
+            event.concepts = Array.from(merged).slice(0, 12);
+          }
+        }
+      } catch {
+        // Summarization failure is non-fatal — server will handle it
+      }
+    }
+
+    try {
+      await this.memoireClient.ingestEvent(event);
+    } catch {
+      // API unreachable — queue for retry
+      this.queue.enqueue([event]);
+    }
+  }
+
+  /** Get the number of pending events in the offline queue */
+  get pendingEvents(): number {
+    return this.queue.pending;
+  }
+
+  destroy(): void {
+    this.dedup.destroy();
+    this.queue.destroy();
+  }
+}
+
+/** Strip private tags from each element of a string array */
+function stripPrivateArray(arr?: string[]): string[] | undefined {
+  if (!arr) return arr;
+  return arr.map(stripPrivateTags).filter(Boolean);
+}
+
+/** Format a structured summary into a readable content string */
+function formatStructuredSummary(summary: SessionSummary): string {
+  const parts: string[] = [];
+
+  parts.push(`Request: ${summary.request}`);
+
+  if (summary.investigated?.length) {
+    parts.push(`Investigated: ${summary.investigated.join('; ')}`);
+  }
+  if (summary.learned?.length) {
+    parts.push(`Learned: ${summary.learned.join('; ')}`);
+  }
+  if (summary.completed?.length) {
+    parts.push(`Completed: ${summary.completed.join('; ')}`);
+  }
+  if (summary.next_steps?.length) {
+    parts.push(`Next steps: ${summary.next_steps.join('; ')}`);
+  }
+  if (summary.notes) {
+    parts.push(`Notes: ${summary.notes}`);
+  }
+
+  return parts.join('\n');
+}

package/src/client-summarizer.test.ts

@@ -0,0 +1,144 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { ClientSummarizer } from './client-summarizer.js';
+
+test('ClientSummarizer parses Anthropic responses', async () => {
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = (async () =>
+    new Response(JSON.stringify({
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            summary: 'Added OAuth2 PKCE flow to auth service',
+            concepts: ['oauth', 'pkce', 'authentication'],
+            details: { title: 'OAuth2 PKCE', subtitle: 'Added PKCE flow' },
+          }),
+        },
+      ],
+    }), {
+      status: 200,
+      headers: { 'content-type': 'application/json' },
+    })) as typeof fetch;
+
+  try {
+    const summarizer = new ClientSummarizer({
+      mode: 'anthropic',
+      apiKey: 'sk-ant-test-fake',
+    });
+
+    const result = await summarizer.summarize('Implemented OAuth2 with PKCE', 'observation');
+
+    assert.ok(result);
+    assert.equal(result.summary, 'Added OAuth2 PKCE flow to auth service');
+    assert.deepEqual(result.concepts, ['oauth', 'pkce', 'authentication']);
+    assert.equal(result.summary_details?.title, 'OAuth2 PKCE');
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test('ClientSummarizer parses native Claude JSON output', async () => {
+  const summarizer = new ClientSummarizer({
+    mode: 'native-claude',
+    runCommand: async ({ command }) => {
+      assert.equal(command, 'claude');
+      return JSON.stringify({
+        result: '{"summary":"Verified auth middleware refresh flow.","concepts":["auth","middleware","refresh"],"details":{"title":"Auth middleware refresh"}}',
+      });
+    },
+  });
+
+  const result = await summarizer.summarize('Inspected auth middleware', 'observation');
+
+  assert.ok(result);
+  assert.equal(result.summary, 'Verified auth middleware refresh flow.');
+  assert.deepEqual(result.concepts, ['auth', 'middleware', 'refresh']);
+});
+
+test('ClientSummarizer parses native Codex JSONL output', async () => {
+  const summarizer = new ClientSummarizer({
+    mode: 'native-codex',
+    runCommand: async ({ command }) => {
+      assert.equal(command, 'codex');
+      return [
+        '{"type":"thread.started","thread_id":"abc"}',
+        '{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"thinking"}}',
+        '{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"{\\"summary\\":\\"Captured OAuth callback fix.\\",\\"concepts\\":[\\"oauth\\",\\"callback\\"],\\"details\\":{\\"title\\":\\"OAuth callback\\"}}"}}',
+      ].join('\n');
+    },
+  });
+
+  const result = await summarizer.summarize('Fixed OAuth callback edge case', 'attempt');
+
+  assert.ok(result);
+  assert.equal(result.summary, 'Captured OAuth callback fix.');
+  assert.deepEqual(result.concepts, ['oauth', 'callback']);
+  assert.equal(result.summary_details?.title, 'OAuth callback');
+});
+
+test('ClientSummarizer returns null on command failure', async () => {
+  const summarizer = new ClientSummarizer({
+    mode: 'native-codex',
+    runCommand: async () => {
+      throw new Error('boom');
+    },
+  });
+
+  const result = await summarizer.summarize('test', 'observation');
+  assert.equal(result, null);
+});
+
+test('ClientSummarizer truncates summary to 180 chars', async () => {
+  const summarizer = new ClientSummarizer({
+    mode: 'native-claude',
+    runCommand: async () => JSON.stringify({
+      result: JSON.stringify({
+        summary: 'A'.repeat(300),
+        concepts: ['test'],
+      }),
+    }),
+  });
+
+  const result = await summarizer.summarize('test', 'observation');
+  assert.ok(result);
+  assert.equal(result.summary.length, 180);
+});
+
+test('ClientSummarizer limits concepts to 12', async () => {
+  const summarizer = new ClientSummarizer({
+    mode: 'native-codex',
+    runCommand: async () => [
+      '{"type":"item.completed","item":{"type":"agent_message","text":"{\\"summary\\":\\"test\\",\\"concepts\\":[' +
+        Array.from({ length: 20 }, (_, i) => `\\"concept${i}\\"`).join(',') +
+        ']}"}}',
+    ].join('\n'),
+  });
+
+  const result = await summarizer.summarize('test', 'observation');
+  assert.ok(result);
+  assert.equal(result.concepts.length, 12);
+});
+
+test('ClientSummarizer handles session_summary details', async () => {
+  const summarizer = new ClientSummarizer({
+    mode: 'native-claude',
+    runCommand: async () => JSON.stringify({
+      result: JSON.stringify({
+        summary: 'Implemented auth middleware',
+        concepts: ['auth', 'middleware'],
+        details: {
+          request: 'Add WorkOS auth',
+          investigated: ['existing auth flow'],
+          completed: ['WorkOS integration'],
+          next_steps: ['add RBAC'],
+        },
+      }),
+    }),
+  });
+
+  const result = await summarizer.summarize('session content', 'session_summary');
+  assert.ok(result);
+  assert.equal(result.summary_details?.request, 'Add WorkOS auth');
+  assert.deepEqual(result.summary_details?.completed, ['WorkOS integration']);
+});