@xiaolei.shawn/mcp-server 0.2.0 → 0.3.0

package/README.md

@@ -24,6 +24,7 @@ You can point the built-in dashboard server to any static bundle via `AL_DASHBOA
 - Session storage on local disk (`AL_SESSIONS_DIR`)
 - Local gateway API for middleware (`/api/gateway/*`)
 - Export session JSON with normalized snapshot (`agentlens export`)
+- Raw log adapter ingestion (`agentlens ingest`, `/api/ingest`) with duplicate suppression
 
 ## Install
 
@@ -55,6 +56,18 @@ agentlens mcp
 - `record_decision`
 - `record_assumption`
 - `record_verification`
+- `record_artifact_created`
+- `record_intent_transition`
+- `record_risk_signal`
+- `record_verification_run`
+- `record_diff_summary`
+- `record_decision_link`
+- `record_assumption_lifecycle`
+- `record_blocker`
+- `record_token_usage_checkpoint`
+- `record_session_quality`
+- `record_replay_bookmark`
+- `record_hotspot`
 - `record_session_end`
 
 ### Gateway tools
@@ -80,6 +93,7 @@ API endpoints:
 - `POST /api/gateway/begin`
 - `POST /api/gateway/act`
 - `POST /api/gateway/end`
+- `POST /api/ingest`
 
 If web assets are available (default `../webapp/dist`), they are served by the same server.
 
@@ -140,6 +154,41 @@ Export by session id:
 agentlens export --session sess_1771256059058_2bd2bd8f --out ./session.json
 ```
 
+## Ingest raw logs via adapters
+
+Example: ingest Codex raw JSONL and convert to canonical events:
+
+```bash
+agentlens ingest --input /path/to/rollout.jsonl --adapter codex_jsonl
+```
+
+Example: ingest Cursor raw logs that contain `<user_query>`, `<think>`, and `Tool call/Tool result` blocks:
+
+```bash
+agentlens ingest --input /path/to/cursor-log.txt --adapter cursor_raw
+```
+
+Auto-detect adapter and merge into an existing session with dedupe:
+
+```bash
+agentlens ingest --input /path/to/raw.jsonl --adapter auto --merge-session sess_123
+```
+
+Notes:
+
+- Ingest writes canonical events to `<session_id>.jsonl`.
+- Original raw content is preserved in `<session_id>.<adapter>.raw.jsonl`.
+- **Merge and dedupe**: When merging into an existing session (e.g. raw log + MCP-canonical events), ingest uses **semantic dedupe** so the same logical event (intent, tool call, artifact, etc.) is not duplicated even if timestamps or payload details differ. Merged events are written in **time order** with contiguous `seq` for accurate recommendations/risk/hotspot analysis.
+- Duplicate events are skipped by default (exact or semantic key depending on merge).
+- Codex adapter preserves user prompts, reasoning summaries, assistant outputs, tool calls/results, and normalized token checkpoints.
+- Cursor adapter preserves user queries, `<think>` reasoning traces, tool call/result traces, and token counters when present.
+- If `--merge-session` is omitted, ingest attempts **fingerprint match** automatically:
+  - Primary signal: normalized user prompt / intent similarity
+  - Secondary signal: timestamp proximity (recent sessions weighted higher)
+  - Min confidence: `AL_INGEST_FINGERPRINT_MIN_CONFIDENCE` (default `0.62`)
+  - Max time window (hours): `AL_INGEST_FINGERPRINT_MAX_WINDOW_HOURS` (default `72`)
+- Ingest output includes `merge_strategy` (`explicit_merge`, `adapted_session_id`, `fingerprint_match`, `new_session`) and optional `merge_confidence`.
+
 ## Publish checklist
 
 1. Update version in `package.json`.

package/dist/__tests__/ingest.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/ingest.test.js

@@ -0,0 +1,144 @@
+/**
+ * Ingest and merge logic tests.
+ * Run from mcp-server: pnpm run build && pnpm test
+ */
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert";
+import { ingestRawContent, ingestRawFile } from "../ingest.js";
+import { readSessionEvents } from "../store.js";
+import { adaptRawContent } from "../adapters/index.js";
+const FIXTURES_DIR = join(process.cwd(), "fixtures");
+function fixturePath(name) {
+    return join(FIXTURES_DIR, name);
+}
+describe("adapters", () => {
+    it("cursor_raw adapts sample and produces session_start, intent, tool_call, session_end", () => {
+        const raw = readFileSync(fixturePath("cursor_sample.txt"), "utf-8");
+        const adapted = adaptRawContent(raw, "cursor_raw");
+        assert.strictEqual(adapted.source, "cursor_raw");
+        assert.ok(adapted.session_id?.startsWith("cursor_"));
+        const kinds = adapted.events.map((e) => e.kind);
+        assert.ok(kinds.includes("session_start"), "has session_start");
+        assert.ok(kinds.includes("intent"), "has intent");
+        assert.ok(kinds.includes("session_end"), "has session_end");
+        const hasToolOrArtifact = kinds.includes("tool_call") || kinds.includes("artifact_created");
+        assert.ok(hasToolOrArtifact, "has tool_call or artifact_created");
+    });
+    it("codex_jsonl adapts sample and produces session_start, intent, session_end", () => {
+        const raw = readFileSync(fixturePath("codex_sample.jsonl"), "utf-8");
+        const adapted = adaptRawContent(raw, "codex_jsonl");
+        assert.strictEqual(adapted.source, "codex_jsonl");
+        const kinds = adapted.events.map((e) => e.kind);
+        assert.ok(kinds.includes("session_start"), "has session_start");
+        assert.ok(kinds.includes("intent"), "has intent");
+        assert.ok(kinds.includes("session_end"), "has session_end");
+    });
+    it("auto adapter selects cursor_raw for cursor-style content", () => {
+        const raw = readFileSync(fixturePath("cursor_sample.txt"), "utf-8");
+        const adapted = adaptRawContent(raw, "auto");
+        assert.strictEqual(adapted.source, "cursor_raw");
+    });
+    it("auto adapter selects codex_jsonl for codex JSONL content", () => {
+        const raw = readFileSync(fixturePath("codex_sample.jsonl"), "utf-8");
+        const adapted = adaptRawContent(raw, "auto");
+        assert.strictEqual(adapted.source, "codex_jsonl");
+    });
+});
+describe("ingest", () => {
+    let sessionsDir;
+    const originalSessionsDir = process.env.AL_SESSIONS_DIR;
+    before(() => {
+        sessionsDir = mkdtempSync(join(tmpdir(), "agentlens-test-"));
+        process.env.AL_SESSIONS_DIR = sessionsDir;
+    });
+    after(() => {
+        process.env.AL_SESSIONS_DIR = originalSessionsDir;
+        rmSync(sessionsDir, { recursive: true, force: true });
+    });
+    it("ingest creates new session and inserts events", () => {
+        const raw = readFileSync(fixturePath("cursor_sample.txt"), "utf-8");
+        const result = ingestRawContent(raw, { adapter: "cursor_raw" });
+        assert.ok(result.session_id);
+        assert.strictEqual(result.adapter, "cursor_raw");
+        assert.strictEqual(result.merge_strategy, "new_session");
+        assert.ok(result.inserted > 0, "inserted > 0");
+        const events = readSessionEvents(result.session_id);
+        assert.strictEqual(events.length, result.inserted);
+    });
+    it("ingest with merge_session_id merges into existing session with semantic dedupe", () => {
+        const raw = readFileSync(fixturePath("cursor_sample.txt"), "utf-8");
+        const first = ingestRawContent(raw, { adapter: "cursor_raw" });
+        const countAfterFirst = readSessionEvents(first.session_id).length;
+        const second = ingestRawContent(raw, {
+            adapter: "cursor_raw",
+            merge_session_id: first.session_id,
+        });
+        assert.strictEqual(second.session_id, first.session_id);
+        assert.strictEqual(second.merge_strategy, "explicit_merge");
+        assert.ok(second.skipped_duplicates > 0, "semantic dedupe skips most events when same content merged");
+        assert.ok(second.inserted <= 1, "at most one new event (e.g. token_usage with different ts) when same content merged");
+        const eventsAfterMerge = readSessionEvents(first.session_id);
+        assert.ok(eventsAfterMerge.length <= countAfterFirst + 1, "event count grows by at most one after merge of same content");
+    });
+    it("merged session is ordered by ts and seq is contiguous", () => {
+        const raw = readFileSync(fixturePath("cursor_sample.txt"), "utf-8");
+        const first = ingestRawContent(raw, { adapter: "cursor_raw" });
+        const sessionId = first.session_id;
+        const events = readSessionEvents(sessionId);
+        for (let i = 1; i < events.length; i++) {
+            const a = events[i - 1];
+            const b = events[i];
+            assert.ok(a.ts <= b.ts || (a.ts === b.ts && (a.seq ?? 0) <= (b.seq ?? 0)), `events ordered: ${a.seq} (${a.ts}) before ${b.seq} (${b.ts})`);
+            assert.strictEqual(b.seq, (a.seq ?? 0) + 1, "seq contiguous");
+        }
+    });
+    it("ingest from file path works", () => {
+        const path = fixturePath("codex_sample.jsonl");
+        const result = ingestRawFile(path, { adapter: "codex_jsonl" });
+        assert.ok(result.session_id);
+        assert.strictEqual(result.adapter, "codex_jsonl");
+        assert.ok(result.inserted > 0);
+    });
+    it("merge raw log from different day: time window filters out all raw events", () => {
+        const sessionId = "sess_merge_target_time_window";
+        const sessionStartTs = "2026-03-02T20:55:12.151Z";
+        const sessionEndTs = "2026-03-02T20:57:42.004Z";
+        const sessionLines = [
+            JSON.stringify({
+                id: `${sessionId}:1:aa`,
+                session_id: sessionId,
+                seq: 1,
+                ts: sessionStartTs,
+                kind: "session_start",
+                actor: { type: "agent" },
+                payload: { goal: "Test" },
+                schema_version: 1,
+            }),
+            JSON.stringify({
+                id: `${sessionId}:2:bb`,
+                session_id: sessionId,
+                seq: 2,
+                ts: sessionEndTs,
+                kind: "session_end",
+                actor: { type: "agent" },
+                payload: { outcome: "completed" },
+                schema_version: 1,
+            }),
+        ].join("\n") + "\n";
+        writeFileSync(join(sessionsDir, `${sessionId}.jsonl`), sessionLines, "utf-8");
+        const raw = readFileSync(fixturePath("codex_sample.jsonl"), "utf-8");
+        const result = ingestRawContent(raw, {
+            adapter: "codex_jsonl",
+            merge_session_id: sessionId,
+        });
+        assert.strictEqual(result.session_id, sessionId);
+        assert.strictEqual(result.merge_strategy, "explicit_merge");
+        assert.strictEqual(result.inserted, 0, "no raw events fall in Mar 2 window");
+        assert.ok(result.filtered_out_by_time_window !== undefined && result.filtered_out_by_time_window > 0, "raw events (Feb 24) were filtered out by time window");
+        const eventsAfter = readSessionEvents(sessionId);
+        assert.strictEqual(eventsAfter.length, 2, "session still has only session_start and session_end");
+    });
+});

package/dist/__tests__/local-analysis.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/local-analysis.test.js

@@ -0,0 +1,118 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { deriveIntentTokenBreakdown, generateFollowupArtifacts } from "../local-analysis.js";
+function baseEvent(overrides) {
+    return {
+        id: "e-1",
+        session_id: "sess-test",
+        seq: 1,
+        ts: "2026-03-03T00:00:00.000Z",
+        kind: "intent",
+        actor: { type: "agent" },
+        payload: {},
+        schema_version: 1,
+        ...overrides,
+    };
+}
+describe("local-analysis", () => {
+    it("generates per-intent artifacts with deterministic template", () => {
+        const events = [
+            baseEvent({
+                id: "i1",
+                seq: 1,
+                kind: "intent",
+                scope: { intent_id: "intent_a" },
+                payload: { intent_id: "intent_a", title: "Implement feature A" },
+            }),
+            baseEvent({
+                id: "t1",
+                seq: 2,
+                kind: "tool_call",
+                scope: { intent_id: "intent_a" },
+                payload: { category: "tool", action: "read_file", target: "src/a.ts" },
+            }),
+            baseEvent({
+                id: "t2",
+                seq: 3,
+                kind: "tool_call",
+                scope: { intent_id: "intent_a" },
+                payload: { category: "tool", action: "read_file", target: "src/a.ts" },
+            }),
+            baseEvent({
+                id: "t3",
+                seq: 4,
+                kind: "tool_call",
+                scope: { intent_id: "intent_a" },
+                payload: { category: "tool", action: "read_file", target: "src/a.ts" },
+            }),
+            baseEvent({
+                id: "v1",
+                seq: 5,
+                kind: "verification",
+                scope: { intent_id: "intent_a" },
+                payload: { type: "test", result: "fail" },
+            }),
+            baseEvent({
+                id: "r1",
+                seq: 6,
+                kind: "risk_signal",
+                scope: { intent_id: "intent_a", file: "src/a.ts" },
+                payload: { level: "high", reasons: ["regression risk"] },
+            }),
+        ];
+        const result = generateFollowupArtifacts(events, {
+            mode: "per_intent",
+            strictness: "soft",
+            focus: "risk",
+        });
+        assert.equal(result.artifacts.length, 1);
+        assert.equal(result.artifacts[0].intent_id, "intent_a");
+        assert.equal(result.artifacts[0].rule_template_id, "high_risk_guardrail");
+        assert.ok(result.artifacts[0].value_claims.risk_mitigation.length > 0);
+    });
+    it("derives token breakdown context/output split", () => {
+        const events = [
+            baseEvent({
+                id: "i1",
+                seq: 1,
+                kind: "intent",
+                scope: { intent_id: "intent_a" },
+                payload: { intent_id: "intent_a", title: "Intent A" },
+            }),
+            baseEvent({
+                id: "ctx1",
+                seq: 2,
+                kind: "tool_call",
+                scope: { intent_id: "intent_a" },
+                payload: { category: "search", action: "search_docs", target: "api docs" },
+            }),
+            baseEvent({
+                id: "tok1",
+                seq: 3,
+                kind: "token_usage_checkpoint",
+                scope: { intent_id: "intent_a" },
+                payload: { usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 } },
+            }),
+            baseEvent({
+                id: "out1",
+                seq: 4,
+                kind: "file_op",
+                scope: { intent_id: "intent_a", file: "src/a.ts" },
+                payload: { category: "file", action: "edit", target: "src/a.ts" },
+            }),
+            baseEvent({
+                id: "tok2",
+                seq: 5,
+                kind: "token_usage_checkpoint",
+                scope: { intent_id: "intent_a" },
+                payload: { usage: { total_tokens: 90 } },
+            }),
+        ];
+        const result = deriveIntentTokenBreakdown(events);
+        assert.equal(result.intent_breakdown.length, 1);
+        assert.equal(result.totals.total_tokens, 240);
+        assert.equal(result.intent_breakdown[0].context_tokens +
+            result.intent_breakdown[0].output_tokens +
+            result.intent_breakdown[0].unknown_tokens, 240);
+    });
+});

package/dist/adapters/codex.d.ts

@@ -0,0 +1,2 @@
+import type { RawAdapter } from "./types.js";
+export declare const codexJsonlAdapter: RawAdapter;