@sanqianx/project-knowledge 4.6.1

package/_site/lib/conversation-exclusions.js

@@ -0,0 +1,204 @@
+const fs = require('fs');
+const path = require('path');
+const { validateProjectId } = require('./contracts');
+const AtomicFile = require('./atomic-file');
+
+const EXCLUSION_SCHEMA = 'conversation-exclusions/v1';
+const EMBEDDED_ASSISTANT_PREFIX = 'embedded-assistant-';
+const EXPLICIT_PREFIX = 'explicit-';
+const CODEX_CONTEXT_TAGS = new Set([
+  'recommended_plugins',
+  'environment_context',
+  'in-app-browser-context',
+]);
+
+/**
+ * T17: legacy embedded Workbench capture exclusion.
+ *
+ * Old builds wrote internal Workbench conversations into Development
+ * Conversation as `explicit-<requirementId>` + `embedded-assistant-<requirementId>`
+ * pairs (rawEventType embedded-claude-result). Those events are excluded from
+ * Development Conversation presentation and from NEW commit binding. Nothing
+ * is deleted: the JSONL lines stay, immutable snapshots stay untouched, and a
+ * manifest is persisted for auditability. A genuine explicit MCP requirement
+ * (no embedded-assistant pair) is never excluded.
+ */
+function isEmbeddedAssistantEvent(event) {
+  if (!event) return false;
+  return String(event.eventId || '').startsWith(EMBEDDED_ASSISTANT_PREFIX)
+    || event.rawEventType === 'embedded-claude-result';
+}
+
+/**
+ * Codex rollout files contain generated context records with role=user. They
+ * are transport/control data, not user-authored prompts. Match only complete,
+ * known envelopes so ordinary prompts containing similar text survive.
+ */
+function isCodexControlUserEvent(event) {
+  if (!event || event.source !== 'codex' || event.role !== 'user' || event.rawEventType !== 'response_item') return false;
+  let remaining = String(event.content || '').trim();
+  if (/^<turn_aborted>\s*[\s\S]*?\s*<\/turn_aborted>$/.test(remaining)) return true;
+
+  let removed = false;
+  const envelope = /^<([A-Za-z0-9_-]+)(?:\s[^>]*)?>[\s\S]*?<\/\1>\s*/;
+  while (remaining) {
+    const match = remaining.match(envelope);
+    if (!match || !CODEX_CONTEXT_TAGS.has(match[1])) break;
+    remaining = remaining.slice(match[0].length).trimStart();
+    removed = true;
+  }
+  return removed && remaining.trim() === '';
+}
+
+function isCodexAbortEvent(event) {
+  return isCodexControlUserEvent(event)
+    && /^<turn_aborted>/.test(String(event.content || '').trim());
+}
+
+function computeConversationExclusions(projectId, events) {
+  validateProjectId(projectId);
+  const paired = new Set();
+  for (const event of events || []) {
+    const eventId = String((event && event.eventId) || '');
+    if (eventId.startsWith(EMBEDDED_ASSISTANT_PREFIX)) {
+      paired.add(eventId.slice(EMBEDDED_ASSISTANT_PREFIX.length));
+    }
+  }
+  const excludedEventIds = [];
+  for (const event of events || []) {
+    if (!event) continue;
+    const eventId = String(event.eventId || '');
+    if (isEmbeddedAssistantEvent(event)) {
+      excludedEventIds.push(eventId);
+      continue;
+    }
+    if (isCodexControlUserEvent(event)) {
+      excludedEventIds.push(eventId);
+      continue;
+    }
+    if (eventId.startsWith(EXPLICIT_PREFIX) && paired.has(eventId.slice(EXPLICIT_PREFIX.length))) {
+      excludedEventIds.push(eventId);
+    }
+  }
+  return {
+    schema: EXCLUSION_SCHEMA,
+    projectId,
+    excludedEventIds: [...new Set(excludedEventIds)].sort(),
+    generatedAt: new Date().toISOString(),
+  };
+}
+
+function eventOrder(left, right) {
+  if (Number.isInteger(left.sequence) && Number.isInteger(right.sequence)) return left.sequence - right.sequence;
+  if (Number.isInteger(left.sequence)) return -1;
+  if (Number.isInteger(right.sequence)) return 1;
+  return String(left.capturedAt || '').localeCompare(String(right.capturedAt || ''))
+    || String(left.eventId || '').localeCompare(String(right.eventId || ''));
+}
+
+/**
+ * Bridge 0.1.1 did not read Codex's nested turn id, so historical assistant
+ * response_item records have turnId=null while each user record received a
+ * generated id. Repair that projection without rewriting the append-only
+ * JSONL: orphan assistants attach to the latest real user in the same Codex
+ * session. A later real prompt or turn_aborted record closes that projected
+ * turn, which also lets commit binding ignore stale Bridge open-turn facts.
+ */
+function projectCodexTurns(events, excludedEventIds) {
+  const excluded = new Set(excludedEventIds || []);
+  const activeBySession = new Map();
+  const closedAtByTurn = new Map();
+  const projected = [];
+  const ordered = [...(events || [])].sort(eventOrder);
+  const sessionKey = event => event && event.source === 'codex' && event.sessionId
+    ? `codex\n${event.sessionId}`
+    : null;
+
+  for (const original of ordered) {
+    const eventId = String((original && original.eventId) || '');
+    const key = sessionKey(original || {});
+    if (excluded.has(eventId)) {
+      if (key && isCodexAbortEvent(original)) {
+        const active = activeBySession.get(key);
+        if (active && Number.isInteger(original.sequence)) closedAtByTurn.set(active, original.sequence);
+        activeBySession.delete(key);
+      }
+      continue;
+    }
+
+    let event = original;
+    if (key && original && original.source === 'codex' && original.role === 'user' && original.turnId) {
+      const previous = activeBySession.get(key);
+      if (previous && previous !== original.turnId && Number.isInteger(original.sequence)) {
+        closedAtByTurn.set(previous, original.sequence);
+      }
+      activeBySession.set(key, original.turnId);
+    } else if (key && original && original.source === 'codex' && original.role === 'assistant' && !original.turnId) {
+      const active = activeBySession.get(key);
+      if (active) event = { ...original, turnId: active };
+    }
+    projected.push(event);
+  }
+
+  return projected.map(event => {
+    const closedAt = event && event.turnId ? closedAtByTurn.get(event.turnId) : null;
+    return Number.isInteger(closedAt)
+      ? { ...event, developmentTurnClosedAtSequence: closedAt }
+      : event;
+  });
+}
+
+function manifestPath(layout, projectId) {
+  return path.join(layout.getProjectMetadataDir(projectId), 'conversation-exclusions.json');
+}
+
+function readManifest(layout, projectId) {
+  try {
+    return AtomicFile.readJsonStrict(manifestPath(layout, projectId), {
+      category: 'conversation-exclusions',
+      validate: value => value,
+    });
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Single source of truth for Development Conversation reads: applies durable
+ * exclusions and the non-destructive Codex turn compatibility projection for
+ * both Conversation Explorer and new CommitConversationBinder runs. The
+ * manifest is refreshed idempotently (best effort) so operators can audit
+ * what is excluded; read failures never break the read itself.
+ */
+function readDevelopmentEvents(conversationStore, projectId) {
+  const events = conversationStore.readEvents(projectId);
+  const manifest = computeConversationExclusions(projectId, events);
+  if (manifest.excludedEventIds.length) {
+    try {
+      const layout = conversationStore.layout;
+      const dir = layout.getProjectMetadataDir(projectId);
+      fs.mkdirSync(dir, { recursive: true });
+      const previous = readManifest(layout, projectId);
+      const unchanged = previous
+        && Array.isArray(previous.excludedEventIds)
+        && previous.excludedEventIds.length === manifest.excludedEventIds.length
+        && previous.excludedEventIds.every((id, index) => id === manifest.excludedEventIds[index]);
+      if (!unchanged) {
+        (conversationStore.atomic || AtomicFile).writeJsonAtomic(manifestPath(layout, projectId), manifest);
+      }
+    } catch {
+      // Manifest persistence is best effort; exclusion itself is computed
+      // deterministically from the durable events on every read.
+    }
+  }
+  return projectCodexTurns(events, manifest.excludedEventIds);
+}
+
+module.exports = {
+  EXCLUSION_SCHEMA,
+  isEmbeddedAssistantEvent,
+  isCodexControlUserEvent,
+  computeConversationExclusions,
+  projectCodexTurns,
+  readDevelopmentEvents,
+};

package/_site/lib/conversation-query-service.js

@@ -0,0 +1,161 @@
+const crypto = require('crypto');
+const { DomainError, validateProjectId } = require('./contracts');
+const { ConversationStore } = require('./conversation-store');
+const { readDevelopmentEvents } = require('./conversation-exclusions');
+
+function localDate(iso) {
+  const date = new Date(iso);
+  if (Number.isNaN(date.getTime())) return '';
+  const pad = value => String(value).padStart(2, '0');
+  return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
+}
+
+function validDate(value) {
+  const date = String(value || '');
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(`${date}T00:00:00`))) {
+    throw new DomainError('INVALID_ARGUMENT', 'date must be YYYY-MM-DD.');
+  }
+  return date;
+}
+
+function fingerprint(projectId, date) {
+  return crypto.createHash('sha256').update(`${projectId}\n${date}`, 'utf8').digest('hex');
+}
+
+function encodeCursor(projectId, date, beforeKey) {
+  return Buffer.from(JSON.stringify({ v: 1, fingerprint: fingerprint(projectId, date), beforeKey }), 'utf8').toString('base64url');
+}
+
+function decodeCursor(value, projectId, date) {
+  if (!value) return null;
+  try {
+    const cursor = JSON.parse(Buffer.from(String(value), 'base64url').toString('utf8'));
+    if (!cursor || cursor.v !== 1 || cursor.fingerprint !== fingerprint(projectId, date) || typeof cursor.beforeKey !== 'string') throw new Error('mismatch');
+    return cursor;
+  } catch (error) {
+    throw new DomainError('INVALID_ARGUMENT', 'Conversation cursor is invalid for this project/date query.', { status: 409, cause: error });
+  }
+}
+
+class ConversationQueryService {
+  constructor(options = {}) {
+    this.registryStore = options.registryStore;
+    this.projectStore = options.projectStore;
+    this.conversationStore = options.conversationStore || new ConversationStore(options);
+    this.logger = options.logger || null;
+  }
+
+  async log(level, event, message, context) {
+    try { if (this.logger && typeof this.logger[level] === 'function') await this.logger[level](event, message, context); } catch {
+      // Read-only conversation queries do not depend on observer availability.
+    }
+  }
+
+  listProjects() {
+    return this.registryStore.listIds().map(projectId => {
+      const config = this.projectStore.readConfig(projectId);
+      const state = this.projectStore.readState(projectId);
+      return {
+        projectId,
+        displayName: config.displayName,
+        enabled: config.enabled !== false,
+        captureStatus: state.conversation && state.conversation.captureStatus || 'unavailable',
+      };
+    });
+  }
+
+  annotations(projectId, events = []) {
+    const byTurn = new Map();
+    const closedAtByTurn = new Map(events
+      .filter(event => event.turnId && Number.isInteger(event.developmentTurnClosedAtSequence))
+      .map(event => [event.turnId, event.developmentTurnClosedAtSequence]));
+    for (const snapshot of this.conversationStore.listSnapshots(projectId)) {
+      for (const turn of snapshot.turns) {
+        const closedAt = closedAtByTurn.get(turn.turnId);
+        if (turn.bindingKind === 'shared-spanning'
+          && Number.isInteger(snapshot.boundaryStartCursor)
+          && Number.isInteger(closedAt)
+          && closedAt <= snapshot.boundaryStartCursor) continue;
+        const current = byTurn.get(turn.turnId) || [];
+        current.push({
+          commitSha: snapshot.commitSha,
+          shortSha: snapshot.commitSha.slice(0, 7),
+          status: turn.bindingKind === 'shared-spanning' ? 'associated' : 'committed',
+          bindingKind: turn.bindingKind,
+          snapshotHash: snapshot.snapshotHash,
+        });
+        byTurn.set(turn.turnId, current);
+      }
+    }
+    for (const values of byTurn.values()) values.sort((left, right) => left.commitSha.localeCompare(right.commitSha));
+    return byTurn;
+  }
+
+  async turns(input = {}) {
+    const projectId = validateProjectId(input.projectId);
+    if (!this.registryStore.readDisplaySnapshot(projectId)) throw new DomainError('PROJECT_NOT_FOUND', 'Project was not found.', { status: 404 });
+    const date = validDate(input.date);
+    const limit = Math.max(1, Math.min(Number(input.limit || 50), 200));
+    const cursor = decodeCursor(input.cursor, projectId, date);
+    const groups = new Map();
+    const events = readDevelopmentEvents(this.conversationStore, projectId);
+    for (const event of events) {
+      const turnId = event.turnId || `event:${event.eventId}`;
+      let group = groups.get(turnId);
+      if (!group) {
+        group = { turnId, source: event.source, sessionId: event.sessionId, events: [] };
+        groups.set(turnId, group);
+      }
+      group.events.push(event);
+    }
+    const annotations = this.annotations(projectId, events);
+    let projected = [...groups.values()].map(group => {
+      group.events.sort((left, right) => String(left.capturedAt).localeCompare(String(right.capturedAt)) || left.eventId.localeCompare(right.eventId));
+      const first = group.events[0];
+      const last = group.events.at(-1);
+      const userEvents = group.events.filter(event => event.role === 'user');
+      const assistantEvents = group.events.filter(event => event.role === 'assistant');
+      const eventDate = localDate((userEvents[0] || first).capturedAt);
+      const turnAnnotations = annotations.get(group.turnId) || [];
+      const status = turnAnnotations.some(annotation => annotation.status === 'committed')
+        ? 'committed'
+        : turnAnnotations.length ? 'associated' : 'uncommitted';
+      return {
+        turnId: group.turnId,
+        source: group.source,
+        sessionId: group.sessionId,
+        date: eventDate,
+        startedAt: first.capturedAt,
+        updatedAt: last.capturedAt,
+        userPrompt: userEvents.map(event => event.content).join('\n\n'),
+        assistantReply: assistantEvents.map(event => event.content).join('\n\n'),
+        userEventIds: userEvents.map(event => event.eventId),
+        assistantEventIds: assistantEvents.map(event => event.eventId),
+        annotation: { status, commits: turnAnnotations },
+        renderAs: 'plain-text',
+        key: `${first.capturedAt}|${group.turnId}`,
+      };
+    }).filter(turn => turn.userEventIds.length > 0 && turn.date === date)
+      .sort((left, right) => right.key.localeCompare(left.key));
+    if (cursor) projected = projected.filter(turn => turn.key < cursor.beforeKey);
+    const page = projected.slice(0, limit);
+    const hasMore = projected.length > page.length;
+    const result = {
+      ok: true,
+      projectId,
+      date,
+      turns: page.map(({ key, ...turn }) => turn),
+      nextCursor: hasMore ? encodeCursor(projectId, date, page.at(-1).key) : null,
+      hasMore,
+      limit,
+    };
+    await this.log('debug', 'conversation.turns_queried', 'Development conversation turns were queried.', {
+      projectId,
+      component: 'conversation-query',
+      context: { date, resultCount: result.turns.length, hasMore },
+    });
+    return result;
+  }
+}
+
+module.exports = { ConversationQueryService, localDate, validDate, encodeCursor, decodeCursor };