@sanqianx/project-knowledge 4.6.1 → 4.8.0

package/_modules/knowledge-engine/src/runtime/contracts.js

@@ -0,0 +1,279 @@
+const crypto = require('crypto');
+
+const SCHEMAS = Object.freeze({
+  settings: 'settings/v2',
+  projectRegistry: 'project-registry/v2',
+  projectConfig: 'project-config/v2',
+  projectState: 'project-state/v2',
+  requirement: 'requirement/v1',
+  aiCodingEvent: 'ai-coding-event/v1',
+  gitCommitBoundary: 'git-commit-boundary/v1',
+  commitConversationSnapshot: 'commit-conversation-snapshot/v1',
+  commitClaim: 'commit-claim/v1',
+  promotionJournal: 'promotion-journal/v1',
+  migrationJournal: 'migration-journal/v1',
+  log: 'log/v2',
+  hookEvent: 'hook-event/v2',
+});
+
+const SCHEMA_VERSIONS = Object.freeze({
+  settings: 2,
+  projectRegistry: 2,
+  projectConfig: 2,
+  projectState: 2,
+  hookManaged: 2,
+  layoutMigration: 'layout-v2',
+});
+
+const TRIGGERS = Object.freeze(['git-hook']);
+const TRIGGER_SET = new Set(TRIGGERS);
+const LOG_LEVELS = Object.freeze(['trace', 'debug', 'info', 'warn', 'error', 'fatal']);
+const LOG_LEVEL_SET = new Set(LOG_LEVELS);
+const ANALYSIS_PHASES = Object.freeze([
+  'idle',
+  'scanning',
+  'claim.created',
+  'requirement.bound',
+  'evidence.prepared',
+  'ai.running',
+  'output.validated',
+  'promotion.prepared',
+  'markdown.promoted',
+  'state.advanced',
+  'index.queued',
+  'index.applied',
+  'history-diverged',
+  'failed',
+]);
+
+const ERROR_CODES = Object.freeze([
+  'INVALID_ARGUMENT',
+  'INVALID_TRIGGER',
+  'SCHEMA_UNSUPPORTED',
+  'DATA_CORRUPT',
+  'IMMUTABLE_FIELD',
+  'PROJECT_BUSY',
+  'PROJECT_NOT_FOUND',
+  'PROJECT_AMBIGUOUS',
+  'HOOK_CONFLICT',
+  'HOOK_INVALID',
+  'HISTORY_DIVERGED',
+  'MIGRATION_SOURCE_CHANGED',
+  'MIGRATION_TARGET_CONFLICT',
+  'MIGRATION_FAILED',
+  'PATH_OUTSIDE_ROOT',
+  'AUTH_REQUIRED',
+  'ORIGIN_FORBIDDEN',
+  'LOGGER_UNAVAILABLE',
+  'LOG_CURSOR_EXPIRED',
+  'EVIDENCE_INTEGRITY_FAILED',
+  'AI_PROFILE_REQUIRED',
+]);
+
+const IMMUTABLE_PROJECT_FIELDS = Object.freeze([
+  'schema',
+  'schemaVersion',
+  'projectId',
+  'storageName',
+  'createdAt',
+  'knowledgePath',
+]);
+const MUTABLE_PROJECT_FIELDS = Object.freeze([
+  'displayName',
+  'enabled',
+  'aiProfileId',
+  'knowledgeLanguage',
+  'relatedProjectIds',
+]);
+
+const REDACTED_VALUE = '[REDACTED]';
+const REDACTION_KEY_PATTERN = /(?:api[-_]?key|auth(?:orization|token)?|access[-_]?token|refresh[-_]?token|bearer|password|passwd|secret|client[-_]?secret|private[-_]?key|cookie|set-cookie)/i;
+const FIELD_LIMITS = Object.freeze({
+  message: 4096,
+  stack: 16384,
+  path: 4096,
+  contextString: 8192,
+  requirement: 131072,
+});
+
+class DomainError extends Error {
+  constructor(code, message, options = {}) {
+    super(String(message || code || 'Domain error'), options.cause ? { cause: options.cause } : undefined);
+    this.name = 'DomainError';
+    this.code = ERROR_CODES.includes(code) ? code : 'INVALID_ARGUMENT';
+    this.status = Number.isInteger(options.status) ? options.status : 400;
+    this.retryable = options.retryable === true;
+    this.operationId = options.operationId || '';
+    this.details = sanitizePublicDetails(options.details);
+  }
+}
+
+function createId(prefix) {
+  const base = typeof crypto.randomUUID === 'function'
+    ? crypto.randomUUID().replace(/-/g, '')
+    : crypto.randomBytes(16).toString('hex');
+  return `${prefix}-${base}`;
+}
+
+function assertSchema(actual, expected, category = 'data') {
+  if (actual !== expected) {
+    throw new DomainError('SCHEMA_UNSUPPORTED', `Unsupported ${category} schema.`, {
+      status: 409,
+      details: { category, expected, actual: String(actual || '') },
+    });
+  }
+  return true;
+}
+
+function validateTrigger(trigger) {
+  if (!TRIGGER_SET.has(trigger)) {
+    throw new DomainError('INVALID_TRIGGER', 'Trigger must be git-hook.', {
+      details: { trigger: String(trigger || '') },
+    });
+  }
+  return trigger;
+}
+
+function validateLogLevel(level) {
+  if (!LOG_LEVEL_SET.has(level)) {
+    throw new DomainError('INVALID_ARGUMENT', 'Invalid log level.', {
+      details: { level: String(level || '') },
+    });
+  }
+  return level;
+}
+
+function validateProjectId(projectId) {
+  const value = String(projectId || '');
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]{2,127}$/.test(value)) {
+    throw new DomainError('INVALID_ARGUMENT', 'Invalid projectId.');
+  }
+  return value;
+}
+
+function validateStateTransition(from, to) {
+  if (!ANALYSIS_PHASES.includes(from) || !ANALYSIS_PHASES.includes(to)) {
+    throw new DomainError('INVALID_ARGUMENT', 'Unknown analysis phase.', { details: { from, to } });
+  }
+  if (from === to) return true;
+  const allowed = {
+    idle: ['scanning', 'index.queued', 'failed'],
+    scanning: ['claim.created', 'idle', 'history-diverged', 'failed'],
+    'claim.created': ['requirement.bound', 'failed'],
+    'requirement.bound': ['evidence.prepared', 'failed'],
+    'evidence.prepared': ['ai.running', 'failed'],
+    'ai.running': ['output.validated', 'failed'],
+    'output.validated': ['promotion.prepared', 'failed'],
+    'promotion.prepared': ['markdown.promoted', 'failed'],
+    'markdown.promoted': ['state.advanced', 'failed'],
+    'state.advanced': ['index.queued', 'scanning', 'idle', 'failed'],
+    'index.queued': ['index.applied', 'failed'],
+    'index.applied': ['idle', 'scanning'],
+    'history-diverged': ['scanning', 'idle'],
+    failed: ['scanning', 'ai.running', 'promotion.prepared', 'index.queued', 'idle'],
+  };
+  if (!(allowed[from] || []).includes(to)) {
+    throw new DomainError('INVALID_ARGUMENT', `Invalid analysis transition: ${from} -> ${to}.`, {
+      details: { from, to },
+    });
+  }
+  return true;
+}
+
+function sanitizePublicDetails(value, seen = new WeakSet()) {
+  if (value == null || typeof value === 'boolean' || typeof value === 'number') return value;
+  if (typeof value === 'string') return value.slice(0, FIELD_LIMITS.contextString);
+  if (value instanceof Error) return { name: value.name, code: value.code || '', message: String(value.message || '').slice(0, FIELD_LIMITS.message) };
+  if (typeof value !== 'object') return String(value).slice(0, FIELD_LIMITS.contextString);
+  if (seen.has(value)) return '[Circular]';
+  seen.add(value);
+  if (Array.isArray(value)) return value.slice(0, 100).map(item => sanitizePublicDetails(item, seen));
+  const out = {};
+  for (const [key, child] of Object.entries(value)) {
+    if (REDACTION_KEY_PATTERN.test(key) || /stack/i.test(key) || /prompt|diff|header/i.test(key)) {
+      continue;
+    } else {
+      out[key] = sanitizePublicDetails(child, seen);
+    }
+  }
+  return out;
+}
+
+function serializeErrorEnvelope(error, operationId = '') {
+  const source = error instanceof DomainError
+    ? error
+    : new DomainError('INVALID_ARGUMENT', 'The operation failed.', { status: 500, operationId });
+  return {
+    ok: false,
+    error: {
+      code: source.code,
+      message: source.message,
+      operationId: source.operationId || operationId || '',
+      retryable: source.retryable === true,
+      details: sanitizePublicDetails(source.details || {}),
+    },
+  };
+}
+
+function publicAiProfileView(profile) {
+  const source = profile && typeof profile === 'object' ? profile : {};
+  const secret = source.apiKey || source.authToken || source.anthropicAuthToken || '';
+  const publicView = {};
+  for (const [key, value] of Object.entries(source)) {
+    if (REDACTION_KEY_PATTERN.test(key)) continue;
+    publicView[key] = value;
+  }
+  publicView.hasApiKey = Boolean(secret);
+  publicView.apiKeyMasked = secret ? `****${String(secret).slice(-4)}` : '';
+  return publicView;
+}
+
+function publicAiProfilesConfig(config) {
+  const source = config && typeof config === 'object' ? config : {};
+  return {
+    ...source,
+    profiles: Array.isArray(source.profiles) ? source.profiles.map(publicAiProfileView) : [],
+  };
+}
+
+function assertMutableProjectPatch(patch, options = {}) {
+  const allowKnowledgePath = options.allowKnowledgePath === true;
+  const allowRepoPath = options.allowRepoPath === true;
+  for (const key of Object.keys(patch && typeof patch === 'object' ? patch : {})) {
+    if (IMMUTABLE_PROJECT_FIELDS.includes(key) && !(allowKnowledgePath && key === 'knowledgePath')) {
+      throw new DomainError('IMMUTABLE_FIELD', `Project field is immutable: ${key}.`, { status: 409, details: { field: key } });
+    }
+    if (!MUTABLE_PROJECT_FIELDS.includes(key)
+      && !(allowKnowledgePath && key === 'knowledgePath')
+      && !(allowRepoPath && key === 'repoPath')) {
+      throw new DomainError('INVALID_ARGUMENT', `Unknown project field: ${key}.`, { details: { field: key } });
+    }
+  }
+  return true;
+}
+
+module.exports = {
+  SCHEMAS,
+  SCHEMA_VERSIONS,
+  TRIGGERS,
+  LOG_LEVELS,
+  ANALYSIS_PHASES,
+  ERROR_CODES,
+  IMMUTABLE_PROJECT_FIELDS,
+  MUTABLE_PROJECT_FIELDS,
+  REDACTED_VALUE,
+  REDACTION_KEY_PATTERN,
+  FIELD_LIMITS,
+  DomainError,
+  createId,
+  assertSchema,
+  validateTrigger,
+  validateLogLevel,
+  validateProjectId,
+  validateStateTransition,
+  sanitizePublicDetails,
+  serializeErrorEnvelope,
+  publicAiProfileView,
+  publicAiProfilesConfig,
+  assertMutableProjectPatch,
+};

package/_modules/knowledge-engine/src/runtime/conversation-exclusions.js

@@ -0,0 +1,204 @@
+const fs = require('fs');
+const path = require('path');
+const { validateProjectId } = require('./contracts');
+const AtomicFile = require('./atomic-file');
+
+const EXCLUSION_SCHEMA = 'conversation-exclusions/v1';
+const EMBEDDED_ASSISTANT_PREFIX = 'embedded-assistant-';
+const EXPLICIT_PREFIX = 'explicit-';
+const CODEX_CONTEXT_TAGS = new Set([
+  'recommended_plugins',
+  'environment_context',
+  'in-app-browser-context',
+]);
+
+/**
+ * T17: legacy embedded Workbench capture exclusion.
+ *
+ * Old builds wrote internal Workbench conversations into Development
+ * Conversation as `explicit-<requirementId>` + `embedded-assistant-<requirementId>`
+ * pairs (rawEventType embedded-claude-result). Those events are excluded from
+ * Development Conversation presentation and from NEW commit binding. Nothing
+ * is deleted: the JSONL lines stay, immutable snapshots stay untouched, and a
+ * manifest is persisted for auditability. A genuine explicit MCP requirement
+ * (no embedded-assistant pair) is never excluded.
+ */
+function isEmbeddedAssistantEvent(event) {
+  if (!event) return false;
+  return String(event.eventId || '').startsWith(EMBEDDED_ASSISTANT_PREFIX)
+    || event.rawEventType === 'embedded-claude-result';
+}
+
+/**
+ * Codex rollout files contain generated context records with role=user. They
+ * are transport/control data, not user-authored prompts. Match only complete,
+ * known envelopes so ordinary prompts containing similar text survive.
+ */
+function isCodexControlUserEvent(event) {
+  if (!event || event.source !== 'codex' || event.role !== 'user' || event.rawEventType !== 'response_item') return false;
+  let remaining = String(event.content || '').trim();
+  if (/^<turn_aborted>\s*[\s\S]*?\s*<\/turn_aborted>$/.test(remaining)) return true;
+
+  let removed = false;
+  const envelope = /^<([A-Za-z0-9_-]+)(?:\s[^>]*)?>[\s\S]*?<\/\1>\s*/;
+  while (remaining) {
+    const match = remaining.match(envelope);
+    if (!match || !CODEX_CONTEXT_TAGS.has(match[1])) break;
+    remaining = remaining.slice(match[0].length).trimStart();
+    removed = true;
+  }
+  return removed && remaining.trim() === '';
+}
+
+function isCodexAbortEvent(event) {
+  return isCodexControlUserEvent(event)
+    && /^<turn_aborted>/.test(String(event.content || '').trim());
+}
+
+function computeConversationExclusions(projectId, events) {
+  validateProjectId(projectId);
+  const paired = new Set();
+  for (const event of events || []) {
+    const eventId = String((event && event.eventId) || '');
+    if (eventId.startsWith(EMBEDDED_ASSISTANT_PREFIX)) {
+      paired.add(eventId.slice(EMBEDDED_ASSISTANT_PREFIX.length));
+    }
+  }
+  const excludedEventIds = [];
+  for (const event of events || []) {
+    if (!event) continue;
+    const eventId = String(event.eventId || '');
+    if (isEmbeddedAssistantEvent(event)) {
+      excludedEventIds.push(eventId);
+      continue;
+    }
+    if (isCodexControlUserEvent(event)) {
+      excludedEventIds.push(eventId);
+      continue;
+    }
+    if (eventId.startsWith(EXPLICIT_PREFIX) && paired.has(eventId.slice(EXPLICIT_PREFIX.length))) {
+      excludedEventIds.push(eventId);
+    }
+  }
+  return {
+    schema: EXCLUSION_SCHEMA,
+    projectId,
+    excludedEventIds: [...new Set(excludedEventIds)].sort(),
+    generatedAt: new Date().toISOString(),
+  };
+}
+
+function eventOrder(left, right) {
+  if (Number.isInteger(left.sequence) && Number.isInteger(right.sequence)) return left.sequence - right.sequence;
+  if (Number.isInteger(left.sequence)) return -1;
+  if (Number.isInteger(right.sequence)) return 1;
+  return String(left.capturedAt || '').localeCompare(String(right.capturedAt || ''))
+    || String(left.eventId || '').localeCompare(String(right.eventId || ''));
+}
+
+/**
+ * Bridge 0.1.1 did not read Codex's nested turn id, so historical assistant
+ * response_item records have turnId=null while each user record received a
+ * generated id. Repair that projection without rewriting the append-only
+ * JSONL: orphan assistants attach to the latest real user in the same Codex
+ * session. A later real prompt or turn_aborted record closes that projected
+ * turn, which also lets commit binding ignore stale Bridge open-turn facts.
+ */
+function projectCodexTurns(events, excludedEventIds) {
+  const excluded = new Set(excludedEventIds || []);
+  const activeBySession = new Map();
+  const closedAtByTurn = new Map();
+  const projected = [];
+  const ordered = [...(events || [])].sort(eventOrder);
+  const sessionKey = event => event && event.source === 'codex' && event.sessionId
+    ? `codex\n${event.sessionId}`
+    : null;
+
+  for (const original of ordered) {
+    const eventId = String((original && original.eventId) || '');
+    const key = sessionKey(original || {});
+    if (excluded.has(eventId)) {
+      if (key && isCodexAbortEvent(original)) {
+        const active = activeBySession.get(key);
+        if (active && Number.isInteger(original.sequence)) closedAtByTurn.set(active, original.sequence);
+        activeBySession.delete(key);
+      }
+      continue;
+    }
+
+    let event = original;
+    if (key && original && original.source === 'codex' && original.role === 'user' && original.turnId) {
+      const previous = activeBySession.get(key);
+      if (previous && previous !== original.turnId && Number.isInteger(original.sequence)) {
+        closedAtByTurn.set(previous, original.sequence);
+      }
+      activeBySession.set(key, original.turnId);
+    } else if (key && original && original.source === 'codex' && original.role === 'assistant' && !original.turnId) {
+      const active = activeBySession.get(key);
+      if (active) event = { ...original, turnId: active };
+    }
+    projected.push(event);
+  }
+
+  return projected.map(event => {
+    const closedAt = event && event.turnId ? closedAtByTurn.get(event.turnId) : null;
+    return Number.isInteger(closedAt)
+      ? { ...event, developmentTurnClosedAtSequence: closedAt }
+      : event;
+  });
+}
+
+function manifestPath(layout, projectId) {
+  return path.join(layout.getProjectMetadataDir(projectId), 'conversation-exclusions.json');
+}
+
+function readManifest(layout, projectId) {
+  try {
+    return AtomicFile.readJsonStrict(manifestPath(layout, projectId), {
+      category: 'conversation-exclusions',
+      validate: value => value,
+    });
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Single source of truth for Development Conversation reads: applies durable
+ * exclusions and the non-destructive Codex turn compatibility projection for
+ * both Conversation Explorer and new CommitConversationBinder runs. The
+ * manifest is refreshed idempotently (best effort) so operators can audit
+ * what is excluded; read failures never break the read itself.
+ */
+function readDevelopmentEvents(conversationStore, projectId) {
+  const events = conversationStore.readEvents(projectId);
+  const manifest = computeConversationExclusions(projectId, events);
+  if (manifest.excludedEventIds.length) {
+    try {
+      const layout = conversationStore.layout;
+      const dir = layout.getProjectMetadataDir(projectId);
+      fs.mkdirSync(dir, { recursive: true });
+      const previous = readManifest(layout, projectId);
+      const unchanged = previous
+        && Array.isArray(previous.excludedEventIds)
+        && previous.excludedEventIds.length === manifest.excludedEventIds.length
+        && previous.excludedEventIds.every((id, index) => id === manifest.excludedEventIds[index]);
+      if (!unchanged) {
+        (conversationStore.atomic || AtomicFile).writeJsonAtomic(manifestPath(layout, projectId), manifest);
+      }
+    } catch {
+      // Manifest persistence is best effort; exclusion itself is computed
+      // deterministically from the durable events on every read.
+    }
+  }
+  return projectCodexTurns(events, manifest.excludedEventIds);
+}
+
+module.exports = {
+  EXCLUSION_SCHEMA,
+  isEmbeddedAssistantEvent,
+  isCodexControlUserEvent,
+  computeConversationExclusions,
+  projectCodexTurns,
+  readDevelopmentEvents,
+};