@codehourra/llm-iwiki 0.1.0

package/src/collectors/util.ts

@@ -0,0 +1,58 @@
+import { tmpdir } from 'node:os'
+
+import type { RawMessage } from './types'
+
+const TITLE_MAX_LENGTH = 120
+
+const EPHEMERAL_PREFIXES = ['/tmp/', '/private/tmp/', '/var/folders/', '/private/var/folders/']
+
+export function isEphemeralPath(path: string | null): boolean {
+  if (!path) return false
+  const temp = tmpdir()
+  if (path === temp) return true
+  const normalized = path.endsWith('/') ? path : `${path}/`
+  const prefixes = [...EPHEMERAL_PREFIXES, `${temp}/`]
+  return prefixes.some((prefix) => normalized.startsWith(prefix))
+}
+
+export function clampTitle(value: string): string {
+  return value.length > TITLE_MAX_LENGTH ? `${value.slice(0, TITLE_MAX_LENGTH)}…` : value
+}
+
+function firstMeaningfulLine(content: string): string | null {
+  for (const line of content.split('\n')) {
+    const trimmed = line.trim()
+    if (trimmed === '' || /^[-=*#>`~]+$/.test(trimmed)) continue
+    return trimmed
+  }
+  return null
+}
+
+export function deriveTitle(messages: RawMessage[]): string | null {
+  for (const message of messages) {
+    if (message.role !== 'user') continue
+    const line = firstMeaningfulLine(message.content)
+    if (line) return clampTitle(line)
+  }
+  return null
+}
+
+export function normalizeContentParts(content: unknown): string {
+  if (typeof content === 'string') return content
+  if (Array.isArray(content)) {
+    return content
+      .map((part) => {
+        if (typeof part === 'string') return part
+        if (part && typeof part === 'object') {
+          const record = part as Record<string, unknown>
+          if (typeof record.text === 'string') return record.text
+          return JSON.stringify(record)
+        }
+        return String(part)
+      })
+      .join('\n')
+      .trim()
+  }
+  if (content == null) return ''
+  return JSON.stringify(content)
+}

package/src/compaction.ts

@@ -0,0 +1,63 @@
+import type { RawMessage } from './collectors'
+
+export interface CompactionOptions {
+  maxPerMessage: number
+  maxMessages: number
+}
+
+const DEFAULTS: CompactionOptions = {
+  maxPerMessage: 1200,
+  maxMessages: 60,
+}
+
+function squashWhitespace(text: string): string {
+  return text.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n').trim()
+}
+
+function truncate(text: string, max: number): string {
+  const cleaned = squashWhitespace(text)
+  if (cleaned.length <= max) return cleaned
+  return `${cleaned.slice(0, max)}\n…[truncated ${cleaned.length - max} chars]`
+}
+
+export interface CompactMessageInput {
+  role: string
+  content: string
+}
+
+/**
+ * Deterministic, dependency-free compaction. Keeps the head and tail of long
+ * conversations and truncates oversized individual messages so the result is a
+ * readable transcript that fits a token budget without calling any model.
+ */
+export function compactTranscript(
+  messages: CompactMessageInput[],
+  options: Partial<CompactionOptions> = {},
+): string {
+  const opts = { ...DEFAULTS, ...options }
+  const meaningful = messages.filter((message) => message.content.trim() !== '')
+
+  let kept = meaningful
+  let omittedNote = ''
+  if (meaningful.length > opts.maxMessages) {
+    const head = Math.ceil(opts.maxMessages * 0.6)
+    const tail = opts.maxMessages - head
+    kept = [...meaningful.slice(0, head), ...meaningful.slice(meaningful.length - tail)]
+    omittedNote = `\n…[omitted ${meaningful.length - opts.maxMessages} middle messages]\n`
+  }
+
+  const lines: string[] = []
+  kept.forEach((message, index) => {
+    if (omittedNote && index === Math.ceil(opts.maxMessages * 0.6)) {
+      lines.push(omittedNote)
+    }
+    const label = message.role === 'user' ? 'User' : message.role === 'assistant' ? 'Assistant' : message.role
+    lines.push(`**${label}:** ${truncate(message.content, opts.maxPerMessage)}`)
+  })
+
+  return lines.join('\n\n')
+}
+
+export function compactRawMessages(messages: RawMessage[], options?: Partial<CompactionOptions>): string {
+  return compactTranscript(messages, options)
+}

package/src/config.ts

@@ -0,0 +1,57 @@
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+
+export interface LlmIwikiConfig {
+  obsidianVault: string | null
+}
+
+const KEY_ALIASES: Record<string, string> = {
+  'obsidian.vault': 'obsidian_vault',
+  obsidian_vault: 'obsidian_vault',
+}
+
+function parseToml(source: string): Record<string, string> {
+  const config: Record<string, string> = {}
+  for (const line of source.split('\n')) {
+    const trimmed = line.trim()
+    if (trimmed === '' || trimmed.startsWith('#') || trimmed.startsWith('[')) continue
+    const match = trimmed.match(/^([\w.]+)\s*=\s*(.*)$/)
+    if (!match) continue
+    const key = match[1]!
+    let value = match[2]!.trim()
+    if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+      value = value.slice(1, -1)
+    }
+    config[key] = value
+  }
+  return config
+}
+
+export function readConfig(configFile: string): LlmIwikiConfig {
+  if (!existsSync(configFile)) return { obsidianVault: null }
+  const raw = parseToml(readFileSync(configFile, 'utf8'))
+  const vault = raw.obsidian_vault
+  return { obsidianVault: vault && vault.trim() !== '' ? vault : null }
+}
+
+export function setConfigValue(configFile: string, key: string, value: string): string {
+  const normalizedKey = KEY_ALIASES[key]
+  if (!normalizedKey) {
+    throw new Error(`Unknown config key: ${key}. Supported: obsidian.vault`)
+  }
+
+  const existing = existsSync(configFile) ? readFileSync(configFile, 'utf8') : ''
+  const lines = existing.split('\n')
+  const line = `${normalizedKey} = "${value}"`
+  let replaced = false
+  const next = lines.map((entry) => {
+    if (entry.trim().startsWith(`${normalizedKey} `) || entry.trim().startsWith(`${normalizedKey}=`)) {
+      replaced = true
+      return line
+    }
+    return entry
+  })
+  if (!replaced) next.push(line)
+
+  writeFileSync(configFile, `${next.filter((entry, index) => !(entry === '' && index === next.length - 1)).join('\n')}\n`)
+  return normalizedKey
+}

package/src/db.ts

@@ -0,0 +1,152 @@
+import { Database } from 'bun:sqlite'
+import { mkdirSync } from 'node:fs'
+import { dirname } from 'node:path'
+
+export type LlmIwikiDatabase = Database
+
+export function openDatabase(databaseFile: string): LlmIwikiDatabase {
+  mkdirSync(dirname(databaseFile), { recursive: true })
+  return new Database(databaseFile)
+}
+
+export function runMigrations(db: LlmIwikiDatabase): void {
+  db.exec(`
+    PRAGMA journal_mode = WAL;
+
+    CREATE TABLE IF NOT EXISTS projects (
+      id TEXT PRIMARY KEY,
+      canonical_name TEXT NOT NULL,
+      display_name TEXT,
+      slug TEXT NOT NULL,
+      canonical_repo_url TEXT,
+      provider TEXT,
+      identity_source TEXT NOT NULL,
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS project_checkouts (
+      id TEXT PRIMARY KEY,
+      project_id TEXT NOT NULL,
+      local_path TEXT NOT NULL,
+      git_root TEXT,
+      remote_url TEXT,
+      canonical_remote_url TEXT,
+      current_branch TEXT,
+      first_seen_at TEXT NOT NULL,
+      last_seen_at TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS project_aliases (
+      id TEXT PRIMARY KEY,
+      project_id TEXT NOT NULL,
+      alias_type TEXT NOT NULL,
+      alias_value TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS sources (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      enabled INTEGER NOT NULL,
+      scan_paths TEXT,
+      config_json TEXT,
+      last_sync_at TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS sessions (
+      id TEXT PRIMARY KEY,
+      source_id TEXT NOT NULL,
+      source_session_id TEXT NOT NULL,
+      project_id TEXT,
+      checkout_id TEXT,
+      raw_project_path TEXT,
+      raw_path TEXT,
+      title TEXT,
+      message_count INTEGER NOT NULL,
+      content_hash TEXT NOT NULL,
+      status TEXT NOT NULL,
+      created_at TEXT,
+      updated_at TEXT,
+      first_seen_at TEXT NOT NULL,
+      last_seen_at TEXT NOT NULL,
+      UNIQUE(source_id, source_session_id, raw_path)
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions (project_id);
+
+    CREATE TABLE IF NOT EXISTS messages (
+      id TEXT PRIMARY KEY,
+      session_id TEXT NOT NULL,
+      role TEXT NOT NULL,
+      content TEXT NOT NULL,
+      timestamp TEXT,
+      tokens_in INTEGER DEFAULT 0,
+      tokens_out INTEGER DEFAULT 0,
+      seq_order INTEGER NOT NULL,
+      content_hash TEXT NOT NULL
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_messages_session ON messages (session_id);
+
+    CREATE TABLE IF NOT EXISTS session_summaries (
+      id TEXT PRIMARY KEY,
+      session_id TEXT NOT NULL,
+      project_id TEXT NOT NULL,
+      title TEXT NOT NULL,
+      value TEXT NOT NULL,
+      summary_markdown TEXT NOT NULL,
+      metadata_json TEXT,
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS experience_candidates (
+      id TEXT PRIMARY KEY,
+      project_id TEXT NOT NULL,
+      proposed_title TEXT NOT NULL,
+      proposed_slug TEXT NOT NULL,
+      proposed_body_markdown TEXT NOT NULL,
+      source_sessions_json TEXT NOT NULL,
+      confidence TEXT,
+      status TEXT NOT NULL,
+      created_at TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS experiences (
+      id TEXT PRIMARY KEY,
+      project_id TEXT,
+      title TEXT NOT NULL,
+      slug TEXT NOT NULL,
+      problem_type TEXT,
+      solution_type TEXT,
+      tech_stack_json TEXT,
+      summary TEXT,
+      body_markdown TEXT NOT NULL,
+      confidence TEXT,
+      status TEXT NOT NULL,
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL,
+      UNIQUE(project_id, slug)
+    );
+
+    CREATE TABLE IF NOT EXISTS session_experience_links (
+      session_id TEXT NOT NULL,
+      experience_id TEXT NOT NULL,
+      relation TEXT NOT NULL,
+      PRIMARY KEY (session_id, experience_id)
+    );
+
+    CREATE TABLE IF NOT EXISTS obsidian_notes (
+      id TEXT PRIMARY KEY,
+      note_type TEXT NOT NULL,
+      entity_id TEXT NOT NULL,
+      file_path TEXT NOT NULL,
+      managed_hash TEXT,
+      frontmatter_hash TEXT,
+      last_exported_at TEXT,
+      last_seen_mtime TEXT,
+      conflict_status TEXT NOT NULL,
+      UNIQUE(note_type, entity_id)
+    );
+  `)
+}

package/src/experiences.ts

@@ -0,0 +1,235 @@
+import type { ParsedExperiencesYaml } from './types'
+import type { LlmIwikiDatabase } from './db'
+import { slugifyProjectName } from './projects'
+
+export type ExperienceScope = 'changed-summaries' | 'all-recent'
+
+export interface PrepareExperiencesResult {
+  markdown: string
+  summaryCount: number
+}
+
+interface SummaryRow {
+  session_id: string
+  title: string
+  value: string
+  summary_markdown: string
+}
+
+const EXPERIENCES_FORMAT = `## 输出格式
+
+基于下面这些会话摘要，提炼出可复用的项目经验，生成 \`experiences.yaml\` 写入
+\`.llm-iwiki/tasks/experiences.yaml\`，再运行 \`llm-iwiki experiences propose\`。
+
+\`\`\`yaml
+project_id: <见下方 project_id>
+experiences:
+  - title: <经验标题>
+    slug: <可选，稳定短标识>
+    summary: |
+      一句话说明这条经验。
+    body_markdown: |
+      ## 背景
+      ## 方案
+      ## 结论
+    source_sessions:
+      - <相关 session_id>
+    confidence: low | medium | high
+\`\`\`
+
+要求：
+- 一条经验可以聚合多个会话，按主题归纳，不要逐会话复述。
+- \`source_sessions\` 填写该经验来自哪些 session_id。`
+
+function fetchSummaries(db: LlmIwikiDatabase, projectId: string, scope: ExperienceScope): SummaryRow[] {
+  const valueClause = scope === 'changed-summaries' ? "AND value IN ('medium', 'high')" : ''
+  return db
+    .query<SummaryRow, [string]>(`
+      SELECT session_id, title, value, summary_markdown
+      FROM session_summaries
+      WHERE project_id = ? ${valueClause}
+      ORDER BY updated_at DESC
+    `)
+    .all(projectId)
+}
+
+export function prepareExperiencesTask(
+  db: LlmIwikiDatabase,
+  projectId: string,
+  scope: ExperienceScope,
+): PrepareExperiencesResult {
+  const summaries = fetchSummaries(db, projectId, scope)
+
+  const blocks = summaries.map((summary) =>
+    [`### ${summary.session_id} — ${summary.title}`, `- value: ${summary.value}`, '', summary.summary_markdown].join('\n'),
+  )
+
+  const markdown = [
+    `# Experiences Task`,
+    '',
+    `project_id: ${projectId}`,
+    `scope: ${scope}`,
+    `summaries: ${blocks.length}`,
+    '',
+    EXPERIENCES_FORMAT,
+    '',
+    '---',
+    '',
+    blocks.join('\n\n---\n\n'),
+    '',
+  ].join('\n')
+
+  return { markdown, summaryCount: blocks.length }
+}
+
+export interface ProposeExperiencesResult {
+  written: number
+}
+
+function hash(value: string): string {
+  return Bun.hash(value).toString(16)
+}
+
+export function proposeExperiences(db: LlmIwikiDatabase, parsed: ParsedExperiencesYaml): ProposeExperiencesResult {
+  const now = new Date().toISOString()
+  let written = 0
+
+  const propose = db.transaction(() => {
+    for (const experience of parsed.experiences) {
+      const slug = experience.slug && experience.slug.trim() !== '' ? experience.slug : slugifyProjectName(experience.title)
+      const id = `cand_${hash(`${parsed.projectId}\u0000${slug}`)}`
+
+      db.query(`
+        INSERT INTO experience_candidates (
+          id, project_id, proposed_title, proposed_slug, proposed_body_markdown, source_sessions_json, confidence, status, created_at
+        ) VALUES ($id, $projectId, $title, $slug, $body, $sources, $confidence, 'proposed', $now)
+        ON CONFLICT(id) DO UPDATE SET
+          proposed_title = excluded.proposed_title,
+          proposed_body_markdown = excluded.proposed_body_markdown,
+          source_sessions_json = excluded.source_sessions_json,
+          confidence = excluded.confidence
+      `).run({
+        $id: id,
+        $projectId: parsed.projectId,
+        $title: experience.title,
+        $slug: slug,
+        $body: experience.bodyMarkdown,
+        $sources: JSON.stringify(experience.sourceSessions),
+        $confidence: experience.confidence,
+        $now: now,
+      })
+      written += 1
+    }
+  })
+  propose()
+
+  return { written }
+}
+
+export interface CandidateRow {
+  id: string
+  project_id: string
+  proposed_title: string
+  proposed_slug: string
+  confidence: string | null
+  status: string
+  source_sessions_json: string
+  created_at: string
+}
+
+export function listCandidates(db: LlmIwikiDatabase, projectId: string | null): CandidateRow[] {
+  if (projectId) {
+    return db
+      .query<CandidateRow, [string]>(
+        'SELECT id, project_id, proposed_title, proposed_slug, confidence, status, source_sessions_json, created_at FROM experience_candidates WHERE project_id = ? ORDER BY created_at DESC',
+      )
+      .all(projectId)
+  }
+  return db
+    .query<CandidateRow, []>(
+      'SELECT id, project_id, proposed_title, proposed_slug, confidence, status, source_sessions_json, created_at FROM experience_candidates ORDER BY created_at DESC',
+    )
+    .all()
+}
+
+export interface AcceptExperienceResult {
+  experienceId: string
+  slug: string
+  linkedSessions: number
+}
+
+export function acceptExperience(db: LlmIwikiDatabase, candidateId: string): AcceptExperienceResult {
+  const candidate = db
+    .query<
+      {
+        project_id: string
+        proposed_title: string
+        proposed_slug: string
+        proposed_body_markdown: string
+        confidence: string | null
+        source_sessions_json: string
+      },
+      [string]
+    >(
+      'SELECT project_id, proposed_title, proposed_slug, proposed_body_markdown, confidence, source_sessions_json FROM experience_candidates WHERE id = ?',
+    )
+    .get(candidateId)
+
+  if (!candidate) throw new Error(`Experience candidate not found: ${candidateId}`)
+
+  let sourceSessions: string[] = []
+  try {
+    const parsed = JSON.parse(candidate.source_sessions_json) as unknown
+    if (Array.isArray(parsed)) sourceSessions = parsed.filter((value): value is string => typeof value === 'string')
+  } catch {
+    sourceSessions = []
+  }
+
+  const now = new Date().toISOString()
+  const experienceId = `exp_${hash(`${candidate.project_id}\u0000${candidate.proposed_slug}`)}`
+
+  const accept = db.transaction(() => {
+    db.query(`
+      INSERT INTO experiences (
+        id, project_id, title, slug, body_markdown, confidence, status, created_at, updated_at
+      ) VALUES ($id, $projectId, $title, $slug, $body, $confidence, 'accepted', $now, $now)
+      ON CONFLICT(project_id, slug) DO UPDATE SET
+        title = excluded.title,
+        body_markdown = excluded.body_markdown,
+        confidence = excluded.confidence,
+        status = 'accepted',
+        updated_at = excluded.updated_at
+    `).run({
+      $id: experienceId,
+      $projectId: candidate.project_id,
+      $title: candidate.proposed_title,
+      $slug: candidate.proposed_slug,
+      $body: candidate.proposed_body_markdown,
+      $confidence: candidate.confidence,
+      $now: now,
+    })
+
+    const resolved = db
+      .query<{ id: string }, [string, string]>('SELECT id FROM experiences WHERE project_id = ? AND slug = ?')
+      .get(candidate.project_id, candidate.proposed_slug)
+    const finalId = resolved?.id ?? experienceId
+
+    for (const sessionId of sourceSessions) {
+      db.query(`
+        INSERT INTO session_experience_links (session_id, experience_id, relation)
+        VALUES (?, ?, 'source')
+        ON CONFLICT(session_id, experience_id) DO NOTHING
+      `).run(sessionId, finalId)
+    }
+
+    db.query("UPDATE experience_candidates SET status = 'accepted' WHERE id = ?").run(candidateId)
+  })
+  accept()
+
+  return { experienceId, slug: candidate.proposed_slug, linkedSessions: sourceSessions.length }
+}
+
+export function rejectExperience(db: LlmIwikiDatabase, candidateId: string): void {
+  const changes = db.query("UPDATE experience_candidates SET status = 'rejected' WHERE id = ?").run(candidateId)
+  if (changes.changes === 0) throw new Error(`Experience candidate not found: ${candidateId}`)
+}

package/src/index.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env bun
+import { runCli } from './cli'
+
+const exitCode = await runCli(process.argv.slice(2), {
+  cwd: process.cwd(),
+  stdout: console.log,
+  stderr: console.error,
+})
+
+process.exit(exitCode)