mixdog 0.9.2 → 0.9.4

package/src/runtime/memory/lib/memory-embed.mjs

@@ -213,6 +213,155 @@ export async function flushEmbeddingDirty(db, options = {}) {
   }
 }
 
+// Embeds raw transcript rows (is_root = 0, non-chunk-member) so the dense
+// recall leg can rank them alongside root/summary entries. Mirrors
+// flushEmbeddingDirty's SKIP LOCKED batch-claim structure but sources the
+// embed text from `content` (raw rows have no element/summary) and honors
+// `limit` as a total cap across the whole call rather than an unbounded loop.
+//
+// Stale-embedding safety when recap toggles back ON: cycle1's promotion
+// UPDATE (memory-cycle1.mjs chunk commit) SETs summary/element, which fires
+// trg_entries_embedding_invalidate (memory.mjs — BEFORE UPDATE OF content,
+// summary, element; NEW.is_root=1 AND summary IS DISTINCT FROM OLD.summary)
+// and NULLs the raw-content embedding written here; cycle1's end-of-cycle
+// flushEmbeddingDirty (is_root=1 AND embedding IS NULL) then re-embeds the
+// root from its fresh element/summary. Member leaves keep their content
+// embedding — the dense recall leg resolves members to their chunk root
+// (memory-recall-store.mjs member→root resolution), so that is leaf-level
+// dense matching, not staleness.
+export async function flushRawEmbeddings(db, options = {}) {
+  const { limit = 200, signal } = options ?? {}
+  throwIfAborted(signal)
+  await ensureEmbCacheTable(db)
+  throwIfAborted(signal)
+
+  let attempted = 0
+  let embedded = 0
+  let cursor = 0
+
+  while (attempted < limit) {
+    throwIfAborted(signal)
+    const batchCap = Math.min(BATCH_SIZE, limit - attempted)
+    const client = await db._pool.connect()
+    let ids
+    try {
+      throwIfAborted(signal)
+      await client.query('BEGIN')
+      const res = await client.query(
+        `SELECT id FROM memory.entries
+         WHERE is_root = 0 AND chunk_root IS NULL AND embedding IS NULL
+           AND NULLIF(btrim(session_id), '') IS NOT NULL
+           AND content IS NOT NULL
+           AND id > $2
+         ORDER BY id
+         LIMIT $1
+         FOR UPDATE SKIP LOCKED`,
+        [batchCap, cursor],
+      )
+      ids = res.rows.map(r => Number(r.id))
+      throwIfAborted(signal)
+    } catch (err) {
+      try { await client.query('ROLLBACK') } catch {}
+      client.release()
+      if (signal?.aborted) throw signal.reason ?? err
+      __mixdogMemoryLog(`[embed] raw flush SKIP LOCKED claim failed: ${err.message}\n`)
+      break
+    }
+
+    if (ids.length === 0) {
+      try { await client.query('COMMIT') } catch {}
+      client.release()
+      break
+    }
+
+    cursor = ids[ids.length - 1]
+    attempted += ids.length
+    let batchDone = false
+    try {
+      throwIfAborted(signal)
+      const writtenIds = await syncRawBatchEmbeddings(client, ids, { signal })
+      embedded += writtenIds.length
+      batchDone = true
+    } catch (err) {
+      if (signal?.aborted) throw signal.reason ?? err
+      __mixdogMemoryLog(`[embed] raw batch failed (ids=${ids[0]}..${ids[ids.length-1]}): ${err.message}\n`)
+    } finally {
+      try {
+        if (batchDone) await client.query('COMMIT')
+        else await client.query('ROLLBACK')
+      } catch {}
+      client.release()
+    }
+
+    if (ids.length < batchCap) break
+  }
+
+  return { attempted, embedded }
+}
+
+// Batch-embed raw transcript rows: content is the embed source text (raw
+// rows have no element/summary). Mirrors syncBatchEmbeddings' shape.
+async function syncRawBatchEmbeddings(db, ids, options = {}) {
+  const signal = options?.signal
+  throwIfAborted(signal)
+  const rows = (await db.query(
+    `SELECT id, content FROM memory.entries
+     WHERE id = ANY($1::bigint[]) AND is_root = 0 AND chunk_root IS NULL`,
+    [ids],
+  )).rows
+  throwIfAborted(signal)
+  if (rows.length === 0) return []
+
+  const dimsRow = (await db.query(`SELECT value FROM memory.meta WHERE key = 'embedding.current_dims'`)).rows[0]
+  throwIfAborted(signal)
+  const expected = Number(dimsRow?.value ?? 0)
+
+  const texts = rows.map(row => (typeof row.content === 'string' ? row.content.trim() : ''))
+  const vectors = await cachedEmbedTextBatch(db, texts.filter(Boolean), { signal })
+  throwIfAborted(signal)
+  let vecIdx = 0
+  const updates = []  // { id, content, vector }
+  for (let i = 0; i < rows.length; i++) {
+    throwIfAborted(signal)
+    const row = rows[i]
+    if (!texts[i]) continue
+    const vector = vectors[vecIdx++]
+    if (!Array.isArray(vector) || vector.length === 0) continue
+    if (Number.isFinite(expected) && expected > 0 && vector.length !== expected) {
+      __mixdogMemoryLog(`[embed] raw dim mismatch (id=${row.id} got=${vector.length} expected=${expected})\n`)
+      continue
+    }
+    updates.push({ id: Number(row.id), content: row.content, vector })
+  }
+
+  if (updates.length === 0) return []
+
+  const valClauses = updates.map((u, i) => {
+    const base = i * 3
+    return `($${base+1}::bigint, $${base+2}::halfvec, $${base+3}::text)`
+  })
+  const params = []
+  for (const u of updates) {
+    params.push(u.id, embeddingToSql(u.vector), u.content)
+  }
+  const res = await db.query(
+    `UPDATE memory.entries AS e
+     SET embedding = t.vector
+     FROM (VALUES ${valClauses.join(',')}) AS t(id, vector, content)
+     WHERE e.id = t.id AND e.is_root = 0 AND e.chunk_root IS NULL
+       AND e.embedding IS NULL
+       AND e.content IS NOT DISTINCT FROM t.content
+     RETURNING e.id`,
+    params,
+  )
+  throwIfAborted(signal)
+  const writtenIds = (res.rows || []).map(r => Number(r.id))
+  if (writtenIds.length < updates.length) {
+    __mixdogMemoryLog(`[embed-sync] ${updates.length - writtenIds.length} raw stale-write(s) skipped — content changed since read\n`)
+  }
+  return writtenIds
+}
+
 // Batch-embed all ids in one DB round-trip for fetching, one API call per unique text,
 // then one VALUES UPDATE for all results.
 // Returns the list of ids that were actually UPDATEd with a fresh embedding.

package/src/runtime/memory/lib/memory-process-lock.mjs

@@ -0,0 +1,162 @@
+// Extracted from index.mjs — single/multi-instance process-lock + owner-election
+// helpers. Pure functions parameterized on lock-file paths so index.mjs retains
+// ownership of DATA_DIR-derived constants and the daemon logger.
+import fs from 'node:fs'
+import { execFileSync } from 'node:child_process'
+
+export function parsePositivePid(value) {
+  const pid = Number(value)
+  return Number.isFinite(pid) && pid > 0 ? pid : null
+}
+
+export function isPidAliveLocal(pid) {
+  if (!Number.isFinite(pid) || pid <= 0) return false
+  try { process.kill(pid, 0); return true }
+  catch (e) { return e.code !== 'ESRCH' }
+}
+
+export function tryAcquireMemoryOwnerLock(ownerLockFile, log = () => {}) {
+  // Returns true on success (this process now owns memory worker for the data
+  // dir), false when a live peer holds the lock. Stale locks (dead PID) are
+  // unlinked and retried atomically. Throws on unexpected fs errors so callers
+  // surface lock-system corruption rather than silently downgrading.
+  //
+  // EPERM/EBUSY/EACCES at openSync are transient — AV scanners (SignKorea /
+  // SKCert / ezPDFWS etc) briefly lock newly-created files during inspection.
+  // The 0.1.x baseline threw immediately and the worker promoted to
+  // permanentlyDegraded, killing memory tools for the rest of the session.
+  // Treat the AV error codes as retryable with bounded backoff (~750ms total)
+  // before giving up and rethrowing.
+  for (let attempt = 0; attempt < 5; attempt++) {
+    try {
+      const fd = fs.openSync(ownerLockFile, 'wx')
+      fs.writeSync(fd, String(process.pid))
+      fs.closeSync(fd)
+      return true
+    } catch (e) {
+      if (e.code === 'EEXIST') {
+        let ownerPid = NaN
+        try { ownerPid = Number(fs.readFileSync(ownerLockFile, 'utf8').trim()) } catch {}
+        if (isPidAliveLocal(ownerPid)) return false
+        // Stale lock: dead owner — unlink and retry exclusive create.
+        try { fs.unlinkSync(ownerLockFile) } catch {}
+        continue
+      }
+      const transient = e.code === 'EPERM' || e.code === 'EBUSY' || e.code === 'EACCES'
+      if (transient && attempt < 4) {
+        // Sync busy-wait acceptable here: this runs on memory worker boot
+        // path, once per process; the parent handler is not blocked.
+        const end = Date.now() + 50 * (attempt + 1)
+        while (Date.now() < end) {}
+        continue
+      }
+      throw e
+    }
+  }
+  return false
+}
+
+export function releaseMemoryOwnerLock(ownerLockFile) {
+  try {
+    const ownerPid = Number(fs.readFileSync(ownerLockFile, 'utf8').trim())
+    if (ownerPid === process.pid) fs.unlinkSync(ownerLockFile)
+  } catch {}
+}
+
+export function killPreviousServer(pid, log = () => {}) {
+  if (pid <= 0 || pid === process.pid) return false
+  if (process.platform === 'win32') {
+    try {
+      execFileSync('taskkill', ['/F', '/T', '/PID', String(pid)], { encoding: 'utf8', timeout: 5000, windowsHide: true })
+      log(`[memory-service] Killed previous server PID ${pid}\n`)
+      return true
+    } catch (e) {
+      // Exit code 128 = process not found; treat stale lock as already-dead = success.
+      // Status 128 reliably means "process not found" regardless of locale; no text match needed.
+      // Status 1 with English text match handles edge cases on some Windows versions.
+      const notFoundText = /not found|no running instance/i.test(e.stdout || '')
+        || /not found|no running instance/i.test(e.stderr || '')
+        || /not found|no running instance/i.test(e.message || '')
+      const alreadyDead = e.status === 128 || (e.status === 1 && notFoundText)
+      if (alreadyDead) {
+        log(`[memory-service] PID ${pid} already dead (stale lock), proceeding\n`)
+        return true
+      }
+      log(`[memory-service] taskkill failed for PID ${pid}: ${e.message}\n`)
+      return false
+    }
+  } else {
+    // Pre-flight: if the process is already gone, treat stale lock as success.
+    try {
+      process.kill(pid, 0)
+    } catch (e) {
+      if (e.code === 'ESRCH') {
+        log(`[memory-service] PID ${pid} already dead (stale lock), proceeding\n`)
+        return true
+      }
+    }
+    try { process.kill(pid, 'SIGTERM') } catch {}
+    try { process.kill(pid, 'SIGKILL') } catch {}
+    // Poll for death up to 2s
+    const deadline = Date.now() + 2000
+    while (Date.now() < deadline) {
+      try {
+        process.kill(pid, 0)
+      } catch (e) {
+        if (e.code === 'ESRCH') {
+          log(`[memory-service] Killed previous server PID ${pid}\n`)
+          return true
+        }
+      }
+      // Synchronous 50ms sleep via shared buffer spin
+      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50)
+    }
+    log(`[memory-service] PID ${pid} still alive after SIGKILL\n`)
+    return false
+  }
+}
+
+export function acquireLock(lockFile, log = () => {}) {
+  // Multi-instance guard. In multi-terminal mode the lock owner is a *peer*
+  // memory worker serving recall for another CC session. killPreviousServer
+  // would taskkill /F that healthy peer mid-flight, then this fork-proxy
+  // mode wouldn't even need a lock anyway. Skip the entire kill-the-previous
+  // protocol; fork-proxy detection in init() takes priority. If neither
+  // proxy nor lock-owner path applies (race window during simultaneous
+  // boot), the worker simply continues without the lock — server-main /
+  // PG / port-listen handle the actual conflict cases.
+  if (process.env.MIXDOG_MULTI_INSTANCE === '1') return
+  try {
+    if (fs.existsSync(lockFile)) {
+      const lockedPid = Number(fs.readFileSync(lockFile, 'utf8').trim())
+      if (lockedPid > 0 && lockedPid !== process.pid) {
+        const killed = killPreviousServer(lockedPid, log)
+        if (!killed) {
+          log(`[memory-service] Could not kill previous server PID ${lockedPid}, aborting\n`)
+          process.exit(1)
+        }
+        try { fs.unlinkSync(lockFile) } catch {}
+      }
+    }
+    const fd = fs.openSync(lockFile, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600)
+    try {
+      fs.writeSync(fd, String(process.pid))
+    } finally {
+      fs.closeSync(fd)
+    }
+  } catch (e) {
+    if (e.code === 'EEXIST') {
+      log(`[memory-service] Lock file exists (EEXIST) — another instance is already running, exiting\n`)
+      process.exit(0)
+    }
+    log(`[memory-service] Lock acquisition failed: ${e.message}\n`)
+    process.exit(1)
+  }
+}
+
+export function releaseLock(lockFile) {
+  try {
+    const content = fs.readFileSync(lockFile, 'utf8').trim()
+    if (Number(content) === process.pid) fs.unlinkSync(lockFile)
+  } catch {}
+}

package/src/runtime/memory/lib/memory-recall-scope-filter.mjs

@@ -18,11 +18,35 @@ export function buildCategoryFilterClause(offset, categories, { tableAlias = ''
   return { clause: `AND (${inner})`, params: [...cats] }
 }
 
+// Shared, param-less predicate excluding promoted/promoting core-candidate
+// roots AND member rows under such a root. Single source of truth so the hybrid
+// recall path (buildRecallScopeFilter) and the query-less browse path
+// (retrieveEntries) can't diverge. tableAlias='' → bare `entries` column refs.
+export function buildPromotedExclusionClauses(tableAlias = '') {
+  const p = `${tableAlias || 'entries'}.`
+  return [
+    // Promoted core-candidate roots have been absorbed into user-curated
+    // core_entries and archived by promoteCoreCandidate. Their content now
+    // lives in the {{USER_CORE}} slot, so surfacing the stale generated root
+    // would double-serve the same fact. 'promoting' (mid-flight or crashed
+    // promote awaiting recovery) is excluded too — its root is already archived
+    // and finalizes to 'promoted'. Member rows whose chunk_root points at a
+    // promoted/promoting root are excluded via EXISTS-on-root (the flag lives
+    // only on the root; members keep NULL). Constant predicates — no bind param.
+    `(${p}core_candidate_status IS NULL OR ${p}core_candidate_status NOT IN ('promoted', 'promoting'))`,
+    `NOT (${p}is_root = 0 AND ${p}chunk_root IS NOT NULL AND ${p}chunk_root <> ${p}id AND EXISTS (
+      SELECT 1 FROM entries r WHERE r.id = ${p}chunk_root AND r.is_root = 1 AND r.core_candidate_status IN ('promoted', 'promoting')
+    ))`,
+  ]
+}
+
 export function buildRecallScopeFilter(offset, options = {}, tableAlias = '') {
   const outerRef = tableAlias || 'entries'
   const p = `${outerRef}.`
   const clauses = [
     `NOT (${p}is_root = 0 AND ${p}chunk_root IS NOT DISTINCT FROM ${p}id AND ${p}status IS NOT DISTINCT FROM 'archived')`,
+    // Exclude promoted/promoting roots + their members (shared predicate).
+    ...buildPromotedExclusionClauses(tableAlias),
   ]
   const params = []
   let next = offset

package/src/runtime/memory/lib/memory-recall-store.mjs

@@ -15,6 +15,18 @@ import { recallReadQuery } from './memory-recall-read-query.mjs'
 const _MV_HOT_ACTIVE_TTL_MS = 60_000
 const _mvHotActiveCache = new WeakMap() // db → { populated: boolean, ts: number }
 const SEMANTIC_ONLY_MIN_SIM = 0.72
+// Cross-language semantic rescue: a query in one language against rows in
+// another rarely clears SEMANTIC_ONLY_MIN_SIM on a small (384-dim) embedding
+// model, yet the TOP dense ranks are still overwhelmingly relevant. Accept
+// semantic-only rows that are both top-ranked in the dense leg AND above a
+// softer floor, instead of dropping everything below 0.72.
+const SEMANTIC_TOP_RANK_MAX = 3
+// Floor for the top-rank rescue. 384-dim MiniLM-class embeddings put UNRELATED
+// text at ~0.65-0.75 sim (measured: gibberish query "zzqqxx ... 999" scored
+// 0.749 against real rows), so anything below ~0.70 is noise regardless of
+// rank; cross-language relevant pairs measured 0.67-0.74. 0.70 keeps most of
+// the cross-language rescue while rejecting pure-noise top ranks.
+const SEMANTIC_TOP_RANK_MIN_SIM = 0.70
 const SHORT_QUERY_TOKEN_MAX = 2
 
 function buildExactTerms(query) {
@@ -62,7 +74,12 @@ function hasFullQueryTextMatch(query, row) {
 function hasQueryTokenCoverage(row, queryTokenCount) {
   if (queryTokenCount <= SHORT_QUERY_TOKEN_MAX) return true
   const hits = Number(row?.exact_hits)
-  return Number.isFinite(hits) && hits >= queryTokenCount
+  // Majority coverage, not full coverage. Requiring EVERY query token to hit
+  // made mixed-language queries ("usage spike 원인") return nothing when one
+  // token has no lexical counterpart in the row — ~60% of tokens matching is
+  // strong enough lexical evidence alongside the semantic leg.
+  const required = Math.max(2, Math.ceil(queryTokenCount * 0.6))
+  return Number.isFinite(hits) && hits >= required
 }
 
 function exactTextBoost(query, row, exactHits) {
@@ -479,7 +496,10 @@ LEFT JOIN exact  x ON x.id = c.id`
   const scored = scoredAll.filter(({ row }) => {
     const hasTextSupport = row.sparse_rank != null || row.trgm_rank != null || row.exact_rank != null
     const sim = Number(row.dense_sim)
-    const hasSemanticSupport = Number.isFinite(sim) && sim >= SEMANTIC_ONLY_MIN_SIM
+    const denseRankNum = row.dense_rank != null ? Number(row.dense_rank) : null
+    const hasSemanticSupport = (Number.isFinite(sim) && sim >= SEMANTIC_ONLY_MIN_SIM)
+      || (Number.isFinite(sim) && sim >= SEMANTIC_TOP_RANK_MIN_SIM
+          && denseRankNum != null && denseRankNum <= SEMANTIC_TOP_RANK_MAX)
     if (!hasTextSupport) {
       if (hasSemanticSupport) return true
       semanticOnlyDropped += 1

package/src/runtime/memory/lib/memory-retrievers.mjs

@@ -1,5 +1,7 @@
 import { recallReadQuery } from './memory-recall-read-query.mjs'
 
+import { buildPromotedExclusionClauses } from './memory-recall-scope-filter.mjs'
+
 const VALID_CATEGORIES_SET = new Set([
   'rule', 'constraint', 'decision', 'fact', 'goal', 'preference', 'task', 'issue',
 ])
@@ -72,6 +74,12 @@ export async function retrieveEntries(db, filters = {}) {
     where.push(`chunk_root IS NULL`)
   }
 
+  // Exclude promoted/promoting core-candidate roots AND members under such a
+  // root — the query-less browse path (includeArchived) would otherwise surface
+  // rows the hybrid recall path already filters. Shared predicate (param-less)
+  // keeps this in lock-step with buildRecallScopeFilter.
+  where.push(...buildPromotedExclusionClauses())
+
   const limit = Math.max(1, Math.min(500, Number(filters.limit ?? 50)))
   const offset = Math.max(0, Number(filters.offset ?? 0))
   const sort = String(filters.sort ?? 'importance').trim().toLowerCase()

package/src/runtime/memory/lib/memory.mjs

@@ -443,6 +443,26 @@ export async function ensureCurrentSchemaExtensions(db, dims) {
 
 
 
+  // Core-candidate promotion pipeline (proposal mode): active entries the
+  // cycle2 nomination pass flags as strong core-memory candidates. Never
+  // auto-inserted into core_entries — a user approves each via the
+  // action:'core' op:'promote' handler. Columns are nullable and the ALTERs
+  // are idempotent (ADD COLUMN IF NOT EXISTS), safe to re-run every boot.
+  //   core_candidate_status: NULL (not a candidate) | 'candidate' | 'promoting'
+  //     (mid-flight promote, recoverable) | 'promoted' | 'dismissed'
+  //   core_candidate_at:     ms timestamp of last nomination/state change
+  // 'dismissed'/'promoted' are terminal for a given root so the pass never
+  // re-nominates the same entry.
+  await db.exec(`ALTER TABLE entries ADD COLUMN IF NOT EXISTS core_candidate_status text`)
+  await db.exec(`ALTER TABLE entries ADD COLUMN IF NOT EXISTS core_candidate_at bigint`)
+  // No index on core_candidate_status by design (round-2 finding #4): the only
+  // readers are listCoreCandidates (user picker, on-demand) and
+  // nominateCoreCandidates (once per cycle2, hourly). Both are rare and the
+  // entries table is small enough that a seq scan is fine — an index isn't
+  // worth the boot-time AccessExclusive build lock on the hot entries table.
+  // Drop it if a previous deploy created it (idempotent no-op otherwise).
+  await db.exec(`DROP INDEX IF EXISTS idx_entries_core_candidate`)
+
   // Dedupe core_entries before creating the unique index — keeps the row with
   // the most recent updated_at (id breaks ties), drops the rest.
   const dedupe = await db.query(`

package/src/runtime/memory/lib/pg/supervisor.mjs

@@ -194,7 +194,7 @@ function patchActiveInstance(fields) {
         else merged[k] = v;
       }
       return merged;
-    }, { compact: true, fsyncDir: true });
+    }, { compact: true, fsyncDir: true, renameFallback: 'truncate' });
   } catch (e) {
     __mixdogMemoryLog(`[supervisor-pg] patchActiveInstance failed: ${e?.message}\n`);
   }

package/src/runtime/memory/lib/promotion-fingerprint.mjs

@@ -0,0 +1,50 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import crypto from 'node:crypto'
+
+// Plugin version + promotion code fingerprint. Extracted from index.mjs
+// (behavior-preserving). Callers pass PLUGIN_ROOT so this stays free of any
+// entry-point path resolution.
+
+export function readPluginVersion(pluginRoot) {
+  try {
+    return JSON.parse(fs.readFileSync(path.join(pluginRoot, 'package.json'), 'utf8')).version || '0.0.1'
+  } catch { return '0.0.1' }
+}
+
+const PROMOTION_FINGERPRINT_ROOTS = ['src/memory']
+
+function collectPromotionFingerprintFiles(pluginRoot) {
+  const out = []
+  const walk = (relDir) => {
+    let entries = []
+    try { entries = fs.readdirSync(path.join(pluginRoot, relDir), { withFileTypes: true }) }
+    catch { return }
+    for (const ent of entries) {
+      const rel = `${relDir}/${ent.name}`.replace(/\\/g, '/')
+      if (ent.isDirectory()) {
+        walk(rel)
+      } else if (ent.isFile() && rel.endsWith('.mjs')) {
+        out.push(rel)
+      }
+    }
+  }
+  for (const root of PROMOTION_FINGERPRINT_ROOTS) walk(root)
+  return out.sort()
+}
+
+export function readPromotionCodeFingerprint(pluginRoot) {
+  const hash = crypto.createHash('sha256')
+  const files = collectPromotionFingerprintFiles(pluginRoot)
+  for (const rel of files) {
+    hash.update(rel)
+    hash.update('\0')
+    try {
+      hash.update(fs.readFileSync(path.join(pluginRoot, rel)))
+    } catch {
+      hash.update('missing')
+    }
+    hash.update('\0')
+  }
+  return `src/memory:${files.length}:${hash.digest('hex').slice(0, 16)}`
+}