mixdog 0.9.3 → 0.9.4

package/src/runtime/memory/lib/memory-embed.mjs

@@ -213,6 +213,155 @@ export async function flushEmbeddingDirty(db, options = {}) {
   }
 }
 
+// Embeds raw transcript rows (is_root = 0, non-chunk-member) so the dense
+// recall leg can rank them alongside root/summary entries. Mirrors
+// flushEmbeddingDirty's SKIP LOCKED batch-claim structure but sources the
+// embed text from `content` (raw rows have no element/summary) and honors
+// `limit` as a total cap across the whole call rather than an unbounded loop.
+//
+// Stale-embedding safety when recap toggles back ON: cycle1's promotion
+// UPDATE (memory-cycle1.mjs chunk commit) SETs summary/element, which fires
+// trg_entries_embedding_invalidate (memory.mjs — BEFORE UPDATE OF content,
+// summary, element; NEW.is_root=1 AND summary IS DISTINCT FROM OLD.summary)
+// and NULLs the raw-content embedding written here; cycle1's end-of-cycle
+// flushEmbeddingDirty (is_root=1 AND embedding IS NULL) then re-embeds the
+// root from its fresh element/summary. Member leaves keep their content
+// embedding — the dense recall leg resolves members to their chunk root
+// (memory-recall-store.mjs member→root resolution), so that is leaf-level
+// dense matching, not staleness.
+export async function flushRawEmbeddings(db, options = {}) {
+  const { limit = 200, signal } = options ?? {}
+  throwIfAborted(signal)
+  await ensureEmbCacheTable(db)
+  throwIfAborted(signal)
+
+  let attempted = 0
+  let embedded = 0
+  let cursor = 0
+
+  while (attempted < limit) {
+    throwIfAborted(signal)
+    const batchCap = Math.min(BATCH_SIZE, limit - attempted)
+    const client = await db._pool.connect()
+    let ids
+    try {
+      throwIfAborted(signal)
+      await client.query('BEGIN')
+      const res = await client.query(
+        `SELECT id FROM memory.entries
+         WHERE is_root = 0 AND chunk_root IS NULL AND embedding IS NULL
+           AND NULLIF(btrim(session_id), '') IS NOT NULL
+           AND content IS NOT NULL
+           AND id > $2
+         ORDER BY id
+         LIMIT $1
+         FOR UPDATE SKIP LOCKED`,
+        [batchCap, cursor],
+      )
+      ids = res.rows.map(r => Number(r.id))
+      throwIfAborted(signal)
+    } catch (err) {
+      try { await client.query('ROLLBACK') } catch {}
+      client.release()
+      if (signal?.aborted) throw signal.reason ?? err
+      __mixdogMemoryLog(`[embed] raw flush SKIP LOCKED claim failed: ${err.message}\n`)
+      break
+    }
+
+    if (ids.length === 0) {
+      try { await client.query('COMMIT') } catch {}
+      client.release()
+      break
+    }
+
+    cursor = ids[ids.length - 1]
+    attempted += ids.length
+    let batchDone = false
+    try {
+      throwIfAborted(signal)
+      const writtenIds = await syncRawBatchEmbeddings(client, ids, { signal })
+      embedded += writtenIds.length
+      batchDone = true
+    } catch (err) {
+      if (signal?.aborted) throw signal.reason ?? err
+      __mixdogMemoryLog(`[embed] raw batch failed (ids=${ids[0]}..${ids[ids.length-1]}): ${err.message}\n`)
+    } finally {
+      try {
+        if (batchDone) await client.query('COMMIT')
+        else await client.query('ROLLBACK')
+      } catch {}
+      client.release()
+    }
+
+    if (ids.length < batchCap) break
+  }
+
+  return { attempted, embedded }
+}
+
+// Batch-embed raw transcript rows: content is the embed source text (raw
+// rows have no element/summary). Mirrors syncBatchEmbeddings' shape.
+async function syncRawBatchEmbeddings(db, ids, options = {}) {
+  const signal = options?.signal
+  throwIfAborted(signal)
+  const rows = (await db.query(
+    `SELECT id, content FROM memory.entries
+     WHERE id = ANY($1::bigint[]) AND is_root = 0 AND chunk_root IS NULL`,
+    [ids],
+  )).rows
+  throwIfAborted(signal)
+  if (rows.length === 0) return []
+
+  const dimsRow = (await db.query(`SELECT value FROM memory.meta WHERE key = 'embedding.current_dims'`)).rows[0]
+  throwIfAborted(signal)
+  const expected = Number(dimsRow?.value ?? 0)
+
+  const texts = rows.map(row => (typeof row.content === 'string' ? row.content.trim() : ''))
+  const vectors = await cachedEmbedTextBatch(db, texts.filter(Boolean), { signal })
+  throwIfAborted(signal)
+  let vecIdx = 0
+  const updates = []  // { id, content, vector }
+  for (let i = 0; i < rows.length; i++) {
+    throwIfAborted(signal)
+    const row = rows[i]
+    if (!texts[i]) continue
+    const vector = vectors[vecIdx++]
+    if (!Array.isArray(vector) || vector.length === 0) continue
+    if (Number.isFinite(expected) && expected > 0 && vector.length !== expected) {
+      __mixdogMemoryLog(`[embed] raw dim mismatch (id=${row.id} got=${vector.length} expected=${expected})\n`)
+      continue
+    }
+    updates.push({ id: Number(row.id), content: row.content, vector })
+  }
+
+  if (updates.length === 0) return []
+
+  const valClauses = updates.map((u, i) => {
+    const base = i * 3
+    return `($${base+1}::bigint, $${base+2}::halfvec, $${base+3}::text)`
+  })
+  const params = []
+  for (const u of updates) {
+    params.push(u.id, embeddingToSql(u.vector), u.content)
+  }
+  const res = await db.query(
+    `UPDATE memory.entries AS e
+     SET embedding = t.vector
+     FROM (VALUES ${valClauses.join(',')}) AS t(id, vector, content)
+     WHERE e.id = t.id AND e.is_root = 0 AND e.chunk_root IS NULL
+       AND e.embedding IS NULL
+       AND e.content IS NOT DISTINCT FROM t.content
+     RETURNING e.id`,
+    params,
+  )
+  throwIfAborted(signal)
+  const writtenIds = (res.rows || []).map(r => Number(r.id))
+  if (writtenIds.length < updates.length) {
+    __mixdogMemoryLog(`[embed-sync] ${updates.length - writtenIds.length} raw stale-write(s) skipped — content changed since read\n`)
+  }
+  return writtenIds
+}
+
 // Batch-embed all ids in one DB round-trip for fetching, one API call per unique text,
 // then one VALUES UPDATE for all results.
 // Returns the list of ids that were actually UPDATEd with a fresh embedding.

package/src/runtime/memory/lib/memory-process-lock.mjs

@@ -0,0 +1,162 @@
+// Extracted from index.mjs — single/multi-instance process-lock + owner-election
+// helpers. Pure functions parameterized on lock-file paths so index.mjs retains
+// ownership of DATA_DIR-derived constants and the daemon logger.
+import fs from 'node:fs'
+import { execFileSync } from 'node:child_process'
+
+export function parsePositivePid(value) {
+  const pid = Number(value)
+  return Number.isFinite(pid) && pid > 0 ? pid : null
+}
+
+export function isPidAliveLocal(pid) {
+  if (!Number.isFinite(pid) || pid <= 0) return false
+  try { process.kill(pid, 0); return true }
+  catch (e) { return e.code !== 'ESRCH' }
+}
+
+export function tryAcquireMemoryOwnerLock(ownerLockFile, log = () => {}) {
+  // Returns true on success (this process now owns memory worker for the data
+  // dir), false when a live peer holds the lock. Stale locks (dead PID) are
+  // unlinked and retried atomically. Throws on unexpected fs errors so callers
+  // surface lock-system corruption rather than silently downgrading.
+  //
+  // EPERM/EBUSY/EACCES at openSync are transient — AV scanners (SignKorea /
+  // SKCert / ezPDFWS etc) briefly lock newly-created files during inspection.
+  // The 0.1.x baseline threw immediately and the worker promoted to
+  // permanentlyDegraded, killing memory tools for the rest of the session.
+  // Treat the AV error codes as retryable with bounded backoff (~750ms total)
+  // before giving up and rethrowing.
+  for (let attempt = 0; attempt < 5; attempt++) {
+    try {
+      const fd = fs.openSync(ownerLockFile, 'wx')
+      fs.writeSync(fd, String(process.pid))
+      fs.closeSync(fd)
+      return true
+    } catch (e) {
+      if (e.code === 'EEXIST') {
+        let ownerPid = NaN
+        try { ownerPid = Number(fs.readFileSync(ownerLockFile, 'utf8').trim()) } catch {}
+        if (isPidAliveLocal(ownerPid)) return false
+        // Stale lock: dead owner — unlink and retry exclusive create.
+        try { fs.unlinkSync(ownerLockFile) } catch {}
+        continue
+      }
+      const transient = e.code === 'EPERM' || e.code === 'EBUSY' || e.code === 'EACCES'
+      if (transient && attempt < 4) {
+        // Sync busy-wait acceptable here: this runs on memory worker boot
+        // path, once per process; the parent handler is not blocked.
+        const end = Date.now() + 50 * (attempt + 1)
+        while (Date.now() < end) {}
+        continue
+      }
+      throw e
+    }
+  }
+  return false
+}
+
+export function releaseMemoryOwnerLock(ownerLockFile) {
+  try {
+    const ownerPid = Number(fs.readFileSync(ownerLockFile, 'utf8').trim())
+    if (ownerPid === process.pid) fs.unlinkSync(ownerLockFile)
+  } catch {}
+}
+
+export function killPreviousServer(pid, log = () => {}) {
+  if (pid <= 0 || pid === process.pid) return false
+  if (process.platform === 'win32') {
+    try {
+      execFileSync('taskkill', ['/F', '/T', '/PID', String(pid)], { encoding: 'utf8', timeout: 5000, windowsHide: true })
+      log(`[memory-service] Killed previous server PID ${pid}\n`)
+      return true
+    } catch (e) {
+      // Exit code 128 = process not found; treat stale lock as already-dead = success.
+      // Status 128 reliably means "process not found" regardless of locale; no text match needed.
+      // Status 1 with English text match handles edge cases on some Windows versions.
+      const notFoundText = /not found|no running instance/i.test(e.stdout || '')
+        || /not found|no running instance/i.test(e.stderr || '')
+        || /not found|no running instance/i.test(e.message || '')
+      const alreadyDead = e.status === 128 || (e.status === 1 && notFoundText)
+      if (alreadyDead) {
+        log(`[memory-service] PID ${pid} already dead (stale lock), proceeding\n`)
+        return true
+      }
+      log(`[memory-service] taskkill failed for PID ${pid}: ${e.message}\n`)
+      return false
+    }
+  } else {
+    // Pre-flight: if the process is already gone, treat stale lock as success.
+    try {
+      process.kill(pid, 0)
+    } catch (e) {
+      if (e.code === 'ESRCH') {
+        log(`[memory-service] PID ${pid} already dead (stale lock), proceeding\n`)
+        return true
+      }
+    }
+    try { process.kill(pid, 'SIGTERM') } catch {}
+    try { process.kill(pid, 'SIGKILL') } catch {}
+    // Poll for death up to 2s
+    const deadline = Date.now() + 2000
+    while (Date.now() < deadline) {
+      try {
+        process.kill(pid, 0)
+      } catch (e) {
+        if (e.code === 'ESRCH') {
+          log(`[memory-service] Killed previous server PID ${pid}\n`)
+          return true
+        }
+      }
+      // Synchronous 50ms sleep via shared buffer spin
+      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50)
+    }
+    log(`[memory-service] PID ${pid} still alive after SIGKILL\n`)
+    return false
+  }
+}
+
+export function acquireLock(lockFile, log = () => {}) {
+  // Multi-instance guard. In multi-terminal mode the lock owner is a *peer*
+  // memory worker serving recall for another CC session. killPreviousServer
+  // would taskkill /F that healthy peer mid-flight, then this fork-proxy
+  // mode wouldn't even need a lock anyway. Skip the entire kill-the-previous
+  // protocol; fork-proxy detection in init() takes priority. If neither
+  // proxy nor lock-owner path applies (race window during simultaneous
+  // boot), the worker simply continues without the lock — server-main /
+  // PG / port-listen handle the actual conflict cases.
+  if (process.env.MIXDOG_MULTI_INSTANCE === '1') return
+  try {
+    if (fs.existsSync(lockFile)) {
+      const lockedPid = Number(fs.readFileSync(lockFile, 'utf8').trim())
+      if (lockedPid > 0 && lockedPid !== process.pid) {
+        const killed = killPreviousServer(lockedPid, log)
+        if (!killed) {
+          log(`[memory-service] Could not kill previous server PID ${lockedPid}, aborting\n`)
+          process.exit(1)
+        }
+        try { fs.unlinkSync(lockFile) } catch {}
+      }
+    }
+    const fd = fs.openSync(lockFile, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600)
+    try {
+      fs.writeSync(fd, String(process.pid))
+    } finally {
+      fs.closeSync(fd)
+    }
+  } catch (e) {
+    if (e.code === 'EEXIST') {
+      log(`[memory-service] Lock file exists (EEXIST) — another instance is already running, exiting\n`)
+      process.exit(0)
+    }
+    log(`[memory-service] Lock acquisition failed: ${e.message}\n`)
+    process.exit(1)
+  }
+}
+
+export function releaseLock(lockFile) {
+  try {
+    const content = fs.readFileSync(lockFile, 'utf8').trim()
+    if (Number(content) === process.pid) fs.unlinkSync(lockFile)
+  } catch {}
+}

package/src/runtime/memory/lib/memory-recall-store.mjs

@@ -15,6 +15,18 @@ import { recallReadQuery } from './memory-recall-read-query.mjs'
 const _MV_HOT_ACTIVE_TTL_MS = 60_000
 const _mvHotActiveCache = new WeakMap() // db → { populated: boolean, ts: number }
 const SEMANTIC_ONLY_MIN_SIM = 0.72
+// Cross-language semantic rescue: a query in one language against rows in
+// another rarely clears SEMANTIC_ONLY_MIN_SIM on a small (384-dim) embedding
+// model, yet the TOP dense ranks are still overwhelmingly relevant. Accept
+// semantic-only rows that are both top-ranked in the dense leg AND above a
+// softer floor, instead of dropping everything below 0.72.
+const SEMANTIC_TOP_RANK_MAX = 3
+// Floor for the top-rank rescue. 384-dim MiniLM-class embeddings put UNRELATED
+// text at ~0.65-0.75 sim (measured: gibberish query "zzqqxx ... 999" scored
+// 0.749 against real rows), so anything below ~0.70 is noise regardless of
+// rank; cross-language relevant pairs measured 0.67-0.74. 0.70 keeps most of
+// the cross-language rescue while rejecting pure-noise top ranks.
+const SEMANTIC_TOP_RANK_MIN_SIM = 0.70
 const SHORT_QUERY_TOKEN_MAX = 2
 
 function buildExactTerms(query) {
@@ -62,7 +74,12 @@ function hasFullQueryTextMatch(query, row) {
 function hasQueryTokenCoverage(row, queryTokenCount) {
   if (queryTokenCount <= SHORT_QUERY_TOKEN_MAX) return true
   const hits = Number(row?.exact_hits)
-  return Number.isFinite(hits) && hits >= queryTokenCount
+  // Majority coverage, not full coverage. Requiring EVERY query token to hit
+  // made mixed-language queries ("usage spike 원인") return nothing when one
+  // token has no lexical counterpart in the row — ~60% of tokens matching is
+  // strong enough lexical evidence alongside the semantic leg.
+  const required = Math.max(2, Math.ceil(queryTokenCount * 0.6))
+  return Number.isFinite(hits) && hits >= required
 }
 
 function exactTextBoost(query, row, exactHits) {
@@ -479,7 +496,10 @@ LEFT JOIN exact  x ON x.id = c.id`
   const scored = scoredAll.filter(({ row }) => {
     const hasTextSupport = row.sparse_rank != null || row.trgm_rank != null || row.exact_rank != null
     const sim = Number(row.dense_sim)
-    const hasSemanticSupport = Number.isFinite(sim) && sim >= SEMANTIC_ONLY_MIN_SIM
+    const denseRankNum = row.dense_rank != null ? Number(row.dense_rank) : null
+    const hasSemanticSupport = (Number.isFinite(sim) && sim >= SEMANTIC_ONLY_MIN_SIM)
+      || (Number.isFinite(sim) && sim >= SEMANTIC_TOP_RANK_MIN_SIM
+          && denseRankNum != null && denseRankNum <= SEMANTIC_TOP_RANK_MAX)
     if (!hasTextSupport) {
       if (hasSemanticSupport) return true
       semanticOnlyDropped += 1

package/src/runtime/memory/lib/pg/supervisor.mjs

@@ -194,7 +194,7 @@ function patchActiveInstance(fields) {
         else merged[k] = v;
       }
       return merged;
-    }, { compact: true, fsyncDir: true });
+    }, { compact: true, fsyncDir: true, renameFallback: 'truncate' });
   } catch (e) {
     __mixdogMemoryLog(`[supervisor-pg] patchActiveInstance failed: ${e?.message}\n`);
   }