mixdog 0.9.78 → 0.9.80

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mixdog",
-  "version": "0.9.78",
+  "version": "0.9.80",
   "private": false,
   "type": "module",
   "description": "Standalone mixdog coding-agent CLI/TUI workspace.",
@@ -74,8 +74,8 @@
     "test:grok-oauth-race": "node --test scripts/grok-oauth-refresh-race-test.mjs",
     "test:atomiclock": "node --test scripts/atomic-lock-tryonce-test.mjs",
     "test:memory-routing": "node --test scripts/memory-cycle-routing-test.mjs scripts/maintenance-default-routes-test.mjs scripts/embedding-worker-exit-test.mjs scripts/embedding-runtime-prune-test.mjs",
-    "test:embedding-runtime": "node --test scripts/embedding-runtime-prune-test.mjs && node scripts/verify-embedding-runtime.mjs",
-    "test:embedding-runtime:core": "node --test scripts/embedding-runtime-prune-test.mjs && node scripts/verify-embedding-runtime.mjs --core",
+    "test:embedding-runtime": "node --test scripts/embedding-runtime-prune-test.mjs scripts/memory-pg-recovery-test.mjs && node scripts/verify-embedding-runtime.mjs",
+    "test:embedding-runtime:core": "node --test scripts/embedding-runtime-prune-test.mjs scripts/memory-pg-recovery-test.mjs && node scripts/verify-embedding-runtime.mjs --core",
     "test:embedding-runtime:warmup": "node scripts/verify-embedding-runtime.mjs --warmup",
     "test:code-graph-dispatch": "node --test scripts/code-graph-dispatch-test.mjs",
     "test:code-graph-clean-cache": "node --test scripts/code-graph-dispatch-test.mjs",

package/src/runtime/memory/lib/memory-recall-store.mjs

@@ -146,12 +146,6 @@ export async function searchRelevantHybrid(db, query, options = {}) {
   const queryTokenCount = countQueryTokens(clean)
   const minExactHits = exactTerms.length >= 8 ? 3 : exactTerms.length >= 4 ? 2 : 1
 
-  // For very short queries (< 3 chars) the trigram operator still works but
-  // we relax the server-side threshold via set_limit() — however that requires
-  // a separate round-trip. Instead we fall back to a plain ILIKE scan for
-  // short text (rare edge case; sequential scan is acceptable for < 3 chars).
-  const isShortQuery = clean.length < 3
-
   // $5 onward are the filter params for the entries legs (non-MV path).
   // Each CTE leg duplicates the same positional params because they live in
   // independent SELECT scopes. When useHotActive=true, the trgm leg still uses
@@ -269,50 +263,26 @@ sparse AS (
 ),`) : `
 sparse AS (SELECT NULL::bigint AS id, NULL::float8 AS lex, NULL::bigint AS sparse_rank WHERE $2::text IS NOT NULL AND false),`
 
-  // trgm CTE: pg_trgm similarity path. For short queries (< 3 chars) the %
-  // operator is unreliable (trigrams need at least 3 chars); use ILIKE instead.
-  // NOTE: trgm always queries entries regardless of useHotActive — mv_hot_active
-  // lacks the content column (trgm/ILIKE) and ts column (short-query ORDER BY).
-  // Uses trgmFilterClause whose $N offsets are aligned to activeBindParams.
-  const trgmCte = isShortQuery ? `
+  // Portable substring leg. The curated Unix PG runtimes include pgvector but
+  // not the optional pg_trgm contrib extension, so fuzzy similarity cannot be
+  // a startup/runtime requirement. FTS and dense vector search retain broad
+  // matching while this leg gives exact substrings a deterministic rescue.
+  // It always queries entries because mv_hot_active omits display text + ts.
+  const trgmCte = `
 trgm AS (
   SELECT id,
-         0.5::float8 AS trg_sim,
+         1.0::float8 AS trg_sim,
          ROW_NUMBER() OVER (ORDER BY ts DESC) AS trgm_rank
   FROM entries
-  WHERE (content ILIKE '%' || $3 || '%' OR element ILIKE '%' || $3 || '%')
+  WHERE (
+      content ILIKE '%' || $3 || '%'
+      OR coalesce(element, '') ILIKE '%' || $3 || '%'
+      OR coalesce(summary, '') ILIKE '%' || $3 || '%'
+    )
     ${trgmFilterClause}
   ORDER BY ts DESC
   LIMIT $4
-),` : `
-trgm AS (
-  SELECT id,
-          GREATEST(
-            CASE WHEN content ILIKE '%' || $3 || '%' THEN 1.0 ELSE similarity(content, $3) END,
-            CASE WHEN coalesce(element, '') ILIKE '%' || $3 || '%' THEN 1.0 ELSE similarity(coalesce(element, ''), $3) END,
-            CASE WHEN coalesce(summary, '') ILIKE '%' || $3 || '%' THEN 1.0 ELSE similarity(coalesce(summary, ''), $3) END
-          ) AS trg_sim,
-          ROW_NUMBER() OVER (ORDER BY GREATEST(
-            CASE WHEN content ILIKE '%' || $3 || '%' THEN 1.0 ELSE similarity(content, $3) END,
-            CASE WHEN coalesce(element, '') ILIKE '%' || $3 || '%' THEN 1.0 ELSE similarity(coalesce(element, ''), $3) END,
-            CASE WHEN coalesce(summary, '') ILIKE '%' || $3 || '%' THEN 1.0 ELSE similarity(coalesce(summary, ''), $3) END
-          ) DESC) AS trgm_rank
-   FROM entries
-   WHERE (
-       content % $3 OR element % $3 OR summary % $3
-       OR content ILIKE '%' || $3 || '%'
-       OR coalesce(element, '') ILIKE '%' || $3 || '%'
-       OR coalesce(summary, '') ILIKE '%' || $3 || '%'
-     )
-     AND GREATEST(
-       CASE WHEN content ILIKE '%' || $3 || '%' THEN 1.0 ELSE similarity(content, $3) END,
-       CASE WHEN coalesce(element, '') ILIKE '%' || $3 || '%' THEN 1.0 ELSE similarity(coalesce(element, ''), $3) END,
-       CASE WHEN coalesce(summary, '') ILIKE '%' || $3 || '%' THEN 1.0 ELSE similarity(coalesce(summary, ''), $3) END
-     ) >= 0.10
-     ${trgmFilterClause}
-   ORDER BY trg_sim DESC
-   LIMIT $4
- ),`
+),`
 
   const exactCte = exactTerms.length > 0 ? `
 exact AS (

package/src/runtime/memory/lib/memory.mjs

@@ -134,17 +134,14 @@ export async function init(db, dims) {
   await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_phase_sweep ON entries(status, is_root, error_count, reviewed_at, id)`)
   await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_promoted_at ON entries(promoted_at) WHERE promoted_at IS NOT NULL`)
   await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_tsv         ON entries USING GIN (search_tsv)`)
-  // Recall CTEs (memory-recall-store.mjs dense/trgm legs) intentionally match
+  // Recall CTEs (memory-recall-store.mjs dense/text legs) intentionally match
   // BOTH root and leaf/chunk rows, so their SQL has NO `is_root = 1` predicate
-  // (only `embedding IS NOT NULL` / content|element|summary text filters).
+  // (only `embedding IS NOT NULL` / portable substring text filters).
   // The old root-only PARTIAL indexes therefore could not be used by those
   // queries — the planner fell back to a Seq Scan + top-N heapsort over every
-  // embedding (verified via EXPLAIN ANALYZE). Broaden the predicates to match
-  // the query shape so HNSW/GIN are actually used. A `summary` trgm index is
-  // added because the trgm leg also filters on `summary` but had no index.
-  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_content_trgm ON entries USING GIN (content gin_trgm_ops)`)
-  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_element_trgm ON entries USING GIN (element gin_trgm_ops) WHERE element IS NOT NULL`)
-  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_summary_trgm ON entries USING GIN (summary gin_trgm_ops) WHERE summary IS NOT NULL`)
+  // embedding (verified via EXPLAIN ANALYZE). Broaden the HNSW predicate to
+  // match the query shape. Substring rescue intentionally stays index-free:
+  // bundled Unix PG runtimes do not include the optional pg_trgm extension.
   await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_embedding_hnsw ON entries USING hnsw (embedding halfvec_cosine_ops) WHERE embedding IS NOT NULL`)
 
   // BEFORE INSERT/UPDATE trigger keeps score in sync with category + last_seen_at
@@ -392,8 +389,6 @@ async function _migrateRecallIndexesIfStale(db) {
   // Each entry: [indexName, newDefTailPredicate] where the presence of
   // "is_root" in the live indexdef signals the stale root-only shape.
   const targets = [
-    { name: 'idx_entries_content_trgm',   create: `CREATE INDEX idx_entries_content_trgm ON entries USING GIN (content gin_trgm_ops)` },
-    { name: 'idx_entries_element_trgm',   create: `CREATE INDEX idx_entries_element_trgm ON entries USING GIN (element gin_trgm_ops) WHERE element IS NOT NULL` },
     { name: 'idx_entries_embedding_hnsw', create: `CREATE INDEX idx_entries_embedding_hnsw ON entries USING hnsw (embedding halfvec_cosine_ops) WHERE embedding IS NOT NULL` },
   ]
   try {
@@ -403,8 +398,7 @@ async function _migrateRecallIndexesIfStale(db) {
         const r = await db.query(`SELECT indexdef FROM pg_indexes WHERE indexname = $1`, [t.name])
         def = r.rows?.[0]?.indexdef ?? null
       } catch { def = null }
-      // Missing index → post-loop IF-NOT-EXISTS ensures below self-heal trgm
-      // indexes; embedding_hnsw is also re-ensured by ensureCurrentSchemaExtensions.
+      // Missing embedding_hnsw is re-ensured by ensureCurrentSchemaExtensions.
       if (def == null) continue
       // Already broadened (no is_root predicate) → no-op, no rebuild.
       if (!/is_root/i.test(def)) continue
@@ -416,26 +410,6 @@ async function _migrateRecallIndexesIfStale(db) {
         __mixdogMemoryLog(`[memory] recall index migration for ${t.name} failed: ${err?.message || err}\n`)
       }
     }
-    // Self-heal trgm recall indexes on already-bootstrapped DBs: init() will
-    // not re-run, and a missing index is not handled by the stale-shape loop
-    // above (def == null → continue). These IF-NOT-EXISTS ensures recreate any
-    // trgm index that never existed or whose migrate-create failed. No-op (cheap
-    // catalog check) when the broadened index is already present.
-    try {
-      await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_content_trgm ON entries USING GIN (content gin_trgm_ops)`)
-    } catch (err) {
-      __mixdogMemoryLog(`[memory] idx_entries_content_trgm ensure failed: ${err?.message || err}\n`)
-    }
-    try {
-      await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_element_trgm ON entries USING GIN (element gin_trgm_ops) WHERE element IS NOT NULL`)
-    } catch (err) {
-      __mixdogMemoryLog(`[memory] idx_entries_element_trgm ensure failed: ${err?.message || err}\n`)
-    }
-    try {
-      await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_summary_trgm ON entries USING GIN (summary gin_trgm_ops) WHERE summary IS NOT NULL`)
-    } catch (err) {
-      __mixdogMemoryLog(`[memory] idx_entries_summary_trgm ensure failed: ${err?.message || err}\n`)
-    }
   } catch (err) {
     __mixdogMemoryLog(`[memory] _migrateRecallIndexesIfStale failed: ${err?.message || err}\n`)
   }

package/src/runtime/memory/lib/pg/adapter.mjs

@@ -139,8 +139,6 @@ async function _initClient(client, schema) {
       ? 'scheduler, public'
       : 'memory, public'
   await client.query(`SET search_path = ${sp}`)
-  // pg_trgm similarity threshold: session-local, must be set per connection.
-  await client.query(`SELECT set_limit(0.10)`)
   await client.query(`SET default_transaction_isolation TO 'read committed'`)
   // Mark seen only after all init statements succeed; failure leaves client
   // unmarked so the next checkout retries init.
@@ -400,7 +398,6 @@ async function bootstrapInstance(pgPool, dataDirKey) {
     const client = await pgPool.connect()
     try {
       await client.query(`CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public`)
-      await client.query(`CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public`)
       await client.query(`CREATE SCHEMA IF NOT EXISTS memory`)
       await client.query(`CREATE SCHEMA IF NOT EXISTS trace`)
       await client.query(`CREATE SCHEMA IF NOT EXISTS scheduler`)

package/src/runtime/memory/lib/pg/process.mjs

@@ -77,18 +77,11 @@ function readPostmasterInfo(pgdataDir) {
   }
 }
 
-function pgIsReady(runtimeDir, env, port) {
-  const probe = spawnSync(pgBin(runtimeDir, 'pg_isready'), ['-h', '127.0.0.1', '-p', String(port)], {
-    env, stdio: 'pipe', timeout: 3_000, windowsHide: true,
-  })
-  return probe.status === 0
-}
-
 async function awaitExistingPostmaster({ runtimeDir, pgdataDir, env, waitMs }) {
   const deadline = Date.now() + Math.max(0, Number(waitMs) || 0)
   let info = readPostmasterInfo(pgdataDir)
   while (info.pid && info.port) {
-    if (pgIsReady(runtimeDir, env, info.port)) return { state: 'ready', ...info }
+    if (await healthcheckPg({ port: info.port })) return { state: 'ready', ...info }
     if (!isPidAlive(info.pid)) return { state: 'dead', ...info }
     if (Date.now() >= deadline) return { state: 'alive-not-ready', ...info }
     await delay(250)
@@ -115,9 +108,8 @@ async function findFreePort(preferred) {
 
 const MIXDOG_CONF_V2_MARKER = '# mixdog overrides v2 — bgwriter / checkpoint distribution'
 
-// Lines emitted into postgresql.conf for v2. effective_io_concurrency is
-// posix_fadvise-only; PG rejects non-zero values on Windows with an invalid-
-// parameter error on every reload. Skip it there.
+// Lines emitted into postgresql.conf for v2. effective_io_concurrency requires
+// posix_fadvise; PostgreSQL rejects non-zero values on Windows and macOS.
 function buildV2Block() {
   const lines = [
     '',
@@ -126,17 +118,16 @@ function buildV2Block() {
     'bgwriter_lru_maxpages = 1000',        // up to 1000 dirty pages per round (default 100)
     'bgwriter_delay = 50ms',               // wake bgwriter every 50ms (default 200ms)
   ]
-  if (process.platform !== 'win32') {
-    lines.push('effective_io_concurrency = 32')  // I/O concurrency hint (POSIX only)
+  if (process.platform === 'linux') {
+    lines.push('effective_io_concurrency = 32')
   }
   return lines.join('\n') + '\n'
 }
 
-// Migration: an earlier v2 always emitted effective_io_concurrency. On Windows
-// that line is invalid — strip it so pg_ctl reload stops logging a parse error
-// every cycle.
-function stripWindowsInvalidLines(conf) {
-  if (process.platform !== 'win32') return conf
+// Migration: an earlier v2 emitted effective_io_concurrency on every POSIX
+// platform. Strip it anywhere except Linux so old macOS clusters can start.
+function stripPlatformInvalidLines(conf) {
+  if (process.platform === 'linux') return conf
   return conf.replace(/^effective_io_concurrency\s*=\s*\d+\s*\r?\n/m, '')
 }
 
@@ -144,7 +135,7 @@ function ensureConfV2(pgdataDir) {
   const confPath = join(pgdataDir, 'postgresql.conf')
   if (!existsSync(confPath)) return false
   const original = readFileSync(confPath, 'utf8')
-  let conf = stripWindowsInvalidLines(original)
+  let conf = stripPlatformInvalidLines(original)
   const stripped = conf !== original
   const hasMarker = conf.includes(MIXDOG_CONF_V2_MARKER)
   if (hasMarker && !stripped) return false
@@ -185,10 +176,6 @@ export async function startPg({
 }) {
   mkdirSync(pgdataDir, { recursive: true })
 
-  if (process.platform === 'darwin') {
-    try { writeFileSync(join(pgdataDir, '.metadata_never_index'), '') } catch {}
-  }
-
   // Idempotent v2 conf reconcile — runs before attach/init. Returns true if
   // the block was just appended (so the attach path can trigger pg_ctl reload).
   // Fresh-init path falls through to confAppend below which already includes v2.
@@ -282,6 +269,12 @@ export async function startPg({
       )
     }
   }
+  // PostgreSQL requires a completely empty target on first init. Creating the
+  // Spotlight marker before initdb makes every fresh macOS cluster fail with
+  // "directory exists but is not empty"; add it only after initialization.
+  if (process.platform === 'darwin') {
+    try { writeFileSync(join(pgdataDir, '.metadata_never_index'), '') } catch {}
+  }
 
   // Choose a free port (guards against stale postmaster from prior crash).
   const port    = await findFreePort(preferredPort)
@@ -350,7 +343,7 @@ export async function startPg({
     const deadline = Date.now() + 30_000
     // eslint-disable-next-line no-constant-condition
     while (true) {
-      if (pgIsReady(runtimeDir, env, port)) {
+      if (await healthcheckPg({ port })) {
         const pid = await confirmPid()
         // pid confirmed with matching port → ready. Otherwise keep polling until
         // the cap (postmaster.pid not yet written or port mismatch).
@@ -383,7 +376,14 @@ export async function startPg({
     }
   }
 
-  const detail = errText || (r.timeout ? '(readiness probe timed out after 30s; no pg_ctl output)' : '(no captured output)')
+  let serverLog = ''
+  try { serverLog = readFileSync(logFile, 'utf8').trim() } catch {}
+  const detail = [
+    errText.trim(),
+    serverLog ? `postgres log:\n${serverLog}` : '',
+  ].filter(Boolean).join('\n') || (
+    r.timeout ? '(readiness probe timed out after 30s; no pg_ctl or postgres log output)' : '(no captured output)'
+  )
   throw new Error(`[pg-process] pg_ctl start failed: ${detail}`)
 }