mixdog 0.8.1 → 0.9.0

package/src/runtime/memory/lib/memory.mjs

@@ -254,23 +254,77 @@ export async function init(db, dims) {
   )
 }
 
-// Validate that the halfvec column dimension stored in the DB matches
-// dimCount from the current model config. Call after schema is confirmed
-// complete and before any embedding operations.
-export async function validateEmbeddingDims(db, dimCount) {
+async function getEmbeddingColumnDims(db, tableName) {
   const r = await db.query(`
-    SELECT atttypmod
+    SELECT a.atttypmod
     FROM pg_attribute a
     JOIN pg_class c ON c.oid = a.attrelid
-    WHERE c.relname = 'entries'
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = current_schema()
+      AND c.relname = $1
       AND a.attname = 'embedding'
       AND a.attnum > 0
       AND NOT a.attisdropped
-  `)
+  `, [tableName])
   const row = r.rows[0]
-  if (!row) return // column absent — pre-schema DB; bootstrapSchema will handle
+  return row ? Number(row.atttypmod) : null
+}
+
+async function ensureHotActiveSearchObjects(db) {
+  await db.exec(`
+    CREATE MATERIALIZED VIEW IF NOT EXISTS mv_hot_active AS
+    SELECT id, element, summary, category, status, score, last_seen_at, promoted_at,
+           project_id, embedding, search_tsv
+    FROM entries
+    WHERE is_root = 1 AND status = 'active' AND embedding IS NOT NULL
+    WITH NO DATA
+  `)
+  await db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS mv_hot_active_id ON mv_hot_active(id)`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS mv_hot_active_hnsw ON mv_hot_active USING hnsw (embedding halfvec_cosine_ops)`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS mv_hot_active_tsv  ON mv_hot_active USING GIN (search_tsv)`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS mv_hot_active_score ON mv_hot_active(score DESC)`)
+}
+
+async function resetEmbeddingColumnsForDims(db, dimCount) {
+  const entriesDims = await getEmbeddingColumnDims(db, 'entries')
+  const coreDims = await getEmbeddingColumnDims(db, 'core_entries')
+  const needsEntriesReset = entriesDims != null && entriesDims !== dimCount
+  const needsCoreReset = coreDims != null && coreDims !== dimCount
+  if (!needsEntriesReset && !needsCoreReset) return false
+
+  __mixdogMemoryLog(
+    `[memory] embedding dimension changed; resetting vectors for halfvec(${dimCount}) ` +
+    `(entries=${entriesDims ?? 'missing'}, core_entries=${coreDims ?? 'missing'})\n`,
+  )
+
+  await db.exec(`DROP MATERIALIZED VIEW IF EXISTS mv_hot_active CASCADE`)
+  await db.exec(`DROP INDEX IF EXISTS idx_entries_embedding_hnsw`)
+  await db.exec(`DROP INDEX IF EXISTS core_entries_embedding_hnsw`)
+
+  if (needsEntriesReset) {
+    await db.exec(`ALTER TABLE entries ALTER COLUMN embedding TYPE halfvec(${dimCount}) USING NULL::halfvec(${dimCount})`)
+    await db.exec(`UPDATE entries SET summary_hash = NULL WHERE summary_hash IS NOT NULL`)
+  }
+  if (needsCoreReset) {
+    await db.exec(`ALTER TABLE core_entries ALTER COLUMN embedding TYPE halfvec(${dimCount}) USING NULL::halfvec(${dimCount})`)
+  }
+
+  await db.exec(`DROP TABLE IF EXISTS memory.embedding_cache`)
+  await db.query(
+    `INSERT INTO meta(key, value) VALUES ($1, $2::jsonb)
+     ON CONFLICT(key) DO UPDATE SET value = EXCLUDED.value`,
+    ['embedding.current_dims', JSON.stringify(dimCount)],
+  )
+  return true
+}
+
+// Validate that the halfvec column dimension stored in the DB matches
+// dimCount from the current model config. Call after schema is confirmed
+// complete and before any embedding operations.
+export async function validateEmbeddingDims(db, dimCount) {
+  const colDims = await getEmbeddingColumnDims(db, 'entries')
+  if (colDims == null) return // column absent — pre-schema DB; bootstrapSchema will handle
   // pgvector halfvec stores dimension as atttypmod directly (unlike varchar which uses dims+4).
-  const colDims = row.atttypmod
   if (colDims !== dimCount) {
     throw new Error(
       `Embedding dimension mismatch: DB column halfvec(${colDims}) vs model config ${dimCount} dims. ` +
@@ -285,6 +339,7 @@ export async function ensureCurrentSchemaExtensions(db, dims) {
   // idempotent and define the current runtime schema.
   if (Number.isInteger(dims) && dims > 0) {
     await db.exec(`ALTER TABLE core_entries ADD COLUMN IF NOT EXISTS embedding halfvec(${dims})`)
+    await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_embedding_hnsw ON entries USING hnsw (embedding halfvec_cosine_ops) WHERE is_root = 1 AND embedding IS NOT NULL`)
     await db.exec(`CREATE INDEX IF NOT EXISTS core_entries_embedding_hnsw ON core_entries USING hnsw (embedding halfvec_cosine_ops) WHERE embedding IS NOT NULL`)
   }
   await db.exec(`ALTER TABLE entries ADD COLUMN IF NOT EXISTS core_summary text`)
@@ -319,6 +374,16 @@ export async function ensureCurrentSchemaExtensions(db, dims) {
     __mixdogMemoryLog(`[memory] ensureCurrentSchemaExtensions: core_entries_unique_proj_elem creation failed — duplicate rows must be deduplicated before this index can be created: ${err?.message || err}\n`)
     throw err
   }
+
+  if (Number.isInteger(dims) && dims > 0) {
+    await db.query(
+      `INSERT INTO meta(key, value) VALUES ($1, $2::jsonb)
+       ON CONFLICT(key) DO UPDATE SET value = EXCLUDED.value`,
+      ['embedding.current_dims', JSON.stringify(dims)],
+    )
+  }
+
+  await ensureHotActiveSearchObjects(db)
 }
 
 export async function openDatabase(dataDir, dims) {
@@ -346,6 +411,9 @@ export async function openDatabase(dataDir, dims) {
         }
       })
     }
+    if (await isBootstrapComplete(db)) {
+      await resetEmbeddingColumnsForDims(db, Number(dims))
+    }
     await ensureCurrentSchemaExtensions(db, Number(dims))
     await validateEmbeddingDims(db, Number(dims))
 
@@ -412,6 +480,17 @@ export async function setMetaValue(db, key, value) {
   )
 }
 
+// Shallow-merge patch into an existing JSON object meta row (jsonb ||). Avoids
+// read-modify-write lost updates when concurrent writers touch different keys.
+export async function mergeMetaValue(db, key, patch) {
+  const patchJson = typeof patch === 'string' ? patch : JSON.stringify(patch ?? {})
+  await db.query(
+    `INSERT INTO meta(key, value) VALUES ($1, $2::jsonb)
+     ON CONFLICT(key) DO UPDATE SET value = COALESCE(meta.value, '{}'::jsonb) || EXCLUDED.value`,
+    [key, patchJson],
+  )
+}
+
 export function embeddingToSql(arr) {
   if (!arr || !Array.isArray(arr)) return null
   return `[${arr.map((n) => Number(n).toFixed(6)).join(',')}]`

package/src/runtime/memory/lib/model-profile.mjs

@@ -4,7 +4,7 @@
  * Writes one JSONL line per measurement point to
  * `history/model-profile.jsonl` (mirrors `retrieval-trace.jsonl`). Meant for
  * the embedding provider. Fire-and-forget: any I/O error is swallowed so
- * instrumentation never breaks a live bridge or memory-cycle call.
+ * instrumentation never breaks a live agent or memory-cycle call.
  *
  * Schema:
  *   {

package/src/runtime/memory/lib/pg/adapter.mjs

@@ -86,7 +86,7 @@ async function _initClient(client, schema) {
 }
 
 // Wrapper around pool.connect() that runs per-client init before returning.
-// Exported so external callers (e.g. trace-store.insertBridgeCalls) can obtain
+// Exported so external callers (e.g. trace-store.insertAgentCalls) can obtain
 // a connection with search_path already set; raw _pool.connect() leaves it at
 // the PG default and unqualified table lookups resolve in the wrong schema.
 export async function checkedConnect(pgPool, schema) {

package/src/runtime/memory/lib/pg/supervisor.mjs

@@ -64,6 +64,15 @@ const LOCK_POLL_MS     = 100;
 const LOCK_WARN_MS     = 5_000;
 const LOCK_WAIT_CODES  = new Set(['EEXIST', 'EPERM', 'EACCES', 'EBUSY']);
 
+function envFlagEnabled(name) {
+  const raw = String(process.env[name] ?? '').trim().toLowerCase();
+  return raw === '1' || raw === 'true' || raw === 'on' || raw === 'yes';
+}
+
+function attachOnlyMode() {
+  return envFlagEnabled('MIXDOG_PG_ATTACH_ONLY') || envFlagEnabled('MIXDOG_MEMORY_SECONDARY');
+}
+
 // ── File lock (O_EXCL pattern from dispatch-persist.mjs / run-mcp.mjs) ───────
 
 /**
@@ -356,6 +365,9 @@ async function _doEnsure(dataDir) {
     source: 'prelock',
   });
   if (prelockReuse) return prelockReuse;
+  if (attachOnlyMode()) {
+    throw new Error('secondary memory runtime requires an existing PG instance')
+  }
 
   // ── Acquire spawn lock to serialize initdb races ─────────────────────────
   const acquired = await acquireSpawnLock(dataDir, () => tryReusePgInstance({

package/src/runtime/memory/lib/runtime-fetcher.mjs

@@ -37,6 +37,7 @@ const BUNDLED_MANIFEST_PATH = fileURLToPath(new URL('../data/runtime-manifest.js
 
 // GitHub raw URL fallback — used only when no cached or bundled manifest exists.
 const MANIFEST_URL = 'https://raw.githubusercontent.com/tribgames/mixdog/main/src/runtime/memory/data/runtime-manifest.json'
+const LEGACY_RUNTIME_RELEASE_REPOSITORY = 'trib-plugin/mixdog'
 
 // ---------------------------------------------------------------------------
 // Platform key
@@ -154,6 +155,26 @@ async function downloadWithRetry(url, destPath) {
   throw lastErr
 }
 
+function runtimeAssetUrlCandidates(url) {
+  const value = String(url || '')
+  const out = []
+  const add = (candidate) => {
+    if (candidate && !out.includes(candidate)) out.push(candidate)
+  }
+  add(value)
+  const match = /^https:\/\/github\.com\/([^/]+\/[^/]+)\/releases\/download\//i.exec(value)
+  if (!match) return out
+  const releaseRepo = match[1]
+  const overrides = String(process.env.MIXDOG_RUNTIME_RELEASE_REPOSITORY || process.env.RUNTIME_RELEASE_REPOSITORY || '')
+    .split(/[\s,;]+/u)
+    .map((repo) => repo.trim())
+    .filter(Boolean)
+  for (const repo of [...overrides, LEGACY_RUNTIME_RELEASE_REPOSITORY]) {
+    if (repo && repo !== releaseRepo) add(value.replace(`/github.com/${releaseRepo}/`, `/github.com/${repo}/`))
+  }
+  return out
+}
+
 // ---------------------------------------------------------------------------
 // Tar entry path validation + extraction
 // ---------------------------------------------------------------------------
@@ -412,10 +433,23 @@ export async function ensureRuntime(dataDir) {
   const stagingLock = acquireStagingLock(stagingDir)
   try {
     let downloadOk = false
+    let lastDownloadErr = null
     try {
-      await downloadWithRetry(url, tarPath)
-      await verifySha256(tarPath, sha256)
-      downloadOk = true
+      for (const candidateUrl of runtimeAssetUrlCandidates(url)) {
+        try {
+          if (candidateUrl !== url) {
+            __mixdogMemoryLog(`[runtime-fetcher] retrying runtime download from fallback release repo — ${candidateUrl}\n`)
+          }
+          await downloadWithRetry(candidateUrl, tarPath)
+          await verifySha256(tarPath, sha256)
+          downloadOk = true
+          break
+        } catch (err) {
+          lastDownloadErr = err
+          try { rmSync(tarPath, { force: true }) } catch {}
+        }
+      }
+      if (!downloadOk && lastDownloadErr) throw lastDownloadErr
       extractTarGz(tarPath, stagingDir, stagingDir)
     } finally {
       try { rmSync(tarPath, { force: true }) } catch {}

package/src/runtime/memory/lib/session-ingest.mjs

@@ -0,0 +1,194 @@
+import crypto from 'node:crypto'
+
+// Side-effect-free helpers for ingest_session (recall fast-track hydration).
+// Extracted from memory/index.mjs so the pure logic (stable identity, sensitive
+// redaction, role/content shaping) can be unit-tested without importing the
+// MCP server entrypoint and its heavy boot-time side effects.
+
+// Roles we persist from an in-memory session transcript. Map provider/runtime
+// role spellings onto the canonical set; unknown roles are dropped.
+const INGEST_SESSION_ROLES = new Set(['user', 'assistant', 'tool', 'system', 'developer'])
+
+export function normalizeIngestRole(role) {
+  const raw = String(role || '').trim().toLowerCase()
+  if (!raw) return null
+  if (raw === 'tool_result' || raw === 'function' || raw === 'tool-result') return 'tool'
+  if (raw === 'human') return 'user'
+  if (raw === 'ai' || raw === 'model') return 'assistant'
+  return INGEST_SESSION_ROLES.has(raw) ? raw : null
+}
+
+// Extract the first textual content block from a message content field.
+export function firstTextContent(content) {
+  if (typeof content === 'string') return content
+  if (!Array.isArray(content)) return ''
+  for (const item of content) {
+    if (typeof item === 'string') return item
+    if (item?.type === 'text' && typeof item.text === 'string') return item.text
+  }
+  return ''
+}
+
+// Collect durable tool-call / tool-result ids for identity + pairing.
+function toolIdentityIds(m) {
+  const ids = []
+  if (Array.isArray(m?.toolCalls)) {
+    for (const tc of m.toolCalls) { if (tc?.id) ids.push(String(tc.id)) }
+  }
+  if (m?.toolCallId) ids.push(String(m.toolCallId))
+  return ids
+}
+
+// Build a stable, content-derived source_ref for a session message so the
+// ON CONFLICT DO NOTHING insert is idempotent across compaction reindexing.
+//
+// Identity hashes ONLY durable/original fields: role, tool-call/result ids, the
+// ORIGINAL message timestamp (m.ts / m.timestamp) when actually present, and the
+// shaped content. It never folds in a synthesized Date.now() fallback ts or the
+// volatile loop index, so the same untimestamped message produces the same ref
+// regardless of its position after compaction shrinks/reindexes the transcript.
+// Two textually identical untimestamped plain messages intentionally dedupe to
+// one row (stable dedupe preferred over positional separation).
+export function stableSessionSourceRef(sessionId, m, role, content) {
+  const toolIds = toolIdentityIds(m)
+  // Only an ORIGINAL, caller-supplied timestamp counts as durable identity.
+  const rawTs = m?.ts ?? m?.timestamp
+  const originalTs = (typeof rawTs === 'number' && Number.isFinite(rawTs)) || (typeof rawTs === 'string' && rawTs.trim())
+    ? String(rawTs)
+    : ''
+  const identity = [role, originalTs, toolIds.join(','), content].join('\u0000')
+  const hash = crypto.createHash('sha256').update(identity).digest('hex').slice(0, 24)
+  return `session:${sessionId}:${hash}`
+}
+
+// Monotonic source_turn allocator for ingest_session. source_turn must be a
+// running continuation order INDEPENDENT of the current (post-compaction)
+// array index: a freshly appended turn must sort AFTER every previously
+// ingested row, even though compaction reset its array index to a low value.
+// Seed `prevMaxTurn` from MAX(source_turn) for the session, then call
+// next() once per actually-inserted row. Re-ingested (ON CONFLICT) rows keep
+// their original turn and must NOT advance the counter, so the caller only
+// invokes next() when a row was truly inserted.
+export function createIngestTurnAllocator(prevMaxTurn = 0) {
+  let turn = Number.isFinite(Number(prevMaxTurn)) ? Math.max(0, Math.floor(Number(prevMaxTurn))) : 0
+  return {
+    peekNext() { return turn + 1 },
+    next() { turn += 1; return turn },
+    current() { return turn },
+  }
+}
+
+// Sensitive tool-arg key matcher (mirrors compact.mjs SENSITIVE_TOOL_ARG_KEY_RE):
+// api_key/authorization/auth/cookie/credential/password/secret/token, etc.
+const SENSITIVE_ARG_KEY_RE = /(?:^|[_-])(?:api[_-]?key|authorization|auth|cookie|credential|passwd|password|refresh[_-]?token|secret|token)(?:$|[_-])/i
+const SENSITIVE_KEY_WORD = '(?:api[_-]?key|authorization|auth|cookie|credential|passwd|password|refresh[_-]?token|secret|token)'
+// Full key matcher: the sensitive WORD may carry a prefix and/or suffix segment
+// joined by `_`/`-` so prefixed variants like `access_token`, `access-token`,
+// `x-api-key`, and `bearer_token` are matched as whole keys (not just the bare
+// word at key start). Mirrors compact.mjs SENSITIVE_TOOL_ARG_KEY_FULL.
+const SENSITIVE_KEY_FULL = `(?:[A-Za-z0-9_-]*[_-])?${SENSITIVE_KEY_WORD}(?:[_-][A-Za-z0-9_-]*)?`
+const TOOL_ARG_MAX_CHARS = 400
+
+// Recursively redact sensitive values by key. Sensitive keys collapse to
+// [redacted] regardless of value shape so nested secrets never survive.
+export function redactToolArgValue(value, key = '', depth = 0) {
+  if (SENSITIVE_ARG_KEY_RE.test(String(key || ''))) return '[redacted]'
+  if (value == null) return value
+  if (typeof value === 'bigint') return String(value)
+  // Defense-in-depth: a non-sensitive KEY can still carry a secret embedded in
+  // its string VALUE (e.g. `{ headers: "authorization: Bearer ..." }`). Run raw
+  // key:value redaction on string values so embedded secrets never survive.
+  if (typeof value === 'string') return redactRawArgString(value)
+  if (typeof value !== 'object') return value
+  if (depth >= 4) return Array.isArray(value) ? `[array:${value.length}]` : '[object]'
+  if (Array.isArray(value)) {
+    return value.slice(0, 8).map((item, idx) => redactToolArgValue(item, String(idx), depth + 1))
+  }
+  const out = {}
+  for (const k of Object.keys(value)) out[k] = redactToolArgValue(value[k], k, depth + 1)
+  return out
+}
+
+// Redact `key: value` / `key=value` pairs inside a raw (non-JSON) arg string.
+// Unlike a single greedy regex, this consumes the WHOLE secret value after the
+// key — including spaces, `Bearer ` prefixes, quoted values with internal
+// spaces, and `;`-separated cookie pairs — so no secret fragment leaks.
+function redactRawArgString(text) {
+  const keyRe = new RegExp(`((?:^|[\\s,{(])["']?${SENSITIVE_KEY_FULL}["']?\\s*[:=]\\s*)`, 'gi')
+  let out = ''
+  let last = 0
+  let match
+  while ((match = keyRe.exec(text)) !== null) {
+    const prefixEnd = match.index + match[0].length
+    out += text.slice(last, prefixEnd)
+    // Determine where the secret value ends. Quoted values run to the closing
+    // quote; unquoted values run to the next separator that ends a field. For
+    // Authorization we also swallow a leading `Bearer `/`Basic ` scheme word so
+    // nothing after it survives.
+    let i = prefixEnd
+    const quote = text[i] === '"' || text[i] === "'" ? text[i] : ''
+    if (quote) {
+      i += 1
+      while (i < text.length && text[i] !== quote) i += 1
+      if (i < text.length) i += 1 // include closing quote
+    } else {
+      // Unquoted: stop at a comma, closing brace/paren, or newline. Spaces and
+      // `;` inside the value are part of the secret (Bearer tokens, cookies).
+      while (i < text.length && !/[,)}\n]/.test(text[i])) i += 1
+    }
+    out += '[redacted]'
+    last = i
+    keyRe.lastIndex = i
+  }
+  out += text.slice(last)
+  return out
+}
+
+// Produce a readable, redacted, length-capped string for tool-call arguments.
+// Object/JSON args are walked key-by-key so nested secrets are caught; a raw
+// non-JSON string has its key:value secret pairs redacted before truncation.
+export function redactToolArgString(rawArgs) {
+  if (rawArgs == null) return ''
+  let parsed = rawArgs
+  if (typeof rawArgs === 'string') {
+    const trimmed = rawArgs.trim()
+    if (/^[[{]/.test(trimmed)) {
+      try { parsed = JSON.parse(trimmed) } catch { parsed = trimmed }
+    } else {
+      parsed = trimmed
+    }
+  }
+  let out
+  if (parsed && typeof parsed === 'object') {
+    try { out = JSON.stringify(redactToolArgValue(parsed)) }
+    catch { out = '[unserializable args]' }
+  } else {
+    out = redactRawArgString(String(parsed))
+  }
+  return out.slice(0, TOOL_ARG_MAX_CHARS)
+}
+
+// Build a readable, role-aware content string for a session message so the
+// structured handoff preserves assistant tool_calls and tool_result bodies
+// (not just plain text). Keeps valid tool-call/tool-result pairing legible by
+// tagging each with its toolCallId, while redacting sensitive argument values.
+export function sessionMessageContent(m) {
+  const parts = []
+  const base = firstTextContent(m?.content)
+  if (base && base.trim()) parts.push(base.trim())
+  if (m?.role === 'assistant' && Array.isArray(m.toolCalls) && m.toolCalls.length) {
+    for (const tc of m.toolCalls.slice(0, 8)) {
+      const name = tc?.name || tc?.function?.name || tc?.id || 'tool'
+      const id = tc?.id ? ` id=${tc.id}` : ''
+      const argStr = redactToolArgString(tc?.arguments ?? tc?.function?.arguments)
+      parts.push(`[tool_call ${name}${id}]${argStr ? ` ${argStr}` : ''}`)
+    }
+  }
+  const isTool = m?.role === 'tool' || normalizeIngestRole(m?.role) === 'tool'
+  if (isTool && m?.toolCallId && parts.length) {
+    parts[0] = `[tool_result id=${m.toolCallId}] ${parts[0]}`
+  } else if (isTool && m?.toolCallId) {
+    parts.push(`[tool_result id=${m.toolCallId}]`)
+  }
+  return parts.join('\n')
+}