dsh-hypatia-auto-memory 0.1.0

package/src/content-policy.js

@@ -0,0 +1,301 @@
+/**
+ * Write-time content policy — pure functions, fully unit-testable.
+ *
+ * Implements the hypatia-memory protocol's write transforms:
+ * secret redaction, relative-to-absolute dates, assistant-message capping,
+ * error one-lining, slug sanitizing, and the chars/4 token estimator.
+ *
+ * @module dsh-hypatia-auto-memory/content-policy
+ */
+
+/**
+ * One redaction rule: pattern plus the label used in the placeholder.
+ *
+ * `keepKey` rules capture four groups — (key)(separator)(quote)(value) — so the
+ * replacement can keep the readable key and any quoting while masking only the
+ * value. The `(?!\[REDACTED)` guard on those values stops a later rule from
+ * re-masking a placeholder an earlier rule already wrote.
+ */
+const SECRET_RULES = [
+  { label: 'secret:key', re: /\bsk-[A-Za-z0-9_-]{8,}\b/g },
+  // JWTs: three base64url segments, the first always starting `eyJ` ('{"').
+  { label: 'secret:jwt', re: /\beyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\b/g },
+  { label: 'secret:bearer', re: /\bBearer\s+[A-Za-z0-9._~+/=-]{8,}\b/gi },
+  // Catch-all for the schemes `secret:bearer` does not cover (Basic, Token, …).
+  // The value runs to end of line, not to the first space: stopping at `\S+`
+  // would mask only the scheme word and leave the credential itself in place.
+  // Runs after `secret:bearer` so the more specific label wins, and the
+  // `(?!\[REDACTED)` guard then makes this a no-op on what that rule wrote.
+  { label: 'secret:authorization', re: /\b(Authorization)(\s*:\s*)()((?!\[REDACTED)[^\s\n][^\n]{3,})/gi, keepKey: true },
+  { label: 'secret:aws', re: /\b(?:AKIA|ASIA)[A-Z0-9]{12,}\b/g },
+  { label: 'secret:github', re: /\bgh[pousr]_[A-Za-z0-9]{8,}\b/g },
+  { label: 'secret:gitlab', re: /\bglpat-[A-Za-z0-9_-]{8,}\b/g },
+  { label: 'secret:slack', re: /\bxox[baprs]-[A-Za-z0-9-]{6,}\b/g },
+  // Accepts `=` or `:` and an optional matching quote pair, so `password="s3cret"`
+  // and `token: s3cret` redact as readily as the bare `token=s3cret` form.
+  // The lookbehind is load-bearing: placeholders are spelled
+  // `[REDACTED:secret:<label>]`, and without it this rule matches the `secret:`
+  // inside one an earlier rule already wrote, nesting placeholders.
+  {
+    label: 'secret:credential',
+    re: /(?<!\[REDACTED:)\b(apiKey|api_key|password|passwd|pwd|token|secret|access_token|refresh_token|client_secret)(\s*[=:]\s*)(["']?)((?!\[REDACTED)[^\s&'",;]{4,})\3/gi,
+    keepKey: true,
+  },
+  { label: 'secret:pem', re: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g },
+]
+
+/**
+ * Redact known credential shapes. Replaces each match with a labeled
+ * placeholder so the surrounding text stays readable and the redaction is
+ * itself auditable. Pure: returns a new string.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function redactSecrets(text) {
+  let out = text
+  for (const rule of SECRET_RULES) {
+    const placeholder = `[REDACTED:${rule.label}]`
+    out = rule.keepKey === true
+      ? out.replace(rule.re, (_match, key, separator, quote) => `${key}${separator}${quote}${placeholder}${quote}`)
+      : out.replace(rule.re, placeholder)
+  }
+  return out
+}
+
+const pad = (n) => String(n).padStart(2, '0')
+
+/** YYYY-MM-DD in local time. */
+export function formatDate(d) {
+  return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`
+}
+
+/** YYYY-MM-DD HH:mm in local time. */
+export function formatDateTime(d) {
+  return `${formatDate(d)} ${pad(d.getHours())}:${pad(d.getMinutes())}`
+}
+
+const DAY_MS = 86_400_000
+const HOUR_MS = 3_600_000
+const MINUTE_MS = 60_000
+
+/**
+ * The moment an event happened, as the base for resolving relative dates in it.
+ *
+ * "明天" means the day after the message was SENT, not the day after it was
+ * written to memory. The two differ by minutes when a span waits for its turn
+ * to end, and by days when a task is deferred until its session is reopened —
+ * absolutizing against write time would then record the wrong date outright.
+ *
+ * @param {{time?: number} | undefined} event
+ * @param {Date} fallback - used when the event carries no usable timestamp.
+ * @returns {Date}
+ */
+export function eventDate(event, fallback) {
+  return typeof event?.time === 'number' && Number.isFinite(event.time) ? new Date(event.time) : fallback
+}
+
+/** Monday of the ISO week containing `d`. */
+function weekMonday(d) {
+  const copy = new Date(d.getTime())
+  copy.setHours(0, 0, 0, 0)
+  copy.setDate(copy.getDate() - ((copy.getDay() + 6) % 7))
+  return copy
+}
+
+function dayOffset(now, days, withTime) {
+  const d = new Date(now.getTime() + days * DAY_MS)
+  return withTime ? formatDateTime(d) : formatDate(d)
+}
+
+function unitOffset(now, count, unitMs, withTime) {
+  return formatDateTime(new Date(now.getTime() + count * unitMs))
+}
+
+/**
+ * Convert relative date/time expressions to absolute ones against `now`.
+ * Conservative: only rewrites high-confidence shapes; unknown phrasings pass
+ * through untouched. Handles both Chinese and English expressions.
+ *
+ * @param {string} text
+ * @param {Date} now - the write time everything is absolutized against.
+ * @returns {string}
+ */
+export function absolutizeDates(text, now) {
+  const thisMonday = weekMonday(now)
+  const monday = (weeks) => formatDate(new Date(thisMonday.getTime() + weeks * 7 * DAY_MS))
+  let out = text
+
+  out = out
+    .replace(/今天/g, formatDate(now))
+    .replace(/昨天/g, () => dayOffset(now, -1))
+    .replace(/明天/g, () => dayOffset(now, 1))
+    .replace(/上周/g, () => `the week of ${monday(-1)}`)
+    .replace(/本周|这周/g, () => `the week of ${monday(0)}`)
+    .replace(/下周/g, () => `the week of ${monday(1)}`)
+    .replace(/刚才|现在/g, () => formatDateTime(now))
+    .replace(/(\d{1,4})\s*天\s*(前)/g, (_m, n) => formatDate(new Date(now.getTime() - Number(n) * DAY_MS)))
+    .replace(/(\d{1,4})\s*天\s*(后)/g, (_m, n) => formatDate(new Date(now.getTime() + Number(n) * DAY_MS)))
+    .replace(/(\d{1,4})\s*小时\s*(前)/g, (_m, n) => unitOffset(now, -Number(n), HOUR_MS))
+    .replace(/(\d{1,4})\s*小时\s*(后)/g, (_m, n) => unitOffset(now, Number(n), HOUR_MS))
+    .replace(/(\d{1,4})\s*分钟\s*(前)/g, (_m, n) => unitOffset(now, -Number(n), MINUTE_MS))
+    .replace(/(\d{1,4})\s*分钟\s*(后)/g, (_m, n) => unitOffset(now, Number(n), MINUTE_MS))
+    .replace(/\btoday\b/gi, () => formatDate(now))
+    .replace(/\byesterday\b/gi, () => dayOffset(now, -1))
+    .replace(/\btomorrow\b/gi, () => dayOffset(now, 1))
+    .replace(/\b(\d{1,4})\s+days?\s+ago\b/gi, (_m, n) => formatDate(new Date(now.getTime() - Number(n) * DAY_MS)))
+    .replace(/\bin\s+(\d{1,4})\s+days?\b/gi, (_m, n) => formatDate(new Date(now.getTime() + Number(n) * DAY_MS)))
+    .replace(/\b(\d{1,4})\s+hours?\s+ago\b/gi, (_m, n) => unitOffset(now, -Number(n), HOUR_MS))
+    .replace(/\b(\d{1,4})\s+minutes?\s+ago\b/gi, (_m, n) => unitOffset(now, -Number(n), MINUTE_MS))
+
+  return out
+}
+
+/** Stack-trace line prefixes stripped by oneLineError. */
+const STACK_LINE = /^\s*(at\s|Traceback|File ".*", line \d+|stack backtrace:|note:|Caused by:|===\s*$|\.\.\.\s*$)/
+
+/**
+ * Collapse an error/tool output to a single-line description: drop stack
+ * frames, keep the first meaningful line, cap the length.
+ *
+ * @param {string} text
+ * @param {number} [cap] - maximum length of the result.
+ * @returns {string}
+ */
+export function oneLineError(text, cap = 200) {
+  const lines = String(text).split(/\r?\n/)
+  let first = ''
+  let last = ''
+  for (const line of lines) {
+    if (STACK_LINE.test(line)) continue
+    const trimmed = line.trim()
+    if (trimmed === '') continue
+    if (first === '') first = trimmed
+    last = trimmed
+  }
+  const chosen = first === last || last === '' ? first : `${first} → ${last}`
+  return chosen.length > cap ? `${chosen.slice(0, cap - 1)}…` : chosen
+}
+
+/**
+ * Sanitize a model-proposed or derived entry name: lowercase, collapse
+ * non-alphanumeric (keeping CJK) runs to single dashes, trim, cap length.
+ *
+ * @param {string} name
+ * @param {number} [cap]
+ * @returns {string}
+ */
+export function sanitizeSlug(name, cap = 60) {
+  const slug = String(name)
+    .toLowerCase()
+    .replace(/[^a-z0-9\u4e00-\u9fff]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, cap)
+    .replace(/-+$/, '')
+  return slug === '' ? 'memory' : slug
+}
+
+/**
+ * Project scope of a session whose directory name is empty — one at `/`. No
+ * directory name contains a slash, so it can never be some real project's.
+ */
+export const ROOT_PROJECT = '/'
+
+/** Unicode White_Space, which Rust's `str::trim` strips — and so hypatia, from each scope. */
+const HYPATIA_TRIM = /^[\t-\r \u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000]+|[\t-\r \u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000]+$/g
+
+/**
+ * Trim `text` as hypatia trims a scope or tag. `String.prototype.trim` is not
+ * the same set: it also strips U+FEFF, which hypatia keeps, and keeps U+0085,
+ * which hypatia strips.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function trimLikeHypatia(text) {
+  return text.replace(HYPATIA_TRIM, '')
+}
+
+/**
+ * The project scope for a directory name, in a form hypatia stores exactly
+ * as queried.
+ *
+ * Every entry is written with `scopes: [project]` and found again with
+ * `["$contains", "scopes", project]`, an exact array-membership test. hypatia
+ * splits a written scope list on commas and trims each item, and stores `""`
+ * as no scope at all, so these names were stored as something the name's own
+ * queries miss: `a,b` as `["a", "b"]` (found under `a` and `b` instead),
+ * `foo,` as `["foo", ""]` (global as well), ` padded ` as `padded`, and the
+ * empty name of `/` as no scope at all. Messages were still summarized — consolidation reads the session log —
+ * but the cascade above the first tier and recall filter by scope and passed
+ * them by. Any other name comes back unchanged, so no scope already in use
+ * moves.
+ *
+ * @param {string} name
+ * @returns {string}
+ */
+export function projectScope(name) {
+  const scope = trimLikeHypatia(String(name ?? '').replaceAll(',', '_'))
+  return scope === '' ? ROOT_PROJECT : scope
+}
+
+/**
+ * Cap assistant message text at `maxChars`, leaving an explicit truncation
+ * marker so later readers know the entry is partial.
+ *
+ * @param {string} text
+ * @param {number} maxChars
+ * @returns {string}
+ */
+export function capAssistant(text, maxChars) {
+  if (text.length <= maxChars) return text
+  const omitted = text.length - maxChars
+  return `${text.slice(0, maxChars)}\n\n[...truncated ${omitted} chars]`
+}
+
+/** The protocol's agent-side token estimator: chars / 4, rounded up. */
+export function estimateTokens(text) {
+  return Math.ceil(String(text).length / 4)
+}
+
+/**
+ * Flatten a session ContentBlock[] to plain text for logging: text blocks
+ * verbatim, images marked, tool-call blocks dropped, other block types
+ * summarized by their type name.
+ *
+ * `tool-result` blocks nest the real payload one level down (a ToolResultBlock
+ * carries its own ContentBlock[]), so they are recursed into rather than
+ * summarized — reading only the outer level yields the empty string for every
+ * tool outcome, which is what the ledger and the consolidation transcript used
+ * to record.
+ *
+ * @param {readonly any[]} blocks
+ * @returns {string}
+ */
+export function blocksToText(blocks) {
+  const parts = []
+  for (const block of blocks ?? []) {
+    if (block.type === 'text') parts.push(block.text)
+    else if (block.type === 'image') parts.push('[image]')
+    else if (block.type === 'thinking') parts.push(`[thinking: ${String(block.text ?? '').slice(0, 200)}]`)
+    else if (block.type === 'tool-result') parts.push(blocksToText(block.content ?? []))
+    // A tool-call block is the model asking for a tool, not something it said.
+    // Rendered as `[tool-call]` it filled the content of half the stored
+    // assistant entries with placeholders; the ledger already lists the calls.
+    else if (block.type === 'tool-call') continue
+    else parts.push(`[${block.type}]`)
+  }
+  return parts.join('\n')
+}
+
+/**
+ * Model-facing text of one `tool/result` event. The payload is a
+ * ToolResultMessage whose single `tool-result` block wraps the tool's own
+ * blocks; `blocksToText` unwraps that nesting.
+ *
+ * @param {any} event - a `tool/result` session event.
+ * @returns {string}
+ */
+export function flattenToolResult(event) {
+  return blocksToText(event?.data?.message?.content ?? [])
+}

package/src/housekeeping.js

@@ -0,0 +1,197 @@
+/**
+ * Startup housekeeping for the progress and task tables.
+ *
+ * A progress row is this plugin's only record of how far a session has been
+ * logged and consolidated. It lives in DSH's storage, not in hypatia, so the two
+ * can drift apart: a shelf that is replaced or wiped leaves every row claiming
+ * work that no longer exists (a live profile had 7 of its 10 rows in that
+ * state), and a session that disappears from DSH leaves a row, and possibly
+ * tasks, that nothing will ever need again.
+ *
+ * Both passes are deliberately conservative, because resetting a row is not
+ * free: the session is re-logged from the start (writes are get-before-create,
+ * so no duplicates, but one CLI round trip per message) and re-consolidated
+ * from the start (a fresh summary under a new name, and a model call). So a row
+ * is only touched on POSITIVE evidence, and any doubt — a failed query, an
+ * empty session listing — leaves it alone.
+ *
+ * @module dsh-hypatia-auto-memory/housekeeping
+ */
+
+import { projectScope } from './content-policy.js'
+import { EMPTY_PROGRESS, advanceProgress } from './progress.js'
+
+/**
+ * Reset every row whose session has no `msg-*` entry left in the shelf.
+ *
+ * Positive evidence only: a row is reset when the watermark says messages were
+ * logged AND the shelf query succeeded AND it returned nothing for that
+ * session. A query that fails proves nothing about the shelf — hypatia could be
+ * missing, or its database down — so that row is left exactly as it was.
+ *
+ * Trade-off, by design: a session whose messages were ALL deliberately deleted
+ * looks the same as one whose shelf was wiped, and will be logged again the next
+ * time it is active. Disable with `housekeeping.reconcileOnStartup: false`.
+ *
+ * @param {{
+ *   progress: any,
+ *   cli: { query: (jse: string) => Promise<any[]> },
+ *   status: import('./status.js').StatusLog,
+ * }} deps
+ * @returns {Promise<{checked: number, reset: string[], unverified: number}>}
+ */
+export async function reconcileProgress({ progress, cli, status }) {
+  if (typeof progress.entries !== 'function') return { checked: 0, reset: [], unverified: 0 }
+  const reset = []
+  let checked = 0
+  let unverified = 0
+  // Snapshot first: resetting rows while iterating the live table would be
+  // iterating a collection that is being written.
+  for (const [sessionId, row] of [...progress.entries()]) {
+    if ((row?.lastLoggedSeq ?? 0) <= 0) continue
+    checked += 1
+    let rows
+    try {
+      rows = await cli.query(JSON.stringify({ $knowledge: [['$like', 'name', `msg-${sessionId}-%`]], limit: 1 }))
+    } catch (error) {
+      unverified += 1
+      status.warn(`progress reconcile skipped ${sessionId}: shelf query failed (${String(error)})`)
+      continue
+    }
+    if (rows.length > 0) continue
+    await advanceProgress(progress, sessionId, () => ({ ...EMPTY_PROGRESS }))
+    reset.push(sessionId)
+  }
+  if (reset.length > 0) {
+    status.info(`progress reconcile: reset ${reset.length} session(s) with no entries left in the shelf; they re-log on their next activity`)
+  }
+  return { checked, reset, unverified }
+}
+
+/**
+ * Remove the progress row and tasks of every session DSH no longer knows.
+ *
+ * `knownSessionIds` must be DSH's full persisted listing. An EMPTY listing is
+ * treated as "unknown", never as "no sessions exist": a persistence layer that
+ * has not finished loading, or failed quietly, would otherwise delete every row
+ * in one pass. Sessions currently live are always kept.
+ *
+ * @param {{
+ *   progress: any,
+ *   queue: { sessionIds: () => Set<string>, forgetSession: (id: string) => Promise<number> },
+ *   knownSessionIds: Set<string>,
+ *   isLive?: (sessionId: string) => boolean,
+ *   status: import('./status.js').StatusLog,
+ * }} deps
+ * @returns {Promise<{removed: string[], skipped: boolean}>}
+ */
+export async function pruneVanishedSessions({ progress, queue, knownSessionIds, isLive = () => false, status }) {
+  if (knownSessionIds.size === 0) {
+    status.warn('progress prune skipped: DSH reported no persisted sessions, which is not proof that none exist')
+    return { removed: [], skipped: true }
+  }
+  const candidates = new Set(typeof progress.entries === 'function' ? [...progress.entries()].map(([id]) => id) : [])
+  for (const id of queue.sessionIds()) candidates.add(id)
+
+  const removed = []
+  for (const sessionId of candidates) {
+    if (knownSessionIds.has(sessionId) || isLive(sessionId)) continue
+    await queue.forgetSession(sessionId)
+    if (progress.get(sessionId) !== undefined) await progress.delete(sessionId)
+    removed.push(sessionId)
+  }
+  if (removed.length > 0) {
+    status.info(`progress prune: removed ${removed.length} session(s) DSH no longer has`)
+  }
+  return { removed, skipped: false }
+}
+
+/**
+ * Consolidate a logged tail that the in-session trigger never reached.
+ *
+ * The session-end trigger cannot survive a process restart. DSH does run its
+ * close path at shutdown — it appends `session/end-seed` to the open session —
+ * but nothing this plugin enqueues there becomes durable before the process is
+ * gone: measured on a live restart, the storage file was not written at all and
+ * the session came back with `lastConsolidatedSeq: 0`. A short session that
+ * never reached the turn/token thresholds thus produced no knowledge, which is
+ * exactly the case the session-end trigger was added for.
+ *
+ * Gated on the SAME floor as the live trigger and read from the row, not by
+ * loading the session: `pendingTokens` is the estimate accumulated up to the
+ * last turn that ended, so a tail too small to be worth a model call while the
+ * session was alive stays too small here. Without that gate every restart would
+ * spend one model call per session carrying any tail at all.
+ *
+ * A session DSH no longer lists is skipped (`cwdFor` returns undefined) — its
+ * row belongs to {@link pruneVanishedSessions}. One that is listed but not
+ * loaded is fine: the executor defers the task until that session is opened.
+ *
+ * @param {{
+ *   progress: any,
+ *   queue: { enqueue: (spec: any) => Promise<void> },
+ *   cwdFor: (sessionId: string) => string | undefined,
+ *   projectForCwd: (cwd: string) => Promise<string>,
+ *   minNewTokens: number,
+ *   status: import('./status.js').StatusLog,
+ * }} deps
+ * @returns {Promise<{enqueued: string[], skippedBelowFloor: number}>}
+ */
+export async function backfillConsolidation({ progress, queue, cwdFor, projectForCwd, minNewTokens, status }) {
+  if (typeof progress.entries !== 'function') return { enqueued: [], skippedBelowFloor: 0 }
+  const enqueued = []
+  let skippedBelowFloor = 0
+  for (const [sessionId, row] of [...progress.entries()]) {
+    const fromSeq = row?.lastConsolidatedSeq ?? 0
+    const toSeq = row?.lastLoggedSeq ?? 0
+    if (toSeq <= fromSeq) continue
+    if ((row?.pendingTokens ?? 0) < minNewTokens) {
+      skippedBelowFloor += 1
+      continue
+    }
+    const cwd = cwdFor(sessionId)
+    if (cwd === undefined) continue
+    const project = await projectForCwd(cwd)
+    await queue.enqueue({ kind: 'consolidate', sessionId, fromSeq, toSeq, project, immediate: true })
+    enqueued.push(sessionId)
+  }
+  if (enqueued.length > 0) {
+    status.info(`consolidation backfill: queued ${enqueued.length} session(s) whose logged tail was never consolidated`)
+  }
+  return { enqueued, skippedBelowFloor }
+}
+
+/**
+ * Rewrite each persisted task's project to the scope `projectScope` makes of
+ * it.
+ *
+ * A task carries the project it was queued under, and its executor writes with
+ * it as stored. One queued before project names became scopes — for a session
+ * at `/`, or in a directory whose name has a comma — would otherwise write one
+ * more span that no query of its project finds. Every kind of task is queued
+ * with a project, so an empty one can only be the `/` case. Run before any
+ * executor is registered, so no task is read mid-rewrite.
+ *
+ * @param {{tasks: any, status: import('./status.js').StatusLog}} deps
+ * @returns {Promise<number>} how many tasks were rewritten.
+ */
+export async function normalizeTaskProjects({ tasks, status }) {
+  if (typeof tasks.entries !== 'function') return 0
+  let rewritten = 0
+  for (const [id, record] of [...tasks.entries()]) {
+    // A failed task never runs again; it is kept, or pruned, as it stands.
+    if (typeof record?.project !== 'string' || record.status === 'failed') continue
+    const scope = projectScope(record.project)
+    if (scope === record.project) continue
+    try {
+      await tasks.update(id, (task) => ({ ...task, project: scope }))
+      rewritten += 1
+    } catch (error) {
+      // Best effort: the task then writes one span under the old name, which
+      // is what it would have done anyway. Not a reason to stop collecting.
+      status.warn(`could not rewrite the project of task ${id}: ${String(error)}`)
+    }
+  }
+  if (rewritten > 0) status.info(`rewrote the project of ${rewritten} queued task(s) to a scope hypatia stores as given`)
+  return rewritten
+}