dsh-hypatia-auto-memory 0.1.0

package/src/shelf.js

@@ -0,0 +1,180 @@
+/**
+ * The shelf auto-memory writes to, and what follows from being able to change it.
+ *
+ * Every hypatia call the plugin makes targets one shelf, read from
+ * `hypatia-auto-memory.shelf` at startup. It is a startup setting — like
+ * `enabled` — because a queue already holding work for one shelf must not
+ * carry it into another halfway through.
+ *
+ * Changing it is a switch, not a migration. The progress and task tables are
+ * this plugin's record of how far each session was written *into a shelf*, so
+ * they are kept per shelf: switching to a new shelf starts its sessions from
+ * zero there, and switching back resumes exactly where the old shelf's
+ * watermarks stopped — nothing is re-logged or re-consolidated into a shelf
+ * that already has it. Rows of the `default` shelf keep their original,
+ * unprefixed keys, so a profile that never changes the setting reads the
+ * tables exactly as before.
+ *
+ * @module dsh-hypatia-auto-memory/shelf
+ */
+
+/** Shelf hypatia itself falls back to when no `--shelf` is given. */
+export const DEFAULT_SHELF = 'default'
+
+/**
+ * Key prefix of a non-default shelf's rows. Legacy keys are session ids and
+ * `<kind>:<sessionId>` task ids, neither of which can start with `@`.
+ */
+const PREFIX = '@shelf/'
+
+function prefixFor(shelf) {
+  return shelf === DEFAULT_SHELF ? '' : `${PREFIX}${encodeURIComponent(shelf)}/`
+}
+
+/**
+ * A storage-domain table narrowed to one shelf's rows.
+ *
+ * Exposes the subset of the table API this plugin uses (`get`, `put`,
+ * `update`, `delete`, `entries`) with keys as callers know them. Create ONE
+ * view per table and share it: `advanceProgress` chains writes per table
+ * object, and two views of one table would not see each other's chain.
+ *
+ * @param {any} table - storage-domain table.
+ * @param {string} shelf - shelf whose rows the view reads and writes.
+ */
+export function shelfTable(table, shelf) {
+  const prefix = prefixFor(shelf)
+  const own = (key) => (prefix === '' ? !key.startsWith(PREFIX) : key.startsWith(prefix))
+  return {
+    get: (key) => table.get(prefix + key),
+    put: (key, value) => table.put(prefix + key, value),
+    update: (key, fn) => table.update(prefix + key, fn),
+    delete: (key) => table.delete(prefix + key),
+    * entries() {
+      for (const [key, value] of table.entries()) {
+        if (own(key)) yield [key.slice(prefix.length), value]
+      }
+    },
+  }
+}
+
+/**
+ * Parse `hypatia list`.
+ *
+ * The CLI prints one aligned row per registered shelf —
+ * `  <name>  <path>  [connected]` — or `No shelves registered.`. A name has no
+ * spaces (it is one clap argument), while a path may, so the name is the first
+ * token, the status the last, and the path everything between.
+ *
+ * @param {string} stdout
+ * @returns {{name: string, path: string, connected: boolean}[]}
+ */
+export function parseShelfList(stdout) {
+  const shelves = []
+  for (const line of stdout.split('\n')) {
+    const match = /^\s*(\S+)\s+(.*?)\s+\[(connected|disconnected)\]\s*$/.exec(line)
+    if (match === null) continue
+    shelves.push({ name: match[1], path: match[2], connected: match[3] === 'connected' })
+  }
+  return shelves
+}
+
+/** How often the Host re-reads `hypatia list` for the settings card. */
+const INVENTORY_INTERVAL_MS = 60_000
+
+/**
+ * Publish `hypatia list` to the settings card through the read-only
+ * inventory namespace (see `INVENTORY_NAMESPACE` in config.js).
+ *
+ * The namespace is owned by a child fiber whose composition layer IS the
+ * listing, so publishing a new listing means disposing that fiber and
+ * registering again. That only happens when the listing changed; an unchanged
+ * poll spawns one CLI call and nothing else. A failed listing keeps the last
+ * shelves it had and says why, so the card can tell "no shelves" from "could
+ * not ask".
+ *
+ * @param {{
+ *   ctx: import('@deepseek-ai/cordis').Context,
+ *   cli: { listShelves: () => Promise<{name: string, path: string, connected: boolean}[]> },
+ *   status: import('./status.js').StatusLog,
+ *   namespace: string,
+ *   schema: any,
+ *   label: string,
+ *   intervalMs?: number,
+ *   now?: () => number,
+ * }} deps
+ * @returns {{refresh: () => Promise<{shelves: any[], error: string}>}}
+ */
+export function publishShelfInventory({ ctx, cli, status, namespace, schema, label, intervalMs = INVENTORY_INTERVAL_MS, now = Date.now }) {
+  let fiber
+  let published = ''
+  let listing = { shelves: [], error: '' }
+  let chain = Promise.resolve()
+  let stopped = false
+  // A failure that repeats every poll is logged once, not once a minute.
+  let publishError = ''
+  const publishFailed = (error) => {
+    const message = String(error)
+    if (message !== publishError) status.warn(`shelf inventory not published: ${message}`)
+    publishError = message
+  }
+
+  async function publish() {
+    let next
+    try {
+      next = { shelves: await cli.listShelves(), error: '' }
+    } catch (error) {
+      next = { shelves: listing.shelves, error: error instanceof Error ? error.message : String(error) }
+    }
+    if (stopped) return listing
+    if (next.error !== '' && next.error !== listing.error) status.warn(`shelf listing failed: ${next.error}`)
+    listing = next
+    const key = JSON.stringify(next)
+    if (key === published) return listing
+    // Claimed up front so an overlapping poll does not publish twice; given
+    // back on any failure below, so the next poll retries the same listing.
+    published = key
+    const base = { ...next, listedAt: now() }
+    try {
+      const previous = fiber
+      fiber = undefined
+      await previous?.dispose()
+      // The profile may be unloading: the parent fiber refuses new children
+      // before the effect that sets `stopped` has run.
+      if (stopped) return listing
+      fiber = ctx.plugin({
+        name: `${label}/shelf-inventory`,
+        inject: ['settings'],
+        apply: (c) => {
+          try {
+            c.settings.register(namespace, schema, { base })
+            publishError = ''
+          } catch (error) {
+            published = ''
+            publishFailed(error)
+          }
+        },
+      })
+    } catch (error) {
+      published = ''
+      if (!stopped) publishFailed(error)
+    }
+    return listing
+  }
+
+  /** Serialized; never rejects, so callers may drop the promise. */
+  function refresh() {
+    const next = chain.then(publish, publish).catch(() => listing)
+    chain = next
+    return next
+  }
+
+  const timer = setInterval(() => { void refresh() }, intervalMs)
+  timer.unref?.()
+  ctx.effect(() => () => {
+    stopped = true
+    clearInterval(timer)
+  }, `${label}: stop shelf inventory`)
+
+  return { refresh }
+}

package/src/skills.js

@@ -0,0 +1,166 @@
+/**
+ * Bundled skill packaging.
+ *
+ * Three skills ship with the plugin:
+ *
+ *   hypatia-memory  this plugin's own variant — the automatic layer does the
+ *                   writing, so the skill is about retrieval and explicit
+ *                   remember/forget, not about logging messages by hand.
+ *   hypatia         verbatim copy of hypatia's CLI reference.
+ *   hypatia-dream   verbatim copy of hypatia's consolidation pass.
+ *
+ * The last two used to arrive with `dsh-hypatia`. This plugin replaces it, so
+ * it carries them itself — otherwise removing `dsh-hypatia` from a profile
+ * would take the CLI reference away with it. Both are vendored by hand from
+ * the hypatia repository's `skills/`; nothing refreshes them automatically.
+ *
+ * @module dsh-hypatia-auto-memory/skills
+ */
+
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+/**
+ * Minimal YAML-frontmatter reader for flat `key: value` pairs — the grammar
+ * every shipped SKILL.md uses.
+ * @param {string} source
+ * @returns {{attributes: Record<string, string>, body: string}}
+ */
+export function parseFrontmatter(source) {
+  const match = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/.exec(source)
+  if (match === null) return { attributes: {}, body: source }
+  const attributes = {}
+  for (const line of match[1].split('\n')) {
+    const pair = /^([A-Za-z-]+):\s*(.*)$/.exec(line)
+    if (pair !== null) attributes[pair[1]] = pair[2].trim()
+  }
+  return { attributes, body: source.slice(match[0].length) }
+}
+
+/** Provider name DSH's filesystem skill provider registers under by default. */
+const FILESYSTEM_PROVIDER = 'filesystem'
+
+/**
+ * What kind of holder an already-registered copy of a skill has.
+ *
+ * - `disk`: a SKILL.md the filesystem provider found — in a project, user,
+ *   custom or bundled root.
+ * - `runtime`: another plugin's registration (dsh-hypatia), or a provider this
+ *   code does not know.
+ *
+ * The distinction matters because the two lose to this plugin differently.
+ * Runtime registrations in one layer are first-come: the registry keeps the
+ * first and ignores a second. A disk copy is subject to DSH's layering: the
+ * skill registry is layered per scope, the nearest layer's same-name entry
+ * wins outright, and rank (project 100/200 < runtime 250 < custom 300 < user
+ * 400/500 < bundled 600) only breaks ties inside one layer. Agent presets —
+ * `st`, `standard`, `cordis`, … — mount their own `skill-filesystem` into the
+ * preset's layer, nearer to the agent than this plugin's global registration.
+ * So in an agent session any disk copy wins, whatever its rank. Seen live:
+ * with `~/.agents/skills/hypatia-dream` present, a session loaded that copy
+ * while the global catalog (what the skill center shows) listed this plugin.
+ *
+ * @param {{provider?: string}} existing
+ * @returns {'disk' | 'runtime'}
+ */
+export function holderKind(existing) {
+  return existing?.provider === FILESYSTEM_PROVIDER ? 'disk' : 'runtime'
+}
+
+/** Directory of an already-registered copy, when it lives on disk. */
+function directoryOf(existing) {
+  if (existing?.resourceBase?.kind === 'directory' && typeof existing.resourceBase.path === 'string') {
+    return existing.resourceBase.path
+  }
+  return typeof existing?.path === 'string' ? dirname(existing.path) : undefined
+}
+
+/**
+ * Register every packaged skill, and say so loudly whenever another copy of
+ * the same name will reach the agent instead.
+ *
+ * Another plugin's registration is yielded to: the registry would ignore this
+ * one anyway. A disk copy is registered over — harmless, and it makes this
+ * plugin's copy the one any global view sees — but it still wins in agent
+ * sessions (see `holderKind`), so the warning names its directory. That
+ * matters most for `hypatia-memory`: `hypatia skill install --agent codex`
+ * writes the canonical one to `~/.agents/skills`, and it is the agent-driven
+ * protocol, which needs host hooks DSH does not have.
+ *
+ * @param {import('@deepseek-ai/cordis').Context} ctx - context injecting `skills`.
+ * @param {string} skillsDir - directory of `<name>/SKILL.md` folders.
+ * @param {import('./status.js').StatusLog} status
+ * @param {string} provider - provider tag recorded on each registration.
+ * @returns {Promise<string[]>} names actually registered.
+ */
+export async function registerSkills(ctx, skillsDir, status, provider) {
+  let entries
+  try {
+    entries = readdirSync(skillsDir, { withFileTypes: true })
+  } catch {
+    status.warn(`skills dir unreadable: ${skillsDir}`)
+    return []
+  }
+  const registered = []
+  for (const entry of entries) {
+    if (entry.isDirectory() === false) continue
+    const skillFile = join(skillsDir, entry.name, 'SKILL.md')
+    if (existsSync(skillFile) === false) continue
+    // Guarded per skill: one unreadable file or one registry rejection used to
+    // throw out of the loop, so every skill after it — a set decided by readdir
+    // order, not by anything the user could reason about — was silently never
+    // registered, with one warning naming only the first casualty.
+    try {
+      const { attributes, body } = parseFrontmatter(readFileSync(skillFile, 'utf8'))
+      const skillName = attributes.name ?? entry.name
+      const existing = await ctx.skills.get(skillName).catch(() => undefined)
+      if (existing !== undefined && existing.provider !== provider) {
+        const why = skillName === 'hypatia-memory'
+          ? ' — the agent-driven memory protocol, which needs host hooks DSH does not have'
+          : ''
+        if (holderKind(existing) === 'runtime') {
+          const holder = existing.source ? `${existing.provider} (${existing.source})` : existing.provider
+          const advice = existing.provider === 'dsh-hypatia'
+            ? 'remove dsh-hypatia from the profile (`dsh plugin --profile <name> remove dsh-hypatia`) '
+              + 'or set `skills: false` on it'
+            : `disable the copy ${existing.provider} registers`
+          status.warn(
+            `skill "${skillName}" is already registered by ${holder}, and DSH keeps the first runtime `
+            + `registration, so the agent gets that copy${why}. To use this plugin's copy, ${advice}`,
+          )
+          continue
+        }
+        const where = directoryOf(existing) ?? 'a skill directory'
+        status.warn(
+          `skill "${skillName}": the ${existing.source ?? 'disk'} copy at ${where} will be used in agent `
+          + `sessions instead of this plugin's${why} — agent presets load skills from disk in a layer `
+          + 'nearer to the agent than plugin registrations. Delete it, or move it out of that skills '
+          + 'directory, to use this plugin\'s copy — renaming it in place is not enough, because DSH '
+          + 'names a skill by its frontmatter `name`, not its directory',
+        )
+      }
+      ctx.skills.register({
+        name: skillName,
+        description: attributes.description ?? '',
+        content: body,
+        path: skillFile,
+        source: 'bundled',
+        provider,
+        // A TAGGED UNION, not a path: DSH validates the loaded skill against
+        // `{kind:'directory',path} | {kind:'url',url} | {kind:'opaque'}` and a
+        // bare string fails that check at LOAD time, not registration — the
+        // skill appears in the registry and every attempt to read it comes back
+        // as `"value.resourceBase" must match exactly one oneOf branch
+        // (matched 0)` — what the agent hit on its first two calls once this
+        // plugin took over the skill names from dsh-hypatia.
+        resourceBase: { kind: 'directory', path: dirname(skillFile) },
+        invocation: { modelInvocable: true, userInvocable: attributes['user-invocable'] !== 'false' },
+      })
+      registered.push(skillName)
+    } catch (error) {
+      status.warn(`skill "${entry.name}" not registered: ${String(error)}`)
+    }
+  }
+  if (registered.length > 0) status.info(`registered bundled skills: ${registered.join(', ')}`)
+  return registered
+}

package/src/state.js

@@ -0,0 +1,87 @@
+/**
+ * Durable state for dsh-hypatia-auto-memory: one storageDomain domain with a
+ * per-session progress watermark table and a task table. Reads are synchronous
+ * from in-memory state, writes resolve only after the backend acknowledges
+ * durability, so the queue never acknowledges work that a crash could lose.
+ *
+ * @module dsh-hypatia-auto-memory/state
+ */
+
+import { defineDomain, domainTable } from '@deepseek-ai/dsh-storage-domain'
+import { z } from 'zod'
+
+export { EMPTY_PROGRESS, advanceProgress } from './progress.js'
+
+/** Per-session watermark: how far logging and consolidation have advanced. */
+export const progressTable = domainTable(z.object({
+  /** Last session-log seq written to hypatia as a msg-* entry. */
+  lastLoggedSeq: z.number().step(1).min(0).default(0),
+  /** Last session-log seq covered by a consolidation run. */
+  lastConsolidatedSeq: z.number().step(1).min(0).default(0),
+  /** Turn when consolidation work was last durably accepted by the queue. */
+  lastCheckTurn: z.number().step(1).min(0).default(0),
+  /** Estimated unconsolidated tokens accumulated since that accepted task. */
+  pendingTokens: z.number().min(0).default(0),
+  /**
+   * 1 once a `session-<id>` entry exists. The protocol builds that node from a
+   * host-supplied session summary (a title, or a compaction summary) and
+   * forbids inventing one, so most sessions never grow it — the flag is what
+   * lets the log path skip the `belongTo` work rather than probing hypatia for
+   * an entry that will not be there.
+   */
+  hasSessionNode: z.number().step(1).min(0).max(1).default(0),
+  /** Message ordinal up to which `belongTo` edges have been asserted. */
+  lastBelongToIndex: z.number().step(1).min(0).default(0),
+}))
+
+/**
+ * One queued unit of work. `kind` is code-validated ('log-message' |
+ * 'consolidate' | 'cascade' | 'session-node'); keeping it a plain string here
+ * avoids schema-enum drift with the storage format. Tasks store seq ranges
+ * only — content is always re-read from the authoritative session log at
+ * execution time, never duplicated here.
+ */
+export const taskTable = domainTable(z.object({
+  kind: z.string(),
+  sessionId: z.string(),
+  /** Half-open [fromSeq, toSeq) session-log range this task covers. */
+  fromSeq: z.number().step(1).min(0),
+  toSeq: z.number().step(1).min(0),
+  /** Project scope the entries are written under. */
+  project: z.string().default(''),
+  /** pending | running | deferred | failed (done tasks are deleted). */
+  status: z.string().default('pending'),
+  attempts: z.number().step(1).min(0).default(0),
+  // Nullable for legacy rows holding null, and DEFAULTED for rows that lack the
+  // key: `persistTask` did not write it, and the storage service validates on
+  // read rather than on write. One such row still in the table at a restart made
+  // the whole domain refuse to open, which silently disabled the plugin for
+  // that run — nothing collected, nothing resumed, one line in a log.
+  error: z.string().nullable().default(null),
+  enqueuedAt: z.number().default(0),
+}))
+
+/** The plugin's whole durable surface: two tables, versioned together.
+ * Domain names must match /^[a-z][a-z0-9_]*$/ — underscores, not hyphens. */
+export const domainSpec = defineDomain({
+  name: 'hypatia_auto_memory',
+  version: 1,
+  tables: { progress: progressTable, tasks: taskTable },
+})
+
+/**
+ * Open the plugin domain through the storageDomain service and wire its
+ * lifecycle to the calling fiber.
+ *
+ * @param {import('@deepseek-ai/cordis').Context} ctx - context injecting `storageDomain`.
+ * @returns {Promise<{progress: any, tasks: any, close: () => Promise<void>}>}
+ */
+export async function openState(ctx) {
+  const domain = await ctx.storageDomain.open(domainSpec)
+  ctx.effect(() => () => {
+    void domain.close().catch(() => {
+      // Closing during teardown must never take the fiber down.
+    })
+  }, 'hypatia-auto-memory: close state domain')
+  return { progress: domain.table('progress'), tasks: domain.table('tasks'), domain }
+}

package/src/status.js

@@ -0,0 +1,53 @@
+/**
+ * Aggregated operational status: structured log lines plus counters that the
+ * README's ops checklist (and any future settings card) can read. Deliberately
+ * in-memory — durable facts (watermarks, failed tasks) live in the state
+ * domain; this is the live dashboard layer.
+ *
+ * @module dsh-hypatia-auto-memory/status
+ */
+
+export function createStatus(ctx, label = 'hypatia-auto-memory') {
+  let log
+  const counters = new Map()
+  let lastError = ''
+  let lastConsolidatedAt = 0
+
+  const logger = () => {
+    if (log === undefined) {
+      try {
+        log = ctx.logger(label)
+      } catch {
+        log = { info: () => {}, warn: () => {}, error: () => {} }
+      }
+    }
+    return log
+  }
+
+  return {
+    info(message) {
+      logger().info(message)
+    },
+    warn(message) {
+      logger().warn(message)
+    },
+    error(message, error) {
+      lastError = error instanceof Error ? `${message}: ${error.message}` : `${message}: ${String(error)}`
+      logger().error(lastError)
+    },
+    /** Bump one counter ('written' | 'duplicate' | 'failed' | ...). */
+    count(name, by = 1) {
+      counters.set(name, (counters.get(name) ?? 0) + by)
+    },
+    markConsolidated() {
+      lastConsolidatedAt = Date.now()
+    },
+    snapshot() {
+      return {
+        counters: Object.fromEntries(counters),
+        lastError,
+        lastConsolidatedAt,
+      }
+    },
+  }
+}