@hiveku-apps/sync 0.4.0 → 0.5.0

package/src/pull.mjs

@@ -1,87 +1,118 @@
 /**
- * `hiveku-sync pull` — fetches memory/rules/skills via the MCP memory_list tool
- * and writes them as local .md files under the current folder.
+ * `hiveku-sync pull` — fetches the account's memory, rules, skills,
+ * shortcuts, specialists and profiles with ONE memory_list call and writes
+ * each entry as a local .md file, filed by its owner (knowledge-files.mjs):
+ *
+ *   rules/sales/<name>.md, skills/seo/<name>.md, memory/helpdesk/<name>.md, ...
+ *   <type folder>/shared/<name>.md for an entry no agent owns
+ *   <type folder>/<owner>/not-followed/<name>.md for an entry filed under an
+ *   owner that no agent reads (the Memory page hides it)
+ *
+ * The owner is the `owner` memory_list returns (the builder's one owner rule),
+ * or the same rule from memory-owner.mjs when an older builder does not return
+ * it. The chief of staff's own `_account:*` rows and Voice and pronunciation
+ * are written where the Memory page shows them, read-only.
+ *
+ * It also refreshes the folder layout and memory rules in CLAUDE.md, the part
+ * between hiveku-sync's markers (claude-md.mjs).
  *
  * The account memory (domains `account` and `account-suggestions`) is not a
  * knowledge entry: it is classified explicitly, never written under memory/,
  * and pulled once from account_memory_get into a read-only
  * hiveku-data/account/ACCOUNT_MEMORY.md (see account-memory.mjs).
  *
+ * A file an earlier pull wrote for an entry this pull files somewhere else
+ * (an older hiveku-sync wrote flat <type>/<name>.md files), or no longer
+ * mirrors, is moved to .hiveku/set-aside/knowledge/<time>/: never deleted.
+ * A file whose entry is gone from Hiveku stays where it is and is listed as
+ * removed. When the listing fails, nothing is moved or reported.
+ *
  * Safety: refuses to overwrite when the .env API key no longer matches the
  * manifest's recorded key preview (prevents pulling Account A's data into a
  * folder previously synced for Account B — the 2026-05-11 cross-tenant bug
  * class).
  */
 
-import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, rename } from 'node:fs/promises';
+import { dirname, join, resolve, sep } from 'node:path';
 
 import { HivekuMcpClient } from './mcp.mjs';
-import {
-  loadDotenv,
-  readEnv,
-  safeSlug,
-  writeFileAtomic,
-  readJson,
-  toFrontmatter,
-  extractDepartmentTag,
-  TYPE_TO_FOLDER,
-  SUPPORTED_TYPES,
-} from './util.mjs';
+import { loadDotenv, readEnv, writeFileAtomic, readJson, SUPPORTED_TYPES } from './util.mjs';
 import {
   ACCOUNT_MEMORY_FILE,
-  classifyMemoryEntry,
   ACCOUNT_MEMORY_TOOL,
   normalizeAccountMemory,
   renderAccountMemoryFile,
   resolveAccountId,
   writeReadOnly,
 } from './account-memory.mjs';
+import { planKnowledgeFiles } from './knowledge-files.mjs';
+import { refreshClaudeMd } from './claude-md.mjs';
 
 const MANIFEST_PATH = '.hiveku/manifest.json';
+export const KNOWLEDGE_SET_ASIDE_DIR = '.hiveku/set-aside/knowledge';
 
 function keyPreview(apiKey) {
   return apiKey.slice(0, 10);
 }
 
-async function fetchType(client, type) {
-  const result = await client.callToolJson('memory_list', { type });
+/** `rel` under `root`, or null when it would land outside it (a manifest path is data, not trusted). */
+function insidePath(root, rel) {
+  const base = resolve(root);
+  const abs = resolve(base, String(rel || '').replace(/^\/+/, ''));
+  return abs.startsWith(base + sep) ? abs : null;
+}
+
+/** Every account-level entry, of every type, in one call. */
+async function fetchAll(client) {
+  const result = await client.callToolJson('memory_list', {});
   if (!result?.data || !Array.isArray(result.data)) {
-    throw new Error(`memory_list(${type}) returned unexpected shape`);
+    throw new Error('memory_list returned unexpected shape');
   }
   return result.data;
 }
 
-function entryFilename(entry) {
-  const name = entry.name || entry.domain.replace(/^_[^:]+:/, '');
-  return `${safeSlug(name)}.md`;
-}
-
-function renderEntry(entry, type) {
-  const body = entry.content ?? '';
-  const department = extractDepartmentTag(body);
-  const frontmatter = toFrontmatter({
-    id: entry.id,
-    name: entry.name,
-    type,
-    domain: entry.domain,
-    department,
-    project_id: entry.project_id,
-    version: entry.version,
-    updated_at: entry.updated_at,
-  });
-  return `${frontmatter}${body}`;
-}
-
 /**
  * Write hiveku-data/account/ACCOUNT_MEMORY.md from account_memory_get, read-only.
  * Returns { file, version, suggestions, accountId }, or throws (the caller
- * records the error and keeps the previous copy).
+ * records the error and keeps the previous copy). Pass `accountId` when it is
+ * already known (null included) to skip looking it up again.
  */
-export async function pullAccountMemory({ client, baseDir, fetchedAt = new Date().toISOString() }) {
+export async function pullAccountMemory({ client, baseDir, fetchedAt = new Date().toISOString(), accountId }) {
   const memory = normalizeAccountMemory(await client.callToolJson(ACCOUNT_MEMORY_TOOL, {}));
-  const accountId = await resolveAccountId(client);
-  await writeReadOnly(join(baseDir, ACCOUNT_MEMORY_FILE), renderAccountMemoryFile({ memory, accountId, fetchedAt }));
-  return { file: ACCOUNT_MEMORY_FILE, version: memory.version, suggestions: memory.suggestions.length, accountId };
+  const id = accountId === undefined ? await resolveAccountId(client) : accountId;
+  await writeReadOnly(join(baseDir, ACCOUNT_MEMORY_FILE), renderAccountMemoryFile({ memory, accountId: id, fetchedAt }));
+  return { file: ACCOUNT_MEMORY_FILE, version: memory.version, suggestions: memory.suggestions.length, accountId: id };
+}
+
+/** The `domain` an earlier pull wrote into a file's front matter, or null. */
+async function domainInFile(path) {
+  try {
+    const text = await readFile(path, 'utf8');
+    const fm = text.match(/^---\n([\s\S]*?)\n---\n/);
+    const line = fm?.[1].match(/^domain: "((?:[^"\\]|\\.)*)"$/m);
+    return line ? line[1].replace(/\\"/g, '"') : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Move files an earlier pull wrote out of the tree, into one timestamped folder. Returns the moved paths. */
+async function setAside(baseDir, relPaths) {
+  if (relPaths.length === 0) return { moved: [], dir: null };
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const dir = join(baseDir, KNOWLEDGE_SET_ASIDE_DIR, stamp);
+  const moved = [];
+  for (const rel of relPaths) {
+    const from = insidePath(baseDir, rel);
+    const to = insidePath(dir, rel);
+    if (!from || !to || !existsSync(from)) continue;
+    await mkdir(dirname(to), { recursive: true });
+    await rename(from, to);
+    moved.push(rel);
+  }
+  return { moved, dir: moved.length ? dir : null };
 }
 
 export async function runPull({ baseDir = process.cwd(), profile, client: injectedClient } = {}) {
@@ -117,35 +148,70 @@ export async function runPull({ baseDir = process.cwd(), profile, client: inject
   const client = injectedClient || new HivekuMcpClient({ baseUrl, apiKey, profile: activeProfile });
   if (!injectedClient) await client.initialize();
 
-  const summary = { total: 0, by_type: {}, errors: [], account_memory: null, account_rows_skipped: 0 };
+  const summary = {
+    total: 0,
+    by_type: {},
+    // Entries an agent reads (or every agent shares), by owner folder.
+    by_owner: {},
+    // Entries filed under <type>/<owner>/not-followed/: no agent reads them.
+    not_followed: 0,
+    errors: [],
+    account_memory: null,
+    account_rows_skipped: 0,
+    not_mirrored: { system: 0, hidden: 0 },
+    set_aside: [],
+    set_aside_dir: null,
+    claude_md: null,
+  };
   const writtenFiles = new Set();
+  const fileDomains = {};
+  const readOnlyFiles = new Set();
+  const previousFiles = Array.isArray(manifest?.files) ? manifest.files : [];
+  const previousDomains = manifest?.file_domains && typeof manifest.file_domains === 'object' ? manifest.file_domains : {};
+
+  // The account id gives the read-only copies their Memory page link.
+  const accountId = await resolveAccountId(client);
+
+  let plan = null;
+  try {
+    plan = planKnowledgeFiles(await fetchAll(client), { accountId });
+  } catch (err) {
+    summary.errors.push({ type: 'memory_list', message: err.message });
+  }
 
-  for (const type of SUPPORTED_TYPES) {
-    const folder = TYPE_TO_FOLDER[type];
-    let entries;
-    try {
-      entries = await fetchType(client, type);
-    } catch (err) {
-      summary.errors.push({ type, message: err.message });
-      continue;
+  if (plan) {
+    for (const type of SUPPORTED_TYPES) summary.by_type[type] = 0;
+    summary.account_rows_skipped = plan.skipped.account;
+    summary.not_mirrored = { system: plan.skipped.system, hidden: plan.skipped.hidden };
+    for (const file of plan.files) {
+      const abs = join(baseDir, file.rel);
+      if (file.readOnly) {
+        await writeReadOnly(abs, file.body);
+        readOnlyFiles.add(file.rel);
+      } else {
+        await writeFileAtomic(abs, file.body);
+      }
+      writtenFiles.add(file.rel);
+      fileDomains[file.rel] = file.domain;
+      summary.by_type[file.type] = (summary.by_type[file.type] ?? 0) + 1;
+      if (file.notFollowed) {
+        summary.not_followed += 1;
+      } else {
+        const owner = file.owner ?? 'shared';
+        summary.by_owner[owner] = (summary.by_owner[owner] ?? 0) + 1;
+      }
+      summary.total += 1;
     }
-    // The two account-memory domains are not knowledge entries: an older
-    // server still lists them, and they would land as memory/account.md and
-    // memory/account-suggestions.md (raw JSON Lines) looking editable. They
-    // are written once, read-only, from account_memory_get below.
-    const knowledge = [];
-    for (const entry of entries) {
-      if (classifyMemoryEntry(entry, folder).kind === 'knowledge') knowledge.push(entry);
-      else summary.account_rows_skipped += 1;
+  } else {
+    // Nothing was listed, so nothing can be called moved or removed: keep
+    // what the last pull recorded, as it was.
+    for (const rel of previousFiles) {
+      if (rel === ACCOUNT_MEMORY_FILE) continue;
+      writtenFiles.add(rel);
+      if (typeof previousDomains[rel] === 'string') fileDomains[rel] = previousDomains[rel];
     }
-    summary.by_type[type] = knowledge.length;
-    summary.total += knowledge.length;
-
-    for (const entry of knowledge) {
-      const filename = entryFilename(entry);
-      const fullPath = join(baseDir, folder, filename);
-      await writeFileAtomic(fullPath, renderEntry(entry, type));
-      writtenFiles.add(`${folder}/${filename}`);
+    for (const rel of Array.isArray(manifest?.read_only) ? manifest.read_only : []) {
+      if (rel !== ACCOUNT_MEMORY_FILE && writtenFiles.has(rel)) readOnlyFiles.add(rel);
     }
   }
 
@@ -153,18 +219,37 @@ export async function runPull({ baseDir = process.cwd(), profile, client: inject
   // A failure keeps the previous copy (and its manifest line), so it is not
   // reported as removed upstream.
   try {
-    summary.account_memory = await pullAccountMemory({ client, baseDir });
+    summary.account_memory = await pullAccountMemory({ client, baseDir, accountId });
     writtenFiles.add(ACCOUNT_MEMORY_FILE);
   } catch (err) {
     summary.errors.push({ type: 'account memory', message: err.message });
-    if ((manifest?.files ?? []).includes(ACCOUNT_MEMORY_FILE)) writtenFiles.add(ACCOUNT_MEMORY_FILE);
+    if (previousFiles.includes(ACCOUNT_MEMORY_FILE)) writtenFiles.add(ACCOUNT_MEMORY_FILE);
   }
 
-  // v1: don't delete stale files; just record them in the manifest so the
-  // user can prune manually if they want. Avoids losing local edits.
-  const previouslySeen = new Set(manifest?.files ?? []);
-  const removedRemotely = [...previouslySeen].filter((f) => !writtenFiles.has(f));
+  // Files the last pull wrote that this one did not. When Hiveku still lists
+  // the entry, its copy is now somewhere else (or no longer mirrored): the
+  // old copy is set aside. Otherwise the entry is gone upstream: the file is
+  // left on disk and reported, never deleted.
+  const notWritten = previousFiles.filter((rel) => !writtenFiles.has(rel));
+  const outOfPlace = [];
+  const removedRemotely = [];
+  for (const rel of notWritten) {
+    const path = insidePath(baseDir, rel);
+    // Already gone from disk (you deleted it): nothing to move, nothing to prune.
+    if (!path || !existsSync(path)) continue;
+    const domain = typeof previousDomains[rel] === 'string' ? previousDomains[rel] : await domainInFile(path);
+    if (plan && domain && plan.listed.has(domain)) outOfPlace.push(rel);
+    else removedRemotely.push(rel);
+  }
+  const tidy = await setAside(baseDir, outOfPlace);
+  summary.set_aside = tidy.moved;
+  summary.set_aside_dir = tidy.dir;
 
+  const sortedDomains = Object.fromEntries(Object.entries(fileDomains).sort(([a], [b]) => (a < b ? -1 : 1)));
+  // Which entry each file holds, when this pull (or the last) knew it. A failed
+  // listing over a folder an older release pulled knows none: leave it out,
+  // so readers (the Codex rules digest) keep reading the files on disk.
+  const knowsDomains = Boolean(plan) || (manifest?.file_domains && typeof manifest.file_domains === 'object');
   await writeFileAtomic(
     manifestPath,
     JSON.stringify(
@@ -173,10 +258,13 @@ export async function runPull({ baseDir = process.cwd(), profile, client: inject
         base_url: baseUrl,
         profile: activeProfile,
         last_sync_at: new Date().toISOString(),
+        // Filed by owner: <type folder>/<owner>/<name>.md (shared/ for no owner).
+        layout: 'by-owner',
         counts: summary.by_type,
         files: [...writtenFiles].sort(),
-        // Pulled for reading only: edited on the dashboard, never uploaded.
-        read_only: [ACCOUNT_MEMORY_FILE],
+        ...(knowsDomains ? { file_domains: sortedDomains } : {}),
+        // Pulled for reading only: changed on the Memory page, never uploaded.
+        read_only: [ACCOUNT_MEMORY_FILE, ...[...readOnlyFiles].sort()],
         removed_remotely_since_last_sync: removedRemotely,
       },
       null,
@@ -184,5 +272,21 @@ export async function runPull({ baseDir = process.cwd(), profile, client: inject
     ),
   );
 
-  return { ...summary, removed_remotely: removedRemotely, profile: activeProfile, baseUrl };
+  // The folder layout and memory rules in CLAUDE.md, which only `init` wrote
+  // before: refreshed, so a folder an older release made stops teaching the
+  // old ones (claude-md.mjs). A failure here is reported, not a failed pull.
+  try {
+    summary.claude_md = await refreshClaudeMd(baseDir);
+  } catch (err) {
+    summary.claude_md = { status: 'failed', message: err.message };
+  }
+
+  let codexLane = false;
+  try {
+    codexLane = (await readFile(join(baseDir, 'AGENTS.md'), 'utf8')).includes('<!-- hiveku:begin -->');
+  } catch {
+    /* no Codex lane */
+  }
+
+  return { ...summary, removed_remotely: removedRemotely, profile: activeProfile, baseUrl, codex_lane: codexLane };
 }

package/src/util.mjs

@@ -87,16 +87,6 @@ export const TYPE_TO_FOLDER = {
 
 export const SUPPORTED_TYPES = Object.keys(TYPE_TO_FOLDER);
 
-/** Extract a department tag from content. Looks for HTML or YAML markers. */
-export function extractDepartmentTag(content) {
-  if (!content) return null;
-  const htmlMatch = content.match(/<!--\s*department:\s*([a-z0-9_-]+)\s*-->/i);
-  if (htmlMatch) return htmlMatch[1];
-  const yamlMatch = content.match(/^department:\s*["']?([a-z0-9_-]+)["']?\s*$/im);
-  if (yamlMatch) return yamlMatch[1];
-  return null;
-}
-
 /** Resolve account base directory (default: ./). */
 export function defaultBaseDir(cwd = process.cwd()) {
   return cwd;