@agfpd/iapeer-memory-core 0.1.1

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@agfpd/iapeer-memory-core",
+  "version": "0.1.1",
+  "description": "iapeer-memory core — host-neutral TypeScript memory primitive: vault schema/taxonomy config, search engine, memoryd, context renderer, role contracts. Consumed by the @agfpd/iapeer-memory facade; version kept in lockstep by its release flow (docs/10-distribution.md).",
+  "license": "MIT",
+  "type": "module",
+  "files": [
+    "src",
+    "tsconfig.json"
+  ],
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "bun test",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "test -z \"$(git status --porcelain)\" || (echo 'release: working tree is dirty — commit or stash before release' >&2 && exit 1)"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "gray-matter": "^4.0.3",
+    "sqlite-vec": "^0.1.9",
+    "zod": "^4.4.3"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.2.0",
+    "typescript": "^5.7.0"
+  }
+}

package/src/config.ts

@@ -0,0 +1,257 @@
+/**
+ * iapeer-memory core runtime config.
+ *
+ * Re-built from the MergeMind reference `config.ts` for the iapeer-memory
+ * namespace: `IAPEER_MEMORY_*` env vars, taxonomy-driven defaults (ADR-002 —
+ * the locale preset supplies folder names instead of hard-coded constants)
+ * and the `~/.iapeer/{cache,…}/iapeer-memory/` path namespace.
+ *
+ * Embedding/reranker stay pluggable and off-by-default: empty endpoint →
+ * BM25-only, a valid working state. Any endpoint speaking the
+ * OpenAI-compatible `/v1/embeddings` + TEI `/rerank` protocols works; no
+ * provider is hard-coded.
+ *
+ * `callerAgent`: at the MCP layer the caller identity comes from the
+ * `X-IAPeer-Identity` header of the http connection (ADR-012) and takes
+ * precedence; the env var is the fallback for CLI/programmatic consumers
+ * outside harness sessions.
+ */
+
+import { statSync } from "node:fs";
+import { isEmbeddingProvider, type EmbeddingProvider } from "./embedding.js";
+import { isRerankerProvider, type RerankerProvider } from "./reranker.js";
+import {
+  defaultExcludeFolders,
+  getTaxonomy,
+  isLocaleId,
+  DEFAULT_CURATOR_SET,
+  DEFAULT_RANKING,
+  type LocaleId,
+  type RankingConfig,
+  type TaxonomyPreset,
+} from "./taxonomy.js";
+
+export type CoreConfig = {
+  vaultPath: string;
+  locale: LocaleId;
+  taxonomy: TaxonomyPreset;
+  ranking: RankingConfig;
+  excludeFolders: string[];
+  /** ADR-006 curator set — personalities exempt from needs_review stamping. */
+  curatorSet: string[];
+  callerAgent: string | null;
+  search: {
+    chunkSize: number;
+    chunkOverlap: number;
+    maxResults: number;
+    rrfK: number;
+  };
+  index: {
+    dbPath: string;
+    fullScanOnStartup: boolean;
+  };
+  /**
+   * MCP-http endpoint of memoryd (ADR-012). The default port 8766 is the
+   * neighbour of the iapeer foundation MCP (8765) — one ecosystem block,
+   * easy to remember, far from the OS ephemeral ranges. `0` = ephemeral
+   * (tests). The adapters' `.mcp.json` must reference the SAME port.
+   */
+  mcp: {
+    port: number;
+  };
+  embedding: {
+    endpoint: string;
+    model: string;
+    dimensions: number;
+    batchSize: number;
+    apiKey: string | null;
+    /** Wire format (ADR-013); configFromEnv always sets it. */
+    provider?: EmbeddingProvider;
+  } | null;
+  reranker: {
+    endpoint: string;
+    model: string;
+    topK: number;
+    weight: number;
+    apiKey: string | null;
+    /** Wire format (ADR-013); configFromEnv always sets it. */
+    provider?: RerankerProvider;
+  } | null;
+};
+
+/**
+ * Keep an operator-configured LOCAL embedding/reranker host off the egress
+ * proxy. Empirically proven failure mode (inherited from the reference):
+ * the process env may carry HTTP(S)_PROXY; NO_PROXY usually excludes LAN via
+ * CIDR, but Bun's fetch does NOT implement CIDR matching in NO_PROXY (exact
+ * host / suffix only). Result: every embedding/reranker call detours to the
+ * proxy, gets a 500, opens the circuit breaker and search silently degrades
+ * to BM25. Adding the exact endpoint host to NO_PROXY is idempotent and
+ * monotonic — it only bypasses the explicitly configured host.
+ */
+function ensureEndpointNotProxied(endpoint: string): void {
+  let host: string;
+  try {
+    host = new URL(endpoint).hostname;
+  } catch {
+    return; // broken URL — let fetch fail loudly, that's a different misconfig
+  }
+  if (!host) return;
+  for (const varName of ["NO_PROXY", "no_proxy"]) {
+    const current = process.env[varName] ?? "";
+    const tokens = current.split(",").map((s) => s.trim()).filter(Boolean);
+    if (tokens.includes(host)) continue;
+    tokens.push(host);
+    process.env[varName] = tokens.join(",");
+  }
+}
+
+function envString(name: string, fallback = ""): string {
+  const value = process.env[name];
+  return typeof value === "string" && value.length > 0 ? value : fallback;
+}
+
+function envNumber(name: string, fallback: number): number {
+  const value = process.env[name];
+  // "" is treated as unset — harness ${VAR} expansion in .mcp.json renders
+  // missing variables as the empty string, not "undefined".
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const n = Number(value);
+  return Number.isFinite(n) ? n : fallback;
+}
+
+function envBoolean(name: string, fallback: boolean): boolean {
+  const value = process.env[name];
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  return value === "1" || value.toLowerCase() === "true";
+}
+
+function envStringArray(name: string, fallback: string[]): string[] {
+  const value = process.env[name];
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  return value.split(",").map((s) => s.trim()).filter(Boolean);
+}
+
+export function configFromEnv(): CoreConfig {
+  const vaultPath = envString("IAPEER_MEMORY_VAULT_PATH");
+  if (!vaultPath) {
+    throw new Error(
+      "IAPEER_MEMORY_VAULT_PATH is not set. Run init to provision the vault, " +
+      "then point IAPEER_MEMORY_VAULT_PATH at its absolute path in the " +
+      "package config and restart.",
+    );
+  }
+
+  // Validate the vault path resolves to a directory. Without this check the
+  // process would start "healthy" on a typo / unmounted drive / offloaded
+  // iCloud root — search would return empty results and agents would assume
+  // an empty vault. Fail loud at startup beats silent degraded mode.
+  try {
+    const stat = statSync(vaultPath);
+    if (!stat.isDirectory()) {
+      throw new Error(
+        `IAPEER_MEMORY_VAULT_PATH (${vaultPath}) is not a directory.`,
+      );
+    }
+  } catch (err) {
+    const e = err as { code?: string; message?: string };
+    if (e.code === "ENOENT") {
+      throw new Error(
+        `IAPEER_MEMORY_VAULT_PATH (${vaultPath}) does not exist. Check that ` +
+        `the vault is provisioned and the path is correct (including a ` +
+        `possibly offloaded iCloud root).`,
+      );
+    }
+    throw err;
+  }
+
+  const rawLocale = envString("IAPEER_MEMORY_LOCALE", "en");
+  if (!isLocaleId(rawLocale)) {
+    throw new Error(
+      `IAPEER_MEMORY_LOCALE (${rawLocale}) is not a known locale preset ` +
+      `(expected "en" or "ru").`,
+    );
+  }
+  const taxonomy = getTaxonomy(rawLocale);
+
+  const dbPath = envString(
+    "IAPEER_MEMORY_DB_PATH",
+    `${process.env.HOME}/.iapeer/cache/iapeer-memory/index.db`,
+  );
+
+  const embeddingEndpoint = envString("IAPEER_MEMORY_EMBEDDING_ENDPOINT");
+  const rerankerEndpoint = envString("IAPEER_MEMORY_RERANKER_ENDPOINT");
+
+  // Provider enums (ADR-013). Unknown value → throw, exactly like an
+  // unknown locale: fail loud at startup beats a silently dead pipeline.
+  // Defaults preserve the inherited behaviour: embedding "openai"
+  // (the reference wire format), reranker "tei" when an endpoint is set.
+  // Reranker "none" is the explicit OFF switch: the block resolves to null
+  // even with an endpoint configured (the pipeline default state — no
+  // reranker — is simply an empty endpoint, as before).
+  const embeddingProviderRaw = envString("IAPEER_MEMORY_EMBEDDING_PROVIDER", "openai");
+  if (!isEmbeddingProvider(embeddingProviderRaw)) {
+    throw new Error(
+      `IAPEER_MEMORY_EMBEDDING_PROVIDER (${embeddingProviderRaw}) is not a known ` +
+      `provider (expected "tei" or "openai").`,
+    );
+  }
+  const rerankerProviderRaw = envString("IAPEER_MEMORY_RERANKER_PROVIDER", "tei");
+  if (rerankerProviderRaw !== "none" && !isRerankerProvider(rerankerProviderRaw)) {
+    throw new Error(
+      `IAPEER_MEMORY_RERANKER_PROVIDER (${rerankerProviderRaw}) is not a known ` +
+      `provider (expected "tei", "cohere", "nvidia", "jina" or "none").`,
+    );
+  }
+
+  // Strip egress proxying from local endpoints before the first fetch
+  // (Bun fetch does not honour CIDR in NO_PROXY — see ensureEndpointNotProxied).
+  if (embeddingEndpoint) ensureEndpointNotProxied(embeddingEndpoint);
+  if (rerankerEndpoint) ensureEndpointNotProxied(rerankerEndpoint);
+
+  return {
+    vaultPath,
+    locale: rawLocale,
+    taxonomy,
+    ranking: { ...DEFAULT_RANKING },
+    callerAgent: envString("IAPEER_MEMORY_AGENT_NAME") || null,
+    curatorSet: envStringArray("IAPEER_MEMORY_CURATOR_SET", [...DEFAULT_CURATOR_SET]),
+    excludeFolders: envStringArray(
+      "IAPEER_MEMORY_EXCLUDE_FOLDERS",
+      defaultExcludeFolders(taxonomy),
+    ),
+    search: {
+      chunkSize: envNumber("IAPEER_MEMORY_CHUNK_SIZE", 500),
+      chunkOverlap: envNumber("IAPEER_MEMORY_CHUNK_OVERLAP", 80),
+      maxResults: envNumber("IAPEER_MEMORY_MAX_RESULTS", 6),
+      rrfK: envNumber("IAPEER_MEMORY_RRF_K", 60),
+    },
+    index: {
+      dbPath,
+      fullScanOnStartup: envBoolean("IAPEER_MEMORY_FULL_SCAN_ON_STARTUP", true),
+    },
+    mcp: {
+      port: envNumber("IAPEER_MEMORY_MCP_PORT", 8766),
+    },
+    embedding: embeddingEndpoint
+      ? {
+          provider: embeddingProviderRaw,
+          endpoint: embeddingEndpoint,
+          model: envString("IAPEER_MEMORY_EMBEDDING_MODEL", "Qwen/Qwen3-Embedding-8B"),
+          dimensions: envNumber("IAPEER_MEMORY_EMBEDDING_DIMENSIONS", 4096),
+          batchSize: envNumber("IAPEER_MEMORY_EMBEDDING_BATCH_SIZE", 32),
+          apiKey: envString("IAPEER_MEMORY_EMBEDDING_API_KEY") || null,
+        }
+      : null,
+    reranker: rerankerEndpoint && rerankerProviderRaw !== "none"
+      ? {
+          provider: rerankerProviderRaw,
+          endpoint: rerankerEndpoint,
+          model: envString("IAPEER_MEMORY_RERANKER_MODEL", "BAAI/bge-reranker-v2-m3"),
+          topK: envNumber("IAPEER_MEMORY_RERANKER_TOP_K", 20),
+          weight: envNumber("IAPEER_MEMORY_RERANKER_WEIGHT", 0.7),
+          apiKey: envString("IAPEER_MEMORY_RERANKER_API_KEY") || null,
+        }
+      : null,
+  };
+}

package/src/context-render.ts

@@ -0,0 +1,185 @@
+/**
+ * context-render — doctrine-fragment assembly for iapeer layer 5 (ADR-001).
+ *
+ * Carries over the LAYER ASSEMBLY from the reference
+ * `mergemind-build-context-shards.py::build_layers` (order of layers, the
+ * Index branch) and DROPS the entire shard mechanics (est_tokens /
+ * pack_shards / write_shards / k-N markers / MAX_SHARDS padding) — per-hook
+ * limits do not exist on the system-prompt path; the fragment is delivered
+ * whole by the iapeer merge.
+ *
+ * Changes against the reference assembly, all sanctioned:
+ * - the writer guide is NOT a per-peer layer anymore — it is the HOST-WIDE
+ *   fragment (`~/.iapeer/fragments/iapeer-memory.md`), written by the
+ *   package at install/update, one copy for the whole fleet;
+ * - the L2/L3 user-override layers are dropped (нюанс 1): user additions
+ *   ride iapeer's own layer 4 (any *.md in the `.iapeer/` roots);
+ * - `capText` is kept as a parity utility (it capped L2/L3 in the
+ *   reference) but the current assembly has nothing left to cap.
+ *
+ * Layer-5 writer contract (iapeer 0.2.8, docs/05-delivery-and-context.md):
+ * fragment files live in `<root>/.iapeer/fragments/<stem>.md`, the stem is
+ * `iapeer-memory.md`, and writes MUST be atomic (temp + rename in the same
+ * directory) — the fragment is re-read on every cold-wake and may race a
+ * peer waking up.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+
+export const FRAGMENT_STEM = "iapeer-memory.md";
+
+export type ContextLayer = [title: string, body: string];
+
+function readFileOrEmpty(filePath: string): string {
+  try {
+    return fs.readFileSync(filePath, "utf-8");
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * Trim text to maxChars at a line boundary, appending a marker when cut.
+ * Parity utility (capped the user-override layers in the reference);
+ * the marker template takes `{label}` and `{max}` placeholders.
+ */
+export function capText(
+  text: string,
+  maxChars: number,
+  label: string,
+  template = "\n\n_[{label} truncated: over the {max} character limit. Shorten the document.]_",
+): string {
+  if (text.length <= maxChars) return text;
+  let cut = text.lastIndexOf("\n", maxChars);
+  if (cut < Math.floor(maxChars / 2)) cut = maxChars;
+  const trimmed = text.slice(0, cut).replace(/\s+$/, "");
+  return trimmed + template.replace("{label}", label).replace("{max}", String(maxChars));
+}
+
+export type FragmentEnv = {
+  /** Peer personality (resolved by the caller — нюанс 10). */
+  agent: string;
+  /** Personality of the Index curator (its branch has a different layout). */
+  indexAgent?: string;
+  paths: {
+    vault: string;
+    db?: string;
+    config?: string;
+    state: string;
+    cache: string;
+    logs?: string;
+  };
+  /** Rendered author index file (capped variant), absolute path. */
+  authorIndexPath: string;
+  /** Tags-dictionary mirror, absolute path (Index branch only). */
+  tagsDictionaryPath?: string;
+};
+
+/**
+ * Assemble the per-peer fragment layers in the reference build_layers
+ * order. Author branch: paths → author index. Index branch: paths → tags
+ * dictionary → own index (the curator gets no writer guide — its contract
+ * is the role doctrine; the guide arrives host-wide by layer mechanics).
+ * Missing/empty sources are skipped gracefully.
+ */
+export function buildLayers(env: FragmentEnv): ContextLayer[] {
+  const layers: ContextLayer[] = [];
+
+  const p = env.paths;
+  const pathsBlock = [
+    "<iapeer-memory-paths>",
+    `vault: ${p.vault ?? ""}`,
+    `db: ${p.db ?? ""}`,
+    `config: ${p.config ?? ""}`,
+    `state: ${p.state ?? ""}`,
+    `cache: ${p.cache ?? ""}`,
+    `logs: ${p.logs ?? ""}`,
+    "</iapeer-memory-paths>",
+  ].join("\n");
+  layers.push(["iapeer-memory paths", pathsBlock]);
+
+  const isIndex = env.agent === (env.indexAgent ?? "index");
+
+  if (isIndex && env.tagsDictionaryPath) {
+    const tags = readFileOrEmpty(env.tagsDictionaryPath);
+    if (tags.trim()) layers.push([path.basename(env.tagsDictionaryPath), tags]);
+  }
+
+  const idx = readFileOrEmpty(env.authorIndexPath);
+  if (idx.trim()) layers.push([path.basename(env.authorIndexPath), idx]);
+
+  return layers;
+}
+
+/** Render layers into the fragment text: `## title` + body per layer. */
+export function renderFragmentText(layers: ContextLayer[]): string {
+  const parts: string[] = [];
+  for (const [title, body] of layers) {
+    parts.push(`## ${title}\n${body.replace(/\s+$/, "")}`);
+  }
+  return parts.join("\n\n") + (parts.length ? "\n" : "");
+}
+
+/**
+ * Atomic fragment write — temp + rename IN THE SAME DIRECTORY (layer-5
+ * writer contract). Creates the fragments directory when missing.
+ */
+export function writeFragmentAtomic(
+  fragmentsDir: string,
+  text: string,
+  stem: string = FRAGMENT_STEM,
+): string {
+  // 0700 — aligned with the iapeer scaffold's fragments/ mode (author's
+  // review note, stage 7): peer doctrine material is owner-only.
+  fs.mkdirSync(fragmentsDir, { recursive: true, mode: 0o700 });
+  const target = path.join(fragmentsDir, stem);
+  const tmp = path.join(
+    fragmentsDir,
+    `.${stem}.${crypto.randomBytes(6).toString("hex")}.tmp`,
+  );
+  try {
+    fs.writeFileSync(tmp, text, "utf-8");
+    fs.renameSync(tmp, target);
+  } catch (err) {
+    try {
+      if (fs.existsSync(tmp)) fs.unlinkSync(tmp);
+    } catch {
+      // best effort
+    }
+    throw err;
+  }
+  return target;
+}
+
+/** `<peerCwd>/.iapeer/fragments/` — the per-peer fragment directory. */
+export function peerFragmentsDir(peerCwd: string): string {
+  return path.join(peerCwd, ".iapeer", "fragments");
+}
+
+/**
+ * Render and atomically write the per-peer fragment for one peer.
+ * Returns the written path.
+ */
+export function renderPeerFragment(opts: {
+  peerCwd: string;
+  env: FragmentEnv;
+}): string {
+  const text = renderFragmentText(buildLayers(opts.env));
+  return writeFragmentAtomic(peerFragmentsDir(opts.peerCwd), text);
+}
+
+/**
+ * Write the HOST-WIDE guide fragment (`<homeIapeerDir>/fragments/`).
+ * The guide content is a package runtime artifact; this function writes
+ * whatever it is given. CAUTION: pointing homeIapeerDir at the production
+ * `~/.iapeer` reaches EVERY peer of the fleet on their next wakes — the
+ * fleet rollout of the guide is a separately sanctioned release step.
+ */
+export function writeHostWideGuideFragment(
+  homeIapeerDir: string,
+  guideText: string,
+): string {
+  return writeFragmentAtomic(path.join(homeIapeerDir, "fragments"), guideText);
+}