@isparling/engram-cli 0.1.0

package/src/markdownRecord.ts

@@ -0,0 +1,265 @@
+// Hand-written parser/serializer for the Markdown record shape:
+//
+//   ---
+//   id: <id>
+//   title: <title>
+//   updated: <YYYY-MM-DD>
+//   ---
+//
+//   ## Active claims
+//
+//   - claim text [source: tag]
+//
+//   ## Evidence log
+//
+//   - YYYY-MM-DD — entry text [source: tag]
+//
+// `## Active claims` and `## Evidence log` are the two sections this
+// module understands and can mutate. Any OTHER `## ...` section in the
+// body is preserved verbatim, in its original position, through parse and
+// serialize — a record is free to carry a `## Notes` section (or anything
+// else) that is never touched. So is any PREAMBLE: free-form content
+// between the frontmatter's closing `---` and the first `##` heading.
+// Losing either on write would be a silent, ungated destructive edit —
+// exactly what the additive/non-additive gate exists to prevent, and
+// exactly what happened here before preambleLines existed: content before
+// the first heading was parsed and then never re-emitted.
+
+import { err, ok, requireDefined, type Result } from "./types.ts";
+
+export type RecordFrontmatter = {
+  id: string;
+  title: string;
+  updated: string;
+};
+
+export type KnownSection =
+  | { kind: "active-claims"; bullets: string[] }
+  | { kind: "evidence-log"; bullets: string[] };
+
+export type OtherSection = { kind: "other"; heading: string; lines: string[] };
+
+export type RecordSection = KnownSection | OtherSection;
+
+export type ParsedRecord = {
+  frontmatter: RecordFrontmatter;
+  /** Raw body lines before the first `## ...` heading, preserved verbatim
+   * (including the mandatory blank separator line after the frontmatter
+   * delimiter, for a record with no real preamble). A record is free to
+   * carry introductory prose here; losing it on the first write would be
+   * a silent, ungated destructive edit. */
+  preambleLines: string[];
+  sections: RecordSection[];
+};
+
+const ACTIVE_CLAIMS_HEADING = "## Active claims";
+const EVIDENCE_LOG_HEADING = "## Evidence log";
+
+export function parseRecord(text: string): Result<ParsedRecord> {
+  const errors: string[] = [];
+  const lines = text.split("\n").map((line) => line.replace(/\r$/, ""));
+
+  if (lines[0] !== "---") {
+    return err(["record must begin with a '---' frontmatter delimiter"]);
+  }
+
+  const closingIndex = lines.indexOf("---", 1);
+  if (closingIndex === -1) {
+    return err(["record frontmatter is missing its closing '---' delimiter"]);
+  }
+
+  const frontmatterLines = lines.slice(1, closingIndex);
+  const frontmatterFields = new Map<string, string>();
+  for (const line of frontmatterLines) {
+    if (line.trim() === "") continue;
+    const match = /^([a-zA-Z_][a-zA-Z0-9_]*):\s?(.*)$/.exec(line);
+    if (!match) {
+      errors.push(`unrecognized frontmatter line: ${JSON.stringify(line)}`);
+      continue;
+    }
+    const key = requireDefined(match[1], "frontmatter regex capture group 1");
+    const value = requireDefined(match[2], "frontmatter regex capture group 2");
+    frontmatterFields.set(key, value.trim());
+  }
+
+  for (const required of ["id", "title", "updated"] as const) {
+    if (!frontmatterFields.has(required)) {
+      errors.push(`frontmatter is missing required field: ${required}`);
+    }
+  }
+  const knownFrontmatterKeys = new Set(["id", "title", "updated"]);
+  for (const key of frontmatterFields.keys()) {
+    if (!knownFrontmatterKeys.has(key)) {
+      errors.push(`frontmatter has unexpected field: ${key}`);
+    }
+  }
+
+  if (errors.length > 0) {
+    return err(errors);
+  }
+
+  const frontmatter: RecordFrontmatter = {
+    id: requireDefined(frontmatterFields.get("id"), "id present after validation"),
+    title: requireDefined(frontmatterFields.get("title"), "title present after validation"),
+    updated: requireDefined(frontmatterFields.get("updated"), "updated present after validation"),
+  };
+
+  const bodyLines = lines.slice(closingIndex + 1);
+  const firstHeadingIndex = bodyLines.findIndex((line) => line.startsWith("## "));
+  const preambleLines = firstHeadingIndex === -1 ? bodyLines.slice() : bodyLines.slice(0, firstHeadingIndex);
+  const sectionLines = firstHeadingIndex === -1 ? [] : bodyLines.slice(firstHeadingIndex);
+  const rawSections = splitIntoSections(sectionLines);
+
+  const sections: RecordSection[] = [];
+  let activeClaimsIndex = -1;
+  let evidenceLogIndex = -1;
+
+  for (const raw of rawSections) {
+    if (raw.heading === ACTIVE_CLAIMS_HEADING) {
+      if (activeClaimsIndex !== -1) {
+        errors.push(`record has more than one "${ACTIVE_CLAIMS_HEADING}" section`);
+        continue;
+      }
+      const bulletsResult = parseBullets(raw.lines, ACTIVE_CLAIMS_HEADING);
+      if (!bulletsResult.ok) {
+        errors.push(...bulletsResult.errors);
+        continue;
+      }
+      activeClaimsIndex = sections.length;
+      sections.push({ kind: "active-claims", bullets: bulletsResult.value });
+    } else if (raw.heading === EVIDENCE_LOG_HEADING) {
+      if (evidenceLogIndex !== -1) {
+        errors.push(`record has more than one "${EVIDENCE_LOG_HEADING}" section`);
+        continue;
+      }
+      const bulletsResult = parseBullets(raw.lines, EVIDENCE_LOG_HEADING);
+      if (!bulletsResult.ok) {
+        errors.push(...bulletsResult.errors);
+        continue;
+      }
+      evidenceLogIndex = sections.length;
+      sections.push({ kind: "evidence-log", bullets: bulletsResult.value });
+    } else {
+      sections.push({ kind: "other", heading: raw.heading, lines: raw.lines.slice() });
+    }
+  }
+
+  if (activeClaimsIndex === -1) {
+    errors.push(`record body is missing the "${ACTIVE_CLAIMS_HEADING}" section`);
+  }
+  if (evidenceLogIndex === -1) {
+    errors.push(`record body is missing the "${EVIDENCE_LOG_HEADING}" section`);
+  }
+  if (activeClaimsIndex !== -1 && evidenceLogIndex !== -1 && activeClaimsIndex > evidenceLogIndex) {
+    errors.push(`"${ACTIVE_CLAIMS_HEADING}" must come before "${EVIDENCE_LOG_HEADING}"`);
+  }
+
+  if (errors.length > 0) {
+    return err(errors);
+  }
+
+  return ok({ frontmatter, preambleLines, sections });
+}
+
+type RawSection = { heading: string; lines: string[] };
+
+function splitIntoSections(bodyLines: string[]): RawSection[] {
+  const sections: RawSection[] = [];
+  let current: RawSection | null = null;
+  for (const line of bodyLines) {
+    if (line.startsWith("## ")) {
+      current = { heading: line.trimEnd(), lines: [] };
+      sections.push(current);
+    } else if (current) {
+      current.lines.push(line);
+    }
+    // lines before the first heading (blank separator lines) are ignored
+  }
+  return sections;
+}
+
+function parseBullets(lines: string[], sectionName: string): Result<string[]> {
+  const bullets: string[] = [];
+  const errors: string[] = [];
+  for (const line of lines) {
+    if (line.trim() === "") continue;
+    const match = /^- (.+)$/.exec(line);
+    if (!match) {
+      errors.push(`unrecognized line in "${sectionName}": ${JSON.stringify(line)}`);
+      continue;
+    }
+    const bulletText = requireDefined(match[1], "bullet regex capture group 1");
+    bullets.push(bulletText.trimEnd());
+  }
+  if (errors.length > 0) return err(errors);
+  return ok(bullets);
+}
+
+export function serializeRecord(record: ParsedRecord): string {
+  const lines: string[] = [];
+  lines.push("---");
+  lines.push(`id: ${record.frontmatter.id}`);
+  lines.push(`title: ${record.frontmatter.title}`);
+  lines.push(`updated: ${record.frontmatter.updated}`);
+  lines.push("---");
+  lines.push(...record.preambleLines);
+
+  for (const section of record.sections) {
+    if (section.kind === "active-claims") {
+      lines.push(ACTIVE_CLAIMS_HEADING);
+      lines.push("");
+      for (const claim of section.bullets) lines.push(`- ${claim}`);
+      lines.push("");
+    } else if (section.kind === "evidence-log") {
+      lines.push(EVIDENCE_LOG_HEADING);
+      lines.push("");
+      for (const evidence of section.bullets) lines.push(`- ${evidence}`);
+      lines.push("");
+    } else {
+      lines.push(section.heading);
+      lines.push(...section.lines);
+    }
+  }
+
+  // Normalize to exactly one trailing newline regardless of how the last
+  // section's captured lines ended.
+  return `${lines.join("\n").replace(/\n*$/, "")}\n`;
+}
+
+function findKnownSection(record: ParsedRecord, kind: "active-claims" | "evidence-log"): KnownSection {
+  for (const section of record.sections) {
+    if (section.kind === kind) return section;
+  }
+  // parseRecord guarantees both known sections exist exactly once before
+  // ever returning ok(...), so reaching here means a ParsedRecord was
+  // constructed some other way. That is a programming error, not a
+  // user-facing validation failure, so it throws rather than returning a
+  // Result.
+  throw new Error(`internal invariant violated: parsed record is missing its "${kind}" section`);
+}
+
+export function getActiveClaims(record: ParsedRecord): string[] {
+  return findKnownSection(record, "active-claims").bullets;
+}
+
+export function getEvidenceLog(record: ParsedRecord): string[] {
+  return findKnownSection(record, "evidence-log").bullets;
+}
+
+/**
+ * Returns a new ParsedRecord with the active-claims and evidence-log
+ * sections replaced and frontmatter updated, preserving the preamble and
+ * every other section (unknown headings, and their relative position)
+ * unchanged — no candidate field can touch either.
+ */
+export function withMutatedContent(
+  record: ParsedRecord,
+  update: { frontmatter: RecordFrontmatter; activeClaims: string[]; evidenceLog: string[] },
+): ParsedRecord {
+  const sections: RecordSection[] = record.sections.map((section) => {
+    if (section.kind === "active-claims") return { kind: "active-claims", bullets: update.activeClaims };
+    if (section.kind === "evidence-log") return { kind: "evidence-log", bullets: update.evidenceLog };
+    return section;
+  });
+  return { frontmatter: update.frontmatter, preambleLines: record.preambleLines, sections };
+}

package/src/packLoader.ts

@@ -0,0 +1,188 @@
+/**
+ * Pack loader — resolves a pack's declared `from` module specifier to a
+ * KnowledgeExtractor or KnowledgePack implementation. Resolution is explicit
+ * and external-only: the loader imports exactly the module the binding named
+ * and enforces that the selected export's declared id and version match the
+ * binding request. There is no bundled registry to fall back to and no silent
+ * substitution.
+ *
+ * The loader does NOT own pack validation, reconciliation, or the transaction
+ * pipeline. It is the one bridge between the binding's declared pack identity
+ * and a loadable module.
+ *
+ * Error contract: every failure is `kind: "validation"` and uses exactly
+ * `pack_from_required`, `pack_load_failed`, `pack_export_invalid`, or
+ * `pack_identity_mismatch`. Messages describe the category and never print
+ * module contents, local paths, or an imported exception stack.
+ *
+ * @module
+ */
+
+import { dirname, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import type { KnowledgeExtractor, KnowledgePack, KnowledgeResult, PresentationPack } from "./knowledgeTypes.ts";
+
+function packError<T>(code: string, message: string): KnowledgeResult<T> {
+  return { ok: false, errors: [{ kind: "validation", code, message }] };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function camelToSnake(name: string): string {
+  return name.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
+}
+
+/**
+ * Locates a candidate export by the requested pack id. Checks, in order:
+ *   1. A named export matching the pack id.
+ *   2. A `default` export.
+ *   3. An export in a `packs` or `packRegistry` object.
+ */
+function findExport(mod: Record<string, unknown>, id: string): unknown {
+  const named = mod[id] ?? mod[camelToSnake(id)];
+  if (named !== undefined) return named;
+
+  const defaultExport = mod.default;
+  if (defaultExport !== undefined) return defaultExport;
+
+  const registry = mod.packs ?? mod.packRegistry;
+  if (isRecord(registry)) {
+    const entry = registry[id];
+    if (entry !== undefined) return entry;
+  }
+
+  return undefined;
+}
+
+function isSourceClassPolicy(value: unknown): boolean {
+  return (
+    isRecord(value) &&
+    Array.isArray(value.allowedSourceClasses) &&
+    value.allowedSourceClasses.every((sourceClass) => typeof sourceClass === "string") &&
+    typeof value.queryStrategy === "function" &&
+    typeof value.classifySource === "function" &&
+    (typeof value.relevanceThreshold === "number" || value.relevanceThreshold === null) &&
+    typeof value.isEligible === "function" &&
+    value.includePresentations === false
+  );
+}
+
+function isView(value: unknown): boolean {
+  return isRecord(value) && typeof value.id === "string" && typeof value.version === "number" &&
+    (value.scope === "search" || value.scope === "space") &&
+    typeof value.retrievalQuery === "function" && typeof value.project === "function";
+}
+
+function isAudience(value: unknown): boolean {
+  return isRecord(value) && typeof value.id === "string" && typeof value.version === "number" &&
+    typeof value.authorize === "function" && typeof value.adapt === "function";
+}
+
+function isDelivery(value: unknown): boolean {
+  return isRecord(value) && typeof value.id === "string" && typeof value.version === "number" &&
+    (value.format === "markdown" || value.format === "plain" || value.format === "json") &&
+    typeof value.maxWords === "number" && typeof value.retain === "boolean";
+}
+
+function isKnowledgePack(value: unknown): value is KnowledgePack & PresentationPack & Record<string, unknown> {
+  return (
+    isRecord(value) &&
+    typeof value.id === "string" &&
+    typeof value.version === "string" &&
+    typeof value.validateEnvelope === "function" &&
+    typeof value.relatedQuery === "function" &&
+    typeof value.reconcile === "function" &&
+    isSourceClassPolicy(value.retrievalPolicy) &&
+    Array.isArray(value.views) && value.views.every(isView) &&
+    Array.isArray(value.audiences) && value.audiences.every(isAudience) &&
+    Array.isArray(value.deliveries) && value.deliveries.every(isDelivery)
+  );
+}
+
+function isKnowledgeExtractor(value: unknown): value is KnowledgeExtractor & Record<string, unknown> {
+  return (
+    isRecord(value) &&
+    typeof value.id === "string" &&
+    typeof value.version === "string" &&
+    typeof value.extractCandidates === "function"
+  );
+}
+
+function moduleSpecifier(from: string, bindingPath: string | undefined): string {
+  if (bindingPath !== undefined && (from.startsWith("./") || from.startsWith("../"))) {
+    return pathToFileURL(resolve(dirname(bindingPath), from)).href;
+  }
+  return from;
+}
+
+/**
+ * Resolves a pack `id`+`version` to its implementation, importing exactly the
+ * binding's `from` module specifier and enforcing that the selected export's
+ * declared identity matches. Refuses:
+ *   - A missing or whitespace-only `from` as `pack_from_required`.
+ *   - An unloadable module as `pack_load_failed`.
+ *   - A module whose candidate export is not a complete KnowledgePack and
+ *     PresentationPack as `pack_export_invalid`.
+ *   - A valid-shaped export whose id or version differs as
+ *     `pack_identity_mismatch`.
+ */
+export async function resolveKnowledgePack(
+  id: string,
+  version: string,
+  from: string | undefined,
+  bindingPath?: string,
+): Promise<KnowledgeResult<KnowledgePack & PresentationPack>> {
+  if (from === undefined || from.trim().length === 0) {
+    return packError("pack_from_required", "a from module specifier is required to resolve the external pack");
+  }
+
+  let mod: Record<string, unknown>;
+  try {
+    mod = await import(moduleSpecifier(from, bindingPath));
+  } catch {
+    return packError("pack_load_failed", "the external pack module could not be loaded");
+  }
+
+  const candidate = findExport(mod, id);
+  if (!isKnowledgePack(candidate)) {
+    return packError("pack_export_invalid", "the external pack module does not export a complete KnowledgePack and PresentationPack");
+  }
+  if (candidate.id !== id || candidate.version !== version) {
+    return packError("pack_identity_mismatch", "the external pack module's id or version does not match the declared pack");
+  }
+  return { ok: true, value: candidate };
+}
+
+/**
+ * Loads a KnowledgeExtractor for the pack `id`+`version` declared with `from`,
+ * with the same from, load, export, and identity checks as
+ * `resolveKnowledgePack`.
+ */
+export async function loadExtractionPack(
+  id: string,
+  version: string,
+  from: string | undefined,
+  bindingPath?: string,
+): Promise<KnowledgeResult<KnowledgeExtractor>> {
+  if (from === undefined || from.trim().length === 0) {
+    return packError("pack_from_required", "a from module specifier is required to load the extraction pack");
+  }
+
+  let mod: Record<string, unknown>;
+  try {
+    mod = await import(moduleSpecifier(from, bindingPath));
+  } catch {
+    return packError("pack_load_failed", "the extraction pack module could not be loaded");
+  }
+
+  const candidate = findExport(mod, id);
+  if (!isKnowledgeExtractor(candidate)) {
+    return packError("pack_export_invalid", "the extraction pack module does not export a complete KnowledgeExtractor");
+  }
+  if (candidate.id !== id || candidate.version !== version) {
+    return packError("pack_identity_mismatch", "the extraction pack module's id or version does not match the declared extractor");
+  }
+  return { ok: true, value: candidate };
+}

package/src/packTypes.ts

@@ -0,0 +1,12 @@
+/**
+ * Public external-pack interface types.
+ *
+ * A normal knowledge operation requires the full KnowledgePack and
+ * PresentationPack surface. KnowledgeExtractor is separately optional and is
+ * selected only by a binding entry with extract: true.
+ */
+export type {
+  KnowledgeExtractor,
+  KnowledgePack,
+  PresentationPack,
+} from "./knowledgeTypes.ts";