@nzpr/kb 0.1.0

package/lib/frontmatter.js

@@ -0,0 +1,66 @@
+import fs from "node:fs";
+import path from "node:path";
+import YAML from "yaml";
+
+export function loadMarkdownWithFrontmatter(filePath) {
+  const raw = fs.readFileSync(filePath, "utf8");
+  if (!raw.startsWith("---\n")) {
+    return parseMarkdownDocument(filePath, {}, raw);
+  }
+  const rest = raw.slice(4);
+  const separator = "\n---\n";
+  const separatorIndex = rest.indexOf(separator);
+  if (separatorIndex === -1) {
+    throw new Error(`${filePath}: malformed frontmatter block`);
+  }
+  const frontmatterText = rest.slice(0, separatorIndex);
+  const body = rest.slice(separatorIndex + separator.length);
+  const metaRaw = YAML.parse(frontmatterText) ?? {};
+  return parseMarkdownDocument(filePath, metaRaw, body);
+}
+
+export function parseMeta(filePath, metaRaw) {
+  const title = "title" in metaRaw ? String(metaRaw.title).trim() : "";
+  if (!title) {
+    throw new Error(`${filePath}: title is required when frontmatter is present`);
+  }
+
+  return {
+    docId: String(metaRaw.id ?? slugFromPath(filePath)),
+    title,
+    path: path.resolve(filePath)
+  };
+}
+
+function parseMarkdownDocument(filePath, metaRaw, rawBody) {
+  const parsed = extractTitleAndBody(rawBody);
+  const meta = parseMeta(filePath, {
+    ...metaRaw,
+    title: metaRaw.title ?? parsed.title
+  });
+  return [meta, parsed.body];
+}
+
+function extractTitleAndBody(rawBody) {
+  const body = String(rawBody ?? "").trim();
+  const match = /^#\s+(.+?)\n+([\s\S]*)$/m.exec(body);
+  if (match) {
+    return {
+      title: match[1].trim(),
+      body: match[2].trim()
+    };
+  }
+  const firstParagraph = body.split(/\n\s*\n/, 1)[0]?.trim() ?? "";
+  return {
+    title: firstParagraph || "Untitled Document",
+    body
+  };
+}
+
+function slugFromPath(filePath) {
+  return path
+    .basename(filePath, path.extname(filePath))
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+}

package/lib/index.js

@@ -0,0 +1,140 @@
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { chunkMarkdown } from "./chunking.js";
+import { connect, initDb, readEmbeddingMetadata, writeEmbeddingMetadata } from "./db.js";
+import { embedText, vectorLiteral } from "./embeddings.js";
+import { loadMarkdownWithFrontmatter } from "./frontmatter.js";
+
+export async function ingestDocuments({ databaseUrl, docsRoot, embeddingProfile }) {
+  const client = await connect(databaseUrl);
+  try {
+    await initDb(client);
+
+    const files = walkMarkdown(docsRoot);
+    const seenDocIds = new Set();
+    const seenPaths = new Set();
+    let indexed = 0;
+    let vectorsWritten = 0;
+    const errors = [];
+    const currentMetadata = await readEmbeddingMetadata(client);
+    const forceReindex = metadataDiffers(currentMetadata, embeddingProfile);
+
+    await client.query("BEGIN");
+    await writeEmbeddingMetadata(client, embeddingProfile);
+    for (const filePath of files) {
+      try {
+        const [meta, body] = loadMarkdownWithFrontmatter(filePath);
+        if (seenDocIds.has(meta.docId)) {
+          errors.push(`${filePath}: duplicate document id ${meta.docId}`);
+          continue;
+        }
+        seenDocIds.add(meta.docId);
+        seenPaths.add(meta.path);
+        const contentHash = sha256(fs.readFileSync(filePath));
+        const existing = await client.query(
+          "SELECT content_hash FROM documents WHERE doc_id = $1",
+          [meta.docId]
+        );
+        if (!forceReindex && existing.rowCount && existing.rows[0].content_hash === contentHash) {
+          indexed += 1;
+          continue;
+        }
+
+        const chunks = chunkMarkdown(meta, body);
+        await replaceDocument(client, meta, contentHash, chunks[0]?.content ?? body, embeddingProfile);
+        indexed += 1;
+        vectorsWritten += chunks.length ? 1 : 0;
+      } catch (error) {
+        errors.push(String(error.message ?? error));
+      }
+    }
+    await removeDeletedDocuments(client, seenDocIds, seenPaths);
+    await client.query("COMMIT");
+
+    if (errors.length) {
+      throw new Error(errors.join("\n"));
+    }
+
+    return {
+      documents: indexed,
+      vectors: vectorsWritten,
+      embeddingMode: embeddingProfile.mode,
+      embeddingModel: embeddingProfile.model,
+      forceReindex
+    };
+  } catch (error) {
+    await client.query("ROLLBACK").catch(() => {});
+    throw error;
+  } finally {
+    await client.end();
+  }
+}
+
+async function replaceDocument(client, meta, contentHash, content, embeddingProfile) {
+  const embedding = vectorLiteral(await embedText(`${meta.title}\n${content}`, embeddingProfile));
+  await client.query(
+    `
+    INSERT INTO documents (
+      doc_id, path, title, content, content_hash, embedding
+    ) VALUES ($1,$2,$3,$4,$5,$6::vector)
+    ON CONFLICT (doc_id) DO UPDATE SET
+      path = EXCLUDED.path,
+      title = EXCLUDED.title,
+      content = EXCLUDED.content,
+      content_hash = EXCLUDED.content_hash,
+      embedding = EXCLUDED.embedding,
+      updated_at = NOW()
+    `,
+    [
+      meta.docId,
+      meta.path,
+      meta.title,
+      content,
+      contentHash,
+      embedding
+    ]
+  );
+}
+
+async function removeDeletedDocuments(client, seenDocIds, seenPaths) {
+  const result = await client.query("SELECT doc_id, path FROM documents");
+  for (const row of result.rows) {
+    if (!seenDocIds.has(row.doc_id) || !seenPaths.has(row.path)) {
+      await client.query("DELETE FROM documents WHERE doc_id = $1", [row.doc_id]);
+    }
+  }
+}
+
+function walkMarkdown(root) {
+  const files = [];
+  const stack = [path.resolve(root)];
+  while (stack.length) {
+    const current = stack.pop();
+    const stats = fs.statSync(current);
+    if (stats.isDirectory()) {
+      const entries = fs.readdirSync(current).sort().reverse();
+      for (const entry of entries) {
+        stack.push(path.join(current, entry));
+      }
+    } else if (current.endsWith(".md")) {
+      files.push(current);
+    }
+  }
+  return files.sort();
+}
+
+function sha256(buffer) {
+  return crypto.createHash("sha256").update(buffer).digest("hex");
+}
+
+function metadataDiffers(metadata, embeddingProfile) {
+  if (!metadata.embedding_mode && !metadata.embedding_dim && !metadata.embedding_model) {
+    return true;
+  }
+  return (
+    metadata.embedding_mode !== embeddingProfile.mode ||
+    metadata.embedding_dim !== String(embeddingProfile.dimensions) ||
+    (metadata.embedding_model ?? "") !== (embeddingProfile.model ?? "")
+  );
+}

package/lib/kb-proposals.js

@@ -0,0 +1,188 @@
+import fs from "node:fs";
+import path from "node:path";
+
+const SECTION_TO_FIELD = new Map([
+  ["Title", "title"],
+  ["Relative Path", "relativePath"],
+  ["Text", "text"],
+  ["Proposed Guidance", "proposedGuidance"]
+]);
+
+export function parseIssueFormBody(body) {
+  const sections = splitIssueSections(body);
+  const values = {};
+  for (const [section, field] of SECTION_TO_FIELD.entries()) {
+    values[field] = sections.get(section) ?? "";
+  }
+
+  const proposal = {
+    title: requiredValue(values.title, "Title"),
+    docId: slugify(requiredValue(values.title, "Title")),
+    relativePath: normalizeRelativePath(values.relativePath.trim() || null),
+    text: requiredValue(values.text || values.proposedGuidance, "Text")
+  };
+
+  return proposal;
+}
+
+export function deriveDocRelativePath(proposal) {
+  if (proposal.relativePath) {
+    return proposal.relativePath;
+  }
+  return path.posix.join("entries", `${slugify(proposal.docId)}.md`);
+}
+
+export function renderDocumentMarkdown(proposal, lastReviewed = currentDateString()) {
+  void lastReviewed;
+  return `# ${proposal.title}\n\n${proposal.text.trim()}\n`;
+}
+
+export function renderProposalIssueBody({ proposal, relativePath }) {
+  const issuePath = relativePath ?? deriveDocRelativePath(proposal);
+  return [
+    "### Title",
+    proposal.title,
+    "",
+    "### Relative Path",
+    issuePath,
+    "",
+    "### Text",
+    proposal.text.trim()
+  ].join("\n");
+}
+
+export function writeProposalDocument({ issueNumber, issueTitle, issueBody, docsRoot }) {
+  const proposal = parseIssueFormBody(issueBody);
+  const relativePath = deriveDocRelativePath(proposal);
+  const absolutePath = path.join(path.resolve(docsRoot), relativePath);
+  fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
+  fs.writeFileSync(absolutePath, renderDocumentMarkdown(proposal), "utf8");
+
+  const branchSlug = slugify(`${proposal.docId}-${issueNumber}`);
+  return {
+    proposal,
+    relativePath,
+    absolutePath,
+    branch: `kb-issue-${branchSlug}`,
+    commitMessage: `kb: materialize issue #${issueNumber} as ${proposal.docId}`,
+    prTitle: `kb: add ${proposal.title}`,
+    prBody: [
+      `Materializes approved KB issue #${issueNumber}.`,
+      "",
+      `Source issue title: ${issueTitle}`,
+      "",
+      `Document path: \`${relativePath}\``
+    ].join("\n")
+  };
+}
+
+export async function createGitHubIssueFromText({
+  title,
+  text,
+  relativePath = null,
+  repo,
+  token,
+  apiBaseUrl = "https://api.github.com"
+}) {
+  if (!repo) {
+    throw new Error("GitHub repository is required, expected OWNER/REPO");
+  }
+  if (!token) {
+    throw new Error("GITHUB_TOKEN is required to create a proposal issue");
+  }
+
+  const proposal = {
+    docId: slugify(requiredValue(title, "Title")),
+    title: requiredValue(title, "Title"),
+    relativePath: normalizeRelativePath(relativePath),
+    text: requiredValue(text, "Text")
+  };
+  const response = await fetch(`${apiBaseUrl.replace(/\/$/, "")}/repos/${repo}/issues`, {
+    method: "POST",
+    headers: {
+      accept: "application/vnd.github+json",
+      authorization: `Bearer ${token}`,
+      "content-type": "application/json",
+      "user-agent": "@nzpr/kb"
+    },
+    body: JSON.stringify({
+      title: `kb: ${proposal.title}`,
+      body: renderProposalIssueBody({
+        proposal,
+        relativePath: proposal.relativePath
+      }),
+      labels: ["kb-entry"]
+    })
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text().catch(() => "");
+    throw new Error(
+      `failed to create GitHub issue: ${response.status} ${response.statusText}${errorText ? ` - ${errorText}` : ""}`
+    );
+  }
+
+  return response.json();
+}
+
+function splitIssueSections(body) {
+  const normalized = String(body ?? "").replace(/\r\n/g, "\n");
+  const lines = normalized.split("\n");
+  const sections = new Map();
+  let currentHeading = null;
+  let currentLines = [];
+
+  const flush = () => {
+    if (!currentHeading) {
+      return;
+    }
+    sections.set(currentHeading, currentLines.join("\n").trim());
+  };
+
+  for (const line of lines) {
+    const match = /^###\s+(.*)$/.exec(line);
+    if (match) {
+      flush();
+      currentHeading = match[1].trim();
+      currentLines = [];
+      continue;
+    }
+    if (currentHeading) {
+      currentLines.push(line);
+    }
+  }
+  flush();
+
+  return sections;
+}
+
+function requiredValue(value, label) {
+  const normalized = String(value ?? "").trim();
+  if (!normalized) {
+    throw new Error(`missing required issue section: ${label}`);
+  }
+  return normalized;
+}
+
+function normalizeRelativePath(value) {
+  if (!value) {
+    return null;
+  }
+  const normalized = value.replace(/\\/g, "/").replace(/^\/+/, "");
+  if (normalized.includes("..")) {
+    throw new Error(`invalid relative path: ${value}`);
+  }
+  return normalized.endsWith(".md") ? normalized : `${normalized}.md`;
+}
+
+function slugify(value) {
+  return String(value ?? "")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .replace(/-{2,}/g, "-");
+}
+
+function currentDateString() {
+  return new Date().toISOString().slice(0, 10);
+}

package/lib/migrations.js

@@ -0,0 +1,149 @@
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { fileURLToPath } from "node:url";
+
+const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const MIGRATIONS_DIR = path.join(PACKAGE_ROOT, "migrations");
+const MIGRATION_FILE_RE = /^(\d+)_([a-z0-9_-]+)\.sql$/i;
+const LEGACY_MIGRATION_CHECKSUMS = new Map([
+  [1, new Set(["594bf296f6c2d897333e424a5c025868beaf662bed279f90fa5f99c75a6c6956"])]
+]);
+
+export function listMigrationFiles() {
+  const entries = fs.readdirSync(MIGRATIONS_DIR).sort();
+  return entries.map(parseMigrationFile);
+}
+
+export async function migrateDatabase(client) {
+  await ensureMigrationTable(client);
+  const applied = await getAppliedMigrations(client);
+  const migrations = listMigrationFiles();
+  const pending = [];
+
+  for (const migration of migrations) {
+    const previous = applied.get(migration.version);
+    if (previous) {
+      if (!checksumMatches(migration.version, previous.checksum, migration.checksum)) {
+        throw new Error(
+          `migration checksum mismatch for ${migration.file}: database=${previous.checksum} current=${migration.checksum}`
+        );
+      }
+      continue;
+    }
+    pending.push(migration);
+  }
+
+  for (const migration of pending) {
+    await client.query("BEGIN");
+    try {
+      await client.query(migration.sql);
+      await client.query(
+        `
+        INSERT INTO schema_migrations (version, name, checksum)
+        VALUES ($1, $2, $3)
+        `,
+        [migration.version, migration.name, migration.checksum]
+      );
+      await client.query("COMMIT");
+    } catch (error) {
+      await client.query("ROLLBACK").catch(() => {});
+      throw new Error(`failed to apply migration ${migration.file}: ${error.message ?? error}`);
+    }
+  }
+
+  const latestVersion = migrations.length ? migrations[migrations.length - 1].version : 0;
+  return {
+    appliedCount: pending.length,
+    latestVersion,
+    currentVersion: latestVersion,
+    pendingCount: 0
+  };
+}
+
+export async function migrationStatus(client) {
+  const applied = await getAppliedMigrationsIfTableExists(client);
+  const migrations = listMigrationFiles();
+  const latestVersion = migrations.length ? migrations[migrations.length - 1].version : 0;
+  let currentVersion = 0;
+
+  for (const migration of migrations) {
+    const previous = applied.get(migration.version);
+    if (!previous) {
+      break;
+    }
+    if (!checksumMatches(migration.version, previous.checksum, migration.checksum)) {
+      throw new Error(
+        `migration checksum mismatch for ${migration.file}: database=${previous.checksum} current=${migration.checksum}`
+      );
+    }
+    currentVersion = migration.version;
+  }
+
+  return {
+    latestVersion,
+    currentVersion,
+    pendingCount: migrations.filter((migration) => !applied.has(migration.version)).length
+  };
+}
+
+async function ensureMigrationTable(client) {
+  await client.query(`
+    CREATE TABLE IF NOT EXISTS schema_migrations (
+      version INTEGER PRIMARY KEY,
+      name TEXT NOT NULL,
+      checksum TEXT NOT NULL,
+      applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    )
+  `);
+}
+
+async function getAppliedMigrations(client) {
+  const result = await client.query(`
+    SELECT version, name, checksum
+    FROM schema_migrations
+    ORDER BY version
+  `);
+  return new Map(result.rows.map((row) => [Number(row.version), row]));
+}
+
+async function getAppliedMigrationsIfTableExists(client) {
+  const result = await client.query(`
+    SELECT to_regclass('public.schema_migrations') AS table_name
+  `);
+  if (!result.rows[0]?.table_name) {
+    return new Map();
+  }
+  return getAppliedMigrations(client);
+}
+
+function parseMigrationFile(file) {
+  const match = MIGRATION_FILE_RE.exec(file);
+  if (!match) {
+    throw new Error(`invalid migration filename: ${file}`);
+  }
+  const version = Number(match[1]);
+  const name = match[2];
+  const fullPath = path.join(MIGRATIONS_DIR, file);
+  const sql = fs.readFileSync(fullPath, "utf8").trim();
+  return {
+    file,
+    version,
+    name,
+    path: fullPath,
+    sql,
+    checksum: sha256(sql)
+  };
+}
+
+function sha256(value) {
+  return crypto.createHash("sha256").update(value).digest("hex");
+}
+
+function checksumMatches(version, databaseChecksum, currentChecksum) {
+  if (databaseChecksum === currentChecksum) {
+    return true;
+  }
+  const legacyChecksums = LEGACY_MIGRATION_CHECKSUMS.get(version);
+  return Boolean(legacyChecksums?.has(databaseChecksum));
+}