@nzpr/kb 0.1.0

package/lib/repo-init.js

@@ -0,0 +1,438 @@
+import fs from "node:fs";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { connect, initDb } from "./db.js";
+import { maskConnection } from "./cli-common.js";
+
+const PACKAGE_NAME = "@nzpr/kb";
+const LABELS = Object.freeze([
+  {
+    name: "kb-entry",
+    color: "0E8A16",
+    description: "Knowledge base proposal or generated change"
+  },
+  {
+    name: "kb-approved",
+    color: "1D76DB",
+    description: "Approved knowledge proposal ready to materialize"
+  }
+]);
+
+export function initializeKnowledgeRepo({ targetDir = process.cwd() } = {}) {
+  const root = path.resolve(targetDir);
+  const files = new Map([
+    [".github/ISSUE_TEMPLATE/config.yml", renderIssueConfig()],
+    [".github/ISSUE_TEMPLATE/knowledge-document.md", renderIssueTemplate()],
+    [".github/workflows/kb-issue-to-pr.yml", renderIssueToPrWorkflow()],
+    [".github/workflows/kb-publish.yml", renderPublishWorkflow()],
+    ["kb/docs/.gitkeep", ""]
+  ]);
+
+  const created = [];
+  const skipped = [];
+
+  for (const [relativePath, content] of files.entries()) {
+    const absolutePath = path.join(root, relativePath);
+    if (fs.existsSync(absolutePath)) {
+      skipped.push(relativePath);
+      continue;
+    }
+    fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
+    fs.writeFileSync(absolutePath, content, "utf8");
+    created.push(relativePath);
+  }
+
+  return {
+    root,
+    created,
+    skipped,
+    configuration: buildConfigurationGuide()
+  };
+}
+
+export async function bootstrapKnowledgeRepo({
+  targetDir = process.cwd(),
+  repo = null,
+  githubToken = null,
+  databaseUrl = null,
+  embeddingMode = null,
+  embeddingApiUrl = null,
+  embeddingModel = null,
+  embeddingApiKey = null,
+  dbConnectTimeoutMs = null,
+  repoAutomationToken = null,
+  runGitHubCommand = defaultRunGitHubCommand,
+  verifyDatabaseReady = defaultVerifyDatabaseReady
+} = {}) {
+  const scaffold = initializeKnowledgeRepo({ targetDir });
+  const result = {
+    ...scaffold,
+    ok: true,
+    database: {
+      status: "pending",
+      message:
+        "rerun with --database-url URL or KB_DATABASE_URL to verify the target database and initialize the schema"
+    },
+    github: {
+      status: "pending",
+      message:
+        "rerun with --repo OWNER/REPO and GITHUB_TOKEN to configure labels, repo settings, and GitHub secrets or variables"
+    }
+  };
+
+  if (databaseUrl) {
+    try {
+      const database = await verifyDatabaseReady({ databaseUrl });
+      result.database = {
+        status: "verified",
+        ...database
+      };
+    } catch (error) {
+      result.ok = false;
+      result.database = {
+        status: "failed",
+        error: String(error?.message ?? error)
+      };
+    }
+  }
+
+  if (repo) {
+    if (!githubToken) {
+      result.ok = false;
+      result.github = {
+        status: "failed",
+        repo,
+        error: "GITHUB_TOKEN is required when configuring a knowledge repo"
+      };
+      return result;
+    }
+
+    if (result.database.status === "failed") {
+      result.github = {
+        status: "skipped",
+        repo,
+        message: "database preflight failed, so repo secrets and variables were not changed"
+      };
+      return result;
+    }
+    const secrets = new Map();
+    const variables = new Map();
+
+    if (databaseUrl) {
+      secrets.set("KB_DATABASE_URL", databaseUrl);
+    }
+    if (embeddingApiKey) {
+      secrets.set("KB_EMBEDDING_API_KEY", embeddingApiKey);
+    }
+    if (repoAutomationToken) {
+      secrets.set("KB_REPO_AUTOMATION_TOKEN", repoAutomationToken);
+    }
+    if (embeddingMode) {
+      variables.set("KB_EMBEDDING_MODE", embeddingMode);
+    }
+    if (embeddingApiUrl) {
+      variables.set("KB_EMBEDDING_API_URL", embeddingApiUrl);
+    }
+    if (embeddingModel) {
+      variables.set("KB_EMBEDDING_MODEL", embeddingModel);
+    }
+    if (dbConnectTimeoutMs) {
+      variables.set("KB_DB_CONNECT_TIMEOUT_MS", String(dbConnectTimeoutMs));
+    }
+
+    try {
+      const github = await configureKnowledgeRepo({
+        repo,
+        githubToken,
+        secrets,
+        variables,
+        runGitHubCommand
+      });
+      result.github = {
+        status: "configured",
+        ...github
+      };
+    } catch (error) {
+      result.ok = false;
+      result.github = {
+        status: "failed",
+        repo,
+        error: String(error?.message ?? error)
+      };
+    }
+  }
+
+  return result;
+}
+
+function buildConfigurationGuide() {
+  return {
+    requiredSecrets: [
+      {
+        name: "KB_DATABASE_URL",
+        purpose: "PostgreSQL connection string for the KB database with write access for publish."
+      }
+    ],
+    optionalSecrets: [
+      {
+        name: "KB_EMBEDDING_API_KEY",
+        purpose: "API key for the embeddings endpoint, if your server requires authentication."
+      },
+      {
+        name: "KB_REPO_AUTOMATION_TOKEN",
+        purpose: "Optional token for issue-to-PR automation if default GitHub token behavior is insufficient."
+      }
+    ],
+    optionalVariables: [
+      {
+        name: "KB_EMBEDDING_MODE",
+        value: "bge-m3-openai",
+        purpose: "Enable high-quality remote embeddings."
+      },
+      {
+        name: "KB_EMBEDDING_API_URL",
+        value: "https://your-embeddings-host/v1/embeddings",
+        purpose: "OpenAI-compatible embeddings endpoint."
+      },
+      {
+        name: "KB_EMBEDDING_MODEL",
+        value: "BAAI/bge-m3",
+        purpose: "Embedding model name expected by the endpoint."
+      },
+      {
+        name: "KB_DB_CONNECT_TIMEOUT_MS",
+        value: "20000",
+        purpose: "Optional database connect timeout override for CI."
+      }
+    ]
+  };
+}
+
+async function configureKnowledgeRepo({
+  repo,
+  githubToken,
+  secrets,
+  variables,
+  runGitHubCommand
+}) {
+  await runGitHubCommand(["repo", "edit", repo, "--enable-issues"], { githubToken });
+
+  for (const label of LABELS) {
+    await runGitHubCommand(
+      [
+        "label",
+        "create",
+        label.name,
+        "--repo",
+        repo,
+        "--color",
+        label.color,
+        "--description",
+        label.description,
+        "--force"
+      ],
+      { githubToken }
+    );
+  }
+
+  for (const [name, value] of secrets.entries()) {
+    await runGitHubCommand(["secret", "set", name, "--repo", repo, "--body", value], {
+      githubToken
+    });
+  }
+
+  for (const [name, value] of variables.entries()) {
+    await runGitHubCommand(["variable", "set", name, "--repo", repo, "--body", value], {
+      githubToken
+    });
+  }
+
+  return {
+    repo,
+    labels: LABELS.map((label) => label.name),
+    secrets: [...secrets.keys()],
+    variables: [...variables.keys()]
+  };
+}
+
+async function defaultVerifyDatabaseReady({ databaseUrl }) {
+  const client = await connect(databaseUrl);
+  try {
+    const result = await initDb(client);
+    return {
+      database: maskConnection(databaseUrl),
+      currentVersion: result.currentVersion,
+      appliedCount: result.appliedCount
+    };
+  } finally {
+    await client.end();
+  }
+}
+
+function defaultRunGitHubCommand(args, { githubToken }) {
+  return execFileSync("gh", args, {
+    encoding: "utf8",
+    env: {
+      ...process.env,
+      GH_TOKEN: githubToken,
+      GITHUB_TOKEN: githubToken
+    }
+  });
+}
+
+function renderIssueConfig() {
+  return ["blank_issues_enabled: false", ""].join("\n");
+}
+
+function renderIssueTemplate() {
+  return [
+    "---",
+    "name: Knowledge Base Document",
+    "about: Propose a new knowledge base document or a substantial update to an existing one",
+    'title: "kb: "',
+    "labels: kb-entry",
+    "---",
+    "",
+    "Use this template to propose a new knowledge base entry.",
+    "",
+    "Keep it minimal. A knowledge entry should be just a title and the text that should be retrieved by search.",
+    "",
+    "### Title",
+    "",
+    "Example Platform Rule",
+    "",
+    "### Relative Path",
+    "",
+    "entries/example-platform-rule.md",
+    "",
+    "### Text",
+    "",
+    "Write the exact knowledge text that should become the document body.",
+    "",
+    "### Review Flow",
+    "",
+    "1. Open the issue with this template.",
+    "2. Review and edit the issue until the title and text are ready.",
+    "3. Add the `kb-approved` label to generate a PR that writes the Markdown file into `kb/docs/`.",
+    ""
+  ].join("\n");
+}
+
+function renderIssueToPrWorkflow() {
+  return [
+    "name: kb-issue-to-pr",
+    "",
+    "on:",
+    "  issues:",
+    "    types:",
+    "      - labeled",
+    "",
+    "permissions:",
+    "  contents: write",
+    "  pull-requests: write",
+    "  issues: write",
+    "",
+    "jobs:",
+    "  materialize:",
+    "    if: github.event.label.name == 'kb-approved'",
+    "    runs-on: ubuntu-latest",
+    "    concurrency:",
+    "      group: kb-issue-${{ github.event.issue.number }}",
+    "      cancel-in-progress: false",
+    "    steps:",
+    "      - name: Checkout",
+    "        uses: actions/checkout@v4",
+    "",
+    "      - name: Setup Node",
+    "        uses: actions/setup-node@v4",
+    "        with:",
+    "          node-version: 24",
+    "",
+    `      - name: Install ${PACKAGE_NAME}`,
+    `        run: npm install -g ${PACKAGE_NAME}`,
+    "",
+    "      - name: Materialize approved issue",
+    "        id: materialize",
+    '        run: kb-admin issue-to-doc --issue-event "$GITHUB_EVENT_PATH" --docs-root ./kb/docs',
+    "",
+    "      - name: Create pull request",
+    "        id: cpr",
+    "        uses: peter-evans/create-pull-request@v8",
+    "        with:",
+    "          token: ${{ secrets.KB_REPO_AUTOMATION_TOKEN || github.token }}",
+    "          branch: ${{ steps.materialize.outputs.branch }}",
+    "          commit-message: ${{ steps.materialize.outputs.commit_message }}",
+    "          title: ${{ steps.materialize.outputs.pr_title }}",
+    "          body: ${{ steps.materialize.outputs.pr_body }}",
+    "          labels: kb-entry",
+    "          add-paths: ${{ steps.materialize.outputs.doc_path }}",
+    "",
+    "      - name: Comment on issue",
+    "        if: steps.cpr.outputs.pull-request-number",
+    "        uses: actions/github-script@v8",
+    "        with:",
+    "          script: |",
+    "            await github.rest.issues.createComment({",
+    "              owner: context.repo.owner,",
+    "              repo: context.repo.repo,",
+    "              issue_number: context.payload.issue.number,",
+    "              body: `Created PR #${{ steps.cpr.outputs.pull-request-number }} for this approved KB entry.`",
+    "            });",
+    ""
+  ].join("\n");
+}
+
+function renderPublishWorkflow() {
+  return [
+    "name: kb-publish",
+    "",
+    "on:",
+    "  push:",
+    "    branches:",
+    "      - main",
+    "    paths:",
+    '      - "kb/docs/**"',
+    '      - ".github/workflows/**"',
+    "  workflow_dispatch:",
+    "",
+    "jobs:",
+    "  publish:",
+    "    runs-on: ubuntu-latest",
+    "    permissions:",
+    "      contents: write",
+    "    concurrency:",
+    "      group: kb-publish",
+    "      cancel-in-progress: false",
+    "    env:",
+    "      KB_DATABASE_URL: ${{ secrets.KB_DATABASE_URL }}",
+    "      KB_GITHUB_REPO: ${{ github.repository }}",
+    "      GITHUB_TOKEN: ${{ github.token }}",
+    "      KB_EMBEDDING_MODE: ${{ secrets.KB_EMBEDDING_MODE || vars.KB_EMBEDDING_MODE }}",
+    "      KB_EMBEDDING_API_URL: ${{ secrets.KB_EMBEDDING_API_URL || vars.KB_EMBEDDING_API_URL }}",
+    "      KB_EMBEDDING_MODEL: ${{ secrets.KB_EMBEDDING_MODEL || vars.KB_EMBEDDING_MODEL }}",
+    "      KB_EMBEDDING_API_KEY: ${{ secrets.KB_EMBEDDING_API_KEY }}",
+    "      KB_DB_CONNECT_TIMEOUT_MS: ${{ secrets.KB_DB_CONNECT_TIMEOUT_MS || vars.KB_DB_CONNECT_TIMEOUT_MS || '20000' }}",
+    "    steps:",
+    "      - name: Checkout",
+    "        uses: actions/checkout@v4",
+    "",
+    "      - name: Ensure publish secret is configured",
+    "        run: |",
+    '          if [ -z "${KB_DATABASE_URL:-}" ]; then',
+    '            echo "KB_DATABASE_URL secret is required for publish" >&2',
+    "            exit 1",
+    "          fi",
+    "",
+    "      - name: Setup Node",
+    "        uses: actions/setup-node@v4",
+    "        with:",
+    "          node-version: 24",
+    "",
+    `      - name: Install ${PACKAGE_NAME}`,
+    `        run: npm install -g ${PACKAGE_NAME}`,
+    "",
+    "      - name: Publish knowledge",
+    "        run: kb publish --docs-root ./kb/docs",
+    ""
+  ].join("\n");
+}

package/lib/search.js

@@ -0,0 +1,206 @@
+import { connect, ensureCompatibility, schemaStatus } from "./db.js";
+import { embedText, tokenOverlap, vectorLiteral } from "./embeddings.js";
+
+export async function searchIndex({
+  databaseUrl,
+  embeddingProfile,
+  query,
+  limit = 5
+}) {
+  const client = await connect(databaseUrl);
+  try {
+    await ensureCompatibility(client, embeddingProfile);
+    const lexicalRows = await lexicalCandidates(client, {
+      query,
+      limit: Math.max(limit * 10, 30)
+    });
+    const semanticRows = await semanticCandidates(client, {
+      embeddingProfile,
+      query,
+      limit: Math.max(limit * 10, 30)
+    });
+
+    const merged = new Map();
+    for (const row of lexicalRows) {
+      merged.set(row.doc_id, {
+        ...row,
+        lexical_score: Number(row.lexical_score),
+        semantic_score: 0
+      });
+    }
+    for (const row of semanticRows) {
+      const existing = merged.get(row.doc_id) ?? {
+        ...row,
+        lexical_score: 0,
+        semantic_score: 0
+      };
+      existing.semantic_score = Math.max(Number(row.semantic_score), existing.semantic_score);
+      merged.set(row.chunk_id, existing);
+    }
+
+    const results = [...merged.values()].map((row) => {
+      const lexical = Math.max(
+        Number(row.lexical_score ?? 0),
+        tokenOverlap(query, `${row.title} ${row.content}`)
+      );
+      const semantic = Number(row.semantic_score ?? 0);
+      const finalScore = combineScores({
+        lexical,
+        semantic,
+        query,
+        title: row.title
+      });
+      return {
+        chunkId: row.doc_id,
+        docId: row.doc_id,
+        title: row.title,
+        heading: row.title,
+        content: row.content,
+        path: row.path,
+        lexicalScore: lexical,
+        semanticScore: semantic,
+        finalScore,
+        lastReviewed: String(row.updated_at)
+      };
+    });
+
+    return results.sort((a, b) => b.finalScore - a.finalScore).slice(0, limit);
+  } finally {
+    await client.end();
+  }
+}
+
+export async function askIndex(options) {
+  const results = await searchIndex(options);
+  if (!results.length) {
+    return { answer: "No matching standards found.", results: [] };
+  }
+  const lines = [`Best guidance for: ${options.query}`, ""];
+  for (const result of results) {
+    lines.push(`- ${result.title}`);
+    lines.push(`  ${snippet(result.content)}`);
+    lines.push(`  Source: ${result.path} | reviewed ${result.lastReviewed}`);
+  }
+  return { answer: lines.join("\n"), results };
+}
+
+export async function listDocuments({ databaseUrl }) {
+  const client = await connect(databaseUrl);
+  try {
+    const result = await client.query(
+      `
+      SELECT doc_id, title, path
+      FROM documents
+      ORDER BY doc_id
+      `
+    );
+    return result.rows;
+  } finally {
+    await client.end();
+  }
+}
+
+export async function knowledgeCatalog({ databaseUrl }) {
+  const client = await connect(databaseUrl);
+  try {
+    const docs = await client.query(`
+      SELECT doc_id, title, path, updated_at
+      FROM documents
+      ORDER BY doc_id
+    `);
+    return {
+      topics: [],
+      projects: [],
+      documents: docs.rows.map((row) => ({
+        doc_id: row.doc_id,
+        title: row.title,
+        path: row.path,
+        updated_at: String(row.updated_at)
+      }))
+    };
+  } finally {
+    await client.end();
+  }
+}
+
+export async function doctor({ databaseUrl, embeddingProfile }) {
+  let client = null;
+  try {
+    client = await connect(databaseUrl);
+    const schema = await schemaStatus(client);
+    await ensureCompatibility(client, embeddingProfile);
+    const docs = await client.query("SELECT COUNT(*)::int AS count FROM documents");
+    return {
+      ok: true,
+      documents: docs.rows[0].count,
+      vectors: docs.rows[0].count,
+      embeddingMode: embeddingProfile.mode,
+      embeddingModel: embeddingProfile.model,
+      embeddingDimensions: embeddingProfile.dimensions,
+      schemaCurrent: schema.currentVersion,
+      schemaLatest: schema.latestVersion,
+      schemaPending: schema.pendingCount
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      documents: 0,
+      vectors: 0,
+      embeddingMode: null,
+      schemaCurrent: null,
+      schemaLatest: null,
+      schemaPending: null,
+      error: String(error.message ?? error)
+    };
+  } finally {
+    if (client) {
+      await client.end();
+    }
+  }
+}
+
+export function snippet(content, limit = 220) {
+  const normalized = content.replace(/\s+/g, " ").trim();
+  if (normalized.length <= limit) {
+    return normalized;
+  }
+  return `${normalized.slice(0, limit - 3).trimEnd()}...`;
+}
+
+async function lexicalCandidates(client, { query, limit }) {
+  const result = await client.query(
+    `
+    SELECT
+      doc_id, title, content, path, updated_at,
+      ts_rank_cd(search_tsv, websearch_to_tsquery('english', $1))::float AS lexical_score
+    FROM documents
+    WHERE search_tsv @@ websearch_to_tsquery('english', $1)
+    ORDER BY lexical_score DESC
+    LIMIT $2
+    `,
+    [query, limit]
+  );
+  return result.rows;
+}
+
+async function semanticCandidates(client, { embeddingProfile, query, limit }) {
+  const queryEmbedding = vectorLiteral(await embedText(query, embeddingProfile));
+  const result = await client.query(
+    `
+    SELECT
+      doc_id, title, content, path, updated_at,
+      (1 - (embedding <=> $1::vector))::float AS semantic_score
+    FROM documents
+    ORDER BY embedding <=> $1::vector
+    LIMIT $2
+    `,
+    [queryEmbedding, limit]
+  );
+  return result.rows;
+}
+
+function combineScores({ lexical, semantic, query, title }) {
+  let score = lexical * 0.68 + Math.max(semantic, 0) * 0.22;
+  score += tokenOverlap(query, title) * 0.08;
+  return score;
+}