harper-knowledge 0.1.0

package/dist/core/search.js

@@ -0,0 +1,306 @@
+/**
+ * Knowledge Base Search
+ *
+ * Supports keyword, semantic (vector), and hybrid search modes.
+ * Applies applicability context filtering to boost/demote results.
+ * Logs all queries to the QueryLog table for analytics.
+ */
+import crypto from "node:crypto";
+import { generateEmbedding } from "./embeddings.js";
+/** Default number of results to return */
+const DEFAULT_LIMIT = 10;
+/** Score boost factor for applicability matches */
+const APPLICABILITY_BOOST = 1.2;
+/** Score penalty factor for applicability mismatches */
+const APPLICABILITY_PENALTY = 0.8;
+/**
+ * Search the knowledge base.
+ *
+ * @param params - Search parameters including query, mode, tags, limit, context
+ * @returns Scored and sorted search results
+ */
+export async function search(params) {
+    const { query, tags, limit = DEFAULT_LIMIT, context, mode = "hybrid", } = params;
+    let results;
+    switch (mode) {
+        case "keyword":
+            results = await keywordSearch(query, tags, limit);
+            break;
+        case "semantic":
+            results = await semanticSearch(query, limit);
+            break;
+        case "hybrid":
+        default:
+            results = await hybridSearch(query, tags, limit);
+            break;
+    }
+    // Apply applicability filtering (boost/demote based on context)
+    if (context) {
+        results = filterByApplicability(results, context);
+    }
+    // Re-sort by score after applicability adjustments
+    results.sort((a, b) => b.score - a.score);
+    // Trim to limit
+    results = results.slice(0, limit);
+    // Log the search query for analytics
+    await logQuery(query, context, results);
+    return results;
+}
+/**
+ * Perform keyword-based search using Harper's condition-based search.
+ */
+async function keywordSearch(query, tags, limit = DEFAULT_LIMIT) {
+    const conditions = [];
+    // Search by title containing the query
+    conditions.push({ attribute: "title", comparator: "contains", value: query });
+    // Filter by tags if provided
+    if (tags && tags.length > 0) {
+        for (const tag of tags) {
+            conditions.push({
+                attribute: "tags",
+                comparator: "contains",
+                value: tag,
+            });
+        }
+    }
+    // Run the title search
+    const titleResults = await collectResults(databases.kb.KnowledgeEntry.search({
+        conditions: [
+            { attribute: "title", comparator: "contains", value: query },
+            { attribute: "deprecated", comparator: "equals", value: false },
+        ],
+        limit: limit * 2, // Fetch extra for merging
+    }));
+    // Also search by content
+    const contentResults = await collectResults(databases.kb.KnowledgeEntry.search({
+        conditions: [
+            { attribute: "content", comparator: "contains", value: query },
+            { attribute: "deprecated", comparator: "equals", value: false },
+        ],
+        limit: limit * 2,
+    }));
+    // Merge and deduplicate
+    const seenIds = new Set();
+    const results = [];
+    // Title matches get higher score
+    for (const entry of titleResults) {
+        const typed = entry;
+        if (typed.id && !seenIds.has(typed.id)) {
+            seenIds.add(typed.id);
+            results.push({
+                ...typed,
+                score: 1.0, // Title match = highest keyword score
+                matchType: "keyword",
+            });
+        }
+    }
+    // Content matches get lower score
+    for (const entry of contentResults) {
+        const typed = entry;
+        if (typed.id && !seenIds.has(typed.id)) {
+            seenIds.add(typed.id);
+            results.push({
+                ...typed,
+                score: 0.7, // Content match = lower keyword score
+                matchType: "keyword",
+            });
+        }
+    }
+    // Filter by tags if needed (post-filter since conditions are ANDed)
+    if (tags && tags.length > 0) {
+        return results.filter((r) => {
+            const entryTags = r.tags || [];
+            return tags.some((tag) => entryTags.includes(tag));
+        });
+    }
+    return results;
+}
+/**
+ * Perform semantic (vector) search using HNSW index.
+ */
+async function semanticSearch(query, limit = DEFAULT_LIMIT) {
+    let queryVector;
+    try {
+        queryVector = await generateEmbedding(query);
+    }
+    catch (error) {
+        logger?.warn?.("Semantic search failed — embedding model not available:", error.message);
+        return [];
+    }
+    const rawResults = await collectResults(databases.kb.KnowledgeEntry.search({
+        sort: { attribute: "embedding", target: queryVector },
+        limit: limit * 2, // Fetch extra to allow for deprecated filtering
+    }));
+    const results = [];
+    for (const entry of rawResults) {
+        const typed = entry;
+        if (typed.deprecated)
+            continue; // Skip deprecated entries
+        // Calculate cosine similarity score (HNSW returns nearest first)
+        // Score decreases with position (1.0 for first result, decreasing)
+        const positionScore = 1.0 - results.length / (limit * 2);
+        results.push({
+            ...typed,
+            score: Math.max(0.1, positionScore),
+            matchType: "semantic",
+        });
+        if (results.length >= limit)
+            break;
+    }
+    return results;
+}
+/**
+ * Perform hybrid search: run both keyword and semantic, merge and deduplicate.
+ */
+async function hybridSearch(query, tags, limit = DEFAULT_LIMIT) {
+    // Run both searches in parallel
+    const [keywordResults, semanticResults] = await Promise.all([
+        keywordSearch(query, tags, limit),
+        semanticSearch(query, limit),
+    ]);
+    // Merge and deduplicate
+    const resultMap = new Map();
+    // Add keyword results
+    for (const result of keywordResults) {
+        resultMap.set(result.id, result);
+    }
+    // Merge semantic results (combine scores if entry appears in both)
+    for (const result of semanticResults) {
+        const existing = resultMap.get(result.id);
+        if (existing) {
+            // Entry found in both — combine scores and mark as hybrid
+            existing.score = ((existing.score + result.score) / 2) * 1.3; // Boost for appearing in both
+            existing.matchType = "hybrid";
+        }
+        else {
+            resultMap.set(result.id, result);
+        }
+    }
+    const merged = Array.from(resultMap.values());
+    merged.sort((a, b) => b.score - a.score);
+    return merged.slice(0, limit);
+}
+/**
+ * Filter and re-score results based on applicability context.
+ *
+ * If the caller provides their environment context (Harper version, storage engine,
+ * Node version, platform), results that match get a score boost, while results
+ * that specify a different scope get a score penalty (but are NOT hidden).
+ */
+export function filterByApplicability(results, context) {
+    return results.map((result) => {
+        const appliesTo = result.appliesTo;
+        if (!appliesTo) {
+            // No applicability scope — no adjustment
+            return result;
+        }
+        let matchCount = 0;
+        let mismatchCount = 0;
+        let totalFields = 0;
+        // Check each applicability field
+        if (appliesTo.harper && context.harper) {
+            totalFields++;
+            if (versionMatches(context.harper, appliesTo.harper)) {
+                matchCount++;
+            }
+            else {
+                mismatchCount++;
+            }
+        }
+        if (appliesTo.storageEngine && context.storageEngine) {
+            totalFields++;
+            if (context.storageEngine === appliesTo.storageEngine) {
+                matchCount++;
+            }
+            else {
+                mismatchCount++;
+            }
+        }
+        if (appliesTo.node && context.node) {
+            totalFields++;
+            if (versionMatches(context.node, appliesTo.node)) {
+                matchCount++;
+            }
+            else {
+                mismatchCount++;
+            }
+        }
+        if (appliesTo.platform && context.platform) {
+            totalFields++;
+            if (context.platform === appliesTo.platform) {
+                matchCount++;
+            }
+            else {
+                mismatchCount++;
+            }
+        }
+        if (totalFields === 0) {
+            return result; // No overlapping fields to compare
+        }
+        // Adjust score based on matches vs mismatches
+        let adjustedScore = result.score;
+        if (matchCount > 0) {
+            adjustedScore *= APPLICABILITY_BOOST;
+        }
+        if (mismatchCount > 0) {
+            adjustedScore *= APPLICABILITY_PENALTY;
+        }
+        return { ...result, score: adjustedScore };
+    });
+}
+/**
+ * Simple version matching.
+ * Supports exact match and basic semver range prefixes (>=, <=, ~, ^).
+ * For production use, consider a proper semver library.
+ */
+function versionMatches(actual, required) {
+    // Strip common prefixes for comparison
+    const cleanActual = actual.replace(/^[v=]/, "");
+    const cleanRequired = required.replace(/^[v=]/, "");
+    // Exact match
+    if (cleanActual === cleanRequired)
+        return true;
+    // Range prefix checks (simplified)
+    if (required.startsWith(">=")) {
+        return cleanActual >= required.slice(2);
+    }
+    if (required.startsWith("<=")) {
+        return cleanActual <= required.slice(2);
+    }
+    // For ~ and ^ ranges, just check major.minor match
+    if (required.startsWith("~") || required.startsWith("^")) {
+        const reqParts = required.slice(1).split(".");
+        const actParts = cleanActual.split(".");
+        return (reqParts[0] === actParts[0] &&
+            (reqParts.length < 2 || reqParts[1] === actParts[1]));
+    }
+    return false;
+}
+/**
+ * Log a search query to the QueryLog table for analytics.
+ */
+async function logQuery(query, context, results) {
+    try {
+        await databases.kb.QueryLog.put({
+            id: crypto.randomUUID(),
+            query,
+            context: context || null,
+            resultCount: results.length,
+            topResultId: results.length > 0 ? results[0].id : null,
+        });
+    }
+    catch (error) {
+        // Don't fail the search if logging fails
+        logger?.warn?.("Failed to log search query:", error.message);
+    }
+}
+/**
+ * Collect all results from an async iterable into an array.
+ */
+async function collectResults(iterable) {
+    const results = [];
+    for await (const item of iterable) {
+        results.push(item);
+    }
+    return results;
+}

package/dist/core/tags.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Tag Management
+ *
+ * Manages knowledge base tags, including listing and synchronizing
+ * tag counts when entries are created, updated, or deleted.
+ */
+import type { KnowledgeTag } from "../types.ts";
+/**
+ * List knowledge tags.
+ *
+ * @param limit - Maximum number of tags to return (default 500)
+ * @returns Tags from the KnowledgeTag table
+ */
+export declare function listTags(limit?: number): Promise<KnowledgeTag[]>;
+/**
+ * Get a single tag by name (ID).
+ *
+ * @param tagName - Tag name (serves as the primary key)
+ * @returns The tag record, or null if not found
+ */
+export declare function getTag(tagName: string): Promise<KnowledgeTag | null>;
+/**
+ * Synchronize tag counts when entries are created, updated, or deleted.
+ *
+ * For tags added (in newTags but not in previousTags), increment entryCount
+ * or create the tag with count 1. For tags removed (in previousTags but not
+ * in newTags), decrement entryCount.
+ *
+ * @param newTags - Tags on the entry after the change
+ * @param previousTags - Tags on the entry before the change (empty for new entries)
+ */
+export declare function syncTags(newTags: string[], previousTags?: string[]): Promise<void>;

package/dist/core/tags.js

@@ -0,0 +1,76 @@
+/**
+ * Tag Management
+ *
+ * Manages knowledge base tags, including listing and synchronizing
+ * tag counts when entries are created, updated, or deleted.
+ */
+/**
+ * List knowledge tags.
+ *
+ * @param limit - Maximum number of tags to return (default 500)
+ * @returns Tags from the KnowledgeTag table
+ */
+export async function listTags(limit = 500) {
+    const results = [];
+    for await (const tag of databases.kb.KnowledgeTag.search({ limit })) {
+        results.push(tag);
+    }
+    return results;
+}
+/**
+ * Get a single tag by name (ID).
+ *
+ * @param tagName - Tag name (serves as the primary key)
+ * @returns The tag record, or null if not found
+ */
+export async function getTag(tagName) {
+    const tag = await databases.kb.KnowledgeTag.get(tagName);
+    return tag;
+}
+/**
+ * Synchronize tag counts when entries are created, updated, or deleted.
+ *
+ * For tags added (in newTags but not in previousTags), increment entryCount
+ * or create the tag with count 1. For tags removed (in previousTags but not
+ * in newTags), decrement entryCount.
+ *
+ * @param newTags - Tags on the entry after the change
+ * @param previousTags - Tags on the entry before the change (empty for new entries)
+ */
+export async function syncTags(newTags, previousTags) {
+    const prev = new Set(previousTags || []);
+    const next = new Set(newTags);
+    // Tags that were added
+    const added = newTags.filter((tag) => !prev.has(tag));
+    // Tags that were removed
+    const removed = (previousTags || []).filter((tag) => !next.has(tag));
+    // Increment counts for added tags
+    for (const tagName of added) {
+        const existing = await databases.kb.KnowledgeTag.get(tagName);
+        if (existing) {
+            await databases.kb.KnowledgeTag.put({
+                ...existing,
+                id: tagName,
+                entryCount: (existing.entryCount || 0) + 1,
+            });
+        }
+        else {
+            await databases.kb.KnowledgeTag.put({
+                id: tagName,
+                entryCount: 1,
+            });
+        }
+    }
+    // Decrement counts for removed tags
+    for (const tagName of removed) {
+        const existing = await databases.kb.KnowledgeTag.get(tagName);
+        if (existing) {
+            const currentCount = existing.entryCount || 0;
+            await databases.kb.KnowledgeTag.put({
+                ...existing,
+                id: tagName,
+                entryCount: Math.max(0, currentCount - 1),
+            });
+        }
+    }
+}

package/dist/core/triage.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Triage Queue Management
+ *
+ * Handles the intake queue for new knowledge submissions from webhooks
+ * and other sources. Items go through pending -> processing -> accepted/dismissed.
+ */
+import type { TriageItem, TriageAction, TriageProcessOptions } from "../types.ts";
+/**
+ * Submit a new item to the triage queue.
+ *
+ * @param source - Source identifier (e.g., "github-webhook", "slack-bot", "manual")
+ * @param summary - Brief summary of the knowledge to triage
+ * @param rawPayload - Original raw payload from the source
+ * @param sourceId - Optional deduplication key from the source system
+ * @returns The created triage item
+ */
+export declare function submitTriage(source: string, summary: string, rawPayload?: unknown, sourceId?: string): Promise<TriageItem>;
+/**
+ * Find a triage item by its source-specific ID for deduplication.
+ *
+ * @param sourceId - The source-specific identifier
+ * @returns The matching triage item, or null if not found
+ */
+export declare function findBySourceId(sourceId: string): Promise<TriageItem | null>;
+/**
+ * Process a triage item with the given action.
+ *
+ * - "accepted": Optionally creates a new knowledge entry from provided data
+ * - "dismissed": Marks the item as dismissed
+ * - "linked": Links the triage item to an existing knowledge entry
+ *
+ * @param id - Triage item ID
+ * @param action - Action to take
+ * @param processedBy - Who processed this item
+ * @param options - Additional options (entry data for accept, linked entry ID)
+ * @returns The updated triage item
+ * @throws Error if the triage item does not exist
+ */
+export declare function processTriage(id: string, action: TriageAction, processedBy: string, options?: TriageProcessOptions): Promise<TriageItem>;
+/**
+ * List pending triage items.
+ *
+ * @param limit - Maximum number of items to return (default 200)
+ * @returns Array of triage items with status "pending"
+ */
+export declare function listPending(limit?: number): Promise<TriageItem[]>;
+/**
+ * Dismiss a triage item.
+ *
+ * Convenience method that calls processTriage with action "dismissed".
+ *
+ * @param id - Triage item ID
+ * @param processedBy - Who dismissed this item
+ */
+export declare function dismissTriage(id: string, processedBy: string): Promise<void>;

package/dist/core/triage.js

@@ -0,0 +1,126 @@
+/**
+ * Triage Queue Management
+ *
+ * Handles the intake queue for new knowledge submissions from webhooks
+ * and other sources. Items go through pending -> processing -> accepted/dismissed.
+ */
+import crypto from "node:crypto";
+import { createEntry } from "./entries.js";
+/**
+ * Submit a new item to the triage queue.
+ *
+ * @param source - Source identifier (e.g., "github-webhook", "slack-bot", "manual")
+ * @param summary - Brief summary of the knowledge to triage
+ * @param rawPayload - Original raw payload from the source
+ * @param sourceId - Optional deduplication key from the source system
+ * @returns The created triage item
+ */
+export async function submitTriage(source, summary, rawPayload, sourceId) {
+    const item = {
+        id: crypto.randomUUID(),
+        source,
+        summary,
+        rawPayload: rawPayload || null,
+        status: "pending",
+        sourceId: sourceId || undefined,
+    };
+    await databases.kb.TriageItem.put(item);
+    logger?.info?.(`Triage item submitted: ${item.id} from ${source}`);
+    return item;
+}
+/**
+ * Find a triage item by its source-specific ID for deduplication.
+ *
+ * @param sourceId - The source-specific identifier
+ * @returns The matching triage item, or null if not found
+ */
+export async function findBySourceId(sourceId) {
+    for await (const item of databases.kb.TriageItem.search({
+        conditions: [
+            { attribute: "sourceId", comparator: "equals", value: sourceId },
+        ],
+        limit: 1,
+    })) {
+        return item;
+    }
+    return null;
+}
+/**
+ * Process a triage item with the given action.
+ *
+ * - "accepted": Optionally creates a new knowledge entry from provided data
+ * - "dismissed": Marks the item as dismissed
+ * - "linked": Links the triage item to an existing knowledge entry
+ *
+ * @param id - Triage item ID
+ * @param action - Action to take
+ * @param processedBy - Who processed this item
+ * @param options - Additional options (entry data for accept, linked entry ID)
+ * @returns The updated triage item
+ * @throws Error if the triage item does not exist
+ */
+export async function processTriage(id, action, processedBy, options) {
+    const existing = await databases.kb.TriageItem.get(id);
+    if (!existing) {
+        throw new Error(`Triage item not found: ${id}`);
+    }
+    const item = existing;
+    const now = new Date();
+    // Update common fields
+    item.status = action;
+    item.action = action;
+    item.processedBy = processedBy;
+    item.processedAt = now;
+    // Handle action-specific logic
+    if (action === "accepted" && options?.entryData) {
+        // Create a new knowledge entry from the triage data
+        const entryData = {
+            ...options.entryData,
+            source: options.entryData.source || item.source,
+            addedBy: options.entryData.addedBy || processedBy,
+        };
+        const entry = await createEntry(entryData);
+        item.draftEntryId = entry.id;
+    }
+    else if (action === "accepted" && options?.linkedEntryId) {
+        // Entry was created externally (e.g., web UI created it first) — link it
+        item.draftEntryId = options.linkedEntryId;
+    }
+    else if (action === "linked" && options?.linkedEntryId) {
+        // Link to an existing knowledge entry
+        item.matchedEntryId = options.linkedEntryId;
+    }
+    // Store the updated triage item
+    await databases.kb.TriageItem.put(item);
+    logger?.info?.(`Triage item ${id} processed: ${action} by ${processedBy}`);
+    return item;
+}
+/**
+ * List pending triage items.
+ *
+ * @param limit - Maximum number of items to return (default 200)
+ * @returns Array of triage items with status "pending"
+ */
+export async function listPending(limit = 200) {
+    const results = [];
+    for await (const item of databases.kb.TriageItem.search({
+        conditions: [
+            { attribute: "status", comparator: "equals", value: "pending" },
+        ],
+        limit,
+    })) {
+        results.push(item);
+    }
+    return results;
+}
+/**
+ * Dismiss a triage item.
+ *
+ * Convenience method that calls processTriage with action "dismissed".
+ *
+ * @param id - Triage item ID
+ * @param processedBy - Who dismissed this item
+ */
+export async function dismissTriage(id, processedBy) {
+    await processTriage(id, "dismissed", processedBy);
+}

package/dist/http-utils.d.ts

@@ -0,0 +1,37 @@
+/**
+ * HTTP Utilities for Harper's stream-based request/response handling.
+ *
+ * Shared by MCP middleware, OAuth middleware, and webhook middleware.
+ */
+import type { HarperRequest } from "./types.ts";
+/**
+ * Read the request body as a string from Harper's stream-based body.
+ *
+ * Harper's request.body is a RequestBody wrapper with .on()/.pipe() methods,
+ * not a parsed object. We need to consume the stream to get the raw text.
+ *
+ * Enforces a maximum body size to prevent memory exhaustion from oversized requests.
+ */
+export declare function readBody(request: HarperRequest): Promise<string>;
+/**
+ * Build Web Standard Headers from Harper's Headers object.
+ *
+ * Harper's request.headers is a custom Headers class (iterable, with .get()),
+ * not a plain Record<string, string>.
+ */
+export declare function buildHeaders(request: HarperRequest): Headers;
+/**
+ * Build the base URL (origin) from a Harper request.
+ *
+ * Uses the request's protocol and host properties. Defaults to
+ * http://localhost:9926 if not available.
+ */
+export declare function getBaseUrl(request: HarperRequest): string;
+/**
+ * Parse an application/x-www-form-urlencoded body into a key-value map.
+ */
+export declare function parseFormBody(body: string): Record<string, string>;
+/**
+ * Get a header value from Harper's request, case-insensitive.
+ */
+export declare function getHeader(request: HarperRequest, name: string): string;