bikky 0.4.2 → 0.4.4

package/dist/daemon/qdrant.js

@@ -8,35 +8,106 @@
  * self-hosted instances.
  */
 import { createHash, randomUUID } from "node:crypto";
-import { loadConfig } from "../config.js";
+import { getEffectiveDestinations, loadConfig } from "../config.js";
 import { embed, initEmbedding, getEmbeddingConfig } from "../llm/index.js";
-import { QdrantClient } from "../lib/qdrant-client.js";
-import { DEFAULT_DOMAIN, QDRANT_INDEXES, categoryForMemorySubtype, layerForMemorySubtype, normalizeCategory, normalizeDomain, normalizeKind, normalizeSource, validateMemorySubtype, } from "../mcp/taxonomy.js";
+import { QdrantPool } from "../lib/qdrant-pool.js";
+import { buildResolver } from "../routing.js";
+import { DEFAULT_DOMAIN, QDRANT_INDEXES, categoryForMemorySubtype, layerForMemorySubtype, normalizeCategory, normalizeDomain, normalizeKind, validateMemorySubtype, } from "../mcp/taxonomy.js";
 import { combineRedactions, redactStorageText, } from "../privacy/redaction.js";
+import { buildOperationOrigin } from "../provenance/origin.js";
+import { buildMemoryRoutingInput, mergeRoutingInputs } from "../routing-context.js";
 // ---------------------------------------------------------------------------
 // State
 // ---------------------------------------------------------------------------
-let qdrantUrl = null;
-let qdrantApiKey = null;
 let collection = "bikky";
 let logFn = () => { };
-let client = null;
+let destinations = [];
+let pool = null;
+let resolver = null;
 const setLogger = (fn) => { logFn = fn; };
 const setEmbeddingConfig = (overrides) => {
     if (overrides && overrides.provider)
         initEmbedding(overrides);
 };
 const clientLogAdapter = (level, msg) => logFn(level, msg);
+const fallbackDestination = () => {
+    if (destinations.length === 0) {
+        throw new Error("Qdrant client not initialized — call init() first");
+    }
+    return destinations.find((destination) => destination.default === true) ?? destinations[0];
+};
+const resolveDestination = (input = {}) => {
+    if (!resolver)
+        return fallbackDestination();
+    return resolver(input);
+};
+const destinationFromRef = (ref) => {
+    if (!ref)
+        return fallbackDestination();
+    if (typeof ref !== "string")
+        return ref;
+    const found = destinations.find((destination) => destination.name === ref);
+    if (!found) {
+        throw new Error(`Unknown Qdrant destination '${ref}'. Configured destinations: ${destinations.map((d) => d.name).join(", ") || "(none)"}`);
+    }
+    return found;
+};
+const pathForDestination = (urlPath, destination) => {
+    if (!urlPath.startsWith("/collections/"))
+        return urlPath;
+    return urlPath.replace(/^\/collections\/[^/]+/, `/collections/${destination.collection}`);
+};
+const routingInputForFact = (fact, normalizedContent, normalizedEntities, extraMetadata = {}) => {
+    const metadata = {
+        ...(fact.metadata ?? {}),
+        ...extraMetadata,
+        category: fact.category,
+        domain: fact.domain ?? DEFAULT_DOMAIN,
+        kind: fact.kind ?? "fact",
+        ...(fact.memory_subtype ? { memory_subtype: fact.memory_subtype } : {}),
+        ...(fact.source ? { source: fact.source } : {}),
+        ...(fact.actor_id ? { actor_id: fact.actor_id } : {}),
+        ...(fact.session_id ? { session_id: fact.session_id } : {}),
+        ...(fact.episode_id ? { episode_id: fact.episode_id } : {}),
+        ...(fact.workstream_key ? { workstream_key: fact.workstream_key } : {}),
+        ...(fact.task_key ? { task_key: fact.task_key } : {}),
+        ...(fact.repo ? { repo: fact.repo } : {}),
+        ...(fact.branch ? { branch: fact.branch } : {}),
+        ...(fact.surface ? { surface: fact.surface } : {}),
+        ...(fact.issue_id ? { issue_id: fact.issue_id } : {}),
+        ...(fact.pr_id ? { pr_id: fact.pr_id } : {}),
+        ...(fact.source_event_ids ? { source_event_ids: fact.source_event_ids } : {}),
+        ...(fact.source_fact_ids ? { source_fact_ids: fact.source_fact_ids } : {}),
+        ...(fact.source_episode_ids ? { source_episode_ids: fact.source_episode_ids } : {}),
+        ...(fact.prompt_version ? { prompt_version: fact.prompt_version } : {}),
+        ...(fact.capture_policy_version ? { capture_policy_version: fact.capture_policy_version } : {}),
+        ...(fact.review_status ? { review_status: fact.review_status } : {}),
+        ...(fact.volatility ? { volatility: fact.volatility } : {}),
+        ...(fact.valid_from ? { valid_from: fact.valid_from } : {}),
+        ...(fact.expires_at ? { expires_at: fact.expires_at } : {}),
+        ...(fact.confidence_reason ? { confidence_reason: fact.confidence_reason } : {}),
+        ...(fact.relation ? {
+            from_entity: fact.relation.from,
+            relation_type: fact.relation.type,
+            to_entity: fact.relation.to,
+        } : {}),
+    };
+    return buildMemoryRoutingInput({
+        content: normalizedContent,
+        entities: normalizedEntities,
+        metadata,
+        extraContent: [fact.origin, fact.last_operation_origin, fact.relation],
+    });
+};
 // ---------------------------------------------------------------------------
 // Init — reads credentials from loadConfig()
 // ---------------------------------------------------------------------------
 const init = () => {
     const cfg = loadConfig();
-    qdrantUrl = cfg.qdrant_url;
-    qdrantApiKey = cfg.qdrant_api_key;
-    collection = cfg.collection || "bikky";
-    if (qdrantUrl)
-        qdrantUrl = qdrantUrl.replace(/\/+$/, "");
+    destinations = getEffectiveDestinations(cfg);
+    collection = (destinations.find((destination) => destination.default === true) ?? destinations[0])?.collection
+        ?? cfg.collection
+        ?? "bikky";
     // Initialize embedding provider from config
     const embCfg = initEmbedding({
         provider: cfg.embedding.provider,
@@ -50,48 +121,72 @@ const init = () => {
         retryBaseDelayMs: cfg.embedding.retry_base_delay_ms,
     });
     logFn("INFO", `Embedding provider: ${embCfg.provider}/${embCfg.model} (${embCfg.dimensions}d) @ ${embCfg.baseUrl}`);
-    const ready = !!qdrantUrl;
+    const ready = destinations.length > 0;
     if (ready) {
-        client = new QdrantClient({
-            url: qdrantUrl,
-            apiKey: qdrantApiKey,
-            collection,
-            timeoutMs: cfg.qdrant_client.timeout_ms,
-            retries: cfg.qdrant_client.retries,
-            retryBaseDelayMs: cfg.qdrant_client.retry_base_delay_ms,
+        pool = new QdrantPool(destinations, {
+            client: cfg.qdrant_client,
             log: clientLogAdapter,
         });
+        resolver = buildResolver(destinations);
+        logFn("INFO", `Qdrant destinations: ${destinations.map((destination) => `${destination.name}/${destination.collection}`).join(", ")}`);
     }
     else {
-        client = null;
+        pool = null;
+        resolver = null;
         logFn("WARN", "Qdrant client: missing URL (some memory features disabled)");
     }
     return ready;
 };
-const isReady = () => !!(qdrantUrl && client);
+const isReady = () => !!(pool && destinations.length > 0);
 const ensureCollection = async () => {
-    if (!client) {
+    if (!pool) {
         throw new Error("Qdrant client not initialized — call init() first");
     }
     const embCfg = getEmbeddingConfig();
-    await client.ensureCollection(embCfg.dimensions, QDRANT_INDEXES);
-    logFn("INFO", `Qdrant collection '${collection}' ready (${QDRANT_INDEXES.length} indexes)`);
+    const results = await Promise.all(destinations.map(async (destination) => {
+        try {
+            await pool.ensureCollection(destination.name, embCfg.dimensions, QDRANT_INDEXES);
+            logFn("INFO", `Qdrant destination '${destination.name}' collection '${destination.collection}' ready (${QDRANT_INDEXES.length} indexes)`);
+            return { destination, ok: true, error: null };
+        }
+        catch (e) {
+            const message = e instanceof Error ? e.message : String(e);
+            logFn("WARN", `Qdrant destination '${destination.name}' readiness check failed: ${message}`);
+            return { destination, ok: false, error: message };
+        }
+    }));
+    if (!results.some((result) => result.ok)) {
+        throw new Error(`No Qdrant destinations ready: ${results.map((result) => `${result.destination.name}: ${result.error}`).join("; ")}`);
+    }
 };
 // ---------------------------------------------------------------------------
 // HTTP requests
 // ---------------------------------------------------------------------------
-const qdrantRequest = async (method, urlPath, body) => {
-    if (!client) {
+const qdrantRequest = async (method, urlPath, body, destinationRef) => {
+    if (!pool) {
         throw new Error(`Qdrant client not initialized — call init() first (${method} ${urlPath})`);
     }
-    const result = await client.request(method, urlPath, body);
+    const destination = destinationFromRef(destinationRef);
+    const result = await pool.client(destination.name).request(method, pathForDestination(urlPath, destination), body);
     // Some Qdrant endpoints return empty bodies on success — preserve old return shape.
     return result ?? {};
 };
+const collectionForDestination = (destinationRef) => destinationFromRef(destinationRef).collection;
+const destinationNames = () => destinations.map((destination) => destination.name);
+const readyDestinations = () => {
+    if (!pool)
+        return [];
+    return destinations.filter((destination) => pool.isCollectionReady(destination.name));
+};
+const activeDestinations = () => {
+    const ready = readyDestinations();
+    return ready.length > 0 ? ready : destinations;
+};
 // ---------------------------------------------------------------------------
 // Read methods
 // ---------------------------------------------------------------------------
-const searchFacts = async (query, filters = {}, limit = 10) => {
+const searchFacts = async (query, filters = {}, limit = 10, destinationRef) => {
+    const destination = destinationFromRef(destinationRef);
     const vector = await embed(query);
     const must = [
         { is_null: { key: "superseded_by" } },
@@ -114,14 +209,15 @@ const searchFacts = async (query, filters = {}, limit = 10) => {
     if (filters.workspaceId) {
         must.push({ key: "workspace_id", match: { value: filters.workspaceId } });
     }
-    const result = await qdrantRequest("POST", `/collections/${collection}/points/search`, {
+    const result = await qdrantRequest("POST", `/collections/${destination.collection}/points/search`, {
         vector,
         filter: { must },
         limit,
         with_payload: true,
-    });
+    }, destination);
     return (result.result || []).map((hit) => ({
         id: hit.id,
+        destination: destination.name,
         score: hit.score,
         content: hit.payload?.content ?? "",
         category: hit.payload?.category ?? "",
@@ -131,7 +227,8 @@ const searchFacts = async (query, filters = {}, limit = 10) => {
         created_at: hit.payload?.created_at ?? "",
     }));
 };
-const scrollFacts = async (filters = {}, limit = 10) => {
+const scrollFacts = async (filters = {}, limit = 10, destinationRef) => {
+    const destination = destinationFromRef(destinationRef);
     const must = [
         { is_null: { key: "superseded_by" } },
     ];
@@ -179,14 +276,15 @@ const scrollFacts = async (filters = {}, limit = 10) => {
     if (filters.workspaceId) {
         must.push({ key: "workspace_id", match: { value: filters.workspaceId } });
     }
-    const result = await qdrantRequest("POST", `/collections/${collection}/points/scroll`, {
+    const result = await qdrantRequest("POST", `/collections/${destination.collection}/points/scroll`, {
         filter: { must, must_not },
         limit,
         ...(filters.orderBy ? { order_by: { key: filters.orderBy.key, direction: filters.orderBy.direction } } : {}),
         with_payload: true,
-    });
+    }, destination);
     return (result.result?.points || []).map((pt) => ({
         id: pt.id,
+        destination: destination.name,
         content: pt.payload?.content ?? "",
         category: pt.payload?.category ?? "",
         entities: pt.payload?.entities || [],
@@ -203,10 +301,17 @@ const scrollFacts = async (filters = {}, limit = 10) => {
         source_fact_ids: pt.payload?.source_fact_ids ?? [],
     }));
 };
+const scrollFactsAcrossDestinations = async (filters = {}, limit = 10) => {
+    const results = await Promise.all(activeDestinations().map((destination) => scrollFacts(filters, limit, destination).catch((e) => {
+        logFn("WARN", `Qdrant scroll failed for destination '${destination.name}': ${e.message}`);
+        return [];
+    })));
+    return results.flat();
+};
 // ---------------------------------------------------------------------------
 // Write methods
 // ---------------------------------------------------------------------------
-const storeFact = async (fact) => {
+const storeFact = async (fact, routeInput) => {
     const normalizedKind = normalizeKind(fact.kind);
     const normalizedSubtype = validateMemorySubtype(normalizedKind, fact.memory_subtype);
     const normalizedCategory = normalizedSubtype
@@ -226,9 +331,18 @@ const storeFact = async (fact) => {
         ...redactedEntities,
         ...(redactedRelation ? [redactedRelation.from, redactedRelation.type, redactedRelation.to] : []),
     ]);
-    const vector = await embed(redactedContent.text);
     const now = new Date().toISOString();
     const id = randomUUID();
+    const origin = fact.origin ?? buildOperationOrigin({
+        interface: "daemon",
+        action: "create",
+        subsystem: "qdrant.store_fact",
+        metadata: {
+            category: normalizedCategory,
+            kind: normalizedKind,
+            ...(normalizedSubtype ? { memory_subtype: normalizedSubtype } : {}),
+        },
+    });
     const payload = {
         content: redactedContent.text,
         category: normalizedCategory,
@@ -236,9 +350,9 @@ const storeFact = async (fact) => {
         kind: normalizedKind,
         ...(normalizedLayer ? { layer: normalizedLayer } : {}),
         ...(normalizedSubtype ? { memory_subtype: normalizedSubtype } : {}),
-        ...(fact.actor_id ? { actor_id: fact.actor_id } : {}),
+        origin,
+        ...(fact.last_operation_origin ? { last_operation_origin: fact.last_operation_origin } : {}),
         entities: redactedEntities.map((entity) => entity.text.toLowerCase()),
-        source: normalizeSource(fact.source ?? "system"),
         confidence: fact.confidence ?? 0.7,
         importance: fact.importance ?? 0.5,
         content_hash: redactedContent.redacted
@@ -281,37 +395,58 @@ const storeFact = async (fact) => {
     if (redaction.redacted) {
         payload.redaction = redaction;
     }
-    await qdrantRequest("PUT", `/collections/${collection}/points`, {
+    const destination = resolveDestination(mergeRoutingInputs(routingInputForFact(fact, redactedContent.text, payload.entities, {
+        category: normalizedCategory,
+        domain: normalizedDomain,
+        kind: normalizedKind,
+        ...(normalizedSubtype ? { memory_subtype: normalizedSubtype } : {}),
+    }), routeInput));
+    const vector = await embed(redactedContent.text);
+    await qdrantRequest("PUT", `/collections/${destination.collection}/points`, {
         points: [{ id, vector, payload }],
-    });
-    logFn("DEBUG", `Qdrant: stored fact ${id} [${normalizedCategory}] ${redactedContent.text.slice(0, 60)}`);
+    }, destination);
+    logFn("DEBUG", `Qdrant: stored fact ${id} in '${destination.name}' [${normalizedCategory}] ${redactedContent.text.slice(0, 60)}`);
     return id;
 };
-const supersedeFact = async (oldFactId, newFactId) => {
+const supersedeFact = async (oldFactId, newFactId, destinationRef, origin) => {
+    const destination = destinationFromRef(destinationRef);
     const now = new Date().toISOString();
-    await qdrantRequest("POST", `/collections/${collection}/points/payload`, {
+    await qdrantRequest("POST", `/collections/${destination.collection}/points/payload`, {
         payload: {
             superseded_by: newFactId,
             superseded_at: now,
             updated_at: now,
+            last_operation_origin: origin ?? buildOperationOrigin({
+                interface: "daemon",
+                action: "supersede",
+                subsystem: "qdrant.supersede_fact",
+                metadata: { new_fact_id: newFactId },
+            }),
         },
         points: [oldFactId],
-    });
-    logFn("DEBUG", `Qdrant: superseded fact ${oldFactId} → ${newFactId}`);
+    }, destination);
+    logFn("DEBUG", `Qdrant: superseded fact ${oldFactId} → ${newFactId} in '${destination.name}'`);
 };
-const reinforceFact = async (factId, currentCount) => {
+const reinforceFact = async (factId, currentCount, destinationRef, origin) => {
+    const destination = destinationFromRef(destinationRef);
     const now = new Date().toISOString();
-    await qdrantRequest("POST", `/collections/${collection}/points/payload`, {
+    await qdrantRequest("POST", `/collections/${destination.collection}/points/payload`, {
         payload: {
             reinforcement_count: (currentCount || 1) + 1,
             last_reinforced_at: now,
             updated_at: now,
+            last_operation_origin: origin ?? buildOperationOrigin({
+                interface: "daemon",
+                action: "reinforce",
+                subsystem: "qdrant.reinforce_fact",
+            }),
         },
         points: [factId],
-    });
-    logFn("DEBUG", `Qdrant: reinforced fact ${factId}`);
+    }, destination);
+    logFn("DEBUG", `Qdrant: reinforced fact ${factId} in '${destination.name}'`);
 };
-const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supersedeThreshold = 0.80 } = {}, workspaceId) => {
+const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supersedeThreshold = 0.80 } = {}, workspaceId, routeInput) => {
+    const destination = resolveDestination(routeInput ?? { content });
     // First: hash-based exact check (fast, no embedding)
     try {
         const must = [
@@ -320,15 +455,16 @@ const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supe
         ];
         if (workspaceId)
             must.push({ key: "workspace_id", match: { value: workspaceId } });
-        const hashResult = await qdrantRequest("POST", `/collections/${collection}/points/scroll`, {
+        const hashResult = await qdrantRequest("POST", `/collections/${destination.collection}/points/scroll`, {
             filter: { must },
             limit: 1,
             with_payload: true,
-        });
+        }, destination);
         const existing = hashResult.result?.points?.[0];
         if (existing) {
             return {
                 action: "skip",
+                destination: destination.name,
                 existingId: existing.id,
                 existingCount: existing.payload?.reinforcement_count || 1,
                 score: 1.0,
@@ -345,18 +481,19 @@ const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supe
         const must = [{ is_null: { key: "superseded_by" } }];
         if (workspaceId)
             must.push({ key: "workspace_id", match: { value: workspaceId } });
-        const searchResult = await qdrantRequest("POST", `/collections/${collection}/points/search`, {
+        const searchResult = await qdrantRequest("POST", `/collections/${destination.collection}/points/search`, {
             vector,
             filter: { must },
             limit: 1,
             with_payload: true,
-        });
+        }, destination);
         const top = searchResult.result?.[0];
         if (!top)
-            return { action: "insert" };
+            return { action: "insert", destination: destination.name };
         if (top.score >= exactThreshold) {
             return {
                 action: "skip",
+                destination: destination.name,
                 existingId: top.id,
                 existingCount: top.payload?.reinforcement_count || 1,
                 score: top.score,
@@ -365,17 +502,18 @@ const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supe
         if (top.score >= supersedeThreshold) {
             return {
                 action: "supersede",
+                destination: destination.name,
                 existingId: top.id,
                 existingCount: top.payload?.reinforcement_count || 1,
                 score: top.score,
             };
         }
-        return { action: "insert" };
+        return { action: "insert", destination: destination.name };
     }
     catch (e) {
         const msg = e instanceof Error ? e.message : String(e);
         logFn("WARN", `Qdrant dedup vector check failed: ${msg}`);
-        return { action: "insert" }; // fail open — better to duplicate than lose
+        return { action: "insert", destination: destination.name }; // fail open — better to duplicate than lose
     }
 };
 /**
@@ -387,7 +525,8 @@ const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supe
  * No hardcoded vocabulary — purely embedding-similarity based, and the
  * exemplar set grows organically every time a user calls memory_forget.
  */
-const badExemplarCheck = async (content, workspaceId) => {
+const badExemplarCheck = async (content, workspaceId, routeInput) => {
+    const destination = resolveDestination(routeInput ?? { content });
     try {
         const vector = await embed(content);
         const must = [
@@ -395,12 +534,12 @@ const badExemplarCheck = async (content, workspaceId) => {
         ];
         if (workspaceId)
             must.push({ key: "workspace_id", match: { value: workspaceId } });
-        const result = await qdrantRequest("POST", `/collections/${collection}/points/search`, {
+        const result = await qdrantRequest("POST", `/collections/${destination.collection}/points/search`, {
             vector,
             filter: { must },
             limit: 1,
             with_payload: true,
-        });
+        }, destination);
         const top = result.result?.[0];
         if (!top)
             return null;
@@ -419,5 +558,5 @@ const badExemplarCheck = async (content, workspaceId) => {
 // ---------------------------------------------------------------------------
 // Exports
 // ---------------------------------------------------------------------------
-export { init, isReady, ensureCollection, setLogger, setEmbeddingConfig, qdrantRequest, embed, searchFacts, scrollFacts, storeFact, supersedeFact, reinforceFact, dedupCheck, badExemplarCheck, collection, };
+export { init, isReady, ensureCollection, setLogger, setEmbeddingConfig, qdrantRequest, resolveDestination, collectionForDestination, destinationNames, readyDestinations, activeDestinations, embed, searchFacts, scrollFacts, scrollFactsAcrossDestinations, storeFact, supersedeFact, reinforceFact, dedupCheck, badExemplarCheck, collection, };
 //# sourceMappingURL=qdrant.js.map

package/dist/daemon/quality-rollups.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Backend aggregation for memory quality telemetry.
+ */
+import type { BikkyConfig, Destination } from "../config.js";
+import type { FactPayload } from "../mcp/types.js";
+import type { LogFn } from "./qdrant.js";
+export type QualityScopeType = "destination" | "repo" | "workstream_key" | "task_key" | "entity" | "origin_user" | "origin_agent";
+export interface QualityPoint {
+    id: string;
+    destination: string;
+    payload: Partial<FactPayload>;
+}
+export interface QualityRollup {
+    destination: string;
+    scope_type: QualityScopeType;
+    scope_value: string;
+    active_fact_count: number;
+    recall_count: number;
+    useful_count: number;
+    misleading_count: number;
+    wrong_count: number;
+    stale_count: number;
+    low_confidence_count: number;
+    generated_at: string;
+    source_fact_ids: string[];
+    source_event_ids: string[];
+}
+export interface QualityRollupResult {
+    destinations_seen: number;
+    facts_seen: number;
+    events_seen: number;
+    rollups_upserted: number;
+    scopes_capped: boolean;
+}
+export interface QualityRollupDeps {
+    isReady: () => boolean;
+    activeDestinations: () => Destination[];
+    qdrantRequest: (method: string, urlPath: string, body?: unknown, destinationRef?: Destination | string | null) => Promise<Record<string, unknown>>;
+    embed: (text: string) => Promise<number[]>;
+}
+export declare const setLogger: (fn: LogFn) => void;
+export declare const buildQualityRollups: (input: {
+    facts: QualityPoint[];
+    events?: QualityPoint[];
+    generatedAt?: Date;
+    staleThresholdDays?: number;
+    lowConfidenceThreshold?: number;
+}) => QualityRollup[];
+export declare const aggregateMemoryQualitySignals: (config: BikkyConfig, deps?: QualityRollupDeps) => Promise<QualityRollupResult>;
+export declare const tick: (config: BikkyConfig, deps?: QualityRollupDeps) => Promise<void>;
+//# sourceMappingURL=quality-rollups.d.ts.map