memo-grafter 0.1.0

package/dist/fleet/types.d.ts

@@ -0,0 +1,38 @@
+import type { AbsorbFromAgentOptions, InjectionResult, MemoGrafterConfig, Message, TopicNode } from "../types.js";
+export interface FleetGraph {
+    id: string;
+    name?: string;
+    agents: FleetAgentInfo[];
+}
+export interface FleetAgentInfo {
+    id: string;
+    sessionId: string;
+    color: string;
+}
+export interface MemoGrafterFleetOptions {
+    id?: string;
+    name?: string;
+}
+export interface WorkerAgentConfig {
+    color: string;
+    sessionId?: string;
+    id?: string;
+}
+export interface ConductorGraftOptions {
+    topicIds?: string[];
+    prompt?: string;
+    minSimilarity?: number;
+    limit?: number;
+}
+export type FleetAbsorbOptions = AbsorbFromAgentOptions;
+export interface FleetWorker {
+    getAgentId(): string;
+    getSessionId(): string;
+    getColor(): string;
+    invoke(userMessage: string): Promise<string>;
+    getHistory(): Message[];
+    getActiveNodes(): Promise<TopicNode[]>;
+    graft(topicIds?: string[]): Promise<InjectionResult>;
+}
+export type { MemoGrafterConfig };
+//# sourceMappingURL=types.d.ts.map

package/dist/fleet/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/fleet/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,eAAe,EAAE,iBAAiB,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAElH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,cAAc,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,uBAAuB;IACtC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,EAAE,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AAExD,MAAM,WAAW,WAAW;IAC1B,UAAU,IAAI,MAAM,CAAC;IACrB,YAAY,IAAI,MAAM,CAAC;IACvB,QAAQ,IAAI,MAAM,CAAC;IACnB,MAAM,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,UAAU,IAAI,OAAO,EAAE,CAAC;IACxB,cAAc,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACvC,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACtD;AAED,YAAY,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/fleet/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/fleet/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/fleet/types.ts"],"names":[],"mappings":""}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export { MemoGrafterAgent } from "./MemoGrafterAgent.js";
+export { MemoGrafter } from "./MemoGrafter.js";
+export { ConductorAgent } from "./fleet/ConductorAgent.js";
+export { MemoGrafterFleet } from "./fleet/MemoGrafterFleet.js";
+export { WorkerAgent } from "./fleet/WorkerAgent.js";
+export { OpenAIEmbedAdapter, OpenAILLMAdapter } from "./adapters/OpenAIAdapter.js";
+export type { AbsorbFromAgentOptions, DriftMode, EmbedAdapter, InjectionResult, LLMAdapter, MemoGrafterConfig, MemoGrafterDriftConfig, MemoGrafterGraphConfig, MemoGrafterInjectConfig, MemoGrafterQueueConfig, Message, TopicEdge, TopicNode, TopicSegment, } from "./types.js";
+export type { ConductorGraftOptions, FleetAbsorbOptions, FleetAgentInfo, FleetGraph, FleetWorker, MemoGrafterFleetOptions, WorkerAgentConfig, } from "./fleet/types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACnF,YAAY,EACV,sBAAsB,EACtB,SAAS,EACT,YAAY,EACZ,eAAe,EACf,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,sBAAsB,EACtB,OAAO,EACP,SAAS,EACT,SAAS,EACT,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,qBAAqB,EACrB,kBAAkB,EAClB,cAAc,EACd,UAAU,EACV,WAAW,EACX,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,kBAAkB,CAAC"}

package/dist/index.js

@@ -0,0 +1,7 @@
+export { MemoGrafterAgent } from "./MemoGrafterAgent.js";
+export { MemoGrafter } from "./MemoGrafter.js";
+export { ConductorAgent } from "./fleet/ConductorAgent.js";
+export { MemoGrafterFleet } from "./fleet/MemoGrafterFleet.js";
+export { WorkerAgent } from "./fleet/WorkerAgent.js";
+export { OpenAIEmbedAdapter, OpenAILLMAdapter } from "./adapters/OpenAIAdapter.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC"}

package/dist/pipeline/GrafterPipeline.d.ts

@@ -0,0 +1,16 @@
+import type { GraphStore } from "../store/GraphStore.js";
+import type { InjectionResult } from "../types.js";
+export declare class GrafterPipeline {
+    private store;
+    private config;
+    constructor(store: GraphStore, config: {
+        hopDepth: number;
+        bufferSize: number;
+        tokenBudget: number;
+    });
+    run(sessionId: string, topicIds: string[]): Promise<InjectionResult>;
+    private assemblePrompt;
+    private formatNode;
+    private countTokens;
+}
+//# sourceMappingURL=GrafterPipeline.d.ts.map

package/dist/pipeline/GrafterPipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GrafterPipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/GrafterPipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAsB,MAAM,aAAa,CAAC;AAEvE,qBAAa,eAAe;IAExB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,MAAM;gBADN,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE;QACd,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;KACrB;IAGG,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC;YA4B5D,cAAc;IAqB5B,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,WAAW;CAGpB"}

package/dist/pipeline/GrafterPipeline.js

@@ -0,0 +1,58 @@
+export class GrafterPipeline {
+    store;
+    config;
+    constructor(store, config) {
+        this.store = store;
+        this.config = config;
+    }
+    async run(sessionId, topicIds) {
+        if (topicIds.length === 0) {
+            return { systemPrompt: "", nodes: [], tokenCount: 0 };
+        }
+        const neighbourhood = await this.store.getNeighbours(topicIds, this.config.hopDepth, sessionId);
+        const nodes = neighbourhood.sort((a, b) => a.messageRange[0] - b.messageRange[0]
+            || a.messageRange[1] - b.messageRange[1]
+            || a.topicOrder - b.topicOrder);
+        const fittedNodes = [...nodes];
+        let systemPrompt = await this.assemblePrompt(sessionId, fittedNodes);
+        let tokenCount = this.countTokens(systemPrompt);
+        while (tokenCount > this.config.tokenBudget && fittedNodes.length > 0) {
+            fittedNodes.pop();
+            systemPrompt = await this.assemblePrompt(sessionId, fittedNodes);
+            tokenCount = this.countTokens(systemPrompt);
+        }
+        return {
+            systemPrompt,
+            nodes: fittedNodes,
+            tokenCount,
+        };
+    }
+    async assemblePrompt(sessionId, nodes) {
+        if (nodes.length === 0)
+            return "";
+        const blocks = [];
+        for (const node of nodes) {
+            const start = Math.max(0, node.messageRange[0] - this.config.bufferSize);
+            const end = node.messageRange[1] + this.config.bufferSize;
+            const messages = await this.store.getBufferMessages(sessionId, start, end);
+            blocks.push(this.formatNode(node, messages));
+        }
+        return [
+            "MemoGrafter retrieved memory context:",
+            "Use these memories as prior conversation context when answering the user.",
+            "If the user asks what you remember, answer from these memories instead of saying you have no record.",
+            "",
+            blocks.join("\n---\n"),
+        ].join("\n");
+    }
+    formatNode(node, messages) {
+        const context = messages
+            .map((message) => `${message.role}: ${message.content}`)
+            .join("\n");
+        return `[Topic: ${node.label}]\nSummary: ${node.summary}\nContext:\n${context}`;
+    }
+    countTokens(prompt) {
+        return Math.ceil(prompt.length / 4);
+    }
+}
+//# sourceMappingURL=GrafterPipeline.js.map

package/dist/pipeline/GrafterPipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GrafterPipeline.js","sourceRoot":"","sources":["../../src/pipeline/GrafterPipeline.ts"],"names":[],"mappings":"AAGA,MAAM,OAAO,eAAe;IAEhB;IACA;IAFV,YACU,KAAiB,EACjB,MAIP;QALO,UAAK,GAAL,KAAK,CAAY;QACjB,WAAM,GAAN,MAAM,CAIb;IACA,CAAC;IAEJ,KAAK,CAAC,GAAG,CAAC,SAAiB,EAAE,QAAkB;QAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,YAAY,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;QACxD,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAChG,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACxC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;eAClC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;eACrC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAC/B,CAAC;QACF,MAAM,WAAW,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QAC/B,IAAI,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACrE,IAAI,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAEhD,OAAO,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtE,WAAW,CAAC,GAAG,EAAE,CAAC;YAClB,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YACjE,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO;YACL,YAAY;YACZ,KAAK,EAAE,WAAW;YAClB,UAAU;SACX,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,KAAkB;QAChE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACzE,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YAC1D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;YAC3E,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,OAAO;YACL,uCAAuC;YACvC,2EAA2E;YAC3E,sGAAsG;YACtG,EAAE;YACF,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;SACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAEO,UAAU,CAAC,IAAe,EAAE,QAAmB;QACrD,MAAM,OAAO,GAAG,QAAQ;aACrB,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;aACvD,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,OAAO,WAAW,IAAI,CAAC,KAAK,eAAe,IAAI,CAAC,OAAO,eAAe,OAAO,EAAE,CAAC;IAClF,CAAC;IAEO,WAAW,CAAC,MAAc;QAChC,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACF"}

package/dist/pipeline/IngestPipeline.d.ts

@@ -0,0 +1,19 @@
+import type { GraphStore } from "../store/GraphStore.js";
+import type { EmbedAdapter, LLMAdapter, Message, TopicNode } from "../types.js";
+export declare class IngestPipeline {
+    private readonly store;
+    private readonly embedder;
+    private readonly config;
+    private readonly driftDetector;
+    private readonly segmentProcessor;
+    constructor(store: GraphStore, llm: LLMAdapter, embedder: EmbedAdapter, config: {
+        windowSize: number;
+        threshold: number;
+        topK: number;
+        mode: "window" | "intent";
+        minSegmentMessages: number;
+    });
+    run(messages: Message[], sessionId: string): Promise<TopicNode[]>;
+    private embedMessage;
+}
+//# sourceMappingURL=IngestPipeline.d.ts.map

package/dist/pipeline/IngestPipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IngestPipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/IngestPipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKhF,qBAAa,cAAc;IAKvB,OAAO,CAAC,QAAQ,CAAC,KAAK;IAEtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAPzB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;gBAGjC,KAAK,EAAE,UAAU,EAClC,GAAG,EAAE,UAAU,EACE,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE;QACvB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;QAC1B,kBAAkB,EAAE,MAAM,CAAC;KAC5B;IAcG,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;YAmBzD,YAAY;CAI3B"}

package/dist/pipeline/IngestPipeline.js

@@ -0,0 +1,44 @@
+import { normalizeText } from "../utils/normalizeText.js";
+import { SegmentProcessor } from "./SegmentProcessor.js";
+import { TopicDriftDetector } from "./TopicDriftDetector.js";
+export class IngestPipeline {
+    store;
+    embedder;
+    config;
+    driftDetector;
+    segmentProcessor;
+    constructor(store, llm, embedder, config) {
+        this.store = store;
+        this.embedder = embedder;
+        this.config = config;
+        this.driftDetector = new TopicDriftDetector({
+            windowSize: config.windowSize,
+            threshold: config.threshold,
+            mode: config.mode,
+            minSegmentMessages: config.minSegmentMessages,
+        });
+        this.segmentProcessor = new SegmentProcessor(store, llm, embedder, {
+            topK: config.topK,
+            semanticThreshold: 0.6,
+        });
+    }
+    async run(messages, sessionId) {
+        if (messages.length === 0)
+            return [];
+        await this.store.saveMessages(sessionId, messages);
+        await this.store.clearSession(sessionId);
+        const embeddings = await Promise.all(messages.map((message) => this.embedMessage(message)));
+        const segments = this.driftDetector.detectSegments(messages, embeddings);
+        const nodes = [];
+        for (const segment of segments) {
+            nodes.push(await this.segmentProcessor.process(segment, messages, sessionId));
+        }
+        await this.store.rebuildEdgesForSession(sessionId, this.config.topK);
+        return nodes;
+    }
+    async embedMessage(message) {
+        const content = normalizeText(message.content) ?? message.content;
+        return this.embedder.embed(content);
+    }
+}
+//# sourceMappingURL=IngestPipeline.js.map

package/dist/pipeline/IngestPipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IngestPipeline.js","sourceRoot":"","sources":["../../src/pipeline/IngestPipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,MAAM,OAAO,cAAc;IAKN;IAEA;IACA;IAPF,aAAa,CAAqB;IAClC,gBAAgB,CAAmB;IAEpD,YACmB,KAAiB,EAClC,GAAe,EACE,QAAsB,EACtB,MAMhB;QATgB,UAAK,GAAL,KAAK,CAAY;QAEjB,aAAQ,GAAR,QAAQ,CAAc;QACtB,WAAM,GAAN,MAAM,CAMtB;QAED,IAAI,CAAC,aAAa,GAAG,IAAI,kBAAkB,CAAC;YAC1C,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;SAC9C,CAAC,CAAC;QACH,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE;YACjE,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,iBAAiB,EAAE,GAAG;SACvB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,QAAmB,EAAE,SAAiB;QAC9C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAErC,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAEzC,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5F,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACzE,MAAM,KAAK,GAAgB,EAAE,CAAC;QAE9B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QAChF,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAErE,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,OAAgB;QACzC,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC;QAClE,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;CACF"}

package/dist/pipeline/SegmentProcessor.d.ts

@@ -0,0 +1,19 @@
+import type { GraphStore } from "../store/GraphStore.js";
+import type { EmbedAdapter, LLMAdapter, Message, TopicNode } from "../types.js";
+import type { DriftSegment } from "./TopicDriftDetector.js";
+export declare class SegmentProcessor {
+    private readonly store;
+    private readonly llm;
+    private readonly embedder;
+    private readonly config;
+    constructor(store: GraphStore, llm: LLMAdapter, embedder: EmbedAdapter, config: {
+        topK: number;
+        semanticThreshold: number;
+    });
+    process(segment: DriftSegment, messages: Message[], sessionId: string): Promise<TopicNode>;
+    private createSegment;
+    private nodeRunner;
+    private nodeProcessor;
+    private parseExtraction;
+}
+//# sourceMappingURL=SegmentProcessor.d.ts.map

package/dist/pipeline/SegmentProcessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SegmentProcessor.d.ts","sourceRoot":"","sources":["../../src/pipeline/SegmentProcessor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAgB,MAAM,aAAa,CAAC;AAE9F,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,qBAAa,gBAAgB;IAEzB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAHN,KAAK,EAAE,UAAU,EACjB,GAAG,EAAE,UAAU,EACf,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE;QACvB,IAAI,EAAE,MAAM,CAAC;QACb,iBAAiB,EAAE,MAAM,CAAC;KAC3B;IAGG,OAAO,CAAC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;YAKlF,aAAa;YAYb,UAAU;YAMV,aAAa;IA8C3B,OAAO,CAAC,eAAe;CAiBxB"}

package/dist/pipeline/SegmentProcessor.js

@@ -0,0 +1,92 @@
+import { randomUUID } from "node:crypto";
+import { normalizeText } from "../utils/normalizeText.js";
+export class SegmentProcessor {
+    store;
+    llm;
+    embedder;
+    config;
+    constructor(store, llm, embedder, config) {
+        this.store = store;
+        this.llm = llm;
+        this.embedder = embedder;
+        this.config = config;
+    }
+    async process(segment, messages, sessionId) {
+        const savedSegment = await this.createSegment(segment, sessionId);
+        return this.nodeRunner(savedSegment, messages);
+    }
+    async createSegment(segment, sessionId) {
+        return this.store.saveSegment({
+            id: randomUUID(),
+            sessionId,
+            startIndex: segment.start,
+            endIndex: segment.end,
+            topicOrder: segment.topicOrder,
+            driftScore: segment.driftScore,
+            createdAt: new Date(),
+        });
+    }
+    async nodeRunner(segment, messages) {
+        const node = await this.nodeProcessor(messages, segment);
+        await this.store.saveNode(node);
+        return node;
+    }
+    async nodeProcessor(messages, segment) {
+        const segmentMessages = messages.slice(segment.startIndex, segment.endIndex + 1);
+        const messageContent = segmentMessages
+            .map((message, index) => `Message ${index + 1}:\n[${message.role}] ${normalizeText(message.content) ?? message.content}`)
+            .join("\n\n");
+        const extractionPrompt = [
+            "Analyze this conversation segment and extract structured memory for a future chatbot.",
+            "",
+            "Identify:",
+            "- The core topic being discussed",
+            "- What the user wanted to know, do, or accomplish",
+            "- What the assistant provided, decided, or concluded",
+            "- Any unresolved questions or open threads",
+            "",
+            "Respond only in this exact format with no extra text:",
+            "LABEL: <3-6 word topic label>",
+            "USER_INTENT: <two-three sentences describing what the user was trying to achieve or understand>",
+            "OUTCOME: <one-two sentences describing what was concluded, decided, or provided>",
+            "OPEN: <one sentence describing any unresolved question or follow-up, or None if fully resolved>",
+            "",
+            "Conversation segment:",
+            messageContent,
+        ].join("\n");
+        const raw = await this.llm.complete([{ role: "user", content: extractionPrompt }]);
+        const extracted = this.parseExtraction(raw);
+        const embedding = await this.embedder.embed(extracted.summary);
+        return {
+            id: randomUUID(),
+            sessionId: segment.sessionId,
+            segmentId: segment.id,
+            label: extracted.label,
+            summary: extracted.summary,
+            embedding,
+            messageRange: [segment.startIndex, segment.endIndex],
+            topicOrder: segment.topicOrder,
+            driftScore: segment.driftScore,
+            agentColor: null,
+            fleetId: null,
+            agentId: null,
+            createdAt: new Date(),
+        };
+    }
+    parseExtraction(raw) {
+        const label = raw.match(/^LABEL:\s*(.+)$/im)?.[1]?.trim() ?? "Unknown";
+        const userIntent = raw.match(/^USER_INTENT:\s*(.+)$/im)?.[1]?.trim() ?? "";
+        const outcome = raw.match(/^OUTCOME:\s*(.+)$/im)?.[1]?.trim() ?? "";
+        const open = raw.match(/^OPEN:\s*(.+)$/im)?.[1]?.trim() ?? "";
+        const parts = [
+            userIntent && `User wanted: ${userIntent}`,
+            outcome && `Outcome: ${outcome}`,
+            open && open.toLowerCase() !== "none" && `Still open: ${open}`,
+        ].filter(Boolean);
+        return {
+            label,
+            summary: parts.join(" "),
+        };
+    }
+}
+//# sourceMappingURL=SegmentProcessor.js.map

package/dist/pipeline/SegmentProcessor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SegmentProcessor.js","sourceRoot":"","sources":["../../src/pipeline/SegmentProcessor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAG1D,MAAM,OAAO,gBAAgB;IAER;IACA;IACA;IACA;IAJnB,YACmB,KAAiB,EACjB,GAAe,EACf,QAAsB,EACtB,MAGhB;QANgB,UAAK,GAAL,KAAK,CAAY;QACjB,QAAG,GAAH,GAAG,CAAY;QACf,aAAQ,GAAR,QAAQ,CAAc;QACtB,WAAM,GAAN,MAAM,CAGtB;IACA,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,OAAqB,EAAE,QAAmB,EAAE,SAAiB;QACzE,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAClE,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAqB,EAAE,SAAiB;QAClE,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;YAC5B,EAAE,EAAE,UAAU,EAAE;YAChB,SAAS;YACT,UAAU,EAAE,OAAO,CAAC,KAAK;YACzB,QAAQ,EAAE,OAAO,CAAC,GAAG;YACrB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,OAAqB,EAAE,QAAmB;QACjE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,QAAmB,EAAE,OAAqB;QACpE,MAAM,eAAe,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;QACjF,MAAM,cAAc,GAAG,eAAe;aACnC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,WAAW,KAAK,GAAG,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;aACxH,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,gBAAgB,GAAG;YACvB,uFAAuF;YACvF,EAAE;YACF,WAAW;YACX,kCAAkC;YAClC,mDAAmD;YACnD,sDAAsD;YACtD,4CAA4C;YAC5C,EAAE;YACF,uDAAuD;YACvD,+BAA+B;YAC/B,iGAAiG;YACjG,kFAAkF;YAClF,iGAAiG;YACjG,EAAE;YACF,uBAAuB;YACvB,cAAc;SACf,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAE/D,OAAO;YACL,EAAE,EAAE,UAAU,EAAE;YAChB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,SAAS;YACT,YAAY,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC;YACpD,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,GAAW;QACjC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;QACvE,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC3E,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACpE,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAE9D,MAAM,KAAK,GAAG;YACZ,UAAU,IAAI,gBAAgB,UAAU,EAAE;YAC1C,OAAO,IAAI,YAAY,OAAO,EAAE;YAChC,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,MAAM,IAAI,eAAe,IAAI,EAAE;SAC/D,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAElB,OAAO;YACL,KAAK;YACL,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;SACzB,CAAC;IACJ,CAAC;CACF"}

package/dist/pipeline/TopicDriftDetector.d.ts

@@ -0,0 +1,22 @@
+import type { DriftMode, Message } from "../types.js";
+export interface DriftSegment {
+    start: number;
+    end: number;
+    topicOrder: number;
+    driftScore: number;
+}
+export declare class TopicDriftDetector {
+    private readonly config;
+    constructor(config: {
+        windowSize: number;
+        threshold: number;
+        mode: DriftMode;
+        minSegmentMessages: number;
+    });
+    detect(messages: Message[], embeddings: number[][]): number[];
+    detectSegments(messages: Message[], embeddings: number[][]): DriftSegment[];
+    private detectBoundaries;
+    private detectIntentBoundaries;
+    private detectWindowBoundaries;
+}
+//# sourceMappingURL=TopicDriftDetector.d.ts.map

package/dist/pipeline/TopicDriftDetector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TopicDriftDetector.d.ts","sourceRoot":"","sources":["../../src/pipeline/TopicDriftDetector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAItD,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAOD,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE;QACvB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,SAAS,CAAC;QAChB,kBAAkB,EAAE,MAAM,CAAC;KAC5B;IAGH,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE;IAI7D,cAAc,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,GAAG,YAAY,EAAE;IA8B3E,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,sBAAsB;IA+B9B,OAAO,CAAC,sBAAsB;CA8B/B"}

package/dist/pipeline/TopicDriftDetector.js

@@ -0,0 +1,96 @@
+import { cosineSimilarity } from "../utils/drift/cosineSimilarity.js";
+import { avg } from "../utils/drift/vectorAvg.js";
+export class TopicDriftDetector {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    detect(messages, embeddings) {
+        return this.detectBoundaries(messages, embeddings).map((boundary) => boundary.index);
+    }
+    detectSegments(messages, embeddings) {
+        if (messages.length === 0)
+            return [];
+        const boundaries = this.detectBoundaries(messages, embeddings);
+        const segments = [];
+        let start = 0;
+        for (const [index, boundary] of boundaries.entries()) {
+            const end = boundary.index - 1;
+            if (end >= start) {
+                segments.push({
+                    start,
+                    end,
+                    topicOrder: index + 1,
+                    driftScore: boundary.score,
+                });
+            }
+            start = boundary.index;
+        }
+        segments.push({
+            start,
+            end: messages.length - 1,
+            topicOrder: segments.length + 1,
+            driftScore: 0,
+        });
+        return segments;
+    }
+    detectBoundaries(messages, embeddings) {
+        if (this.config.mode === "intent") {
+            return this.detectIntentBoundaries(messages, embeddings);
+        }
+        return this.detectWindowBoundaries(messages, embeddings);
+    }
+    detectIntentBoundaries(messages, embeddings) {
+        const boundaries = [];
+        let segmentStart = 0;
+        let topicEmbeddings = [];
+        for (let index = 0; index < messages.length; index += 1) {
+            const message = messages[index];
+            const embedding = embeddings[index];
+            if (!message || !embedding || message.role !== "user")
+                continue;
+            if (topicEmbeddings.length === 0) {
+                topicEmbeddings = [embedding];
+                continue;
+            }
+            const driftScore = 1 - cosineSimilarity(avg(topicEmbeddings), embedding);
+            const segmentLength = index - segmentStart;
+            if (driftScore > this.config.threshold && segmentLength >= this.config.minSegmentMessages) {
+                boundaries.push({ index, score: driftScore });
+                segmentStart = index;
+                topicEmbeddings = [embedding];
+                continue;
+            }
+            topicEmbeddings.push(embedding);
+        }
+        return boundaries;
+    }
+    detectWindowBoundaries(messages, embeddings) {
+        const boundaries = [];
+        const halfWindowSize = Math.floor(this.config.windowSize / 2);
+        if (halfWindowSize < 1)
+            return boundaries;
+        let segmentStart = 0;
+        let window = [];
+        for (let index = 0; index < messages.length; index += 1) {
+            const embedding = embeddings[index];
+            if (!embedding)
+                continue;
+            window = [...window, { index, embedding }].slice(-this.config.windowSize);
+            if (window.length < this.config.windowSize)
+                continue;
+            const previous = window.slice(0, halfWindowSize).map((item) => item.embedding);
+            const current = window.slice(-halfWindowSize).map((item) => item.embedding);
+            const driftScore = 1 - cosineSimilarity(avg(previous), avg(current));
+            const boundaryIndex = window.at(-halfWindowSize)?.index ?? index;
+            const segmentLength = boundaryIndex - segmentStart;
+            if (driftScore > this.config.threshold && segmentLength >= this.config.minSegmentMessages) {
+                boundaries.push({ index: boundaryIndex, score: driftScore });
+                segmentStart = boundaryIndex;
+                window = window.filter((item) => item.index >= boundaryIndex);
+            }
+        }
+        return boundaries;
+    }
+}
+//# sourceMappingURL=TopicDriftDetector.js.map

package/dist/pipeline/TopicDriftDetector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TopicDriftDetector.js","sourceRoot":"","sources":["../../src/pipeline/TopicDriftDetector.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,GAAG,EAAE,MAAM,6BAA6B,CAAC;AAclD,MAAM,OAAO,kBAAkB;IAEV;IADnB,YACmB,MAKhB;QALgB,WAAM,GAAN,MAAM,CAKtB;IACA,CAAC;IAEJ,MAAM,CAAC,QAAmB,EAAE,UAAsB;QAChD,OAAO,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACvF,CAAC;IAED,cAAc,CAAC,QAAmB,EAAE,UAAsB;QACxD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAErC,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;YACrD,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC;YAC/B,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,CAAC;oBACZ,KAAK;oBACL,GAAG;oBACH,UAAU,EAAE,KAAK,GAAG,CAAC;oBACrB,UAAU,EAAE,QAAQ,CAAC,KAAK;iBAC3B,CAAC,CAAC;YACL,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QACzB,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK;YACL,GAAG,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC;YACxB,UAAU,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC;YAC/B,UAAU,EAAE,CAAC;SACd,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,gBAAgB,CAAC,QAAmB,EAAE,UAAsB;QAClE,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC3D,CAAC;IAEO,sBAAsB,CAAC,QAAmB,EAAE,UAAsB;QACxE,MAAM,UAAU,GAAoB,EAAE,CAAC;QACvC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,eAAe,GAAe,EAAE,CAAC;QAErC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;YAChC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM;gBAAE,SAAS;YAEhE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,eAAe,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC9B,SAAS;YACX,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,SAAS,CAAC,CAAC;YACzE,MAAM,aAAa,GAAG,KAAK,GAAG,YAAY,CAAC;YAE3C,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;gBAC1F,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gBAC9C,YAAY,GAAG,KAAK,CAAC;gBACrB,eAAe,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC9B,SAAS;YACX,CAAC;YAED,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,sBAAsB,CAAC,QAAmB,EAAE,UAAsB;QACxE,MAAM,UAAU,GAAoB,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QAC9D,IAAI,cAAc,GAAG,CAAC;YAAE,OAAO,UAAU,CAAC;QAE1C,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,MAAM,GAAkD,EAAE,CAAC;QAE/D,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,CAAC,SAAS;gBAAE,SAAS;YAEzB,MAAM,GAAG,CAAC,GAAG,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAC1E,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU;gBAAE,SAAS;YAErD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/E,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5E,MAAM,UAAU,GAAG,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;YACrE,MAAM,aAAa,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,cAAc,CAAC,EAAE,KAAK,IAAI,KAAK,CAAC;YACjE,MAAM,aAAa,GAAG,aAAa,GAAG,YAAY,CAAC;YAEnD,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;gBAC1F,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gBAC7D,YAAY,GAAG,aAAa,CAAC;gBAC7B,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}

package/dist/queue/IngestQueue.d.ts

@@ -0,0 +1,16 @@
+import type { IngestPipeline } from "../pipeline/IngestPipeline.js";
+import type { MemoGrafterQueueConfig, Message } from "../types.js";
+export declare class IngestQueue {
+    private readonly pipeline;
+    private readonly connection;
+    private readonly queue;
+    private worker;
+    private readonly defaultJobOptions;
+    private readonly queueName;
+    constructor(pipeline: IngestPipeline, config: MemoGrafterQueueConfig);
+    enqueue(messages: Message[], sessionId: string): Promise<void>;
+    close(): Promise<void>;
+    private ensureWorker;
+    private withTimeout;
+}
+//# sourceMappingURL=IngestQueue.d.ts.map

package/dist/queue/IngestQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IngestQueue.d.ts","sourceRoot":"","sources":["../../src/queue/IngestQueue.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,KAAK,EAAE,sBAAsB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAOnE,qBAAa,WAAW;IAQpB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAP3B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuB;IAC7C,OAAO,CAAC,MAAM,CAAsC;IACpD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAc;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAGhB,QAAQ,EAAE,cAAc,EACzC,MAAM,EAAE,sBAAsB;IAgC1B,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoB9D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB5B,OAAO,CAAC,YAAY;YAwBN,WAAW;CAY1B"}

package/dist/queue/IngestQueue.js

@@ -0,0 +1,103 @@
+import { randomUUID } from "node:crypto";
+import { Queue, Worker } from "bullmq";
+import { Redis } from "ioredis";
+export class IngestQueue {
+    pipeline;
+    connection;
+    queue;
+    worker = null;
+    defaultJobOptions;
+    queueName;
+    constructor(pipeline, config) {
+        this.pipeline = pipeline;
+        this.queueName = config.queueName ?? `mg-ingest-${randomUUID()}`;
+        this.connection = new Redis(config.redisUrl, {
+            enableOfflineQueue: false,
+            maxRetriesPerRequest: null,
+            retryStrategy: () => null,
+        });
+        this.connection.on("error", (error) => {
+            console.warn("MemoGrafter ingest queue Redis warning:", error.message);
+        });
+        this.defaultJobOptions = {
+            attempts: 3,
+            backoff: {
+                type: "exponential",
+                delay: 1000,
+            },
+            removeOnComplete: config.removeOnComplete ?? true,
+            removeOnFail: config.removeOnFail ?? true,
+        };
+        this.queue = new Queue(this.queueName, {
+            connection: this.connection,
+            defaultJobOptions: this.defaultJobOptions,
+        });
+        this.queue.on("error", (error) => {
+            console.warn("MemoGrafter ingest queue warning:", error.message);
+        });
+    }
+    async enqueue(messages, sessionId) {
+        try {
+            await this.withTimeout(this.queue.add("ingest", {
+                messages: [...messages],
+                sessionId,
+            }, this.defaultJobOptions), 1000, "MemoGrafter ingest queue enqueue timed out.");
+            this.ensureWorker();
+        }
+        catch (error) {
+            console.warn("MemoGrafter ingest queue enqueue failed:", error);
+        }
+    }
+    async close() {
+        if (this.worker) {
+            await this.withTimeout(this.worker.close(false), 10000, "MemoGrafter ingest queue worker close timed out.").catch((error) => {
+                console.warn("MemoGrafter ingest queue worker close warning:", error);
+            });
+        }
+        await this.withTimeout(this.queue.close(), 1000, "MemoGrafter ingest queue close timed out.").catch((error) => {
+            console.warn("MemoGrafter ingest queue close warning:", error);
+        });
+        if (this.worker) {
+            await Promise.resolve(this.worker.disconnect()).catch((error) => {
+                console.warn("MemoGrafter ingest queue worker disconnect warning:", error);
+            });
+        }
+        await Promise.resolve(this.queue.disconnect()).catch((error) => {
+            console.warn("MemoGrafter ingest queue disconnect warning:", error);
+        });
+        this.connection.disconnect();
+    }
+    ensureWorker() {
+        if (this.worker)
+            return;
+        this.worker = new Worker(this.queueName, async (job) => {
+            try {
+                await this.pipeline.run(job.data.messages, job.data.sessionId);
+            }
+            catch (error) {
+                console.warn("MemoGrafter background ingest failed:", error);
+                throw error;
+            }
+        }, { connection: this.connection });
+        this.worker.on("failed", (_job, error) => {
+            console.warn("MemoGrafter ingest queue worker warning:", error.message);
+        });
+        this.worker.on("error", (error) => {
+            console.warn("MemoGrafter ingest queue worker warning:", error.message);
+        });
+    }
+    async withTimeout(promise, milliseconds, message) {
+        let timeout;
+        const timeoutPromise = new Promise((_, reject) => {
+            timeout = setTimeout(() => reject(new Error(message)), milliseconds);
+        });
+        try {
+            return await Promise.race([promise, timeoutPromise]);
+        }
+        finally {
+            if (timeout)
+                clearTimeout(timeout);
+        }
+    }
+}
+//# sourceMappingURL=IngestQueue.js.map

package/dist/queue/IngestQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IngestQueue.js","sourceRoot":"","sources":["../../src/queue/IngestQueue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAoB,MAAM,QAAQ,CAAC;AACzD,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAShC,MAAM,OAAO,WAAW;IAQH;IAPF,UAAU,CAAQ;IAClB,KAAK,CAAuB;IACrC,MAAM,GAAiC,IAAI,CAAC;IACnC,iBAAiB,CAAc;IAC/B,SAAS,CAAS;IAEnC,YACmB,QAAwB,EACzC,MAA8B;QADb,aAAQ,GAAR,QAAQ,CAAgB;QAGzC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,aAAa,UAAU,EAAE,EAAE,CAAC;QAEjE,IAAI,CAAC,UAAU,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC3C,kBAAkB,EAAE,KAAK;YACzB,oBAAoB,EAAE,IAAI;YAC1B,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YAC3C,OAAO,CAAC,IAAI,CAAC,yCAAyC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,GAAG;YACvB,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE;gBACP,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,IAAI;aACZ;YACD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,IAAI;YACjD,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI;SAC1C,CAAC;QAEF,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAgB,IAAI,CAAC,SAAS,EAAE;YACpD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;SAC1C,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YACtC,OAAO,CAAC,IAAI,CAAC,mCAAmC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,QAAmB,EAAE,SAAiB;QAClD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CACpB,IAAI,CAAC,KAAK,CAAC,GAAG,CACZ,QAAQ,EACR;gBACE,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC;gBACvB,SAAS;aACV,EACD,IAAI,CAAC,iBAAiB,CACvB,EACD,IAAI,EACJ,6CAA6C,CAC9C,CAAC;YACF,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,0CAA0C,EAAE,KAAK,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,kDAAkD,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBACnI,OAAO,CAAC,IAAI,CAAC,gDAAgD,EAAE,KAAK,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,2CAA2C,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YACrH,OAAO,CAAC,IAAI,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBACvE,OAAO,CAAC,IAAI,CAAC,qDAAqD,EAAE,KAAK,CAAC,CAAC;YAC7E,CAAC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YACtE,OAAO,CAAC,IAAI,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;IAC/B,CAAC;IAEO,YAAY;QAClB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAExB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CACtB,IAAI,CAAC,SAAS,EACd,KAAK,EAAE,GAAG,EAAE,EAAE;YACZ,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACjE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;gBAC7D,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,EACD,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAChC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YACvC,OAAO,CAAC,IAAI,CAAC,0CAA0C,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAChC,OAAO,CAAC,IAAI,CAAC,0CAA0C,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,WAAW,CAAI,OAAmB,EAAE,YAAoB,EAAE,OAAe;QACrF,IAAI,OAAkD,CAAC;QACvD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;QACvD,CAAC;gBAAS,CAAC;YACT,IAAI,OAAO;gBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;CACF"}