@unblocklabs/unblock-memory 0.1.2 → 0.2.1

package/dist/src/manager.js

@@ -2,11 +2,24 @@ import { mkdir } from "node:fs/promises";
 import { dirname } from "node:path";
 import chokidar from "chokidar";
 import { ensureMemoryAnalysisSchema, latestAnalysisRunId, markMemoryAnalysisStale, readAnalysisSummary, readCluster, readClusters, runAnalysisWorker, } from "./analysis.js";
+import { readSessionManifest, sessionMetadataByPath, syncSessionProjections, } from "./session-sync.js";
 import { parseSafeVirtualPath } from "./sources.js";
 const DEFAULT_READ_LINES = 120;
 const MAX_READ_CHARS = 12_000;
 const WATCH_DEBOUNCE_MS = 250;
 const qmdModule = import("@unblocklabs/qmd");
+function completedEmbeddingCount(result) {
+    if (result.errors > 0) {
+        throw new Error(`QMD failed to embed ${result.errors} chunk${result.errors === 1 ? "" : "s"}`);
+    }
+    return result.chunksEmbedded;
+}
+async function ensureSemanticChunking(store) {
+    const configured = store.internal.db.prepare("SELECT value FROM store_config WHERE key = 'embedding_chunk_strategy'").get();
+    if (configured?.value === "semantic")
+        return;
+    completedEmbeddingCount(await store.embed({ chunkStrategy: "semantic" }));
+}
 export function enableSecureDelete(store) {
     store.internal.db.exec("PRAGMA secure_delete = ON");
 }
@@ -78,7 +91,7 @@ function lineSpan(result) {
     const endLine = startLine + Math.max(0, result.bestChunk.split("\n").length - 1);
     return { startLine, endLine };
 }
-function lexicalResult(hit) {
+function lexicalResult(hit, corpus, session) {
     const body = hit.body ?? hit.title;
     const endLine = Math.max(1, body.split("\n").length);
     return {
@@ -89,9 +102,36 @@ function lexicalResult(hit) {
         textScore: hit.score,
         snippet: body,
         source: "memory",
+        corpus,
+        ...(session ? { session } : {}),
         citation: `${hit.displayPath}#L1-L${endLine}`,
     };
 }
+function sessionAllowedPaths(metadataByPath, collection, filter) {
+    const startedFrom = filter.startedFrom === undefined ? undefined : Date.parse(filter.startedFrom);
+    const startedTo = filter.startedTo === undefined ? undefined : Date.parse(filter.startedTo);
+    if (startedFrom !== undefined && !Number.isFinite(startedFrom)) {
+        throw new Error("memory_search sessionFilter.startedFrom must be an ISO 8601 timestamp");
+    }
+    if (startedTo !== undefined && !Number.isFinite(startedTo)) {
+        throw new Error("memory_search sessionFilter.startedTo must be an ISO 8601 timestamp");
+    }
+    if (startedFrom !== undefined && startedTo !== undefined && startedFrom > startedTo) {
+        throw new Error("memory_search sessionFilter.startedFrom must not be after startedTo");
+    }
+    const provider = filter.provider?.trim().toLowerCase();
+    const accountId = filter.accountId?.trim();
+    const conversationId = filter.conversationId?.trim();
+    const paths = [...metadataByPath].flatMap(([path, metadata]) => (startedFrom === undefined || metadata.startedAt >= startedFrom) &&
+        (startedTo === undefined || metadata.startedAt <= startedTo) &&
+        (provider === undefined || metadata.provider?.trim().toLowerCase() === provider) &&
+        (filter.chatType === undefined || metadata.chatType === filter.chatType) &&
+        (accountId === undefined || metadata.accountId?.trim() === accountId) &&
+        (conversationId === undefined || metadata.conversationId?.trim() === conversationId)
+        ? [path]
+        : []);
+    return { [collection]: paths };
+}
 export class QmdMemoryManager {
     #dbPath;
     #workspaceDir;
@@ -99,6 +139,7 @@ export class QmdMemoryManager {
     #storeFactory;
     #analysisExecutable;
     #analysisRunner;
+    #sessions;
     #store;
     #cleanupRemovedDocuments;
     #operationChain;
@@ -109,6 +150,7 @@ export class QmdMemoryManager {
     #closed = false;
     #files = 0;
     #dirty = true;
+    #sessionMetadata = new Map();
     constructor(params) {
         this.#dbPath = params.dbPath;
         this.#workspaceDir = params.workspaceDir;
@@ -116,14 +158,20 @@ export class QmdMemoryManager {
         this.#storeFactory = params.storeFactory;
         this.#analysisExecutable = params.analysisExecutable;
         this.#analysisRunner = params.analysisRunner ?? runAnalysisWorker;
+        this.#sessions = params.sessions;
     }
     async start() {
+        if (this.#sessions) {
+            this.#sessionMetadata = sessionMetadataByPath(await readSessionManifest(this.#sessions.manifestPath));
+        }
         this.#startWatcher();
         await this.sync({ reason: "first-use" });
         await this.#watchReady;
     }
     #startWatcher() {
-        const paths = [...new Set([...this.#sources.values()].map((source) => source.watchPath))];
+        const paths = [...new Set([...this.#sources.values()]
+                .filter((source) => source.kind === "files")
+                .map((source) => source.watchPath))];
         if (paths.length === 0 || this.#watcher)
             return;
         this.#watcher = chokidar.watch(paths, {
@@ -176,20 +224,46 @@ export class QmdMemoryManager {
         const prunedDocuments = await pruneStaleCollections(store, new Set(this.#collectionNames()));
         if (prunedDocuments > 0)
             markMemoryAnalysisStale(store.internal.db);
+        try {
+            await ensureSemanticChunking(store);
+        }
+        catch (error) {
+            await store.close();
+            throw error;
+        }
         this.#cleanupRemovedDocuments = (changedDocuments) => {
             cleanupRemovedDocuments(store, changedDocuments);
         };
         this.#store = store;
         return store;
     }
-    #collectionNames() {
-        return [...this.#sources.keys()];
+    #collectionNames(corpora) {
+        if (corpora === undefined)
+            return [...this.#sources.keys()];
+        if (corpora.length === 0)
+            throw new Error("memory_search corpora must not be empty");
+        const selected = new Set(corpora);
+        if (selected.has("all")) {
+            if (selected.size > 1)
+                throw new Error('memory_search corpus "all" must be used alone');
+            return [...this.#sources.keys()];
+        }
+        const known = new Set([...this.#sources.values()].map((source) => source.corpus));
+        const unknown = [...selected].find((corpus) => !known.has(corpus));
+        if (unknown)
+            throw new Error(`memory_search unknown corpus: ${unknown}`);
+        return [...this.#sources.values()]
+            .filter((source) => selected.has(source.corpus))
+            .map((source) => source.collection);
     }
     sync(params) {
         const run = async () => {
             const store = await this.#getStore();
             this.#dirty = true;
-            const update = await store.update();
+            const collections = [...this.#sources.values()]
+                .filter((source) => source.kind === "files")
+                .map((source) => source.collection);
+            const update = await store.update({ collections });
             this.#cleanupRemovedDocuments?.(update.updated + update.removed);
             const analysisStore = store;
             const invalidatesAnalysis = update.indexed + update.updated + update.removed > 0 ||
@@ -198,17 +272,62 @@ export class QmdMemoryManager {
             if (invalidatesAnalysis && analysisStore.internal) {
                 markMemoryAnalysisStale(analysisStore.internal.db);
             }
-            const embed = await store.embed({ force: params?.force, chunkStrategy: "semantic" });
-            if (!invalidatesAnalysis && embed.chunksEmbedded > 0 && analysisStore.internal) {
+            let chunksEmbedded = 0;
+            for (const collection of collections.length > 0 ? collections : [undefined]) {
+                const embed = await store.embed({
+                    ...(collection ? { collection } : {}),
+                    force: params?.force,
+                    chunkStrategy: "semantic",
+                });
+                chunksEmbedded += completedEmbeddingCount(embed);
+            }
+            if (!invalidatesAnalysis && chunksEmbedded > 0 && analysisStore.internal) {
                 markMemoryAnalysisStale(analysisStore.internal.db);
             }
             const status = await store.getStatus();
-            const collections = await store.listCollections();
-            this.#files = collections.reduce((total, collection) => total + collection.active_count, 0);
+            const indexedCollections = await store.listCollections();
+            this.#files = indexedCollections.reduce((total, collection) => total + collection.active_count, 0);
             this.#dirty = status.needsEmbedding > 0;
         };
         return this.#enqueue(run);
     }
+    syncSessions(force = false) {
+        return this.#enqueue(async () => {
+            const sessions = this.#sessions;
+            if (!sessions)
+                throw new Error('memory session sync requires a configured "sessions" corpus');
+            const store = await this.#getStore();
+            const synced = await syncSessionProjections({
+                ...sessions,
+                force,
+                index: async () => {
+                    const update = await store.update({ collections: [sessions.collection] });
+                    this.#cleanupRemovedDocuments?.(update.updated + update.removed);
+                    const analysisStore = store;
+                    const invalidatesAnalysis = update.indexed + update.updated + update.removed > 0 ||
+                        update.needsEmbedding > 0;
+                    if (invalidatesAnalysis && analysisStore.internal) {
+                        markMemoryAnalysisStale(analysisStore.internal.db);
+                    }
+                    const embed = await store.embed({
+                        collection: sessions.collection,
+                        chunkStrategy: "semantic",
+                    });
+                    const chunksEmbedded = completedEmbeddingCount(embed);
+                    if (!invalidatesAnalysis && chunksEmbedded > 0 && analysisStore.internal) {
+                        markMemoryAnalysisStale(analysisStore.internal.db);
+                    }
+                    return chunksEmbedded;
+                },
+            });
+            this.#sessionMetadata = sessionMetadataByPath(synced.manifest);
+            const status = await store.getStatus();
+            const collections = await store.listCollections();
+            this.#files = collections.reduce((total, collection) => total + collection.active_count, 0);
+            this.#dirty = status.needsEmbedding > 0;
+            return synced.result;
+        });
+    }
     recluster(options, signal) {
         return this.#enqueue(async () => {
             if (!this.#analysisExecutable) {
@@ -257,34 +376,57 @@ export class QmdMemoryManager {
             return [];
         if (this.#sources.size === 0)
             return [];
+        const collections = this.#collectionNames(opts?.corpora);
         opts?.signal?.throwIfAborted();
         await this.#operationChain;
+        const sessions = this.#sessions;
+        const allowedPaths = opts?.sessionFilter && sessions && collections.includes(sessions.collection)
+            ? sessionAllowedPaths(this.#sessionMetadata, sessions.collection, opts.sessionFilter)
+            : undefined;
         const store = await this.#getStore();
         if (opts?.lexicalOnly) {
             const hits = await store.searchLex(query, {
                 limit: opts.maxResults ?? 5,
-                collection: this.#collectionNames(),
+                collection: collections,
+            });
+            return hits.flatMap((hit) => {
+                const corpus = this.#sources.get(hit.collectionName)?.corpus;
+                const prefix = `qmd://${hit.collectionName}/`;
+                const session = corpus === "sessions" && hit.filepath.startsWith(prefix)
+                    ? this.#sessionMetadata.get(hit.filepath.slice(prefix.length))
+                    : undefined;
+                return hit.score >= (opts.minScore ?? 0) && corpus ? [lexicalResult(hit, corpus, session)] : [];
             });
-            return hits
-                .filter((hit) => hit.score >= (opts.minScore ?? 0))
-                .map(lexicalResult);
         }
         const hits = await store.vsearch(query, {
-            collection: this.#collectionNames(),
+            collection: collections,
             limit: opts?.maxResults ?? 5,
             minScore: opts?.minScore ?? 0.3,
+            allowedPaths,
         });
-        return hits.map((hit) => {
+        return hits.flatMap((hit) => {
+            const collection = /^qmd:\/\/([^/]+)\//.exec(hit.file)?.[1];
+            const corpus = collection ? this.#sources.get(collection)?.corpus : undefined;
+            if (!corpus)
+                return [];
             const span = lineSpan(hit);
-            return {
-                path: hit.file,
-                ...span,
-                score: hit.score,
-                vectorScore: hit.score,
-                snippet: hit.bestChunk,
-                source: "memory",
-                citation: `${hit.displayPath}#L${span.startLine}-L${span.endLine}`,
-            };
+            const relativePath = collection && hit.file.startsWith(`qmd://${collection}/`)
+                ? hit.file.slice(`qmd://${collection}/`.length)
+                : undefined;
+            const session = corpus === "sessions" && relativePath
+                ? this.#sessionMetadata.get(relativePath)
+                : undefined;
+            return [{
+                    path: hit.file,
+                    ...span,
+                    score: hit.score,
+                    vectorScore: hit.score,
+                    snippet: hit.bestChunk,
+                    source: "memory",
+                    corpus,
+                    ...(session ? { session } : {}),
+                    citation: `${hit.displayPath}#L${span.startLine}-L${span.endLine}`,
+                }];
         });
     }
     async readFile(params) {
@@ -308,6 +450,12 @@ export class QmdMemoryManager {
         });
     }
     status() {
+        const corpora = new Map();
+        for (const source of this.#sources.values()) {
+            const sources = corpora.get(source.corpus) ?? [];
+            sources.push(source);
+            corpora.set(source.corpus, sources);
+        }
         return {
             backend: "builtin",
             provider: "unblock-memory",
@@ -318,7 +466,9 @@ export class QmdMemoryManager {
             sources: ["memory"],
             vector: { enabled: true, available: !this.#dirty },
             custom: {
-                paths: [...this.#sources.values()].map((source) => source.configuredPath),
+                corpora: [...corpora].map(([name, sources]) => sources[0]?.kind === "sessions"
+                    ? { name, kind: "sessions", chatTypes: sources[0].chatTypes }
+                    : { name, kind: "files", paths: sources.map((source) => source.configuredPath) }),
                 ...(this.#watchError ? { watchError: this.#watchError } : {}),
             },
         };

package/dist/src/plugin.js

@@ -11,6 +11,19 @@ function getContext(ctx) {
 }
 const searchParameters = Type.Object({
     query: Type.String({ pattern: "\\S" }),
+    corpora: Type.Optional(Type.Array(Type.String({ pattern: "\\S" }), { minItems: 1 })),
+    sessionFilter: Type.Optional(Type.Object({
+        startedFrom: Type.Optional(Type.String({ pattern: "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" })),
+        startedTo: Type.Optional(Type.String({ pattern: "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" })),
+        provider: Type.Optional(Type.String({ pattern: "\\S" })),
+        chatType: Type.Optional(Type.Union([
+            Type.Literal("channel"),
+            Type.Literal("group"),
+            Type.Literal("direct"),
+        ])),
+        accountId: Type.Optional(Type.String({ pattern: "\\S" })),
+        conversationId: Type.Optional(Type.String({ pattern: "\\S" })),
+    }, { additionalProperties: false })),
     maxResults: Type.Optional(Type.Integer({ minimum: 1, maximum: 20 })),
     minScore: Type.Optional(Type.Number({ minimum: 0, maximum: 1 })),
 }, { additionalProperties: false });
@@ -19,6 +32,9 @@ const getParameters = Type.Object({
     from: Type.Optional(Type.Integer({ minimum: 1 })),
     lines: Type.Optional(Type.Integer({ minimum: 1, maximum: 1000 })),
 }, { additionalProperties: false });
+const syncSessionsParameters = Type.Object({
+    force: Type.Optional(Type.Boolean()),
+}, { additionalProperties: false });
 function createSearchTool(runtime, ctx) {
     const active = getContext(ctx);
     if (!active)
@@ -26,15 +42,21 @@ function createSearchTool(runtime, ctx) {
     return {
         name: "memory_search",
         label: "Memory Search",
-        description: "Search canonical Markdown memory with semantic vector retrieval.",
+        description: "Search configured Markdown corpora with semantic vector retrieval. Omit corpora to search all of them.",
         parameters: searchParameters,
         async execute(_toolCallId, params, signal) {
-            const { query: untrimmedQuery, maxResults, minScore } = Value.Parse(searchParameters, params);
+            const { query: untrimmedQuery, corpora, sessionFilter, maxResults, minScore } = Value.Parse(searchParameters, params);
             const query = untrimmedQuery.trim();
             const { manager, error } = await runtime.getMemorySearchManager(active);
             if (!manager)
                 return jsonResult({ results: [], error: error ?? "memory unavailable" });
-            const results = await manager.search(query, { maxResults, minScore, signal });
+            const results = await manager.search(query, {
+                corpora: corpora?.map((corpus) => corpus.trim()),
+                sessionFilter,
+                maxResults,
+                minScore,
+                signal,
+            });
             return jsonResult({ results, provider: "unblock-memory" });
         },
     };
@@ -62,6 +84,32 @@ function createGetTool(runtime, ctx) {
         },
     };
 }
+function createSyncSessionsTool(runtime, ctx) {
+    const active = getContext(ctx);
+    if (!active)
+        return null;
+    return {
+        name: "memory_sync_sessions",
+        label: "Sync Memory Sessions",
+        description: "Project and index this agent's configured OpenClaw session transcripts.",
+        parameters: syncSessionsParameters,
+        async execute(_toolCallId, params) {
+            const { force } = Value.Parse(syncSessionsParameters, params);
+            const { manager, error } = await runtime.getMemorySearchManager(active);
+            if (!manager)
+                return jsonResult({ status: "unavailable", error: error ?? "memory unavailable" });
+            try {
+                return jsonResult(await manager.syncSessions(force));
+            }
+            catch (syncError) {
+                return jsonResult({
+                    status: "unavailable",
+                    error: syncError instanceof Error ? syncError.message : String(syncError),
+                });
+            }
+        },
+    };
+}
 const reclusterParameters = Type.Object({
     space: Type.Optional(Type.Object({
         method: Type.Optional(Type.Union([Type.Literal("umap"), Type.Literal("none")])),
@@ -219,7 +267,7 @@ export function resolveFlushPlan(params = {}) {
 }
 export function registerUnblockMemory(api) {
     const config = resolveConfig(api.pluginConfig);
-    const runtime = new QmdMemoryRuntime(config.paths, config.analysis.executable);
+    const runtime = new QmdMemoryRuntime(config.corpora, config.analysis.executable);
     const capability = {
         deterministicRecallToolName: "memory_search",
         supportsPrivateTranscriptRecall: false,
@@ -232,6 +280,7 @@ export function registerUnblockMemory(api) {
     api.registerMemoryCapability(capability);
     api.registerTool((ctx) => createSearchTool(runtime, ctx), { names: ["memory_search"] });
     api.registerTool((ctx) => createGetTool(runtime, ctx), { names: ["memory_get"] });
+    api.registerTool((ctx) => createSyncSessionsTool(runtime, ctx), { names: ["memory_sync_sessions"] });
     api.registerTool((ctx) => createReclusterTool(runtime, ctx), { names: ["memory_recluster"] });
     api.registerTool((ctx) => createListClustersTool(runtime, ctx), { names: ["memory_list_clusters"] });
     api.registerTool((ctx) => createFetchClusterTool(runtime, ctx), { names: ["memory_fetch_cluster"] });

package/dist/src/runtime.d.ts

@@ -1,9 +1,10 @@
 import type { OpenClawConfig } from "openclaw/plugin-sdk/plugin-entry";
+import type { CorpusConfig } from "./config.js";
 import type { MemoryPluginRuntimeContract } from "./contracts.js";
 import { QmdMemoryManager } from "./manager.js";
 export declare class QmdMemoryRuntime implements MemoryPluginRuntimeContract {
     #private;
-    constructor(paths: readonly string[], analysisExecutable?: string);
+    constructor(corpora: readonly CorpusConfig[], analysisExecutable?: string);
     getMemorySearchManager(params: {
         cfg: OpenClawConfig;
         agentId: string;

package/dist/src/runtime.js

@@ -1,13 +1,15 @@
 import { join } from "node:path";
-import { resolveAgentWorkspaceDir, resolveStateDir, } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
+import { resolveAgentDir, resolveAgentWorkspaceDir, resolveStateDir, } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
+import { resolveAgentIdentity } from "openclaw/plugin-sdk/agent-runtime";
 import { QmdMemoryManager } from "./manager.js";
-import { resolveSource } from "./sources.js";
+import { resolveTimezone } from "./session-projector.js";
+import { resolveSessionSource, resolveSources } from "./sources.js";
 export class QmdMemoryRuntime {
-    #paths;
+    #corpora;
     #analysisExecutable;
     #managers = new Map();
-    constructor(paths, analysisExecutable) {
-        this.#paths = paths;
+    constructor(corpora, analysisExecutable) {
+        this.#corpora = corpora;
         this.#analysisExecutable = analysisExecutable;
     }
     async getMemorySearchManager(params) {
@@ -39,11 +41,32 @@ export class QmdMemoryRuntime {
     }
     async #createManager(cfg, agentId) {
         const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId);
+        const stateDir = join(resolveStateDir(), "agents", agentId, "unblock-memory");
+        const fileCorpora = this.#corpora.filter((corpus) => corpus.kind === "files");
+        const sessionCorpus = this.#corpora.find((corpus) => corpus.kind === "sessions");
+        const sources = resolveSources(workspaceDir, fileCorpora);
+        const sessionSource = sessionCorpus
+            ? resolveSessionSource(join(stateDir, "sessions"), sessionCorpus.chatTypes)
+            : undefined;
+        if (sessionSource)
+            sources.push(sessionSource);
         const manager = new QmdMemoryManager({
             workspaceDir,
-            dbPath: join(resolveStateDir(), "agents", agentId, "unblock-memory", "index.sqlite"),
-            sources: this.#paths.map((source) => resolveSource(workspaceDir, source)),
+            dbPath: join(stateDir, "index.sqlite"),
+            sources,
             analysisExecutable: this.#analysisExecutable,
+            ...(sessionCorpus && sessionSource ? {
+                sessions: {
+                    agentId,
+                    agentName: resolveAgentIdentity(cfg, agentId)?.name?.trim() || agentId,
+                    chatTypes: sessionCorpus.chatTypes,
+                    collection: sessionSource.collection,
+                    databasePath: join(resolveAgentDir(cfg, agentId), "openclaw-agent.sqlite"),
+                    manifestPath: join(stateDir, "sessions-manifest.json"),
+                    outputDir: sessionSource.root,
+                    timezone: resolveTimezone(cfg.agents?.defaults?.userTimezone?.trim()),
+                },
+            } : {}),
         });
         await manager.start();
         return manager;

package/dist/src/session-projector.d.ts

@@ -0,0 +1,21 @@
+import type { ChatType } from "./config.js";
+export type SessionMetadata = {
+    sessionId: string;
+    provider?: string;
+    chatType: ChatType;
+    accountId?: string;
+    conversationId?: string;
+    startedAt: number;
+};
+export type SessionProjectionInput = SessionMetadata & {
+    label?: string;
+    agentName: string;
+    timezone: string;
+    events: readonly {
+        eventJson: string;
+        createdAt: number;
+    }[];
+};
+export declare function projectSession(input: SessionProjectionInput): string | undefined;
+export declare function sessionDocumentPath(metadata: SessionMetadata): string;
+export declare function resolveTimezone(configured?: string): string;