@andre.li/memoark 0.3.1

package/dist/core/dedup.d.ts

@@ -0,0 +1,45 @@
+/**
+ * DedupStore — Message deduplication tracking
+ * Checkpoints source identity and content hashes to detect new/modified/unchanged messages
+ */
+import type { RawMessage } from "./types.js";
+export declare class DedupStore {
+    private checkpointPath;
+    private entries;
+    constructor(checkpointPath: string);
+    /**
+     * Load existing entries from checkpointPath (JSONL format)
+     * Each line: {"source_hash":"...","content_hash":"..."}
+     */
+    load(): void;
+    /**
+     * Generate source identity hash for a message
+     * SHA-256(platform:channel:identityKey:timestamp)
+     *
+     * Identity key priority:
+     * 1. metadata.message_id
+     * 2. metadata.uuid
+     * 3. metadata.thread_id
+     * 4. metadata.session_id ? `${session_id}:${index ?? 0}` : undefined
+     * 5. `${timestamp}:${index ?? 0}` (fallback)
+     */
+    sourceIdentityHash(msg: RawMessage): string;
+    /**
+     * Generate content hash for a message
+     * SHA-256(content:attachment_ids_joined_by_comma)
+     */
+    contentHash(msg: RawMessage): string;
+    /**
+     * Check message status
+     * - new: source_hash not in entries
+     * - unchanged: source_hash exists AND content_hash matches
+     * - modified: source_hash exists AND content_hash differs
+     */
+    check(msg: RawMessage): "new" | "unchanged" | "modified";
+    /**
+     * Commit messages to checkpoint file (append JSONL)
+     * Only called by pipeline on success — NEVER in finally block
+     */
+    commit(msgs: RawMessage[]): void;
+}
+//# sourceMappingURL=dedup.d.ts.map

package/dist/core/dedup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/core/dedup.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAc,UAAU,EAAE,MAAM,YAAY,CAAC;AAEzD,qBAAa,UAAU;IAGT,OAAO,CAAC,cAAc;IAFlC,OAAO,CAAC,OAAO,CAAsB;gBAEjB,cAAc,EAAE,MAAM;IAI1C;;;OAGG;IACH,IAAI,IAAI,IAAI;IAkBZ;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM;IAuB3C;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM;IAMpC;;;;;OAKG;IACH,KAAK,CAAC,GAAG,EAAE,UAAU,GAAG,KAAK,GAAG,WAAW,GAAG,UAAU;IAYxD;;;OAGG;IACH,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,IAAI;CAkBjC"}

package/dist/core/dedup.js

@@ -0,0 +1,111 @@
+/**
+ * DedupStore — Message deduplication tracking
+ * Checkpoints source identity and content hashes to detect new/modified/unchanged messages
+ */
+import { createHash } from "node:crypto";
+import { appendFileSync, existsSync, readFileSync } from "node:fs";
+export class DedupStore {
+    checkpointPath;
+    entries; // source_hash → content_hash
+    constructor(checkpointPath) {
+        this.checkpointPath = checkpointPath;
+        this.entries = new Map();
+    }
+    /**
+     * Load existing entries from checkpointPath (JSONL format)
+     * Each line: {"source_hash":"...","content_hash":"..."}
+     */
+    load() {
+        if (!existsSync(this.checkpointPath)) {
+            return;
+        }
+        const content = readFileSync(this.checkpointPath, "utf-8");
+        const lines = content.split("\n").filter((line) => line.trim());
+        for (const line of lines) {
+            try {
+                const entry = JSON.parse(line);
+                this.entries.set(entry.source_hash, entry.content_hash);
+            }
+            catch {
+                // Skip malformed lines
+            }
+        }
+    }
+    /**
+     * Generate source identity hash for a message
+     * SHA-256(platform:channel:identityKey:timestamp)
+     *
+     * Identity key priority:
+     * 1. metadata.message_id
+     * 2. metadata.uuid
+     * 3. metadata.thread_id
+     * 4. metadata.session_id ? `${session_id}:${index ?? 0}` : undefined
+     * 5. `${timestamp}:${index ?? 0}` (fallback)
+     */
+    sourceIdentityHash(msg) {
+        const metadata = msg.metadata || {};
+        let identityKey;
+        if (metadata.message_id) {
+            identityKey = String(metadata.message_id);
+        }
+        else if (metadata.uuid) {
+            identityKey = String(metadata.uuid);
+        }
+        else if (metadata.thread_id) {
+            identityKey = String(metadata.thread_id);
+        }
+        else if (metadata.session_id) {
+            const index = metadata.index ?? 0;
+            identityKey = `${metadata.session_id}:${index}`;
+        }
+        else {
+            const index = metadata.index ?? 0;
+            identityKey = `${msg.timestamp}:${index}`;
+        }
+        const composite = `${msg.platform}:${msg.channel}:${identityKey}:${msg.timestamp}`;
+        return createHash("sha256").update(composite).digest("hex");
+    }
+    /**
+     * Generate content hash for a message
+     * SHA-256(content:attachment_ids_joined_by_comma)
+     */
+    contentHash(msg) {
+        const attachmentIds = msg.attachments?.map((a) => a.id).join(",") || "";
+        const composite = `${msg.content}:${attachmentIds}`;
+        return createHash("sha256").update(composite).digest("hex");
+    }
+    /**
+     * Check message status
+     * - new: source_hash not in entries
+     * - unchanged: source_hash exists AND content_hash matches
+     * - modified: source_hash exists AND content_hash differs
+     */
+    check(msg) {
+        const sourceHash = this.sourceIdentityHash(msg);
+        const existingContentHash = this.entries.get(sourceHash);
+        if (!existingContentHash) {
+            return "new";
+        }
+        const currentContentHash = this.contentHash(msg);
+        return currentContentHash === existingContentHash ? "unchanged" : "modified";
+    }
+    /**
+     * Commit messages to checkpoint file (append JSONL)
+     * Only called by pipeline on success — NEVER in finally block
+     */
+    commit(msgs) {
+        if (msgs.length === 0) {
+            return;
+        }
+        const lines = msgs.map((msg) => {
+            const sourceHash = this.sourceIdentityHash(msg);
+            const contentHash = this.contentHash(msg);
+            const entry = { source_hash: sourceHash, content_hash: contentHash };
+            // Update in-memory map
+            this.entries.set(sourceHash, contentHash);
+            return JSON.stringify(entry);
+        });
+        appendFileSync(this.checkpointPath, `${lines.join("\n")}\n`, "utf-8");
+    }
+}
+//# sourceMappingURL=dedup.js.map

package/dist/core/dedup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/core/dedup.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAGnE,MAAM,OAAO,UAAU;IAGD;IAFZ,OAAO,CAAsB,CAAC,6BAA6B;IAEnE,YAAoB,cAAsB;QAAtB,mBAAc,GAAd,cAAc,CAAQ;QACxC,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,IAAI;QACF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACrC,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAEhE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAe,CAAC;gBAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;YAC1D,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,GAAe;QAChC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEpC,IAAI,WAAmB,CAAC;QAExB,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;aAAM,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzB,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC9B,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;YAClC,WAAW,GAAG,GAAG,QAAQ,CAAC,UAAU,IAAI,KAAK,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;YAClC,WAAW,GAAG,GAAG,GAAG,CAAC,SAAS,IAAI,KAAK,EAAE,CAAC;QAC5C,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,OAAO,IAAI,WAAW,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;QACnF,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,GAAe;QACzB,MAAM,aAAa,GAAG,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACxE,MAAM,SAAS,GAAG,GAAG,GAAG,CAAC,OAAO,IAAI,aAAa,EAAE,CAAC;QACpD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,GAAe;QACnB,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEzD,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACjD,OAAO,kBAAkB,KAAK,mBAAmB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/E,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,IAAkB;QACvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAChD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,KAAK,GAAe,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC;YAEjF,uBAAuB;YACvB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;YAE1C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACxE,CAAC;CACF"}

package/dist/core/entity-extract.d.ts

@@ -0,0 +1,3 @@
+import type { QuickEntity } from "./types.js";
+export declare function extractQuickEntities(text: string): QuickEntity[];
+//# sourceMappingURL=entity-extract.d.ts.map

package/dist/core/entity-extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-extract.d.ts","sourceRoot":"","sources":["../../src/core/entity-extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAa9C,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,CAgDhE"}

package/dist/core/entity-extract.js

@@ -0,0 +1,54 @@
+const PATTERNS = [
+    { type: "url", regex: /https?:\/\/[^\s<>\]）》]+/g },
+    { type: "email", regex: /[\w.+-]+@[\w.-]+\.\w{2,}/g },
+    // Negative lookbehind blocks matching the `@domain` tail of an email (e.g. `alice@example.com`
+    // would otherwise also yield handle `@example.com`).
+    { type: "handle", regex: /(?<![\w.+-])@[\w.-]{2,}/g },
+    { type: "hashtag", regex: /#[\w一-鿿]+/g },
+    { type: "phone", regex: /(?:\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3,4}[-.\s]?\d{4}/g },
+    { type: "ticket_id", regex: /\b\d{12,15}\b/g },
+];
+export function extractQuickEntities(text) {
+    if (!text)
+        return [];
+    const seen = new Set();
+    const results = [];
+    // Extract URLs first to mask them from email pattern
+    const urlRanges = [];
+    const urlRegex = PATTERNS[0].regex;
+    let match;
+    match = urlRegex.exec(text);
+    while (match !== null) {
+        urlRanges.push([match.index, match.index + match[0].length]);
+        const key = `url:${match[0]}`;
+        if (!seen.has(key)) {
+            seen.add(key);
+            results.push({ type: "url", value: match[0] });
+        }
+        match = urlRegex.exec(text);
+    }
+    // Helper: check if position is inside a URL
+    const isInsideUrl = (pos) => urlRanges.some(([start, end]) => pos >= start && pos < end);
+    // Extract remaining patterns (skip URLs, already done)
+    for (let i = 1; i < PATTERNS.length; i++) {
+        const { type, regex } = PATTERNS[i];
+        const re = new RegExp(regex.source, regex.flags);
+        match = re.exec(text);
+        while (match !== null) {
+            if (type === "email" && isInsideUrl(match.index)) {
+                // /g regex is stateful via lastIndex — any `continue` must re-call re.exec(text)
+                // first or this loop will spin forever on the same match.
+                match = re.exec(text);
+                continue;
+            }
+            const key = `${type}:${match[0]}`;
+            if (!seen.has(key)) {
+                seen.add(key);
+                results.push({ type, value: match[0] });
+            }
+            match = re.exec(text);
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=entity-extract.js.map

package/dist/core/entity-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-extract.js","sourceRoot":"","sources":["../../src/core/entity-extract.ts"],"names":[],"mappings":"AAEA,MAAM,QAAQ,GAAwD;IACpE,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE;IAClD,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,2BAA2B,EAAE;IACrD,+FAA+F;IAC/F,qDAAqD;IACrD,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACrD,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE;IACxC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,8DAA8D,EAAE;IACxF,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,gBAAgB,EAAE;CAC/C,CAAC;AAEF,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,qDAAqD;IACrD,MAAM,SAAS,GAA4B,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACnC,IAAI,KAA6B,CAAC;IAElC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;QACtB,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,4CAA4C;IAC5C,MAAM,WAAW,GAAG,CAAC,GAAW,EAAW,EAAE,CAC3C,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;IAE9D,uDAAuD;IACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACjD,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;YACtB,IAAI,IAAI,KAAK,OAAO,IAAI,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjD,iFAAiF;gBACjF,0DAA0D;gBAC1D,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1C,CAAC;YACD,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/core/helpers.d.ts

@@ -0,0 +1,3 @@
+import type { ExtractionResult } from "./types.js";
+export declare function isEmptyExtraction(result: ExtractionResult): boolean;
+//# sourceMappingURL=helpers.d.ts.map

package/dist/core/helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/core/helpers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAUnE"}

package/dist/core/helpers.js

@@ -0,0 +1,10 @@
+export function isEmptyExtraction(result) {
+    return (result.entities.length === 0 &&
+        result.timeline.length === 0 &&
+        result.links.length === 0 &&
+        result.decisions.length === 0 &&
+        result.tasks.length === 0 &&
+        result.discoveries.length === 0 &&
+        result.knowledge.length === 0);
+}
+//# sourceMappingURL=helpers.js.map

package/dist/core/helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.js","sourceRoot":"","sources":["../../src/core/helpers.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,iBAAiB,CAAC,MAAwB;IACxD,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAC5B,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAC5B,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QACzB,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QACzB,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;QAC/B,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAC9B,CAAC;AACJ,CAAC"}

package/dist/core/identity-resolver.d.ts

@@ -0,0 +1,21 @@
+import type { PGlite } from "@electric-sql/pglite";
+import type { RawMessage } from "./types.js";
+export interface IdentityBackend {
+    resolveFeishuOpenId(openId: string): Promise<{
+        name: string;
+        slugHint?: string;
+    } | null>;
+}
+export declare class IdentityResolver {
+    private db;
+    private backend?;
+    constructor(db: PGlite, backend?: IdentityBackend | undefined);
+    enrichBatch(messages: RawMessage[]): Promise<RawMessage[]>;
+    private resolve;
+    private isUnresolvedId;
+    canonicalizeExtractionResult(result: import("./types.js").ExtractionResult): Promise<{
+        result: import("./types.js").ExtractionResult;
+        aliases: Map<string, string[]>;
+    }>;
+}
+//# sourceMappingURL=identity-resolver.d.ts.map

package/dist/core/identity-resolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"identity-resolver.d.ts","sourceRoot":"","sources":["../../src/core/identity-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,MAAM,WAAW,eAAe;IAC9B,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;CAC1F;AAOD,qBAAa,gBAAgB;IAEzB,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,OAAO,CAAC;gBADR,EAAE,EAAE,MAAM,EACV,OAAO,CAAC,EAAE,eAAe,YAAA;IAG7B,WAAW,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;YAwBlD,OAAO;IAkCrB,OAAO,CAAC,cAAc;IAIhB,4BAA4B,CAChC,MAAM,EAAE,OAAO,YAAY,EAAE,gBAAgB,GAC5C,OAAO,CAAC;QAAE,MAAM,EAAE,OAAO,YAAY,EAAE,gBAAgB,CAAC;QAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;KAAE,CAAC;CAG9F"}

package/dist/core/identity-resolver.js

@@ -0,0 +1,62 @@
+export class IdentityResolver {
+    db;
+    backend;
+    constructor(db, backend) {
+        this.db = db;
+        this.backend = backend;
+    }
+    async enrichBatch(messages) {
+        const unresolvedIds = new Set();
+        for (const msg of messages) {
+            if (this.isUnresolvedId(msg.contact)) {
+                unresolvedIds.add(msg.contact);
+            }
+        }
+        if (unresolvedIds.size === 0)
+            return messages;
+        const resolved = new Map();
+        for (const id of unresolvedIds) {
+            const name = await this.resolve("feishu", id);
+            if (name)
+                resolved.set(id, name);
+        }
+        return messages.map((msg) => {
+            const name = resolved.get(msg.contact);
+            if (!name)
+                return msg;
+            return { ...msg, contact: name };
+        });
+    }
+    async resolve(platform, externalId) {
+        const cached = await this.db.query("SELECT display_name, slug_hint FROM identity_cache WHERE platform = $1 AND external_id = $2", [platform, externalId]);
+        if (cached.rows.length > 0) {
+            const row = cached.rows[0];
+            return row.slug_hint ? `${row.display_name} (${row.slug_hint})` : row.display_name;
+        }
+        if (!this.backend)
+            return null;
+        try {
+            const result = await this.backend.resolveFeishuOpenId(externalId);
+            if (!result)
+                return null;
+            await this.db.query(`INSERT INTO identity_cache (platform, external_id, display_name, slug_hint)
+         VALUES ($1, $2, $3, $4)
+         ON CONFLICT (platform, external_id) DO UPDATE SET
+           display_name = EXCLUDED.display_name,
+           slug_hint = EXCLUDED.slug_hint,
+           resolved_at = NOW()`, [platform, externalId, result.name, result.slugHint ?? null]);
+            return result.slugHint ? `${result.name} (${result.slugHint})` : result.name;
+        }
+        catch (error) {
+            console.warn(`[IdentityResolver] Failed to resolve ${externalId}:`, error);
+            return null;
+        }
+    }
+    isUnresolvedId(contact) {
+        return contact.startsWith("ou_");
+    }
+    async canonicalizeExtractionResult(result) {
+        return { result, aliases: new Map() };
+    }
+}
+//# sourceMappingURL=identity-resolver.js.map

package/dist/core/identity-resolver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"identity-resolver.js","sourceRoot":"","sources":["../../src/core/identity-resolver.ts"],"names":[],"mappings":"AAYA,MAAM,OAAO,gBAAgB;IAEjB;IACA;IAFV,YACU,EAAU,EACV,OAAyB;QADzB,OAAE,GAAF,EAAE,CAAQ;QACV,YAAO,GAAP,OAAO,CAAkB;IAChC,CAAC;IAEJ,KAAK,CAAC,WAAW,CAAC,QAAsB;QACtC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,IAAI,aAAa,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,QAAQ,CAAC;QAE9C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE3C,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC9C,IAAI,IAAI;gBAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACvC,IAAI,CAAC,IAAI;gBAAE,OAAO,GAAG,CAAC;YACtB,OAAO,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACnC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,QAAgB,EAAE,UAAkB;QACxD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAChC,6FAA6F,EAC7F,CAAC,QAAQ,EAAE,UAAU,CAAC,CACvB,CAAC;QAEF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3B,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,YAAY,KAAK,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC;QACrF,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAC;YAEzB,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACjB;;;;;+BAKuB,EACvB,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,CAC7D,CAAC;YAEF,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;QAC/E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,wCAAwC,UAAU,GAAG,EAAE,KAAK,CAAC,CAAC;YAC3E,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,OAAe;QACpC,OAAO,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,4BAA4B,CAChC,MAA6C;QAE7C,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IACxC,CAAC;CACF"}

package/dist/core/pipeline-factory.d.ts

@@ -0,0 +1,13 @@
+import type { PGlite } from "@electric-sql/pglite";
+import type { LLMProvider } from "../extractors/providers/types.js";
+import type { Config } from "./config.js";
+import { IdentityResolver } from "./identity-resolver.js";
+import type { PipelineConfig } from "./pipeline.js";
+export interface PipelineRuntime {
+    config: PipelineConfig;
+    provider: LLMProvider;
+    identity_resolver?: IdentityResolver;
+}
+export declare function buildPipelineConfig(config: Config, output_dir: string): PipelineConfig;
+export declare function createPipelineRuntime(config: Config, pg: PGlite, output_dir: string): Promise<PipelineRuntime>;
+//# sourceMappingURL=pipeline-factory.d.ts.map

package/dist/core/pipeline-factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline-factory.d.ts","sourceRoot":"","sources":["../../src/core/pipeline-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AACpE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAGpD,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,CAAC;IACvB,QAAQ,EAAE,WAAW,CAAC;IACtB,iBAAiB,CAAC,EAAE,gBAAgB,CAAC;CACtC;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,cAAc,CAWtF;AAED,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,eAAe,CAAC,CAmB1B"}

package/dist/core/pipeline-factory.js

@@ -0,0 +1,32 @@
+import { createLLMProvider } from "../extractors/providers/index.js";
+import { IdentityResolver } from "./identity-resolver.js";
+import { statePath } from "./state.js";
+export function buildPipelineConfig(config, output_dir) {
+    return {
+        dedup_checkpoint: statePath("dedup.jsonl"),
+        cursor_checkpoint: statePath("cursors.yaml"),
+        block_gap_minutes: config.block_builder.block_gap_minutes,
+        max_block_tokens: config.block_builder.max_block_tokens,
+        max_block_messages: config.block_builder.max_block_messages,
+        privacy: config.privacy,
+        output_dir,
+        block_concurrency: config.pipeline?.block_concurrency,
+    };
+}
+export async function createPipelineRuntime(config, pg, output_dir) {
+    const llmConfig = { ...config.llm };
+    if (!llmConfig.api_key && !process.env.OPENAI_API_KEY) {
+        throw new Error("No API key configured. Set api_key in memoark.yaml or OPENAI_API_KEY env var.");
+    }
+    if (!llmConfig.api_key) {
+        llmConfig.api_key = process.env.OPENAI_API_KEY;
+    }
+    const provider = createLLMProvider(llmConfig);
+    const identity_resolver = new IdentityResolver(pg);
+    return {
+        config: buildPipelineConfig(config, output_dir),
+        provider,
+        identity_resolver,
+    };
+}
+//# sourceMappingURL=pipeline-factory.js.map

package/dist/core/pipeline-factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline-factory.js","sourceRoot":"","sources":["../../src/core/pipeline-factory.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAGrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAQvC,MAAM,UAAU,mBAAmB,CAAC,MAAc,EAAE,UAAkB;IACpE,OAAO;QACL,gBAAgB,EAAE,SAAS,CAAC,aAAa,CAAC;QAC1C,iBAAiB,EAAE,SAAS,CAAC,cAAc,CAAC;QAC5C,iBAAiB,EAAE,MAAM,CAAC,aAAa,CAAC,iBAAiB;QACzD,gBAAgB,EAAE,MAAM,CAAC,aAAa,CAAC,gBAAgB;QACvD,kBAAkB,EAAE,MAAM,CAAC,aAAa,CAAC,kBAAkB;QAC3D,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU;QACV,iBAAiB,EAAE,MAAM,CAAC,QAAQ,EAAE,iBAAiB;KACtD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,MAAc,EACd,EAAU,EACV,UAAkB;IAElB,MAAM,SAAS,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;IACpC,IAAI,CAAC,SAAS,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACvB,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IACjD,CAAC;IACD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE9C,MAAM,iBAAiB,GAAG,IAAI,gBAAgB,CAAC,EAAE,CAAC,CAAC;IAEnD,OAAO;QACL,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC;QAC/C,QAAQ;QACR,iBAAiB;KAClB,CAAC;AACJ,CAAC"}

package/dist/core/pipeline.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Pipeline Orchestration
+ * Chains all components for complete data flow:
+ * Collector → Dedup → BlockBuilder → NoiseFilter → Extractor → Privacy → Formatter → Adapter
+ */
+import { type StoreAdapterContext } from "../adapters/store.js";
+import type { LLMProvider } from "../extractors/providers/types.js";
+import type { PrivacyConfig } from "./config.js";
+import type { Collector, RawMessage } from "./types.js";
+/**
+ * Pipeline configuration
+ */
+export interface PipelineConfig {
+    dedup_checkpoint: string;
+    cursor_checkpoint: string;
+    block_gap_minutes: number;
+    max_block_tokens: number;
+    max_block_messages: number;
+    privacy: PrivacyConfig;
+    output_dir: string;
+    block_concurrency?: number;
+}
+/**
+ * Pipeline execution options
+ */
+export interface PipelineOpts {
+    source: Collector;
+    provider?: LLMProvider;
+    format: "json" | "markdown";
+    adapter: "store" | "file" | "gbrain" | "stdout";
+    stores?: StoreAdapterContext;
+    dryRun?: boolean;
+    since?: string;
+    limit?: number;
+}
+/**
+ * Pipeline execution result
+ */
+export interface PipelineResult {
+    fatal: boolean;
+    error?: string;
+    totalMessages: number;
+    totalBlocks: number;
+    okBlocks: number;
+    skippedBlocks: number;
+    failedBlocks: number;
+    okMessages: RawMessage[];
+    skippedMessages: RawMessage[];
+    failedMessages: RawMessage[];
+    lastSuccessMessage?: RawMessage;
+    warnings: string[];
+}
+/**
+ * Run the complete extraction pipeline
+ *
+ * @param config - Pipeline configuration
+ * @param opts - Execution options
+ * @returns Pipeline result
+ */
+export declare function runPipeline(config: PipelineConfig, opts: PipelineOpts): Promise<PipelineResult>;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/core/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/core/pipeline.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAM9E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAKpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAIjD,OAAO,KAAK,EAGV,SAAS,EAGT,UAAU,EACX,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,MAAM,EAAE,MAAM,GAAG,UAAU,CAAC;IAC5B,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAChD,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,EAAE,CAAC;IACzB,eAAe,EAAE,UAAU,EAAE,CAAC;IAC9B,cAAc,EAAE,UAAU,EAAE,CAAC;IAC7B,kBAAkB,CAAC,EAAE,UAAU,CAAC;IAChC,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,cAAc,CAAC,CA0QzB"}