memkin 0.4.0

package/dist/core/config.js

@@ -0,0 +1,247 @@
+/**
+ * Configuration loader for Memkin
+ * Loads YAML config files with environment variable interpolation
+ * and recursive merging with defaults
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { parse } from "yaml";
+/**
+ * Validates StoreConfig at runtime.
+ * @param s - The store config section.
+ * @param missingEnv - Names of env vars that were NOT resolved (still placeholders).
+ * @throws Error if the config is invalid.
+ */
+export function validateStoreConfig(s, missingEnv = []) {
+    const engine = s.engine ?? "pglite";
+    if (engine !== "pglite" && engine !== "postgres" && engine !== "managed") {
+        throw new Error(`Invalid store.engine "${engine}". Supported values: pglite, postgres, managed.`);
+    }
+    if (engine === "postgres") {
+        if (!s.database_url) {
+            throw new Error("store.database_url is required when store.engine is postgres.");
+        }
+        // Check for unresolved env placeholder like ${DATABASE_URL}
+        const placeholderMatch = s.database_url.match(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/);
+        if (placeholderMatch) {
+            const varName = placeholderMatch[1];
+            if (missingEnv.includes(varName)) {
+                throw new Error(`store.database_url contains 未解析的环境变量 (unresolved env var) \${${varName}}. ` +
+                    `Set the ${varName} environment variable before starting.`);
+            }
+            // Placeholder is present but env var was resolved — treat as valid
+        }
+        else {
+            // No placeholder — must be a literal postgres(ql):// URL
+            if (!/^postgres(ql)?:\/\//.test(s.database_url)) {
+                throw new Error(`store.database_url must be a valid postgres(ql):// connection url. ` +
+                    `Got: "${s.database_url}"`);
+            }
+        }
+        if (s.pool_size !== undefined) {
+            if (!Number.isInteger(s.pool_size) || s.pool_size < 1) {
+                throw new Error(`store.pool_size must be an integer ≥ 1. Got: ${s.pool_size}`);
+            }
+        }
+    }
+    if (engine === "managed" && s.managed !== undefined) {
+        const m = s.managed;
+        if (m.runtime_dir !== undefined) {
+            if (typeof m.runtime_dir !== "string" || m.runtime_dir.trim() === "") {
+                throw new Error(`store.managed.runtime_dir must be a non-empty string. Got: "${m.runtime_dir}"`);
+            }
+        }
+    }
+}
+/**
+ * Default configuration values
+ */
+const DEFAULT_CONFIG = {
+    privacy: {
+        enabled: true,
+        mode: "reversible",
+        redact_phone: true,
+        redact_id_card: true,
+        redact_bank_card: true,
+        redact_email: false,
+        redact_url: false,
+        blocked_words: [],
+        replacement: "[REDACTED]",
+    },
+    llm: {
+        provider: "openai",
+        model: "gpt-4o-mini",
+    },
+    block_builder: {
+        block_gap_minutes: 30,
+        max_block_tokens: 4000,
+        max_block_messages: 100,
+    },
+    adapters: {},
+    sources: {
+        "claude-code": { enabled: true },
+        codex: { enabled: true },
+        hermes: { enabled: true },
+    },
+    store: {
+        data_dir: "~/.memkin/data",
+    },
+    embedding: {
+        provider: "openai",
+        model: "text-embedding-3-large",
+        dimensions: 768,
+    },
+    server: {
+        http_port: 3927,
+        mcp_transport: "stdio",
+    },
+    mcp: {
+        expose_legacy_tools: false,
+        http: {
+            enabled: false,
+            bind_host: "127.0.0.1",
+            port: 3928,
+            allowed_origins: ["http://127.0.0.1:3928", "http://localhost:3928"],
+            allowed_hosts: ["127.0.0.1:3928", "localhost:3928"],
+            auth_token_env: "",
+            read_only: true,
+        },
+    },
+    profile: {
+        enabled: false,
+        allow: [],
+        deny: [],
+        min_sample_size: 20,
+        tz_offset_hours: 8,
+    },
+    search: {
+        pool_by_page: true,
+        llm_rewrite: false,
+    },
+};
+/**
+ * Recursively interpolate environment variables in an object
+ * Replaces ${VAR_NAME} with process.env.VAR_NAME
+ * If VAR_NAME is not found, keeps the placeholder and records the variable name
+ *
+ * @param obj - Object to interpolate
+ * @returns Object with interpolated values and missing variable names
+ */
+function interpolateEnv(obj) {
+    const missing = new Set();
+    function walk(value) {
+        if (typeof value === "string") {
+            return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (match, varName) => {
+                const envValue = process.env[varName];
+                if (envValue === undefined) {
+                    missing.add(varName);
+                    return match;
+                }
+                return envValue;
+            });
+        }
+        if (Array.isArray(value)) {
+            return value.map((item) => walk(item));
+        }
+        if (value !== null && typeof value === "object") {
+            const result = {};
+            for (const [key, nestedValue] of Object.entries(value)) {
+                result[key] = walk(nestedValue);
+            }
+            return result;
+        }
+        return value;
+    }
+    return { result: walk(obj), missing: Array.from(missing).sort() };
+}
+function attachContext(config, context) {
+    return Object.defineProperty(config, "__context", {
+        value: context,
+        enumerable: false,
+        configurable: false,
+        writable: false,
+    });
+}
+export function resolveConfigPath(explicit) {
+    if (explicit)
+        return resolve(explicit);
+    const envPath = process.env.MEMKIN_CONFIG;
+    if (envPath)
+        return resolve(envPath);
+    let dir = process.cwd();
+    while (true) {
+        const candidate = join(dir, "memkin.yaml");
+        if (existsSync(candidate))
+            return candidate;
+        const parent = dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return resolve(process.cwd(), "memkin.yaml");
+}
+/**
+ * Deep merge user config into defaults
+ * User values override defaults at all levels
+ *
+ * @param defaults - Default configuration
+ * @param user - User-provided configuration
+ * @returns Merged configuration
+ */
+function mergeConfig(defaults, user) {
+    const result = { ...defaults };
+    for (const key in user) {
+        const userValue = user[key];
+        const defaultValue = defaults[key];
+        // If user value is an object and default exists and is an object, merge recursively
+        if (userValue !== null &&
+            typeof userValue === "object" &&
+            !Array.isArray(userValue) &&
+            defaultValue !== null &&
+            typeof defaultValue === "object" &&
+            !Array.isArray(defaultValue)) {
+            result[key] = mergeConfig(defaultValue, userValue);
+        }
+        else {
+            // Otherwise, user value overrides default
+            result[key] = userValue;
+        }
+    }
+    return result;
+}
+/**
+ * Load configuration from YAML file
+ * Performs environment variable interpolation and merges with defaults
+ *
+ * @param filePath - Path to YAML config file (default: discovered memkin.yaml)
+ * @returns Loaded and merged configuration
+ * @throws Error if file cannot be read or parsed
+ */
+export function loadConfig(filePath) {
+    const configPath = resolveConfigPath(filePath);
+    const projectRoot = dirname(configPath);
+    let userConfig = {};
+    try {
+        const content = readFileSync(configPath, "utf-8");
+        const parsed = parse(content);
+        userConfig = parsed || {};
+    }
+    catch (error) {
+        // If file doesn't exist or can't be read, use empty config (will use defaults)
+        if (error.code !== "ENOENT") {
+            throw error;
+        }
+    }
+    const interpolated = interpolateEnv(userConfig);
+    userConfig = interpolated.result;
+    // Merge with defaults
+    const merged = mergeConfig(DEFAULT_CONFIG, userConfig);
+    // Validate store config after env interpolation
+    validateStoreConfig(merged.store ?? {}, interpolated.missing);
+    return attachContext(merged, {
+        configPath,
+        projectRoot,
+        missingEnvVars: interpolated.missing,
+    });
+}
+//# sourceMappingURL=config.js.map

package/dist/core/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/core/config.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAkP7B;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,CAAc,EAAE,aAAuB,EAAE;IAC3E,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC;IAEpC,IAAI,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CACb,yBAAyB,MAAM,iDAAiD,CACjF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACnF,CAAC;QAED,4DAA4D;QAC5D,MAAM,gBAAgB,GAAG,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChF,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CACb,gEAAgE,OAAO,KAAK;oBAC1E,WAAW,OAAO,wCAAwC,CAC7D,CAAC;YACJ,CAAC;YACD,mEAAmE;QACrE,CAAC;aAAM,CAAC;YACN,yDAAyD;YACzD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC;gBAChD,MAAM,IAAI,KAAK,CACb,qEAAqE;oBACnE,SAAS,CAAC,CAAC,YAAY,GAAG,CAC7B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;gBACtD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;YACjF,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACpD,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBACrE,MAAM,IAAI,KAAK,CACb,+DAA+D,CAAC,CAAC,WAAW,GAAG,CAChF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAYD;;GAEG;AACH,MAAM,cAAc,GAAW;IAC7B,OAAO,EAAE;QACP,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,YAAY;QAClB,YAAY,EAAE,IAAI;QAClB,cAAc,EAAE,IAAI;QACpB,gBAAgB,EAAE,IAAI;QACtB,YAAY,EAAE,KAAK;QACnB,UAAU,EAAE,KAAK;QACjB,aAAa,EAAE,EAAE;QACjB,WAAW,EAAE,YAAY;KAC1B;IACD,GAAG,EAAE;QACH,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,aAAa;KACrB;IACD,aAAa,EAAE;QACb,iBAAiB,EAAE,EAAE;QACrB,gBAAgB,EAAE,IAAI;QACtB,kBAAkB,EAAE,GAAG;KACxB;IACD,QAAQ,EAAE,EAAE;IACZ,OAAO,EAAE;QACP,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;QAChC,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;QACxB,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAC1B;IACD,KAAK,EAAE;QACL,QAAQ,EAAE,gBAAgB;KAC3B;IACD,SAAS,EAAE;QACT,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,wBAAwB;QAC/B,UAAU,EAAE,GAAG;KAChB;IACD,MAAM,EAAE;QACN,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,OAAO;KACvB;IACD,GAAG,EAAE;QACH,mBAAmB,EAAE,KAAK;QAC1B,IAAI,EAAE;YACJ,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,IAAI;YACV,eAAe,EAAE,CAAC,uBAAuB,EAAE,uBAAuB,CAAC;YACnE,aAAa,EAAE,CAAC,gBAAgB,EAAE,gBAAgB,CAAC;YACnD,cAAc,EAAE,EAAE;YAClB,SAAS,EAAE,IAAI;SAChB;KACF;IACD,OAAO,EAAE;QACP,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,EAAE;QACT,IAAI,EAAE,EAAE;QACR,eAAe,EAAE,EAAE;QACnB,eAAe,EAAE,CAAC;KACnB;IACD,MAAM,EAAE;QACN,YAAY,EAAE,IAAI;QAClB,WAAW,EAAE,KAAK;KACnB;CACF,CAAC;AAEF;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,GAAY;IAClC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,SAAS,IAAI,CAAC,KAAc;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC,OAAO,CAAC,iCAAiC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBACzE,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACrB,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAChD,MAAM,MAAM,GAA4B,EAAE,CAAC;YAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvD,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;YAClC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;AACpE,CAAC;AAED,SAAS,aAAa,CAAC,MAAc,EAAE,OAAsB;IAC3D,OAAO,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,EAAE;QAChD,KAAK,EAAE,OAAO;QACd,UAAU,EAAE,KAAK;QACjB,YAAY,EAAE,KAAK;QACnB,QAAQ,EAAE,KAAK;KAChB,CAAiB,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAiB;IACjD,IAAI,QAAQ;QAAE,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEvC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IAC1C,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC;IAErC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IACxB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAC3C,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,KAAK,GAAG;YAAE,MAAM;QAC1B,GAAG,GAAG,MAAM,CAAC;IACf,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAClB,QAAiC,EACjC,IAA6B;IAE7B,MAAM,MAAM,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;IAE/B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAEnC,oFAAoF;QACpF,IACE,SAAS,KAAK,IAAI;YAClB,OAAO,SAAS,KAAK,QAAQ;YAC7B,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;YACzB,YAAY,KAAK,IAAI;YACrB,OAAO,YAAY,KAAK,QAAQ;YAChC,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,EAC5B,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,GAAG,WAAW,CACvB,YAAuC,EACvC,SAAoC,CACrC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,0CAA0C;YAC1C,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,UAAU,CAAC,QAAiB;IAC1C,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAExC,IAAI,UAAU,GAA4B,EAAE,CAAC;IAE7C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,UAAU,GAAG,MAAM,IAAI,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,+EAA+E;QAC/E,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAChD,UAAU,GAAG,YAAY,CAAC,MAAiC,CAAC;IAE5D,sBAAsB;IACtB,MAAM,MAAM,GAAG,WAAW,CAAC,cAAoD,EAAE,UAAU,CAAC,CAAC;IAE7F,gDAAgD;IAChD,mBAAmB,CAAE,MAA4B,CAAC,KAAK,IAAI,EAAE,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAErF,OAAO,aAAa,CAAC,MAA2B,EAAE;QAChD,UAAU;QACV,WAAW;QACX,cAAc,EAAE,YAAY,CAAC,OAAO;KACrC,CAAC,CAAC;AACL,CAAC"}

package/dist/core/cursors.d.ts

@@ -0,0 +1,36 @@
+/**
+ * CursorStore — Collector cursor persistence
+ * Stores continuation tokens for incremental collection
+ */
+export declare class CursorStore {
+    private cursorPath;
+    private cursors;
+    private dirty;
+    constructor(cursorPath: string);
+    /**
+     * Load cursors from YAML file (key: value pairs)
+     */
+    load(): void;
+    /**
+     * Get cursor for a collector
+     */
+    get(collectorId: string): string | undefined;
+    /**
+     * Set cursor in memory only, mark dirty
+     */
+    set(collectorId: string, cursor: string): void;
+    /**
+     * Store a structured cursor as JSON string
+     */
+    setJSON(collectorId: string, data: Record<string, unknown>): void;
+    /**
+     * Retrieve a structured cursor, returns undefined if missing or not valid JSON
+     */
+    getJSON<T = Record<string, unknown>>(collectorId: string): T | undefined;
+    /**
+     * Write all cursors to YAML file (only if dirty)
+     * Only called by pipeline when failedBlocks === 0
+     */
+    commit(): void;
+}
+//# sourceMappingURL=cursors.d.ts.map

package/dist/core/cursors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cursors.d.ts","sourceRoot":"","sources":["../../src/core/cursors.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,qBAAa,WAAW;IAIV,OAAO,CAAC,UAAU;IAH9B,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,KAAK,CAAkB;gBAEX,UAAU,EAAE,MAAM;IAItC;;OAEG;IACH,IAAI,IAAI,IAAI;IAqBZ;;OAEG;IACH,GAAG,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAI5C;;OAEG;IACH,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAK9C;;OAEG;IACH,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAIjE;;OAEG;IACH,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAYxE;;;OAGG;IACH,MAAM,IAAI,IAAI;CAcf"}

package/dist/core/cursors.js

@@ -0,0 +1,90 @@
+/**
+ * CursorStore — Collector cursor persistence
+ * Stores continuation tokens for incremental collection
+ */
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { parse, stringify } from "yaml";
+export class CursorStore {
+    cursorPath;
+    cursors; // collectorId → cursor value
+    dirty = false;
+    constructor(cursorPath) {
+        this.cursorPath = cursorPath;
+        this.cursors = new Map();
+    }
+    /**
+     * Load cursors from YAML file (key: value pairs)
+     */
+    load() {
+        if (!existsSync(this.cursorPath)) {
+            return;
+        }
+        try {
+            const content = readFileSync(this.cursorPath, "utf-8");
+            const data = parse(content);
+            if (data && typeof data === "object") {
+                for (const [key, value] of Object.entries(data)) {
+                    if (typeof value === "string") {
+                        this.cursors.set(key, value);
+                    }
+                }
+            }
+        }
+        catch {
+            // Skip malformed or empty YAML files
+        }
+    }
+    /**
+     * Get cursor for a collector
+     */
+    get(collectorId) {
+        return this.cursors.get(collectorId);
+    }
+    /**
+     * Set cursor in memory only, mark dirty
+     */
+    set(collectorId, cursor) {
+        this.cursors.set(collectorId, cursor);
+        this.dirty = true;
+    }
+    /**
+     * Store a structured cursor as JSON string
+     */
+    setJSON(collectorId, data) {
+        this.set(collectorId, JSON.stringify(data));
+    }
+    /**
+     * Retrieve a structured cursor, returns undefined if missing or not valid JSON
+     */
+    getJSON(collectorId) {
+        const raw = this.get(collectorId);
+        if (!raw)
+            return undefined;
+        try {
+            const parsed = JSON.parse(raw);
+            if (typeof parsed === "object" && parsed !== null)
+                return parsed;
+            return undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Write all cursors to YAML file (only if dirty)
+     * Only called by pipeline when failedBlocks === 0
+     */
+    commit() {
+        if (!this.dirty) {
+            return;
+        }
+        const data = {};
+        for (const [key, value] of this.cursors.entries()) {
+            data[key] = value;
+        }
+        const yaml = stringify(data);
+        writeFileSync(this.cursorPath, yaml, "utf-8");
+        this.dirty = false;
+    }
+}
+//# sourceMappingURL=cursors.js.map

package/dist/core/cursors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cursors.js","sourceRoot":"","sources":["../../src/core/cursors.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAExC,MAAM,OAAO,WAAW;IAIF;IAHZ,OAAO,CAAsB,CAAC,6BAA6B;IAC3D,KAAK,GAAY,KAAK,CAAC;IAE/B,YAAoB,UAAkB;QAAlB,eAAU,GAAV,UAAU,CAAQ;QACpC,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,IAAI;QACF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAkC,CAAC;YAE7D,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACrC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;wBAC9B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBAC/B,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;QACvC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,WAAmB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,WAAmB,EAAE,MAAc;QACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,WAAmB,EAAE,IAA6B;QACxD,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,OAAO,CAA8B,WAAmB;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;gBAAE,OAAO,MAAW,CAAC;YACtE,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,MAAM;QACJ,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAA2B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACpB,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF"}

package/dist/core/dedup.d.ts

@@ -0,0 +1,45 @@
+/**
+ * DedupStore — Message deduplication tracking
+ * Checkpoints source identity and content hashes to detect new/modified/unchanged messages
+ */
+import type { RawMessage } from "./types.js";
+export declare class DedupStore {
+    private checkpointPath;
+    private entries;
+    constructor(checkpointPath: string);
+    /**
+     * Load existing entries from checkpointPath (JSONL format)
+     * Each line: {"source_hash":"...","content_hash":"..."}
+     */
+    load(): void;
+    /**
+     * Generate source identity hash for a message
+     * SHA-256(platform:channel:identityKey:timestamp)
+     *
+     * Identity key priority:
+     * 1. metadata.message_id
+     * 2. metadata.uuid
+     * 3. metadata.thread_id
+     * 4. metadata.session_id ? `${session_id}:${index ?? 0}` : undefined
+     * 5. `${timestamp}:${index ?? 0}` (fallback)
+     */
+    sourceIdentityHash(msg: RawMessage): string;
+    /**
+     * Generate content hash for a message
+     * SHA-256(content:attachment_ids_joined_by_comma)
+     */
+    contentHash(msg: RawMessage): string;
+    /**
+     * Check message status
+     * - new: source_hash not in entries
+     * - unchanged: source_hash exists AND content_hash matches
+     * - modified: source_hash exists AND content_hash differs
+     */
+    check(msg: RawMessage): "new" | "unchanged" | "modified";
+    /**
+     * Commit messages to checkpoint file (append JSONL)
+     * Only called by pipeline on success — NEVER in finally block
+     */
+    commit(msgs: RawMessage[]): void;
+}
+//# sourceMappingURL=dedup.d.ts.map

package/dist/core/dedup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/core/dedup.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAc,UAAU,EAAE,MAAM,YAAY,CAAC;AAEzD,qBAAa,UAAU;IAGT,OAAO,CAAC,cAAc;IAFlC,OAAO,CAAC,OAAO,CAAsB;gBAEjB,cAAc,EAAE,MAAM;IAI1C;;;OAGG;IACH,IAAI,IAAI,IAAI;IAkBZ;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM;IAuB3C;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM;IAMpC;;;;;OAKG;IACH,KAAK,CAAC,GAAG,EAAE,UAAU,GAAG,KAAK,GAAG,WAAW,GAAG,UAAU;IAYxD;;;OAGG;IACH,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,IAAI;CAkBjC"}

package/dist/core/dedup.js

@@ -0,0 +1,111 @@
+/**
+ * DedupStore — Message deduplication tracking
+ * Checkpoints source identity and content hashes to detect new/modified/unchanged messages
+ */
+import { createHash } from "node:crypto";
+import { appendFileSync, existsSync, readFileSync } from "node:fs";
+export class DedupStore {
+    checkpointPath;
+    entries; // source_hash → content_hash
+    constructor(checkpointPath) {
+        this.checkpointPath = checkpointPath;
+        this.entries = new Map();
+    }
+    /**
+     * Load existing entries from checkpointPath (JSONL format)
+     * Each line: {"source_hash":"...","content_hash":"..."}
+     */
+    load() {
+        if (!existsSync(this.checkpointPath)) {
+            return;
+        }
+        const content = readFileSync(this.checkpointPath, "utf-8");
+        const lines = content.split("\n").filter((line) => line.trim());
+        for (const line of lines) {
+            try {
+                const entry = JSON.parse(line);
+                this.entries.set(entry.source_hash, entry.content_hash);
+            }
+            catch {
+                // Skip malformed lines
+            }
+        }
+    }
+    /**
+     * Generate source identity hash for a message
+     * SHA-256(platform:channel:identityKey:timestamp)
+     *
+     * Identity key priority:
+     * 1. metadata.message_id
+     * 2. metadata.uuid
+     * 3. metadata.thread_id
+     * 4. metadata.session_id ? `${session_id}:${index ?? 0}` : undefined
+     * 5. `${timestamp}:${index ?? 0}` (fallback)
+     */
+    sourceIdentityHash(msg) {
+        const metadata = msg.metadata || {};
+        let identityKey;
+        if (metadata.message_id) {
+            identityKey = String(metadata.message_id);
+        }
+        else if (metadata.uuid) {
+            identityKey = String(metadata.uuid);
+        }
+        else if (metadata.thread_id) {
+            identityKey = String(metadata.thread_id);
+        }
+        else if (metadata.session_id) {
+            const index = metadata.index ?? 0;
+            identityKey = `${metadata.session_id}:${index}`;
+        }
+        else {
+            const index = metadata.index ?? 0;
+            identityKey = `${msg.timestamp}:${index}`;
+        }
+        const composite = `${msg.platform}:${msg.channel}:${identityKey}:${msg.timestamp}`;
+        return createHash("sha256").update(composite).digest("hex");
+    }
+    /**
+     * Generate content hash for a message
+     * SHA-256(content:attachment_ids_joined_by_comma)
+     */
+    contentHash(msg) {
+        const attachmentIds = msg.attachments?.map((a) => a.id).join(",") || "";
+        const composite = `${msg.content}:${attachmentIds}`;
+        return createHash("sha256").update(composite).digest("hex");
+    }
+    /**
+     * Check message status
+     * - new: source_hash not in entries
+     * - unchanged: source_hash exists AND content_hash matches
+     * - modified: source_hash exists AND content_hash differs
+     */
+    check(msg) {
+        const sourceHash = this.sourceIdentityHash(msg);
+        const existingContentHash = this.entries.get(sourceHash);
+        if (!existingContentHash) {
+            return "new";
+        }
+        const currentContentHash = this.contentHash(msg);
+        return currentContentHash === existingContentHash ? "unchanged" : "modified";
+    }
+    /**
+     * Commit messages to checkpoint file (append JSONL)
+     * Only called by pipeline on success — NEVER in finally block
+     */
+    commit(msgs) {
+        if (msgs.length === 0) {
+            return;
+        }
+        const lines = msgs.map((msg) => {
+            const sourceHash = this.sourceIdentityHash(msg);
+            const contentHash = this.contentHash(msg);
+            const entry = { source_hash: sourceHash, content_hash: contentHash };
+            // Update in-memory map
+            this.entries.set(sourceHash, contentHash);
+            return JSON.stringify(entry);
+        });
+        appendFileSync(this.checkpointPath, `${lines.join("\n")}\n`, "utf-8");
+    }
+}
+//# sourceMappingURL=dedup.js.map

package/dist/core/dedup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/core/dedup.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAGnE,MAAM,OAAO,UAAU;IAGD;IAFZ,OAAO,CAAsB,CAAC,6BAA6B;IAEnE,YAAoB,cAAsB;QAAtB,mBAAc,GAAd,cAAc,CAAQ;QACxC,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,IAAI;QACF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACrC,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAEhE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAe,CAAC;gBAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;YAC1D,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,GAAe;QAChC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEpC,IAAI,WAAmB,CAAC;QAExB,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;aAAM,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzB,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC9B,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;YAClC,WAAW,GAAG,GAAG,QAAQ,CAAC,UAAU,IAAI,KAAK,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;YAClC,WAAW,GAAG,GAAG,GAAG,CAAC,SAAS,IAAI,KAAK,EAAE,CAAC;QAC5C,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,OAAO,IAAI,WAAW,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;QACnF,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,GAAe;QACzB,MAAM,aAAa,GAAG,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACxE,MAAM,SAAS,GAAG,GAAG,GAAG,CAAC,OAAO,IAAI,aAAa,EAAE,CAAC;QACpD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,GAAe;QACnB,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEzD,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACjD,OAAO,kBAAkB,KAAK,mBAAmB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/E,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,IAAkB;QACvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAChD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,KAAK,GAAe,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC;YAEjF,uBAAuB;YACvB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;YAE1C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACxE,CAAC;CACF"}

package/dist/core/entity-extract.d.ts

@@ -0,0 +1,3 @@
+import type { QuickEntity } from "./types.js";
+export declare function extractQuickEntities(text: string): QuickEntity[];
+//# sourceMappingURL=entity-extract.d.ts.map

package/dist/core/entity-extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-extract.d.ts","sourceRoot":"","sources":["../../src/core/entity-extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAa9C,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,CAgDhE"}

package/dist/core/entity-extract.js

@@ -0,0 +1,54 @@
+const PATTERNS = [
+    { type: "url", regex: /https?:\/\/[^\s<>\]）》]+/g },
+    { type: "email", regex: /[\w.+-]+@[\w.-]+\.\w{2,}/g },
+    // Negative lookbehind blocks matching the `@domain` tail of an email (e.g. `alice@example.com`
+    // would otherwise also yield handle `@example.com`).
+    { type: "handle", regex: /(?<![\w.+-])@[\w.-]{2,}/g },
+    { type: "hashtag", regex: /#[\w一-鿿]+/g },
+    { type: "phone", regex: /(?:\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3,4}[-.\s]?\d{4}/g },
+    { type: "ticket_id", regex: /\b\d{12,15}\b/g },
+];
+export function extractQuickEntities(text) {
+    if (!text)
+        return [];
+    const seen = new Set();
+    const results = [];
+    // Extract URLs first to mask them from email pattern
+    const urlRanges = [];
+    const urlRegex = PATTERNS[0].regex;
+    let match;
+    match = urlRegex.exec(text);
+    while (match !== null) {
+        urlRanges.push([match.index, match.index + match[0].length]);
+        const key = `url:${match[0]}`;
+        if (!seen.has(key)) {
+            seen.add(key);
+            results.push({ type: "url", value: match[0] });
+        }
+        match = urlRegex.exec(text);
+    }
+    // Helper: check if position is inside a URL
+    const isInsideUrl = (pos) => urlRanges.some(([start, end]) => pos >= start && pos < end);
+    // Extract remaining patterns (skip URLs, already done)
+    for (let i = 1; i < PATTERNS.length; i++) {
+        const { type, regex } = PATTERNS[i];
+        const re = new RegExp(regex.source, regex.flags);
+        match = re.exec(text);
+        while (match !== null) {
+            if (type === "email" && isInsideUrl(match.index)) {
+                // /g regex is stateful via lastIndex — any `continue` must re-call re.exec(text)
+                // first or this loop will spin forever on the same match.
+                match = re.exec(text);
+                continue;
+            }
+            const key = `${type}:${match[0]}`;
+            if (!seen.has(key)) {
+                seen.add(key);
+                results.push({ type, value: match[0] });
+            }
+            match = re.exec(text);
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=entity-extract.js.map

package/dist/core/entity-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-extract.js","sourceRoot":"","sources":["../../src/core/entity-extract.ts"],"names":[],"mappings":"AAEA,MAAM,QAAQ,GAAwD;IACpE,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE;IAClD,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,2BAA2B,EAAE;IACrD,+FAA+F;IAC/F,qDAAqD;IACrD,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACrD,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE;IACxC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,8DAA8D,EAAE;IACxF,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,gBAAgB,EAAE;CAC/C,CAAC;AAEF,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,qDAAqD;IACrD,MAAM,SAAS,GAA4B,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACnC,IAAI,KAA6B,CAAC;IAElC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;QACtB,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,4CAA4C;IAC5C,MAAM,WAAW,GAAG,CAAC,GAAW,EAAW,EAAE,CAC3C,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;IAE9D,uDAAuD;IACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACjD,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;YACtB,IAAI,IAAI,KAAK,OAAO,IAAI,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjD,iFAAiF;gBACjF,0DAA0D;gBAC1D,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1C,CAAC;YACD,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/core/env-validation.d.ts

@@ -0,0 +1,9 @@
+import type { LoadedConfig } from "./config.js";
+export type EnvValidationCommand = "extract" | "embed" | "search" | "serve" | "doctor";
+export declare function getMissingEnvVarsForCommand(config: LoadedConfig, command: EnvValidationCommand, opts?: {
+    searchMode?: string;
+}): string[];
+export declare function validateEnvForCommand(config: LoadedConfig, command: Exclude<EnvValidationCommand, "doctor" | "serve">, opts?: {
+    searchMode?: string;
+}): void;
+//# sourceMappingURL=env-validation.d.ts.map

package/dist/core/env-validation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"env-validation.d.ts","sourceRoot":"","sources":["../../src/core/env-validation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AAiEvF,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,oBAAoB,EAC7B,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GACjC,MAAM,EAAE,CAoBV;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,OAAO,CAAC,oBAAoB,EAAE,QAAQ,GAAG,OAAO,CAAC,EAC1D,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GACjC,IAAI,CAUN"}