@wirecat/cli-core 0.592.0 → 0.594.0

package/dist/mail/marks.js

@@ -1,7 +1,7 @@
 import { CliError } from "@wirecat/cli-base";
 import { mailOwnText } from "../conversations/mail-text.js";
 import { isBusy } from "../store/sqlite/transaction.js";
-import { BULK_TRIAGE_DEFAULTS, newsletterSorter } from "./newsletter.js";
+import { BULK_TRIAGE_DEFAULTS, newsletterSorter, transactionalCue, } from "./newsletter.js";
 /** A holder's name: what `--holder` takes and the mark's key ends with. */
 export const HOLDER = /^[a-z0-9][a-z0-9-]{0,39}$/;
 /** At most this many emails are read past the mark in one go; the rest come with the next read. */
@@ -65,13 +65,15 @@ export const marksOf = async (store, mailbox) => {
         .flatMap(({ key, value, at }) => parsed(key.slice(PREFIX.length), value, at) ?? [])
         .sort((a, b) => a.holder.localeCompare(b.holder));
 };
+/** Sent to many and not an order, an invoice or a shipment: those carry List-Unsubscribe too, and are for the holder. */
+const isBulk = (email) => email.bulk && (email.listed || transactionalCue({ subject: email.subject, from: email.from }) === null);
 /** Why an email is not for the holder, or `held` while it is partial and young, or `read`. */
 const verdict = (email, now) => {
     if (email.draft)
         return "draft";
     if (email.envelope)
         return "envelope";
-    if (email.bulk)
+    if (isBulk(email))
         return "bulk";
     if (email.demoted)
         return "suspicious";
@@ -102,14 +104,14 @@ export const readPastMark = async (store, mailbox, holder, options = {}) => {
     const { wanted, sorted, oldestStoredAt: _, ...read } = await examine(store, mailbox, holder, options);
     const excerpts = wanted.flatMap((id) => {
         const one = sorted.get(id);
-        return one ? [{ email: one.email, excerpt: one.excerpt, cues: one.cues }] : [];
+        return one ? [{ email: one.email, excerpt: one.excerpt, cues: one.cues, partial: one.partial }] : [];
     });
     const emails = await store.mail.emailHits(wanted.filter((id) => !sorted.has(id)));
     return { emails, excerpts, ...read };
 };
 /**
  * The batch's bulk emails a holder might read, sorted by `newsletterSorter`, and where the batch ends so it holds at
- * most `perRead` excerpts. Drafts, envelopes, suspicious senders and partial mail keep their own verdicts.
+ * most `perRead` excerpts. Drafts, envelopes and suspicious senders keep their own verdicts; partial bulk mail is sorted too, and flagged.
  */
 const sortBulk = async (store, batch, triage) => {
     const sorted = new Map();
@@ -118,7 +120,7 @@ const sortBulk = async (store, batch, triage) => {
     const sort = newsletterSorter(triage.cues);
     let excerpts = 0;
     for (const [index, marked] of batch.entries()) {
-        if (!marked.bulk || marked.draft || marked.envelope || marked.demoted || marked.partial)
+        if (!isBulk(marked) || marked.draft || marked.envelope || marked.demoted)
             continue;
         const [email] = await store.mail.emailHits([marked.id]);
         if (!email)
@@ -130,7 +132,7 @@ const sortBulk = async (store, batch, triage) => {
             return { sorted, end: index };
         if (one.verdict === "excerpt")
             excerpts += 1;
-        sorted.set(marked.id, { ...one, email });
+        sorted.set(marked.id, { ...one, email, partial: marked.partial });
     }
     return { sorted, end: batch.length };
 };

package/dist/mail/newsletter.d.ts

@@ -27,6 +27,16 @@ export interface BulkSorted {
     cues: string[];
     excerpt: string;
 }
+/**
+ * What makes a bulk email transactional (an order, an invoice, a receipt, a shipment), or `null`. List-Unsubscribe
+ * sits on those too, so it proves nothing alone. Subject and sender only, so the store can ask it of every email.
+ * An order's number settles it; an amount with its currency or a word of the lists below does not, when the subject
+ * also advertises or the sender is a newsletter's ("Order now, save 20%").
+ */
+export declare const transactionalCue: ({ subject, from }: {
+    subject: string | null;
+    from: string | null;
+}) => string | null;
 /** The first lines of an email's own text, at most `chars`, cut at a word. */
 export declare const excerptOf: (text: string, chars: number) => string;
 /**

package/dist/mail/newsletter.js

@@ -130,6 +130,76 @@ const matcherOf = (words) => {
     const pattern = new RegExp(`(?<![\\p{L}\\p{N}])(?:${list.map(escaped).join("|")})`, "giu");
     return (text) => [...new Set([...text.toLowerCase().matchAll(pattern)].map(([found]) => found))];
 };
+/** Words that, in a subject, say "a record of something of yours": a purchase, a payment, a shipment. */
+const TRANSACTIONAL_WORDS = [
+    "заказ",
+    "счёт",
+    "счет",
+    "чек",
+    "квитанци",
+    "оплат",
+    "доставк",
+    "посылк",
+    "отправлен",
+    "order",
+    "invoice",
+    "receipt",
+    "payment",
+    "shipped",
+    "shipping",
+    "delivered",
+    "delivery",
+    "tracking",
+    "refund",
+    "pedido",
+    "factura",
+    "recibo",
+    "pago",
+    "envío",
+    "reembolso",
+];
+/** An amount with its currency, either side: `1 250 ₽`, `49,90 EUR`, `$12.50`. */
+const AMOUNT = /\d[\d\s.,]*\s?(?:₽|€|\$|£|руб|р\.|rub|eur|usd|gbp)(?![\p{L}])|(?:₽|€|\$|£|rub|eur|usd|gbp)\s?\d/iu;
+/** An order number as shops print it, such as Amazon's `406-3547475-4115561`. */
+const ORDER_ID = /(?<!\d)\d{3}-\d{7}-\d{7}(?!\d)/;
+/** Parts of a sender's address that say "orders, billing or shipping". */
+const TRANSACTIONAL_SENDERS = new Set([
+    "order",
+    "orders",
+    "billing",
+    "invoice",
+    "invoices",
+    "receipt",
+    "receipts",
+    "payment",
+    "payments",
+    "shipping",
+    "tracking",
+]);
+const transactionalWords = matcherOf(TRANSACTIONAL_WORDS);
+const advertisingWords = matcherOf(NEWSLETTER_WORDS);
+const NEWSLETTER_SENDER_PARTS = new Set(NEWSLETTER_SENDERS.filter((one) => !one.includes(".")));
+/**
+ * What makes a bulk email transactional (an order, an invoice, a receipt, a shipment), or `null`. List-Unsubscribe
+ * sits on those too, so it proves nothing alone. Subject and sender only, so the store can ask it of every email.
+ * An order's number settles it; an amount with its currency or a word of the lists below does not, when the subject
+ * also advertises or the sender is a newsletter's ("Order now, save 20%").
+ */
+export const transactionalCue = ({ subject, from }) => {
+    const title = subject ?? "";
+    const number = REFERENCE_NUMBER.exec(title) ?? ORDER_ID.exec(title);
+    if (number)
+        return number[0];
+    const parts = (from ?? "").toLowerCase().split(/[.@_+-]/);
+    if (advertisingWords(title).length > 0 || parts.some((part) => NEWSLETTER_SENDER_PARTS.has(part)))
+        return null;
+    const [word] = transactionalWords(title);
+    if (word)
+        return word;
+    if (AMOUNT.test(title))
+        return "an amount";
+    return parts.find((part) => TRANSACTIONAL_SENDERS.has(part)) ?? null;
+};
 /** The first lines of an email's own text, at most `chars`, cut at a word. */
 export const excerptOf = (text, chars) => {
     const flat = text

package/dist/mail/review.d.ts

@@ -22,6 +22,7 @@ export interface MailReviewOptions {
 export type MailHit = StoredHit & {
     marks?: EmailMarks;
     excerptOnly?: true;
+    partialExcerpt?: true;
 };
 export interface MailReview extends Review {
     chats: (Omit<ReviewChat, "messages"> & {

package/dist/services/index.d.ts

@@ -154,6 +154,7 @@ export declare const servicesFor: (deps: ServiceDeps) => Services;
 export { entriesOfList, type ListContext, type ListedEntry } from "../search/account-list.js";
 export { FIELD_VERSION, QUERY_FIELDS, QUERY_OPERATORS, validateAst } from "../search/lucene/registry.js";
 export { QUERY_LIMITS, QUERY_VERSION } from "../search/lucene/types.js";
+export { questionLookup } from "../search/question-plan.js";
 export { type SearchTotal, TOTAL_CAP } from "../search/total.js";
 export { TASK_KINDS, TASK_ORIGINS, type TaskKind, type TaskOrigin } from "../tasks/index.js";
 export type { AdminFound, AdminQuery, AdminSelection, AdminStatisticsService } from "./admin-statistics.js";

package/dist/services/index.js

@@ -119,6 +119,7 @@ export const servicesFor = (deps) => {
 export { entriesOfList } from "../search/account-list.js";
 export { FIELD_VERSION, QUERY_FIELDS, QUERY_OPERATORS, validateAst } from "../search/lucene/registry.js";
 export { QUERY_LIMITS, QUERY_VERSION } from "../search/lucene/types.js";
+export { questionLookup } from "../search/question-plan.js";
 export { TOTAL_CAP } from "../search/total.js";
 export { TASK_KINDS, TASK_ORIGINS } from "../tasks/index.js";
 export { adminStatisticsService } from "./admin-statistics.js";

package/dist/store/sqlite/crm.d.ts

@@ -6,7 +6,7 @@ import type { StoreContext } from "./open.js";
 /** What the store could not keep, and why: an item whose stage is not stored or whose field it refuses, or a pipeline. */
 export interface CrmSkipped {
     externalId: string;
-    reason: "crm.stage-unknown" | "crm.item-invalid" | "crm.pipeline-clash";
+    reason: "crm.stage-unknown" | "crm.item-invalid";
     message: string;
 }
 export interface CrmSaved {
@@ -39,6 +39,8 @@ export interface CrmCopies {
     }): Promise<number>;
     /** How many of the kind the account holds, not deleted; every contact it ever saved. */
     count(key: AccountKey, kind: CrmKind): Promise<number>;
+    /** The stored copy's `deal:<id>` or `lead:<id>`, deleted or not; `null` when the store holds none. */
+    refOf(key: AccountKey, kind: DealKind, externalId: string): Promise<string | null>;
 }
 /**
  * A contact's identity id: identities are unique per provider, and two portals of one CRM both have a contact 5.

package/dist/store/sqlite/crm.js

@@ -36,24 +36,14 @@ export const crmCopiesOver = (context, people) => {
     const pipelineOf = (accountId, kind, externalId) => database
         .prepare("SELECT * FROM pipelines WHERE account_id = ? AND kind = ? AND external_id = ?")
         .get(accountId, kind, externalId);
-    // Store version 52 keeps a pipeline's id unique per account, not per kind: a lead pipeline sharing a deal pipeline's
-    // id cannot be stored until a migration widens that key, so it is skipped and named, not written over the other.
-    const otherKindHolds = (accountId, kind, externalId) => database
-        .prepare("SELECT 1 FROM pipelines WHERE account_id = ? AND kind <> ? AND external_id = ?")
-        .get(accountId, kind, externalId) !== undefined;
+    /**
+     * One kind's whole list: a pipeline it lacks is marked deleted, one listed again comes back. An empty list marks
+     * nothing: a sign-in that lost its CRM rights lists no pipeline at all.
+     */
     const savePipelines = (key, kind, pipelines) => write(database, () => {
         const accountId = account(key);
         const at = now();
-        const skipped = [];
         for (const pipeline of pipelines) {
-            if (otherKindHolds(accountId, kind, pipeline.externalId)) {
-                skipped.push({
-                    externalId: pipeline.externalId,
-                    reason: "crm.pipeline-clash",
-                    message: `${kind} pipeline ${pipeline.externalId} has the id of a stored pipeline of the other kind`,
-                });
-                continue;
-            }
             const name = textOf(pipeline.name, 80) ?? pipeline.externalId;
             const held = pipelineOf(accountId, kind, pipeline.externalId);
             const id = held
@@ -62,13 +52,22 @@ export const crmCopiesOver = (context, people) => {
                     .prepare("INSERT INTO pipelines (account_id, external_id, kind, name, name_folded, created_at, updated_at) " +
                     "VALUES (?, ?, ?, ?, ?, ?, ?) RETURNING id")
                     .get(accountId, pipeline.externalId, kind, name, fold(name), at, at)?.id);
-            if (held && held.name !== name)
+            if (held && (held.name !== name || held.deleted_at != null))
                 database
-                    .prepare("UPDATE pipelines SET name = ?, name_folded = ?, updated_at = ? WHERE id = ?")
+                    .prepare("UPDATE pipelines SET name = ?, name_folded = ?, updated_at = ?, deleted_at = NULL WHERE id = ?")
                     .run(name, fold(name), at, id);
             saveStages(id, pipeline, at);
         }
-        return { saved: pipelines.length - skipped.length, skipped };
+        if (pipelines.length === 0)
+            return { saved: 0, skipped: [] };
+        const listed = new Set(pipelines.map(({ externalId }) => externalId));
+        const mark = database.prepare("UPDATE pipelines SET deleted_at = ?, updated_at = ? WHERE id = ?");
+        for (const row of database
+            .prepare("SELECT id, external_id FROM pipelines WHERE account_id = ? AND kind = ? AND deleted_at IS NULL")
+            .all(accountId, kind))
+            if (!listed.has(String(row.external_id)))
+                mark.run(at, at, Number(row.id));
+        return { saved: pipelines.length, skipped: [] };
     });
     /**
      * Stage names are unique in a pipeline, but a CRM may swap two, or give two stages one name: listed stages first
@@ -90,18 +89,23 @@ export const crmCopiesOver = (context, people) => {
         const listed = new Set(wanted.map(({ externalId }) => externalId));
         const taken = new Set(wanted.map(({ folded }) => folded));
         const rename = database.prepare("UPDATE pipeline_stages SET name_folded = ? WHERE id = ?");
+        const gone = database.prepare("UPDATE pipeline_stages SET deleted_at = ? WHERE id = ? AND deleted_at IS NULL");
         for (const row of held) {
-            if (row.external_id != null && listed.has(String(row.external_id)))
+            const kept = row.external_id != null && listed.has(String(row.external_id));
+            if (kept)
                 rename.run(`\u0000${row.id}`, Number(row.id));
             else if (taken.has(String(row.name_folded)))
                 rename.run(`${row.name_folded} (${row.external_id ?? row.id})`, Number(row.id));
+            if (!kept)
+                gone.run(at, Number(row.id));
         }
         const byId = new Map(held.map((row) => [String(row.external_id), row]));
         for (const stage of wanted) {
             const row = byId.get(stage.externalId);
             if (row)
                 database
-                    .prepare("UPDATE pipeline_stages SET name = ?, name_folded = ?, position = ?, outcome = ?, updated_at = ? WHERE id = ?")
+                    .prepare("UPDATE pipeline_stages SET name = ?, name_folded = ?, position = ?, outcome = ?, updated_at = ?, " +
+                    "deleted_at = NULL WHERE id = ?")
                     .run(stage.name, stage.folded, stage.position, stage.outcome, at, Number(row.id));
             else
                 database
@@ -151,7 +155,11 @@ export const crmCopiesOver = (context, people) => {
         const deal = kind === "lead" && item.deal
             ? database.prepare("SELECT id FROM deals WHERE account_id = ? AND external_id = ?").get(accountId, item.deal)
             : undefined;
-        const metadata = { ...(item.metadata ?? {}), ...(item.url ? { url: item.url } : {}) };
+        const metadata = {
+            ...(item.metadata ?? {}),
+            ...(item.url ? { url: item.url } : {}),
+            ...(item.closedReason && status !== "open" ? { closed_reason: textOf(item.closedReason, 2000) } : {}),
+        };
         const values = {
             title,
             organization: item.organization ? organizationIdOf(accountId, item.organization) : null,
@@ -314,5 +322,14 @@ export const crmCopiesOver = (context, people) => {
                 .prepare("SELECT count(*) AS n FROM identities WHERE provider = ? AND substr(external_id, 1, ?) = ?")
                 .get(key.provider, key.account.length + 1, `${key.account}/`)?.n)
             : stored(key, kind).length,
+        refOf: async (key, kind, externalId) => {
+            const accountId = findAccountPk(context, key);
+            const row = accountId === undefined
+                ? undefined
+                : database
+                    .prepare(`SELECT id FROM ${TABLES[kind]} WHERE account_id = ? AND external_id = ?`)
+                    .get(accountId, externalId);
+            return row ? `${kind}:${row.id}` : null;
+        },
     };
 };

package/dist/store/sqlite/deals.js

@@ -94,9 +94,11 @@ export const dealOperationsOver = (context) => {
             ? row.account_id == null
             : row.account_provider === scope.provider && row.account_external_id === scope.account);
     const kindOf = (kind) => (kind === undefined ? "deal" : oneOf(kind, DEAL_KINDS, "a kind"));
-    const pipelineRows = (kind, scope) => database
+    /** A pipeline the CRM deleted is left out, unless named by its reference: its deals still stand in it. */
+    const pipelineRows = (kind, scope, { deleted = false } = {}) => database
         .prepare("SELECT p.*, a.provider AS account_provider, a.external_id AS account_external_id FROM pipelines p " +
-        "LEFT JOIN accounts a ON a.id = p.account_id WHERE (? IS NULL OR p.kind = ?) ORDER BY p.name_folded, p.id")
+        "LEFT JOIN accounts a ON a.id = p.account_id WHERE (? IS NULL OR p.kind = ?) " +
+        `${deleted ? "" : "AND p.deleted_at IS NULL "}ORDER BY p.name_folded, p.id`)
         .all(kind ?? null, kind ?? null)
         .filter((row) => inScope(scope, row));
     const label = (row) => `pipeline:${row.id} (${row.name}${row.account_provider == null ? "" : `, ${row.account_provider}:${row.account_external_id}`})`;
@@ -104,7 +106,7 @@ export const dealOperationsOver = (context) => {
     const pipelineRow = (reference, kind, scope) => {
         const named = reference.trim();
         if (/^pipeline:\d+$/.test(named)) {
-            const row = pipelineRows(kind, scope).find((one) => `pipeline:${one.id}` === named);
+            const row = pipelineRows(kind, scope, { deleted: true }).find((one) => `pipeline:${one.id}` === named);
             if (!row)
                 throw new CliError("not_found", `no ${kind ?? ""} pipeline ${named}`, { reason: "pipeline.not-found" });
             return row;
@@ -267,7 +269,7 @@ export const dealOperationsOver = (context) => {
                 });
                 return made;
             });
-            return pipelinesOf(pipelineRows(pipelineKind).filter((row) => Number(row.id) === id))[0];
+            return pipelinesOf(pipelineRows(pipelineKind, undefined, { deleted: true }).filter((row) => Number(row.id) === id))[0];
         },
         pipelines: (filter = {}) => pipelinesOf(pipelineRows(filter.kind === undefined ? undefined : kindOf(filter.kind), filter.account)),
         pipeline: (reference, options = {}) => {

package/dist/store/sqlite/email-activity.d.ts

@@ -54,6 +54,10 @@ export interface MarkedEmail {
     changedAt: number;
     /** Sent to many: the bulk mailbox or a List-Id. */
     bulk: boolean;
+    /** Its own List-Id: a real mailing list, as opposed to List-Unsubscribe or Precedence alone. */
+    listed: boolean;
+    subject: string | null;
+    from: string | null;
     envelope: boolean;
     /** Stored from its first bytes, not whole yet. */
     partial: boolean;

package/dist/store/sqlite/email-activity.js

@@ -100,7 +100,8 @@ export const writtenTo = (database, addresses) => {
 };
 const MARKED = `
   SELECT e.id, e.external_id, t.external_id AS thread, coalesce(e.received_at, e.sent_at) AS at, e.created_at,
-    e.updated_at, coalesce(e.draft, 0) AS draft, json_extract(e.metadata, '$.partial') IS NOT NULL AS partial,
+    e.updated_at, e.subject, e.from_address, coalesce(e.draft, 0) AS draft,
+    coalesce(json_extract(e.headers, '$."list-id"'), json_extract(e.metadata, '$.listId')) IS NOT NULL AS listed, json_extract(e.metadata, '$.partial') IS NOT NULL AS partial,
     coalesce(json_extract(e.headers, '$."list-id"'), json_extract(e.metadata, '$.listId')) IS NOT NULL
       OR EXISTS (SELECT 1 FROM email_mailboxes m JOIN mailboxes b ON b.id = m.mailbox_id
         WHERE m.email_id = e.id AND b.kind = 'bulk') AS bulk,
@@ -117,6 +118,9 @@ const markedOf = (row) => ({
     storedAt: Number(row.created_at),
     changedAt: Number(row.updated_at),
     bulk: Boolean(row.bulk),
+    listed: Boolean(row.listed),
+    subject: row.subject === null || row.subject === undefined ? null : String(row.subject),
+    from: row.from_address === null || row.from_address === undefined ? null : String(row.from_address),
     envelope: Boolean(row.envelope),
     partial: Boolean(row.partial),
     draft: Boolean(row.draft),

package/dist/store/sqlite/emails.js

@@ -8,6 +8,7 @@ import { markEmailsDeleted, markEmailsRestored, recountEmailThread as recount, s
 import { bool, flag, fromJson, int, page, str, toJson } from "./events.js";
 import { folded } from "./folded.js";
 import { ensurePerson } from "./identities.js";
+import { nearestMailPieces } from "./mail-nearest.js";
 import { mailRulesOver } from "./mail-rules.js";
 import { attachmentOf, saveAttachment, wordsQuery } from "./meetings.js";
 import { stemmerCache } from "./stems.js";
@@ -15,7 +16,6 @@ import { writeState } from "./sync.js";
 import { threadRows } from "./thread-columns.js";
 import { write } from "./transaction.js";
 import { dropUnusedVectors } from "./vector-index.js";
-import { dot } from "./vectors.js";
 /** A thread as a conversation: the summary `search conversations` and `related` answer for an email. */
 export const threadConversation = (place, builtAt) => ({
     id: `email:${place.emailId}`,
@@ -590,28 +590,10 @@ export const mailStoreOver = ({ database, orm, now }, stemmerFor = stemmerCache(
     },
     async nearest(model, query, { accountId, limit, threadExternalId, since }) {
         drainEmailIndex(database, stemmerFor);
-        const page = database.prepare(`SELECT p.id, p.type, p.owner, p.position, p.email_id AS email, v.vector FROM ${MAIL_PIECES}
-         JOIN embeddings v ON v.model = ? AND v.content_hash = p.content_hash
-        WHERE p.id > ? AND p.account_id = ?
-          AND (? IS NULL OR p.thread_id IN (SELECT id FROM email_threads WHERE account_id = p.account_id AND external_id = ?))
-          AND (? IS NULL OR p.occurred_at >= ?)
-        ORDER BY p.id LIMIT 500`);
-        const best = new Map();
-        for (let after = 0;;) {
-            const rows = page.all(model, after, accountId, threadExternalId ?? null, threadExternalId ?? null, since ?? null, since ?? null);
-            for (const row of rows) {
-                const score = dot(query, row.vector);
-                const key = row.type === "email_thread" ? `${row.owner}:${row.position}` : `email:${row.email}`;
-                if (score > (best.get(key) ?? Number.NEGATIVE_INFINITY))
-                    best.set(key, score);
-            }
-            if (rows.length < 500)
-                break;
-            after = Number(rows.at(-1)?.id);
-        }
+        const best = nearestMailPieces(database, model, query, { accountId, threadExternalId, since }, limit * 2);
         const threads = new Map();
         const emails = new Map();
-        for (const [key, score] of [...best.entries()].sort(([, a], [, b]) => b - a)) {
+        for (const [key, score] of best) {
             if (emails.size >= limit)
                 break;
             const [owner, position] = key.split(":");

package/dist/store/sqlite/mail-nearest.d.ts

@@ -0,0 +1,12 @@
+import type { CacheDatabase } from "../driver.js";
+interface MailPieceFilter {
+    accountId: number;
+    threadExternalId?: string | undefined;
+    since?: number | undefined;
+}
+/**
+ * Each mail piece of the account (a thread's piece or an email's attachment) by its best score against `query`, best
+ * first. Without a thread the index gives the best `depth` of each kind; a thread, or an index not ready, scans.
+ */
+export declare const nearestMailPieces: (database: CacheDatabase, model: string, query: Float32Array, filter: MailPieceFilter, depth: number) => (readonly [string, number])[];
+export {};

package/dist/store/sqlite/mail-nearest.js

@@ -0,0 +1,73 @@
+import { MAIL_PIECES } from "./email-index.js";
+import { searchIndex } from "./vector-index.js";
+import { dot } from "./vectors.js";
+const MAIL_PIECE_ROWS = `SELECT p.id, p.content_hash AS hash, p.type, p.owner, p.position, p.email_id AS email, v.vector
+    FROM ${MAIL_PIECES} JOIN embeddings v ON v.model = ? AND v.content_hash = p.content_hash
+   WHERE p.account_id = ?
+     AND (? IS NULL OR p.thread_id IN (SELECT id FROM email_threads WHERE account_id = p.account_id AND external_id = ?))
+     AND (? IS NULL OR p.occurred_at >= ?)`;
+const pieceKey = (row) => row.type === "email_thread" ? `${row.owner}:${row.position}` : `email:${row.email}`;
+const filterParams = (model, { accountId, threadExternalId, since }) => [
+    model,
+    accountId,
+    threadExternalId ?? null,
+    threadExternalId ?? null,
+    since ?? null,
+    since ?? null,
+];
+const byScore = (best) => [...best.entries()].map(([key, { score }]) => [key, score]).sort(([, a], [, b]) => b - a);
+/** Every mail piece of the account scored against `query`: each thread piece or attachment keeps its best score. */
+const scannedMailPieces = (database, model, query, filter) => {
+    const page = database.prepare(`${MAIL_PIECE_ROWS} AND p.id > ? ORDER BY p.id LIMIT 500`);
+    const best = new Map();
+    for (let after = 0;;) {
+        const rows = page.all(...filterParams(model, filter), after);
+        for (const row of rows) {
+            const score = dot(query, row.vector);
+            const key = pieceKey(row);
+            if (score > (best.get(key)?.score ?? Number.NEGATIVE_INFINITY))
+                best.set(key, { score });
+        }
+        if (rows.length < 500)
+            break;
+        after = Number(rows.at(-1)?.id);
+    }
+    return byScore(best);
+};
+/**
+ * The best `depth` pieces of each kind by the model's vector index, or `undefined` when the index cannot answer and
+ * the caller scans. Reading every mail vector of a large mailbox took most of a `search all`.
+ */
+const indexedMailPieces = (database, model, query, filter, depth) => {
+    const candidates = database.prepare(`${MAIL_PIECE_ROWS} AND p.id IN (SELECT value FROM json_each(?)) ORDER BY p.id`);
+    const merged = new Map();
+    for (const kind of ["email_thread", "attachment"]) {
+        const found = searchIndex(database, model, { kind, account: filter.accountId, ...(filter.since === undefined ? {} : { since: filter.since }) }, query, depth, (near) => {
+            const hashes = new Map(near.map(({ id, hash }) => [id, hash]));
+            const best = new Map();
+            for (const row of candidates.all(...filterParams(model, filter), JSON.stringify([...hashes.keys()]))) {
+                if (hashes.get(Number(row.id)) !== row.hash)
+                    continue;
+                const score = dot(query, row.vector);
+                const key = pieceKey(row);
+                if (score > (best.get(key)?.score ?? Number.NEGATIVE_INFINITY))
+                    best.set(key, { score, chunk: Number(row.id) });
+            }
+            const results = byScore(best)
+                .slice(0, depth)
+                .map(([key]) => [key, best.get(key)]);
+            return { results, best: results.map(([, { chunk }]) => chunk) };
+        });
+        if (!found)
+            return undefined;
+        for (const [key, held] of found)
+            merged.set(key, held);
+    }
+    return byScore(merged);
+};
+/**
+ * Each mail piece of the account (a thread's piece or an email's attachment) by its best score against `query`, best
+ * first. Without a thread the index gives the best `depth` of each kind; a thread, or an index not ready, scans.
+ */
+export const nearestMailPieces = (database, model, query, filter, depth) => (filter.threadExternalId === undefined ? indexedMailPieces(database, model, query, filter, depth) : undefined) ??
+    scannedMailPieces(database, model, query, filter);

package/dist/store/sqlite/manifest.js

@@ -70,6 +70,7 @@ export const MANIFEST = [
     { name: "20261105144549_version-51-task-links", version: 51, minCompatible: 34 },
     { name: "20261106010504_version-52-deals", version: 52, minCompatible: 34 },
     { name: "20261106010507_version-52-deals-rules", version: 52, minCompatible: 34 },
+    { name: "20261106165009_version-53-crm-pipelines", version: 53, minCompatible: 34 },
 ];
 export const generatedMigrations = (manifest = MANIFEST, generated = GENERATED) => {
     const migrations = [];

package/dist/store/sqlite/migrations.generated.js

@@ -1033,5 +1033,14 @@ export const GENERATED = [
             "-- The default lead pipeline, so a lead can be recorded before the owner or an agent makes one.\nINSERT INTO pipelines (kind, name, name_folded, created_at, updated_at)\n  VALUES ('lead', 'Leads', 'leads', CAST(strftime('%s', 'now') AS INTEGER) * 1000, CAST(strftime('%s', 'now') AS INTEGER) * 1000);",
             "INSERT INTO pipeline_stages (pipeline_id, name, name_folded, position, outcome, created_at, updated_at)\n  SELECT p.id, v.name, v.name, v.position, v.outcome, p.created_at, p.created_at FROM pipelines p,\n    (SELECT 'new' AS name, 1 AS position, 'open' AS outcome UNION ALL SELECT 'contacted', 2, 'open'\n      UNION ALL SELECT 'qualified', 3, 'open' UNION ALL SELECT 'won', 4, 'won' UNION ALL SELECT 'lost', 5, 'lost') v\n  WHERE p.kind = 'lead' AND p.name_folded = 'leads' AND p.account_id IS NULL;"
         ]
+    },
+    {
+        "name": "20261106165009_version-53-crm-pipelines",
+        "statements": [
+            "ALTER TABLE `pipeline_stages` ADD `deleted_at` integer;",
+            "ALTER TABLE `pipelines` ADD `deleted_at` integer;",
+            "DROP INDEX IF EXISTS `pipelines_by_account_id_external_id`;",
+            "CREATE UNIQUE INDEX `pipelines_by_account_id_kind_external_id` ON `pipelines` (`account_id`,`kind`,`external_id`) WHERE account_id IS NOT NULL AND external_id IS NOT NULL;"
+        ]
     }
 ];

package/dist/store/sqlite/schema.d.ts

@@ -15527,6 +15527,22 @@ export declare const pipelines: import("drizzle-orm/sqlite-core").SQLiteTableWit
             identity: undefined;
             generated: undefined;
         }, {}>;
+        deletedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "pipelines";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
     };
     dialect: 'sqlite';
     isAlias: false;
@@ -15679,6 +15695,22 @@ export declare const pipelineStages: import("drizzle-orm/sqlite-core").SQLiteTab
             identity: undefined;
             generated: undefined;
         }, {}>;
+        deletedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "pipeline_stages";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
     };
     dialect: 'sqlite';
     isAlias: false;

package/dist/store/sqlite/schema.js

@@ -1545,9 +1545,10 @@ export const pipelines = sqliteTable("pipelines", {
     nameFolded: text("name_folded").notNull(),
     createdAt: integer("created_at").notNull(),
     updatedAt: integer("updated_at").notNull(),
+    deletedAt: integer("deleted_at"),
 }, (table) => [
-    uniqueIndex("pipelines_by_account_id_external_id")
-        .on(table.accountId, table.externalId)
+    uniqueIndex("pipelines_by_account_id_kind_external_id")
+        .on(table.accountId, table.kind, table.externalId)
         .where(sql `account_id IS NOT NULL AND external_id IS NOT NULL`),
     uniqueIndex("pipelines_by_name_folded_kind").on(table.nameFolded, table.kind).where(sql `account_id IS NULL`),
 ]);
@@ -1563,6 +1564,7 @@ export const pipelineStages = sqliteTable("pipeline_stages", {
     externalId: text("external_id"),
     createdAt: integer("created_at").notNull(),
     updatedAt: integer("updated_at").notNull(),
+    deletedAt: integer("deleted_at"),
 }, (table) => [
     unique().on(table.pipelineId, table.nameFolded),
     uniqueIndex("pipeline_stages_by_pipeline_id_external_id")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.592.0",
+  "version": "0.594.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",
@@ -270,7 +270,7 @@
     "@types/node": "^22.20.5",
     "@types/pngjs": "6.0.5",
     "@vitest/coverage-v8": "^5.0.5",
-    "@wirecat/cli-base": "0.80.0",
+    "@wirecat/cli-base": "0.80.2",
     "@wirecat/cli-core-legacy-tags": "npm:@wirecat/cli-core@0.414.0",
     "commander": "15.0.0",
     "cspell": "10.4.0",