@remit/web-client 0.0.85 → 0.0.87

package/src/lib/search-tokens.ts

@@ -4,11 +4,34 @@
  * into removable chips. Only tokens the search APIs can actually honor are
  * recognized — everything else is left untouched as ordinary free-text words.
  *
+ * The vocabulary is the search parameters `threadOperationsSearchThreads` and
+ * `semanticSearchOperationsSemanticSearch` take, said in words:
+ *
+ * | token                  | thread search      | semantic search        |
+ * | ---------------------- | ------------------ | ---------------------- |
+ * | `from:`                | `from`             | —                      |
+ * | `subject:`             | `subject`          | —                      |
+ * | `category:`            | `category[]`       | `category`             |
+ * | `is:unread` / `is:read`| `unread`           | `isRead`               |
+ * | `is:starred`           | `starred`          | `hasStars`             |
+ * | `has:attachment`       | `attachments`      | `hasAttachment`        |
+ * | `before:` / `after:`   | —                  | `sentDateTo` / `From`  |
+ * | `in:`                  | route scope        | `mailboxId`            |
+ * | `account:`             | —                  | —                      |
+ *
+ * A token an engine has no parameter for is applied client-side over the rows
+ * that engine returned (`matchesSearchTokens` in `lib/brief.ts`), never dropped.
+ *
  * `in:<mailbox>` and `account:<name>` resolve against a name index the caller
  * supplies via `SearchTokenContext` (built from the loaded mailboxes/accounts —
  * see `lib/search-token-index.ts`). An unresolvable name (no index, or no
  * match in it) is left as free text rather than becoming a chip for a filter
- * that can't actually apply.
+ * that can't actually apply. `category:` resolves the same way against a fixed
+ * vocabulary, so `category:nonsense` stays free text too.
+ *
+ * Token names are case-insensitive. A value containing whitespace is written in
+ * double quotes — `in:"Sent Items"` — and an unterminated quote runs to the end
+ * of the input, so a value stays readable while it is still being typed.
  *
  * Pure functions only: no React, no fetch — the name index is injected, not
  * fetched here. `MailListHeader` renders the chips and the callers that issue
@@ -17,10 +40,22 @@
  * actually supports.
  */
 
+import type { RemitImapMessageCategory } from "@remit/api-http-client/types.gen.ts";
+import { MessageCategory } from "@remit/domain-enums";
+
 export type SearchToken =
 	| { type: "from"; raw: string; value: string }
+	| { type: "subject"; raw: string; value: string }
+	| {
+			type: "category";
+			raw: string;
+			value: string;
+			category: RemitImapMessageCategory;
+	  }
 	| { type: "hasAttachment"; raw: string }
 	| { type: "isUnread"; raw: string }
+	| { type: "isRead"; raw: string }
+	| { type: "isStarred"; raw: string }
 	| { type: "before"; raw: string; value: string; epochSeconds: number }
 	| { type: "after"; raw: string; value: string; epochSeconds: number }
 	| { type: "in"; raw: string; value: string; mailboxId: string }
@@ -40,8 +75,200 @@ export interface SearchTokenContext {
 	accountsByName?: ReadonlyMap<string, string>;
 }
 
+/** The word before the colon. `is`/`has` carry their meaning in the value. */
+export type SearchTokenName =
+	| "from"
+	| "subject"
+	| "category"
+	| "is"
+	| "has"
+	| "before"
+	| "after"
+	| "in"
+	| "account";
+
+/** One option of a token's fixed value vocabulary. */
+export interface SearchTokenValueOption {
+	value: string;
+	/** What the value reads as in prose. Defaults to the value. */
+	label?: string;
+}
+
+/**
+ * Where a token's values come from. `fixed` is the vocabulary carried here;
+ * the rest are looked up by the caller (mailboxes, accounts, the address
+ * search) or typed freely.
+ */
+export type SearchTokenValueSource =
+	| "fixed"
+	| "contact"
+	| "mailbox"
+	| "account"
+	| "date"
+	| "text";
+
+/** A token name, what it narrows by, and where its values come from. */
+export interface SearchTokenSpec {
+	name: SearchTokenName;
+	/** What the token narrows by, in the user's words. */
+	label: string;
+	values: SearchTokenValueSource;
+	/** The vocabulary, when `values` is `fixed`. */
+	options?: readonly SearchTokenValueOption[];
+}
+
+/**
+ * Category labels, keyed by the API enum. `uncategorized` reads "Unclassified"
+ * — the same word the filter chips use (issue #45): mail the classifier has not
+ * reached is a pending state with a name, never the absence of one.
+ */
+const CATEGORY_LABELS: Record<RemitImapMessageCategory, string> = {
+	personal: "Personal",
+	uncategorized: "Unclassified",
+	transactional: "Transactional",
+	newsletter: "Newsletter",
+	marketing: "Marketing",
+	social: "Social",
+	automated: "Automated",
+};
+
+const CATEGORY_OPTIONS: readonly SearchTokenValueOption[] = Object.values(
+	MessageCategory,
+).map((value) => ({ value, label: CATEGORY_LABELS[value] }));
+
+/**
+ * Spellings that resolve to a category beyond the enum values themselves — the
+ * label the chips show, so what the user reads is also what the user can type.
+ */
+const CATEGORY_ALIASES: Record<string, RemitImapMessageCategory> = {
+	unclassified: "uncategorized",
+};
+
+/** The whole vocabulary, in the order a suggestion list offers it. */
+export const SEARCH_TOKEN_SPECS: readonly SearchTokenSpec[] = [
+	{ name: "from", label: "Sender", values: "contact" },
+	{ name: "subject", label: "Subject", values: "text" },
+	{
+		name: "category",
+		label: "Category",
+		values: "fixed",
+		options: CATEGORY_OPTIONS,
+	},
+	{
+		name: "is",
+		label: "State",
+		values: "fixed",
+		options: [
+			{ value: "unread", label: "Unread" },
+			{ value: "read", label: "Read" },
+			{ value: "starred", label: "Starred" },
+		],
+	},
+	{
+		name: "has",
+		label: "Contents",
+		values: "fixed",
+		options: [{ value: "attachment", label: "Has attachment" }],
+	},
+	{ name: "in", label: "Folder", values: "mailbox" },
+	{ name: "account", label: "Account", values: "account" },
+	{ name: "after", label: "Sent on or after", values: "date" },
+	{ name: "before", label: "Sent before", values: "date" },
+];
+
+const SPECS_BY_NAME = new Map<string, SearchTokenSpec>(
+	SEARCH_TOKEN_SPECS.map((spec) => [spec.name, spec]),
+);
+
+/** The spec for a token name, or `undefined` when the name is not one of ours. */
+export const searchTokenSpec = (name: string): SearchTokenSpec | undefined =>
+	SPECS_BY_NAME.get(name.toLowerCase());
+
 const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
 
+const needsQuotes = (value: string): boolean => /[\s"]/.test(value);
+
+/**
+ * A value as it is written in a query: quoted when it carries whitespace, bare
+ * otherwise. The inverse of the unquoting the parser does, so a suggestion the
+ * user picks parses back to the value it was built from.
+ */
+export const quoteSearchTokenValue = (value: string): string =>
+	needsQuotes(value) ? `"${value.replace(/"/g, "")}"` : value;
+
+/** `name:value`, quoted as needed — the text a query carries for one token. */
+export const searchTokenTerm = (name: string, value: string): string =>
+	`${name}:${quoteSearchTokenValue(value)}`;
+
+const unquote = (value: string): string => {
+	if (!value.startsWith('"')) return value;
+	const inner = value.slice(1);
+	return inner.endsWith('"') ? inner.slice(0, -1) : inner;
+};
+
+/** One whitespace-separated term of a query, with where it sits in the input. */
+export interface SearchQueryWord {
+	/** The term exactly as typed, quotes included. */
+	raw: string;
+	/** Index of the term's first character in the query. */
+	start: number;
+	/** Index just past the term's last character. */
+	end: number;
+}
+
+/**
+ * Split a query into terms on whitespace, except inside double quotes, so
+ * `in:"Sent Items"` is one term. An unterminated quote runs to the end of the
+ * input — a half-typed value is still one term, not a broken one.
+ */
+export function splitSearchWords(query: string): SearchQueryWord[] {
+	const words: SearchQueryWord[] = [];
+	let start = -1;
+	let quoted = false;
+	for (let i = 0; i < query.length; i++) {
+		const char = query[i] as string;
+		if (char === '"') {
+			quoted = !quoted;
+			if (start < 0) start = i;
+			continue;
+		}
+		if (!quoted && /\s/.test(char)) {
+			if (start >= 0) words.push({ raw: query.slice(start, i), start, end: i });
+			start = -1;
+			continue;
+		}
+		if (start < 0) start = i;
+	}
+	if (start >= 0) {
+		words.push({ raw: query.slice(start), start, end: query.length });
+	}
+	return words;
+}
+
+/** A term split at its first colon: the token name and the value as typed. */
+export interface SearchTermParts {
+	name: string;
+	/** The value with its quotes removed. */
+	value: string;
+	/** The value exactly as typed, quotes included. */
+	rawValue: string;
+}
+
+/**
+ * Split `name:value` at the first colon. A term with no colon, or one starting
+ * with a colon, is not a token attempt and returns `undefined`.
+ */
+export function splitSearchTerm(word: string): SearchTermParts | undefined {
+	const colon = word.indexOf(":");
+	if (colon <= 0) return undefined;
+	const rawValue = word.slice(colon + 1);
+	return {
+		name: word.slice(0, colon).toLowerCase(),
+		value: unquote(rawValue),
+		rawValue,
+	};
+}
+
 function parseDateToken(
 	type: "before" | "after",
 	raw: string,
@@ -53,39 +280,72 @@ function parseDateToken(
 	return { type, raw, value, epochSeconds: Math.floor(epochMs / 1000) };
 }
 
+/** The category a `category:` value names, by enum value or by chip label. */
+export function resolveCategory(
+	value: string,
+): RemitImapMessageCategory | undefined {
+	const lower = value.toLowerCase();
+	const alias = CATEGORY_ALIASES[lower];
+	if (alias) return alias;
+	const known = Object.values(MessageCategory).find((c) => c === lower);
+	return known;
+}
+
+function parseStateToken(raw: string, value: string): SearchToken | undefined {
+	switch (value.toLowerCase()) {
+		case "unread":
+			return { type: "isUnread", raw };
+		case "read":
+			return { type: "isRead", raw };
+		// `flagged` is the wire name for IMAP \Flagged; "starred" is the word the
+		// chips use. Both are typed, so both parse.
+		case "starred":
+		case "flagged":
+			return { type: "isStarred", raw };
+		default:
+			return undefined;
+	}
+}
+
 function parseWord(
 	word: string,
 	context: SearchTokenContext,
 ): SearchToken | undefined {
-	const lower = word.toLowerCase();
-	if (lower.startsWith("from:") && word.length > 5) {
-		return { type: "from", raw: word, value: word.slice(5) };
-	}
-	if (lower === "has:attachment") {
-		return { type: "hasAttachment", raw: word };
-	}
-	if (lower === "is:unread") {
-		return { type: "isUnread", raw: word };
-	}
-	if (lower.startsWith("before:") && word.length > 7) {
-		return parseDateToken("before", word, word.slice(7));
-	}
-	if (lower.startsWith("after:") && word.length > 6) {
-		return parseDateToken("after", word, word.slice(6));
-	}
-	if (lower.startsWith("in:") && word.length > 3) {
-		const value = word.slice(3);
-		const mailboxId = context.mailboxesByName?.get(value.toLowerCase());
-		if (!mailboxId) return undefined;
-		return { type: "in", raw: word, value, mailboxId };
-	}
-	if (lower.startsWith("account:") && word.length > 8) {
-		const value = word.slice(8);
-		const accountId = context.accountsByName?.get(value.toLowerCase());
-		if (!accountId) return undefined;
-		return { type: "account", raw: word, value, accountId };
+	const parts = splitSearchTerm(word);
+	if (!parts || parts.value.length === 0) return undefined;
+	const { name, value } = parts;
+	switch (name) {
+		case "from":
+			return { type: "from", raw: word, value };
+		case "subject":
+			return { type: "subject", raw: word, value };
+		case "category": {
+			const category = resolveCategory(value);
+			if (!category) return undefined;
+			return { type: "category", raw: word, value, category };
+		}
+		case "is":
+			return parseStateToken(word, value);
+		case "has":
+			return value.toLowerCase() === "attachment"
+				? { type: "hasAttachment", raw: word }
+				: undefined;
+		case "before":
+		case "after":
+			return parseDateToken(name, word, value);
+		case "in": {
+			const mailboxId = context.mailboxesByName?.get(value.toLowerCase());
+			if (!mailboxId) return undefined;
+			return { type: "in", raw: word, value, mailboxId };
+		}
+		case "account": {
+			const accountId = context.accountsByName?.get(value.toLowerCase());
+			if (!accountId) return undefined;
+			return { type: "account", raw: word, value, accountId };
+		}
+		default:
+			return undefined;
 	}
-	return undefined;
 }
 
 /** Split a query into recognized filter tokens and the remaining free text. */
@@ -93,22 +353,21 @@ export function parseSearchTokens(
 	query: string,
 	context: SearchTokenContext = {},
 ): ParsedSearchQuery {
-	const words = query.split(/\s+/).filter((w) => w.length > 0);
 	const tokens: SearchToken[] = [];
 	const freeWords: string[] = [];
-	for (const word of words) {
-		const token = parseWord(word, context);
+	for (const word of splitSearchWords(query)) {
+		const token = parseWord(word.raw, context);
 		if (token) tokens.push(token);
-		else freeWords.push(word);
+		else freeWords.push(word.raw);
 	}
 	return { freeText: freeWords.join(" "), tokens };
 }
 
 /** Remove one parsed token's raw text from the original query string. */
 export function removeSearchToken(query: string, token: SearchToken): string {
-	return query
-		.split(/\s+/)
-		.filter((w) => w.length > 0 && w !== token.raw)
+	return splitSearchWords(query)
+		.filter((word) => word.raw !== token.raw)
+		.map((word) => word.raw)
 		.join(" ");
 }
 
@@ -117,10 +376,18 @@ export function searchTokenLabel(token: SearchToken): string {
 	switch (token.type) {
 		case "from":
 			return `From: ${token.value}`;
+		case "subject":
+			return `Subject: ${token.value}`;
+		case "category":
+			return `Category: ${CATEGORY_LABELS[token.category]}`;
 		case "hasAttachment":
 			return "Has attachment";
 		case "isUnread":
 			return "Unread";
+		case "isRead":
+			return "Read";
+		case "isStarred":
+			return "Starred";
 		case "before":
 			return `Before ${token.value}`;
 		case "after":

package/src/lib/thread-cache.ts

@@ -37,7 +37,7 @@ export const isInfiniteThreadData = (
 	data !== null &&
 	Array.isArray((data as { pages?: unknown }).pages);
 
-const isThreadItemsPage = (data: unknown): data is ThreadItemsPage =>
+export const isThreadItemsPage = (data: unknown): data is ThreadItemsPage =>
 	typeof data === "object" &&
 	data !== null &&
 	Array.isArray((data as { items?: unknown }).items);

package/src/lib/thread-search-tokens.test.ts

@@ -0,0 +1,193 @@
+import assert from "node:assert/strict";
+import { describe, test } from "node:test";
+import type { RemitImapThreadMessageResponse } from "@remit/api-http-client/types.gen.ts";
+import type { InboxFilterParams } from "./inbox-filters.js";
+import { parseSearchTokens, type SearchToken } from "./search-tokens.js";
+import {
+	applyResidualTokens,
+	threadSearchTokens,
+} from "./thread-search-tokens.js";
+
+const tokensOf = (query: string): SearchToken[] =>
+	parseSearchTokens(query, {
+		accountsByName: new Map([["alice", "acc_1"]]),
+		mailboxesByName: new Map([["sent", "mb2"]]),
+	}).tokens;
+
+const split = (query: string, chips: InboxFilterParams = {}) =>
+	threadSearchTokens(tokensOf(query), chips);
+
+const residualTypes = (query: string, chips: InboxFilterParams = {}) =>
+	split(query, chips).residual.map((token) => token.type);
+
+const thread = (
+	overrides: Partial<RemitImapThreadMessageResponse> = {},
+): RemitImapThreadMessageResponse => ({
+	threadId: "t1",
+	threadMessageId: "tm1",
+	messageId: "m1",
+	accountConfigId: "cfg_1",
+	accountId: "acc_1",
+	mailboxId: "mb1",
+	fromName: "Sender",
+	fromEmail: "sender@example.com",
+	subject: "Subject",
+	snippet: "Snippet",
+	category: "uncategorized",
+	sentDate: Date.parse("2026-03-01T00:00:00Z"),
+	isRead: false,
+	isDeleted: false,
+	hasAttachment: false,
+	hasStars: false,
+	star: "none",
+	senderTrust: "unknown",
+	muted: false,
+	createdAt: 0,
+	updatedAt: 0,
+	...overrides,
+});
+
+describe("threadSearchTokens", () => {
+	test("every token the endpoint has a parameter for reaches the request", () => {
+		const { params, residual } = split(
+			'from:dhl subject:"invoice due" category:newsletter is:starred has:attachment',
+		);
+		assert.deepEqual(params, {
+			from: "dhl",
+			subject: "invoice due",
+			category: ["newsletter"],
+			starred: true,
+			attachments: true,
+		});
+		assert.deepEqual(residual, []);
+	});
+
+	test("is:unread and is:read are the two sides of `unread`", () => {
+		assert.equal(split("is:unread").params.unread, true);
+		assert.equal(split("is:read").params.unread, false);
+	});
+
+	test("is:flagged is the same filter as is:starred", () => {
+		assert.equal(split("is:flagged").params.starred, true);
+	});
+
+	test("a category label resolves to the enum the request carries", () => {
+		assert.deepEqual(split("category:unclassified").params.category, [
+			"uncategorized",
+		]);
+	});
+
+	test("no tokens set no parameters", () => {
+		assert.deepEqual(split("just words").params, {});
+		assert.deepEqual(split("just words").residual, []);
+	});
+});
+
+describe("threadSearchTokens residue", () => {
+	test("dates and the account have no parameter on this endpoint", () => {
+		assert.deepEqual(residualTypes("after:2026-01-01 before:2026-02-01"), [
+			"after",
+			"before",
+		]);
+		assert.deepEqual(residualTypes("account:alice"), ["account"]);
+	});
+
+	test("a mailbox term is residue where it resolves at all", () => {
+		assert.deepEqual(residualTypes("in:sent"), ["in"]);
+	});
+
+	test("a second value for a single-valued parameter is residue", () => {
+		const { params, residual } = split("from:dhl from:ups");
+		assert.equal(params.from, "dhl");
+		assert.deepEqual(
+			residual.map((token) => token.raw),
+			["from:ups"],
+		);
+		assert.deepEqual(split("subject:one subject:two").params.subject, "one");
+		assert.deepEqual(residualTypes("subject:one subject:two"), ["subject"]);
+		assert.deepEqual(residualTypes("category:social category:personal"), [
+			"category",
+		]);
+	});
+
+	test("repeating a token asks nothing new, so nothing is left over", () => {
+		assert.deepEqual(residualTypes("from:dhl from:DHL"), []);
+		assert.deepEqual(residualTypes("is:starred is:starred"), []);
+	});
+
+	test("contradicting state tokens cannot both be sent", () => {
+		const { params, residual } = split("is:unread is:read");
+		assert.equal(params.unread, true);
+		assert.deepEqual(
+			residual.map((token) => token.type),
+			["isRead"],
+		);
+	});
+});
+
+describe("threadSearchTokens against the filter chips", () => {
+	test("a chip that already asks for it leaves nothing over", () => {
+		assert.deepEqual(residualTypes("is:starred", { starred: true }), []);
+		assert.deepEqual(residualTypes("is:unread", { unread: true }), []);
+		assert.deepEqual(
+			residualTypes("category:personal", { category: ["personal"] }),
+			[],
+		);
+	});
+
+	test("the visible chip wins the parameter and the token becomes residue", () => {
+		assert.deepEqual(residualTypes("is:read", { unread: true }), ["isRead"]);
+		assert.deepEqual(
+			residualTypes("category:social", { category: ["personal"] }),
+			["category"],
+		);
+	});
+});
+
+describe("applyResidualTokens", () => {
+	test("no residue leaves the rows untouched", () => {
+		const rows = [thread()];
+		assert.equal(applyResidualTokens(rows, []), rows);
+	});
+
+	test("a date the request could not carry is applied over the rows", () => {
+		const rows = [
+			thread({
+				messageId: "old",
+				sentDate: Date.parse("2026-01-01T00:00:00Z"),
+			}),
+			thread({
+				messageId: "new",
+				sentDate: Date.parse("2026-04-01T00:00:00Z"),
+			}),
+		];
+		const { residual } = split("after:2026-02-01");
+		assert.deepEqual(
+			applyResidualTokens(rows, residual).map((row) => row.messageId),
+			["new"],
+		);
+	});
+
+	test("a contradiction the request could not express matches nothing", () => {
+		const rows = [thread({ isRead: false })];
+		const { residual } = split("is:unread is:read");
+		assert.deepEqual(applyResidualTokens(rows, residual), []);
+	});
+
+	test("an account other than the row's drops the row", () => {
+		const { residual } = split("account:alice");
+		assert.equal(applyResidualTokens([thread()], residual).length, 1);
+		assert.equal(
+			applyResidualTokens([thread({ accountId: "acc_2" })], residual).length,
+			0,
+		);
+	});
+
+	test("a per-mailbox row takes the account of the view it is listed in", () => {
+		const rows = [thread({ accountId: undefined })];
+		const { residual } = split("account:alice");
+		assert.equal(applyResidualTokens(rows, residual, "acc_1").length, 1);
+		assert.equal(applyResidualTokens(rows, residual, "acc_2").length, 0);
+		assert.equal(applyResidualTokens(rows, residual).length, 0);
+	});
+});