@remit/drizzle-service 0.0.31 → 0.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import type { ThreadMessageItem } from "@remit/data-ports";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The measured category shape of the owner's INBOX, shared by the sqlite and
|
|
5
|
+
* Postgres category suites (#304).
|
|
6
|
+
*
|
|
7
|
+
* It lives in one module because the shape is the load-bearing part of the
|
|
8
|
+
* regression: a filter resolved over the page the server returned is empty at
|
|
9
|
+
* every page size precisely because the rare categories sit outside the newest
|
|
10
|
+
* page. Two copies of those numbers can drift apart, and the one that drifts
|
|
11
|
+
* stops being the argument.
|
|
12
|
+
*
|
|
13
|
+
* Measured on the live instance (docs/architecture/mail-list-boundary.md).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export type Category =
|
|
17
|
+
| "personal"
|
|
18
|
+
| "marketing"
|
|
19
|
+
| "automated"
|
|
20
|
+
| "newsletter"
|
|
21
|
+
| "transactional"
|
|
22
|
+
| "social";
|
|
23
|
+
|
|
24
|
+
export type CategoryTotals = Record<Category, number>;
|
|
25
|
+
|
|
26
|
+
/** INBOX as measured: 14,187 non-deleted rows. */
|
|
27
|
+
export const LIVE_TOTALS: CategoryTotals = {
|
|
28
|
+
personal: 4753,
|
|
29
|
+
marketing: 3942,
|
|
30
|
+
automated: 2680,
|
|
31
|
+
newsletter: 2295,
|
|
32
|
+
transactional: 429,
|
|
33
|
+
social: 88,
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The same shape with the four common categories trimmed, for the Postgres twin.
|
|
38
|
+
*
|
|
39
|
+
* The rare tail is untouched — `social` stays at its measured 88 — so every
|
|
40
|
+
* assertion the suites make about a rare category holds identically. What
|
|
41
|
+
* shrinks is the bulk that only exists to make the common categories common,
|
|
42
|
+
* which the Postgres suite does not assert on and which costs ~20s of seeding.
|
|
43
|
+
*/
|
|
44
|
+
export const TRIMMED_TOTALS: CategoryTotals = {
|
|
45
|
+
personal: 700,
|
|
46
|
+
marketing: 500,
|
|
47
|
+
automated: 400,
|
|
48
|
+
newsletter: 200,
|
|
49
|
+
transactional: 130,
|
|
50
|
+
social: 88,
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The newest 100 rows of that INBOX, as measured. Two of the 4,753 personal
|
|
55
|
+
* messages and two of the 88 social ones are in it, which is why selecting
|
|
56
|
+
* either chip over the newest page returns almost nothing.
|
|
57
|
+
*/
|
|
58
|
+
export const NEWEST_100: CategoryTotals = {
|
|
59
|
+
automated: 77,
|
|
60
|
+
newsletter: 14,
|
|
61
|
+
marketing: 3,
|
|
62
|
+
social: 2,
|
|
63
|
+
transactional: 2,
|
|
64
|
+
personal: 2,
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
export const BASE_DATE = 1_767_225_600_000;
|
|
68
|
+
|
|
69
|
+
export const totalRows = (totals: CategoryTotals): number =>
|
|
70
|
+
Object.values(totals).reduce((sum, count) => sum + count, 0);
|
|
71
|
+
|
|
72
|
+
// mulberry32: a real 32-bit generator, so the stream is the one the shuffle
|
|
73
|
+
// claims. The interleaving is part of the fixture — a failure has to be
|
|
74
|
+
// reproducible — so this is seeded rather than Math.random.
|
|
75
|
+
const mulberry32 = (seed: number): (() => number) => {
|
|
76
|
+
let state = seed >>> 0;
|
|
77
|
+
return () => {
|
|
78
|
+
state = (state + 0x6d2b79f5) >>> 0;
|
|
79
|
+
let t = state;
|
|
80
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
81
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
82
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
83
|
+
};
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
const shuffled = (values: Category[], seed: number): Category[] => {
|
|
87
|
+
const random = mulberry32(seed);
|
|
88
|
+
const out = [...values];
|
|
89
|
+
for (let i = out.length - 1; i > 0; i--) {
|
|
90
|
+
const j = Math.floor(random() * (i + 1));
|
|
91
|
+
[out[i], out[j]] = [out[j], out[i]];
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
const expand = (counts: Partial<CategoryTotals>): Category[] => {
|
|
97
|
+
const out: Category[] = [];
|
|
98
|
+
for (const [category, count] of Object.entries(counts) as Array<
|
|
99
|
+
[Category, number]
|
|
100
|
+
>) {
|
|
101
|
+
for (let i = 0; i < count; i++) out.push(category);
|
|
102
|
+
}
|
|
103
|
+
return out;
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
type FixtureRow = Omit<ThreadMessageItem, "star"> & { star: "none" };
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* One mailbox of thread-message rows in the given category distribution,
|
|
110
|
+
* newest first, with the measured mix in the newest 100.
|
|
111
|
+
*
|
|
112
|
+
* Every seventh row repeats its predecessor's `sentDate`, so a keyset walk over
|
|
113
|
+
* the fixture crosses tie groups instead of a strictly distinct sequence.
|
|
114
|
+
*/
|
|
115
|
+
export const categoryFixtureRows = (options: {
|
|
116
|
+
totals: CategoryTotals;
|
|
117
|
+
accountConfigId: string;
|
|
118
|
+
mailboxId: string;
|
|
119
|
+
}): FixtureRow[] => {
|
|
120
|
+
const { totals, accountConfigId, mailboxId } = options;
|
|
121
|
+
const tail = Object.fromEntries(
|
|
122
|
+
(Object.keys(totals) as Category[]).map((category) => [
|
|
123
|
+
category,
|
|
124
|
+
totals[category] - NEWEST_100[category],
|
|
125
|
+
]),
|
|
126
|
+
) as CategoryTotals;
|
|
127
|
+
|
|
128
|
+
const byRecency = [
|
|
129
|
+
...shuffled(expand(NEWEST_100), 11),
|
|
130
|
+
...shuffled(expand(tail), 29),
|
|
131
|
+
];
|
|
132
|
+
|
|
133
|
+
return byRecency.map((category, index) => {
|
|
134
|
+
const step = index - (index % 7 === 0 && index > 0 ? 1 : 0);
|
|
135
|
+
const sentDate = BASE_DATE - step * 1000;
|
|
136
|
+
return {
|
|
137
|
+
threadMessageId: `tm-category-${index}`,
|
|
138
|
+
accountConfigId,
|
|
139
|
+
threadId: `t-category-${index}`,
|
|
140
|
+
messageId: `m-category-${index}`,
|
|
141
|
+
mailboxId,
|
|
142
|
+
uid: index + 1,
|
|
143
|
+
referenceOrder: 0,
|
|
144
|
+
internalDate: sentDate,
|
|
145
|
+
sentDate,
|
|
146
|
+
isRead: false,
|
|
147
|
+
hasAttachment: false,
|
|
148
|
+
star: "none",
|
|
149
|
+
hasStars: false,
|
|
150
|
+
isDeleted: false,
|
|
151
|
+
category,
|
|
152
|
+
createdAt: 0,
|
|
153
|
+
updatedAt: 0,
|
|
154
|
+
};
|
|
155
|
+
});
|
|
156
|
+
};
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { after, before, describe, test } from "node:test";
|
|
3
|
+
import { drizzle } from "drizzle-orm/node-postgres";
|
|
4
|
+
import type pg from "pg";
|
|
5
|
+
import type { Db } from "../db.js";
|
|
6
|
+
import { threadMessageTable } from "../schema/thread-message.js";
|
|
7
|
+
import * as schema from "../schema.js";
|
|
8
|
+
import { createTestDb } from "../test-db.js";
|
|
9
|
+
import { categoryFixtureRows, TRIMMED_TOTALS } from "./category-fixture.js";
|
|
10
|
+
import { DrizzleThreadMessageRepository } from "./thread-message.js";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The Postgres half of the category-predicate guard (#304). The behavioural
|
|
14
|
+
* coverage is in thread-message.sqlite.test.ts against the full measured shape;
|
|
15
|
+
* what this file exists for is the query plan, which is dialect-specific:
|
|
16
|
+
* `EXPLAIN QUERY PLAN` is SQLite syntax and Postgres needs its own `EXPLAIN`.
|
|
17
|
+
*
|
|
18
|
+
* It runs against the embedded Postgres harness, which pushes the generated
|
|
19
|
+
* drizzle schema, so `tm_by_mailbox_category_date` is present here for the same
|
|
20
|
+
* reason it is present in the sqlite harness: the index is declared in TypeSpec
|
|
21
|
+
* and emitted into the schema both dialects derive from. That is what makes the
|
|
22
|
+
* assertion a real guard rather than a restatement of the predicate — remove the
|
|
23
|
+
* index from the schema and it fails.
|
|
24
|
+
*
|
|
25
|
+
* The fixture is the measured shape with its common categories trimmed
|
|
26
|
+
* (TRIMMED_TOTALS): the rare tail the assertions use is untouched, and the
|
|
27
|
+
* planner needs only enough rows to prefer an index over a sequential scan. It
|
|
28
|
+
* also needs statistics before it will, so the fixture is ANALYZEd; a production
|
|
29
|
+
* instance gets that from autovacuum.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
const ACCOUNT = "acct-category-pg";
|
|
33
|
+
const MAILBOX = "mbx-category-pg";
|
|
34
|
+
|
|
35
|
+
describe("thread-message category predicate (postgres)", () => {
|
|
36
|
+
let pool: pg.Pool;
|
|
37
|
+
let close: () => Promise<void>;
|
|
38
|
+
let repo: DrizzleThreadMessageRepository;
|
|
39
|
+
const logged: Array<{ text: string; params: unknown[] }> = [];
|
|
40
|
+
|
|
41
|
+
before(async () => {
|
|
42
|
+
({ pool, close } = await createTestDb());
|
|
43
|
+
|
|
44
|
+
const db = drizzle(pool, {
|
|
45
|
+
schema,
|
|
46
|
+
logger: {
|
|
47
|
+
logQuery(text: string, params: unknown[]) {
|
|
48
|
+
logged.push({ text, params });
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
}) as unknown as Db<Record<string, unknown>>;
|
|
52
|
+
repo = new DrizzleThreadMessageRepository(db);
|
|
53
|
+
|
|
54
|
+
const rows = categoryFixtureRows({
|
|
55
|
+
totals: TRIMMED_TOTALS,
|
|
56
|
+
accountConfigId: ACCOUNT,
|
|
57
|
+
mailboxId: MAILBOX,
|
|
58
|
+
});
|
|
59
|
+
const insertDb = drizzle(pool, { schema });
|
|
60
|
+
for (let i = 0; i < rows.length; i += 500) {
|
|
61
|
+
await insertDb.insert(threadMessageTable).values(rows.slice(i, i + 500));
|
|
62
|
+
}
|
|
63
|
+
await pool.query("ANALYZE thread_message");
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
after(async () => {
|
|
67
|
+
await close();
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test("a filtered page is a full page of matches, however far back they sit", async () => {
|
|
71
|
+
const social = await repo.searchByMailboxWindow(
|
|
72
|
+
ACCOUNT,
|
|
73
|
+
MAILBOX,
|
|
74
|
+
{ category: ["social"] },
|
|
75
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
76
|
+
);
|
|
77
|
+
assert.equal(social.items.length, 50);
|
|
78
|
+
assert.ok(social.items.every((item) => item.category === "social"));
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test("a continuation token walks the whole match set without repeats or gaps", async () => {
|
|
82
|
+
const seen: string[] = [];
|
|
83
|
+
let continuationToken: string | undefined;
|
|
84
|
+
do {
|
|
85
|
+
const page = await repo.searchByMailboxWindow(
|
|
86
|
+
ACCOUNT,
|
|
87
|
+
MAILBOX,
|
|
88
|
+
{ category: ["social"] },
|
|
89
|
+
{
|
|
90
|
+
limit: 25,
|
|
91
|
+
order: "desc",
|
|
92
|
+
excludeDeleted: true,
|
|
93
|
+
continuationToken,
|
|
94
|
+
},
|
|
95
|
+
);
|
|
96
|
+
seen.push(...page.items.map((item) => item.threadMessageId));
|
|
97
|
+
continuationToken = page.continuationToken;
|
|
98
|
+
} while (continuationToken);
|
|
99
|
+
assert.equal(seen.length, TRIMMED_TOTALS.social);
|
|
100
|
+
assert.equal(new Set(seen).size, TRIMMED_TOTALS.social);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
describe("query plan", () => {
|
|
104
|
+
// Each assertion names the category column as well as the index: both
|
|
105
|
+
// engines can reach this index on its (account_config_id, mailbox_id)
|
|
106
|
+
// prefix alone, so an index-only assertion would still pass with the
|
|
107
|
+
// predicate removed.
|
|
108
|
+
const assertServedByIndex = async (
|
|
109
|
+
run: () => Promise<unknown>,
|
|
110
|
+
): Promise<void> => {
|
|
111
|
+
logged.length = 0;
|
|
112
|
+
await run();
|
|
113
|
+
const selects = logged.filter((entry) => /^\s*select/i.test(entry.text));
|
|
114
|
+
assert.ok(selects.length > 0, "the repo issued a select");
|
|
115
|
+
const plan: string[] = [];
|
|
116
|
+
for (const entry of selects) {
|
|
117
|
+
const explained = await pool.query<{ "QUERY PLAN": string }>({
|
|
118
|
+
text: `EXPLAIN ${entry.text}`,
|
|
119
|
+
values: entry.params,
|
|
120
|
+
});
|
|
121
|
+
plan.push(...explained.rows.map((row) => row["QUERY PLAN"]));
|
|
122
|
+
}
|
|
123
|
+
const text = plan.join("\n");
|
|
124
|
+
assert.ok(
|
|
125
|
+
text.includes("tm_by_mailbox_category_date"),
|
|
126
|
+
`no plan node used the index:\n${text}`,
|
|
127
|
+
);
|
|
128
|
+
assert.ok(
|
|
129
|
+
/Index Cond:[^\n]*category/.test(text),
|
|
130
|
+
`the index was used but not on category:\n${text}`,
|
|
131
|
+
);
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
test("the filtered window is served by tm_by_mailbox_category_date", async () => {
|
|
135
|
+
await assertServedByIndex(() =>
|
|
136
|
+
repo.searchByMailboxWindow(
|
|
137
|
+
ACCOUNT,
|
|
138
|
+
MAILBOX,
|
|
139
|
+
{ category: ["social"] },
|
|
140
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
141
|
+
),
|
|
142
|
+
);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
test("the filtered count is served by tm_by_mailbox_category_date", async () => {
|
|
146
|
+
await assertServedByIndex(() =>
|
|
147
|
+
repo.countByMailbox(
|
|
148
|
+
ACCOUNT,
|
|
149
|
+
MAILBOX,
|
|
150
|
+
{ category: ["social"] },
|
|
151
|
+
{ excludeDeleted: true },
|
|
152
|
+
),
|
|
153
|
+
);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
});
|
|
@@ -3,6 +3,13 @@ import { after, before, describe, test } from "node:test";
|
|
|
3
3
|
import type { CreateThreadMessageInput } from "@remit/data-ports";
|
|
4
4
|
import { threadMessageTable } from "../schema/thread-message.js";
|
|
5
5
|
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
6
|
+
import {
|
|
7
|
+
BASE_DATE,
|
|
8
|
+
type Category,
|
|
9
|
+
categoryFixtureRows,
|
|
10
|
+
LIVE_TOTALS,
|
|
11
|
+
totalRows,
|
|
12
|
+
} from "./category-fixture.js";
|
|
6
13
|
import { DrizzleThreadMessageRepository } from "./thread-message.js";
|
|
7
14
|
|
|
8
15
|
// The thread-message repo on sqlite (RFC 036 D1): CRUD, keyset pagination, and
|
|
@@ -36,11 +43,12 @@ function makeInput(
|
|
|
36
43
|
|
|
37
44
|
describe("DrizzleThreadMessageRepository (sqlite)", () => {
|
|
38
45
|
let db: Awaited<ReturnType<typeof createSqliteTestDb>>["db"];
|
|
46
|
+
let sqlite: Awaited<ReturnType<typeof createSqliteTestDb>>["sqlite"];
|
|
39
47
|
let close: () => Promise<void>;
|
|
40
48
|
let repo: DrizzleThreadMessageRepository;
|
|
41
49
|
|
|
42
50
|
before(async () => {
|
|
43
|
-
({ db, close } = await createSqliteTestDb(
|
|
51
|
+
({ db, sqlite, close } = await createSqliteTestDb(
|
|
44
52
|
{
|
|
45
53
|
threadMessage: threadMessageTable,
|
|
46
54
|
},
|
|
@@ -608,4 +616,360 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
|
|
|
608
616
|
);
|
|
609
617
|
});
|
|
610
618
|
});
|
|
619
|
+
|
|
620
|
+
// ─── category as a SQL predicate (#304) ───────────────────────────────────
|
|
621
|
+
//
|
|
622
|
+
// The shape is the owner's instance, not a convenience fixture — see
|
|
623
|
+
// ./category-fixture.ts. A filter applied to the page the server happens to
|
|
624
|
+
// return is empty or near-empty on that shape whatever the page size, which
|
|
625
|
+
// is the reported bug.
|
|
626
|
+
describe("category filter over the whole mailbox", () => {
|
|
627
|
+
const CAT_ACCOUNT = "acct-category";
|
|
628
|
+
const CAT_MAILBOX = "mbx-category";
|
|
629
|
+
|
|
630
|
+
before(async () => {
|
|
631
|
+
const rows = categoryFixtureRows({
|
|
632
|
+
totals: LIVE_TOTALS,
|
|
633
|
+
accountConfigId: CAT_ACCOUNT,
|
|
634
|
+
mailboxId: CAT_MAILBOX,
|
|
635
|
+
});
|
|
636
|
+
assert.equal(rows.length, totalRows(LIVE_TOTALS));
|
|
637
|
+
for (let i = 0; i < rows.length; i += 500) {
|
|
638
|
+
await db.insert(threadMessageTable).values(rows.slice(i, i + 500));
|
|
639
|
+
}
|
|
640
|
+
});
|
|
641
|
+
|
|
642
|
+
const walk = async (
|
|
643
|
+
category: Category[],
|
|
644
|
+
pageSize: number,
|
|
645
|
+
): Promise<string[]> => {
|
|
646
|
+
const seen: string[] = [];
|
|
647
|
+
let continuationToken: string | undefined;
|
|
648
|
+
do {
|
|
649
|
+
const page = await repo.searchByMailboxWindow(
|
|
650
|
+
CAT_ACCOUNT,
|
|
651
|
+
CAT_MAILBOX,
|
|
652
|
+
{ category },
|
|
653
|
+
{
|
|
654
|
+
limit: pageSize,
|
|
655
|
+
order: "desc",
|
|
656
|
+
excludeDeleted: true,
|
|
657
|
+
continuationToken,
|
|
658
|
+
},
|
|
659
|
+
);
|
|
660
|
+
seen.push(...page.items.map((item) => item.threadMessageId));
|
|
661
|
+
continuationToken = page.continuationToken;
|
|
662
|
+
} while (continuationToken);
|
|
663
|
+
return seen;
|
|
664
|
+
};
|
|
665
|
+
|
|
666
|
+
test("the newest page holds almost none of the rare categories", async () => {
|
|
667
|
+
const newest = await repo.searchByMailboxWindow(
|
|
668
|
+
CAT_ACCOUNT,
|
|
669
|
+
CAT_MAILBOX,
|
|
670
|
+
{},
|
|
671
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
672
|
+
);
|
|
673
|
+
assert.equal(newest.items.length, 50);
|
|
674
|
+
const count = (category: Category) =>
|
|
675
|
+
newest.items.filter((item) => item.category === category).length;
|
|
676
|
+
assert.ok(
|
|
677
|
+
count("social") <= 2,
|
|
678
|
+
"at most 2 social rows in the newest page",
|
|
679
|
+
);
|
|
680
|
+
assert.ok(
|
|
681
|
+
count("personal") <= 2,
|
|
682
|
+
"at most 2 personal rows in the newest page",
|
|
683
|
+
);
|
|
684
|
+
});
|
|
685
|
+
|
|
686
|
+
// The regression the epic turns on. A filter resolved over the returned
|
|
687
|
+
// page cannot produce this answer: on this mailbox the newest 50 rows hold
|
|
688
|
+
// two personal messages, so the old path returned two rows (or none, for
|
|
689
|
+
// social) with a continuation token attached.
|
|
690
|
+
test("a filtered page is a full page of matches, however far back they sit", async () => {
|
|
691
|
+
const personal = await repo.searchByMailboxWindow(
|
|
692
|
+
CAT_ACCOUNT,
|
|
693
|
+
CAT_MAILBOX,
|
|
694
|
+
{ category: ["personal"] },
|
|
695
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
696
|
+
);
|
|
697
|
+
assert.equal(personal.items.length, 50);
|
|
698
|
+
assert.ok(
|
|
699
|
+
personal.items.every((item) => item.category === "personal"),
|
|
700
|
+
"every row on the page matches the filter",
|
|
701
|
+
);
|
|
702
|
+
|
|
703
|
+
const social = await repo.searchByMailboxWindow(
|
|
704
|
+
CAT_ACCOUNT,
|
|
705
|
+
CAT_MAILBOX,
|
|
706
|
+
{ category: ["social"] },
|
|
707
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
708
|
+
);
|
|
709
|
+
assert.equal(social.items.length, 50);
|
|
710
|
+
assert.ok(social.items.every((item) => item.category === "social"));
|
|
711
|
+
});
|
|
712
|
+
|
|
713
|
+
test("the filtered page keeps newest-first order and the id tiebreak", async () => {
|
|
714
|
+
const page = await repo.searchByMailboxWindow(
|
|
715
|
+
CAT_ACCOUNT,
|
|
716
|
+
CAT_MAILBOX,
|
|
717
|
+
{ category: ["social"] },
|
|
718
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
719
|
+
);
|
|
720
|
+
for (let i = 1; i < page.items.length; i++) {
|
|
721
|
+
const previous = page.items[i - 1];
|
|
722
|
+
const current = page.items[i];
|
|
723
|
+
assert.ok(
|
|
724
|
+
previous.sentDate > current.sentDate ||
|
|
725
|
+
(previous.sentDate === current.sentDate &&
|
|
726
|
+
previous.threadMessageId < current.threadMessageId),
|
|
727
|
+
"rows descend by sentDate, ascending by id inside a tie group",
|
|
728
|
+
);
|
|
729
|
+
}
|
|
730
|
+
});
|
|
731
|
+
|
|
732
|
+
test("a continuation token walks the whole match set without repeats or gaps", async () => {
|
|
733
|
+
const seen = await walk(["social"], 25);
|
|
734
|
+
assert.equal(seen.length, LIVE_TOTALS.social);
|
|
735
|
+
assert.equal(new Set(seen).size, LIVE_TOTALS.social);
|
|
736
|
+
});
|
|
737
|
+
|
|
738
|
+
test("multiple categories behave as a union", async () => {
|
|
739
|
+
const page = await repo.searchByMailboxWindow(
|
|
740
|
+
CAT_ACCOUNT,
|
|
741
|
+
CAT_MAILBOX,
|
|
742
|
+
{ category: ["social", "transactional"] },
|
|
743
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
744
|
+
);
|
|
745
|
+
assert.equal(page.items.length, 50);
|
|
746
|
+
assert.ok(
|
|
747
|
+
page.items.every(
|
|
748
|
+
(item) =>
|
|
749
|
+
item.category === "social" || item.category === "transactional",
|
|
750
|
+
),
|
|
751
|
+
);
|
|
752
|
+
|
|
753
|
+
const seen = await walk(["social", "transactional"], 200);
|
|
754
|
+
assert.equal(seen.length, LIVE_TOTALS.social + LIVE_TOTALS.transactional);
|
|
755
|
+
});
|
|
756
|
+
|
|
757
|
+
// No `limit`: countByMailbox still clamps its answer to the clamped limit,
|
|
758
|
+
// which is what #305 changes. What this asserts is the predicate — the
|
|
759
|
+
// count is over the mailbox rather than over a page of it.
|
|
760
|
+
test("countByMailbox counts the matches in the mailbox, not the page", async () => {
|
|
761
|
+
assert.equal(
|
|
762
|
+
await repo.countByMailbox(
|
|
763
|
+
CAT_ACCOUNT,
|
|
764
|
+
CAT_MAILBOX,
|
|
765
|
+
{ category: ["social"] },
|
|
766
|
+
{ excludeDeleted: true },
|
|
767
|
+
),
|
|
768
|
+
LIVE_TOTALS.social,
|
|
769
|
+
);
|
|
770
|
+
});
|
|
771
|
+
|
|
772
|
+
test("the category predicate composes with the other filters", async () => {
|
|
773
|
+
const page = await repo.searchByMailboxWindow(
|
|
774
|
+
CAT_ACCOUNT,
|
|
775
|
+
CAT_MAILBOX,
|
|
776
|
+
{ category: ["social"], unread: true },
|
|
777
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
778
|
+
);
|
|
779
|
+
assert.equal(page.items.length, 50);
|
|
780
|
+
assert.ok(
|
|
781
|
+
page.items.every(
|
|
782
|
+
(item) => item.category === "social" && item.isRead === false,
|
|
783
|
+
),
|
|
784
|
+
);
|
|
785
|
+
|
|
786
|
+
const none = await repo.searchByMailboxWindow(
|
|
787
|
+
CAT_ACCOUNT,
|
|
788
|
+
CAT_MAILBOX,
|
|
789
|
+
{ category: ["social"], starred: true },
|
|
790
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
791
|
+
);
|
|
792
|
+
assert.equal(none.items.length, 0);
|
|
793
|
+
});
|
|
794
|
+
|
|
795
|
+
test("an empty category set is not a filter", async () => {
|
|
796
|
+
const page = await repo.searchByMailboxWindow(
|
|
797
|
+
CAT_ACCOUNT,
|
|
798
|
+
CAT_MAILBOX,
|
|
799
|
+
{ category: [] },
|
|
800
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
801
|
+
);
|
|
802
|
+
assert.equal(page.items.length, 50);
|
|
803
|
+
assert.ok(
|
|
804
|
+
page.items.some((item) => item.category !== page.items[0]?.category),
|
|
805
|
+
"the page is unfiltered, so it holds more than one category",
|
|
806
|
+
);
|
|
807
|
+
});
|
|
808
|
+
|
|
809
|
+
// #45: `uncategorized` is the not-yet-classified state as a named value.
|
|
810
|
+
// Folding it into `personal` made a classification gap read as a large
|
|
811
|
+
// personal inbox, so the filter has to be able to ask for it and must
|
|
812
|
+
// never answer with the other.
|
|
813
|
+
test("uncategorized is its own filterable value", async () => {
|
|
814
|
+
const account = "acct-category-default";
|
|
815
|
+
const mailbox = "mbx-category-default";
|
|
816
|
+
const base = BASE_DATE;
|
|
817
|
+
await repo.create(
|
|
818
|
+
makeInput({
|
|
819
|
+
accountConfigId: account,
|
|
820
|
+
mailboxId: mailbox,
|
|
821
|
+
messageId: "m-default",
|
|
822
|
+
subject: "not yet classified",
|
|
823
|
+
sentDate: base,
|
|
824
|
+
internalDate: base,
|
|
825
|
+
}),
|
|
826
|
+
);
|
|
827
|
+
await repo.create(
|
|
828
|
+
makeInput({
|
|
829
|
+
accountConfigId: account,
|
|
830
|
+
mailboxId: mailbox,
|
|
831
|
+
messageId: "m-personal",
|
|
832
|
+
subject: "classified personal",
|
|
833
|
+
category: "personal",
|
|
834
|
+
sentDate: base - 1000,
|
|
835
|
+
internalDate: base - 1000,
|
|
836
|
+
}),
|
|
837
|
+
);
|
|
838
|
+
|
|
839
|
+
const uncategorized = await repo.searchByMailboxWindow(
|
|
840
|
+
account,
|
|
841
|
+
mailbox,
|
|
842
|
+
{ category: ["uncategorized"] },
|
|
843
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
844
|
+
);
|
|
845
|
+
assert.deepEqual(
|
|
846
|
+
uncategorized.items.map((item) => item.subject),
|
|
847
|
+
["not yet classified"],
|
|
848
|
+
);
|
|
849
|
+
|
|
850
|
+
const personal = await repo.searchByMailboxWindow(
|
|
851
|
+
account,
|
|
852
|
+
mailbox,
|
|
853
|
+
{ category: ["personal"] },
|
|
854
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
855
|
+
);
|
|
856
|
+
assert.deepEqual(
|
|
857
|
+
personal.items.map((item) => item.subject),
|
|
858
|
+
["classified personal"],
|
|
859
|
+
);
|
|
860
|
+
});
|
|
861
|
+
|
|
862
|
+
test("soft-deleted rows stay out of a filtered page", async () => {
|
|
863
|
+
const account = "acct-category-deleted";
|
|
864
|
+
const mailbox = "mbx-category-deleted";
|
|
865
|
+
await repo.create(
|
|
866
|
+
makeInput({
|
|
867
|
+
accountConfigId: account,
|
|
868
|
+
mailboxId: mailbox,
|
|
869
|
+
messageId: "m-live-social",
|
|
870
|
+
subject: "live",
|
|
871
|
+
category: "social",
|
|
872
|
+
}),
|
|
873
|
+
);
|
|
874
|
+
await repo.create(
|
|
875
|
+
makeInput({
|
|
876
|
+
accountConfigId: account,
|
|
877
|
+
mailboxId: mailbox,
|
|
878
|
+
messageId: "m-trashed-social",
|
|
879
|
+
subject: "trashed",
|
|
880
|
+
category: "social",
|
|
881
|
+
isDeleted: true,
|
|
882
|
+
}),
|
|
883
|
+
);
|
|
884
|
+
|
|
885
|
+
const page = await repo.searchByMailboxWindow(
|
|
886
|
+
account,
|
|
887
|
+
mailbox,
|
|
888
|
+
{ category: ["social"] },
|
|
889
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
890
|
+
);
|
|
891
|
+
assert.deepEqual(
|
|
892
|
+
page.items.map((item) => item.subject),
|
|
893
|
+
["live"],
|
|
894
|
+
);
|
|
895
|
+
});
|
|
896
|
+
|
|
897
|
+
// The guard on I1. Without it a rare-category page silently costs a
|
|
898
|
+
// mailbox scan and no test notices: the index is invisible to
|
|
899
|
+
// vps-migrations-drift.sqlite.test.ts, which compares the committed
|
|
900
|
+
// migration set against the same generated schema the index comes from.
|
|
901
|
+
//
|
|
902
|
+
// The statements are the ones the repo itself issued, captured from the
|
|
903
|
+
// driver, so this cannot pass against a query the repo does not run. Each
|
|
904
|
+
// assertion names the category column as well as the index: SQLite picks
|
|
905
|
+
// this index on its (account_config_id, mailbox_id) prefix alone, so an
|
|
906
|
+
// index-only assertion would still pass with the predicate removed.
|
|
907
|
+
describe("query plan", () => {
|
|
908
|
+
const selectsDuring = async (
|
|
909
|
+
run: () => Promise<unknown>,
|
|
910
|
+
): Promise<string[]> => {
|
|
911
|
+
const captured: string[] = [];
|
|
912
|
+
const original = sqlite.prepare.bind(sqlite);
|
|
913
|
+
const record = (source: string) => {
|
|
914
|
+
captured.push(source);
|
|
915
|
+
return original(source);
|
|
916
|
+
};
|
|
917
|
+
sqlite.prepare = record as typeof sqlite.prepare;
|
|
918
|
+
try {
|
|
919
|
+
await run();
|
|
920
|
+
} finally {
|
|
921
|
+
sqlite.prepare = original as typeof sqlite.prepare;
|
|
922
|
+
}
|
|
923
|
+
return captured.filter((source) => /^\s*select/i.test(source));
|
|
924
|
+
};
|
|
925
|
+
|
|
926
|
+
const plansFor = (statements: string[]): string[] => {
|
|
927
|
+
assert.ok(statements.length > 0, "the repo issued a select");
|
|
928
|
+
return statements.flatMap((source) => {
|
|
929
|
+
const parameters = new Array((source.match(/\?/g) ?? []).length).fill(
|
|
930
|
+
0,
|
|
931
|
+
);
|
|
932
|
+
const rows = sqlite
|
|
933
|
+
.prepare(`EXPLAIN QUERY PLAN ${source}`)
|
|
934
|
+
.all(...parameters) as Array<{ detail: string }>;
|
|
935
|
+
return rows.map((row) => row.detail);
|
|
936
|
+
});
|
|
937
|
+
};
|
|
938
|
+
|
|
939
|
+
const assertServedByIndex = (plan: string[]): void => {
|
|
940
|
+
assert.ok(
|
|
941
|
+
plan.some(
|
|
942
|
+
(detail) =>
|
|
943
|
+
detail.includes("tm_by_mailbox_category_date") &&
|
|
944
|
+
detail.includes("category=?"),
|
|
945
|
+
),
|
|
946
|
+
`no plan step matched on category through the index: ${plan.join(" | ")}`,
|
|
947
|
+
);
|
|
948
|
+
};
|
|
949
|
+
|
|
950
|
+
test("the filtered window is served by tm_by_mailbox_category_date", async () => {
|
|
951
|
+
const statements = await selectsDuring(() =>
|
|
952
|
+
repo.searchByMailboxWindow(
|
|
953
|
+
CAT_ACCOUNT,
|
|
954
|
+
CAT_MAILBOX,
|
|
955
|
+
{ category: ["social"] },
|
|
956
|
+
{ limit: 50, order: "desc", excludeDeleted: true },
|
|
957
|
+
),
|
|
958
|
+
);
|
|
959
|
+
assertServedByIndex(plansFor(statements));
|
|
960
|
+
});
|
|
961
|
+
|
|
962
|
+
test("the filtered count is served by tm_by_mailbox_category_date", async () => {
|
|
963
|
+
const statements = await selectsDuring(() =>
|
|
964
|
+
repo.countByMailbox(
|
|
965
|
+
CAT_ACCOUNT,
|
|
966
|
+
CAT_MAILBOX,
|
|
967
|
+
{ category: ["social"] },
|
|
968
|
+
{ excludeDeleted: true },
|
|
969
|
+
),
|
|
970
|
+
);
|
|
971
|
+
assertServedByIndex(plansFor(statements));
|
|
972
|
+
});
|
|
973
|
+
});
|
|
974
|
+
});
|
|
611
975
|
});
|
|
@@ -158,6 +158,13 @@ function buildSearchConditions(search: SearchOptions): SQL[] {
|
|
|
158
158
|
if (search.attachments !== undefined) {
|
|
159
159
|
conditions.push(eq(threadMessageTable.hasAttachment, search.attachments));
|
|
160
160
|
}
|
|
161
|
+
// An equality over a column on the row, so it sits inside the keyset window
|
|
162
|
+
// rather than filtering what the window returned: a page is a full page of
|
|
163
|
+
// matches however rare the category is. Served by
|
|
164
|
+
// tm_by_mailbox_category_date.
|
|
165
|
+
if (search.category?.length) {
|
|
166
|
+
conditions.push(inArray(threadMessageTable.category, search.category));
|
|
167
|
+
}
|
|
161
168
|
|
|
162
169
|
return conditions;
|
|
163
170
|
}
|