@remit/backend 0.0.87 → 0.0.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/service/organize.test.ts +56 -243
- package/src/service/organize.ts +14 -217
package/package.json
CHANGED
|
@@ -101,7 +101,7 @@ const matchAccepted = async (
|
|
|
101
101
|
return result;
|
|
102
102
|
};
|
|
103
103
|
|
|
104
|
-
/** A standing filter fixture — the
|
|
104
|
+
/** A standing filter fixture — the account's already-existing rules. */
|
|
105
105
|
const filterItem = (over: Partial<FilterItem> = {}): FilterItem => ({
|
|
106
106
|
filterId: "filter-other",
|
|
107
107
|
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
@@ -126,10 +126,9 @@ const filterItem = (over: Partial<FilterItem> = {}): FilterItem => ({
|
|
|
126
126
|
* persists a standing rule.
|
|
127
127
|
*
|
|
128
128
|
* `activeFilters`/`filterAnchorRows`/`threadMessages` model the account's
|
|
129
|
-
* *other*, already-existing standing filters and message rows
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
* precedence check is a same-length no-op.
|
|
129
|
+
* *other*, already-existing standing filters and message rows. A back-apply
|
|
130
|
+
* never arbitrates against them (reader #497), so `filterReads` pins that the
|
|
131
|
+
* Active filter set is not even consulted.
|
|
133
132
|
*/
|
|
134
133
|
const trackingClient = (
|
|
135
134
|
seed: {
|
|
@@ -143,6 +142,7 @@ const trackingClient = (
|
|
|
143
142
|
) => {
|
|
144
143
|
const labeled: Array<{ messageId: string; labelId: string }> = [];
|
|
145
144
|
let filterWrites = 0;
|
|
145
|
+
let filterReads = 0;
|
|
146
146
|
let filterAnchorWrites = 0;
|
|
147
147
|
const anchorPuts: CreateFilterAnchorInput[] = [];
|
|
148
148
|
const activeFilters = seed.activeFilters ?? [];
|
|
@@ -190,7 +190,10 @@ const trackingClient = (
|
|
|
190
190
|
filterWrites += 1;
|
|
191
191
|
return {} as never;
|
|
192
192
|
},
|
|
193
|
-
listByAccountAndState: async () =>
|
|
193
|
+
listByAccountAndState: async () => {
|
|
194
|
+
filterReads += 1;
|
|
195
|
+
return activeFilters;
|
|
196
|
+
},
|
|
194
197
|
refreshExpiry: async (filter: FilterItem) => filter,
|
|
195
198
|
},
|
|
196
199
|
filterAnchor: {
|
|
@@ -214,6 +217,7 @@ const trackingClient = (
|
|
|
214
217
|
client,
|
|
215
218
|
labeled,
|
|
216
219
|
filterWrites: () => filterWrites,
|
|
220
|
+
filterReads: () => filterReads,
|
|
217
221
|
filterAnchorWrites: () => filterAnchorWrites,
|
|
218
222
|
anchorPuts,
|
|
219
223
|
};
|
|
@@ -728,7 +732,7 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
728
732
|
|
|
729
733
|
const tracked = trackingClient();
|
|
730
734
|
const result = await applyOrganize(
|
|
731
|
-
{ client: tracked.client
|
|
735
|
+
{ client: tracked.client },
|
|
732
736
|
ACCOUNT_CONFIG_ID,
|
|
733
737
|
applied.messageIds,
|
|
734
738
|
p,
|
|
@@ -764,7 +768,7 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
764
768
|
);
|
|
765
769
|
const tracked = trackingClient();
|
|
766
770
|
const result = await applyOrganize(
|
|
767
|
-
{ client: tracked.client
|
|
771
|
+
{ client: tracked.client },
|
|
768
772
|
ACCOUNT_CONFIG_ID,
|
|
769
773
|
matched,
|
|
770
774
|
p,
|
|
@@ -794,7 +798,6 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
794
798
|
{
|
|
795
799
|
client: tracked.client,
|
|
796
800
|
moveService: mover.moveService,
|
|
797
|
-
match: matchDeps(store),
|
|
798
801
|
},
|
|
799
802
|
ACCOUNT_CONFIG_ID,
|
|
800
803
|
matched,
|
|
@@ -844,7 +847,6 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
844
847
|
{
|
|
845
848
|
client: tracked.client,
|
|
846
849
|
moveService: mover.moveService,
|
|
847
|
-
match: matchDeps(store),
|
|
848
850
|
},
|
|
849
851
|
ACCOUNT_CONFIG_ID,
|
|
850
852
|
matched,
|
|
@@ -880,7 +882,6 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
880
882
|
{
|
|
881
883
|
client: tracked.client,
|
|
882
884
|
moveService: mover.moveService,
|
|
883
|
-
match: matchDeps(store),
|
|
884
885
|
},
|
|
885
886
|
ACCOUNT_CONFIG_ID,
|
|
886
887
|
matched,
|
|
@@ -890,7 +891,6 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
890
891
|
{
|
|
891
892
|
client: tracked.client,
|
|
892
893
|
moveService: mover.moveService,
|
|
893
|
-
match: matchDeps(store),
|
|
894
894
|
},
|
|
895
895
|
ACCOUNT_CONFIG_ID,
|
|
896
896
|
matched,
|
|
@@ -912,22 +912,25 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
912
912
|
});
|
|
913
913
|
});
|
|
914
914
|
|
|
915
|
-
|
|
916
|
-
|
|
915
|
+
/**
|
|
916
|
+
* A standing filter acts when a message is first seen and when the user presses
|
|
917
|
+
* "run rules now", and holds no claim afterwards. A later apply is the newer
|
|
918
|
+
* event, so it moves the mail — and because nothing is suppressed, `applied`
|
|
919
|
+
* counts moves that happened rather than moves that were skipped (reader #497).
|
|
920
|
+
*/
|
|
921
|
+
describe("a user-initiated apply outranks every standing filter (reader #497)", () => {
|
|
922
|
+
it("moves a hand-picked selection a standing filter claims for a different mailbox", async () => {
|
|
917
923
|
const store = createMemoryVectorStore();
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
//
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
});
|
|
926
|
-
const newerFilter = filterItem({
|
|
927
|
-
filterId: "filter-newer",
|
|
924
|
+
const picked = ["msg-1", "msg-2", "msg-3"];
|
|
925
|
+
await store.upsert(picked.map((id) => bodyChunk(id, ANCHOR_VECTOR)));
|
|
926
|
+
const p = predicate({ actionMailboxId: "mbox-projects" });
|
|
927
|
+
// The standing rule that filed all three into Archive in the first place —
|
|
928
|
+
// moving them back out is the whole point of the button.
|
|
929
|
+
const standing = filterItem({
|
|
930
|
+
filterId: "filter-github",
|
|
928
931
|
ruleChangedAt: 1_000,
|
|
929
932
|
actionChangedAt: 1_000,
|
|
930
|
-
actionMailboxId: "mbox-
|
|
933
|
+
actionMailboxId: "mbox-archive",
|
|
931
934
|
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
932
935
|
});
|
|
933
936
|
|
|
@@ -937,128 +940,40 @@ describe("applyOrganize resolves move precedence against current Active filters
|
|
|
937
940
|
p,
|
|
938
941
|
);
|
|
939
942
|
const tracked = trackingClient({
|
|
940
|
-
activeFilters: [
|
|
941
|
-
threadMessages:
|
|
943
|
+
activeFilters: [standing],
|
|
944
|
+
threadMessages: Object.fromEntries(
|
|
945
|
+
picked.map((id) => [id, { subject: "Dinner reservation" }]),
|
|
946
|
+
),
|
|
942
947
|
});
|
|
943
948
|
const mover = trackingMoveService();
|
|
944
949
|
const result = await applyOrganize(
|
|
945
|
-
{
|
|
946
|
-
client: tracked.client,
|
|
947
|
-
moveService: mover.moveService,
|
|
948
|
-
match: matchDeps(store),
|
|
949
|
-
},
|
|
950
|
+
{ client: tracked.client, moveService: mover.moveService },
|
|
950
951
|
ACCOUNT_CONFIG_ID,
|
|
951
952
|
matched,
|
|
952
953
|
p,
|
|
953
954
|
);
|
|
954
955
|
|
|
955
|
-
assert.equal(result.applied, 1, "a suppressed move is not a failure");
|
|
956
|
-
assert.equal(result.failed, 0);
|
|
957
|
-
assert.deepEqual(
|
|
958
|
-
tracked.labeled,
|
|
959
|
-
[{ messageId: "msg-1", labelId: "lbl-1" }],
|
|
960
|
-
"the additive label still applies even though the move is suppressed",
|
|
961
|
-
);
|
|
962
956
|
assert.deepEqual(
|
|
963
|
-
mover.moves,
|
|
964
|
-
|
|
965
|
-
"
|
|
957
|
+
mover.moves.map((move) => move.messageId).sort(),
|
|
958
|
+
picked,
|
|
959
|
+
"every hand-picked message moves",
|
|
966
960
|
);
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
const p = predicate({ actionMailboxId: "mbox-target" });
|
|
973
|
-
// A different standing filter matches this message too, but agrees on the
|
|
974
|
-
// same destination — nothing to defer to.
|
|
975
|
-
const agreeingFilter = filterItem({
|
|
976
|
-
filterId: "filter-agrees",
|
|
977
|
-
ruleChangedAt: 1_000,
|
|
978
|
-
actionChangedAt: 1_000,
|
|
979
|
-
actionMailboxId: "mbox-target",
|
|
980
|
-
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
981
|
-
});
|
|
982
|
-
|
|
983
|
-
const { messageIds: matched } = await matchAccepted(
|
|
984
|
-
matchDeps(store),
|
|
985
|
-
ACCOUNT_CONFIG_ID,
|
|
986
|
-
p,
|
|
961
|
+
assert.ok(
|
|
962
|
+
mover.moves.every(
|
|
963
|
+
(move) => move.destinationMailboxId === "mbox-projects",
|
|
964
|
+
),
|
|
965
|
+
"the destination is the one the user picked, not the standing filter's",
|
|
987
966
|
);
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
const mover = trackingMoveService();
|
|
993
|
-
const result = await applyOrganize(
|
|
994
|
-
{
|
|
995
|
-
client: tracked.client,
|
|
996
|
-
moveService: mover.moveService,
|
|
997
|
-
match: matchDeps(store),
|
|
998
|
-
},
|
|
999
|
-
ACCOUNT_CONFIG_ID,
|
|
1000
|
-
matched,
|
|
1001
|
-
p,
|
|
967
|
+
assert.equal(
|
|
968
|
+
result.applied,
|
|
969
|
+
mover.moves.length,
|
|
970
|
+
"the reported count equals the moves that actually happened",
|
|
1002
971
|
);
|
|
1003
|
-
|
|
1004
|
-
assert.equal(result.applied, 1);
|
|
1005
972
|
assert.equal(result.failed, 0);
|
|
1006
|
-
assert.
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
"the
|
|
1010
|
-
);
|
|
1011
|
-
});
|
|
1012
|
-
|
|
1013
|
-
it("suppresses an out-ranked move by a newer *semantic* filter's persisted anchor", async () => {
|
|
1014
|
-
const store = createMemoryVectorStore();
|
|
1015
|
-
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
1016
|
-
const p = predicate({ actionMailboxId: "mbox-old" });
|
|
1017
|
-
const newerSemanticFilter = filterItem({
|
|
1018
|
-
filterId: "filter-newer-semantic",
|
|
1019
|
-
ruleChangedAt: 1_000,
|
|
1020
|
-
actionChangedAt: 1_000,
|
|
1021
|
-
actionMailboxId: "mbox-new",
|
|
1022
|
-
hasAnchor: true,
|
|
1023
|
-
});
|
|
1024
|
-
const persistedAnchor: FilterAnchorItem = {
|
|
1025
|
-
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
1026
|
-
filterId: "filter-newer-semantic",
|
|
1027
|
-
anchorEmbedding: ANCHOR_VECTOR,
|
|
1028
|
-
anchorEmbeddingId: CURRENT_EMBEDDING_ID,
|
|
1029
|
-
anchorSourceText: ANCHOR_SOURCE_TEXT,
|
|
1030
|
-
anchorMessageId: "msg-anchor-2",
|
|
1031
|
-
createdAt: 0,
|
|
1032
|
-
updatedAt: 0,
|
|
1033
|
-
};
|
|
1034
|
-
|
|
1035
|
-
const { messageIds: matched } = await matchAccepted(
|
|
1036
|
-
matchDeps(store),
|
|
1037
|
-
ACCOUNT_CONFIG_ID,
|
|
1038
|
-
p,
|
|
1039
|
-
);
|
|
1040
|
-
const tracked = trackingClient({
|
|
1041
|
-
activeFilters: [newerSemanticFilter],
|
|
1042
|
-
filterAnchorRows: [persistedAnchor],
|
|
1043
|
-
threadMessages: { "msg-1": { subject: "Dinner reservation" } },
|
|
1044
|
-
});
|
|
1045
|
-
const mover = trackingMoveService();
|
|
1046
|
-
const result = await applyOrganize(
|
|
1047
|
-
{
|
|
1048
|
-
client: tracked.client,
|
|
1049
|
-
moveService: mover.moveService,
|
|
1050
|
-
match: matchDeps(store, [], [persistedAnchor]),
|
|
1051
|
-
},
|
|
1052
|
-
ACCOUNT_CONFIG_ID,
|
|
1053
|
-
matched,
|
|
1054
|
-
p,
|
|
1055
|
-
);
|
|
1056
|
-
|
|
1057
|
-
assert.equal(result.applied, 1);
|
|
1058
|
-
assert.deepEqual(
|
|
1059
|
-
mover.moves,
|
|
1060
|
-
[],
|
|
1061
|
-
"a newer semantic filter's own persisted anchor outranks the move",
|
|
973
|
+
assert.equal(
|
|
974
|
+
tracked.filterReads(),
|
|
975
|
+
0,
|
|
976
|
+
"the account's standing filters are not consulted at all",
|
|
1062
977
|
);
|
|
1063
978
|
});
|
|
1064
979
|
});
|
|
@@ -1067,20 +982,16 @@ describe("applyOrganize resolves move precedence against current Active filters
|
|
|
1067
982
|
* A same-dimension embedding-model swap leaves every persisted FilterAnchor
|
|
1068
983
|
* stamped with the old model's id and its vector in the old model's space.
|
|
1069
984
|
* Index-time matching repairs that lazily (RFC 039 Decision 1a); the
|
|
1070
|
-
* back-apply
|
|
1071
|
-
*
|
|
1072
|
-
* claim to mirror (reader #399).
|
|
985
|
+
* back-apply widen must do the same, or it queries with a stale-model vector
|
|
986
|
+
* and silently disagrees with the pipeline it claims to mirror (reader #399).
|
|
1073
987
|
*/
|
|
1074
988
|
describe("back-apply repairs a drifted anchor the way index-time matching does (reader #399)", () => {
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
// nothing but the id stamp distinguishes it from a usable anchor.
|
|
1079
|
-
anchorEmbedding: number[] = ORTHOGONAL_VECTOR,
|
|
1080
|
-
): FilterAnchorItem => ({
|
|
989
|
+
// Written under the previous model: same dimensions, different space, so
|
|
990
|
+
// nothing but the id stamp distinguishes it from a usable anchor.
|
|
991
|
+
const staleAnchor = (filterId: string): FilterAnchorItem => ({
|
|
1081
992
|
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
1082
993
|
filterId,
|
|
1083
|
-
anchorEmbedding,
|
|
994
|
+
anchorEmbedding: ORTHOGONAL_VECTOR,
|
|
1084
995
|
anchorEmbeddingId: STALE_EMBEDDING_ID,
|
|
1085
996
|
anchorSourceText: ANCHOR_SOURCE_TEXT,
|
|
1086
997
|
anchorMessageId: "msg-anchor",
|
|
@@ -1143,104 +1054,6 @@ describe("back-apply repairs a drifted anchor the way index-time matching does (
|
|
|
1143
1054
|
assert.deepEqual(deps.anchorPuts[0].anchorEmbedding, ANCHOR_VECTOR);
|
|
1144
1055
|
assert.equal(deps.anchorPuts[0].anchorSourceText, ANCHOR_SOURCE_TEXT);
|
|
1145
1056
|
});
|
|
1146
|
-
|
|
1147
|
-
it("filterCurrentlyMatches re-embeds a drifted anchor, so the drifted filter still outranks the move", async () => {
|
|
1148
|
-
const store = createMemoryVectorStore();
|
|
1149
|
-
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
1150
|
-
const p = predicate({
|
|
1151
|
-
actionLabelId: "lbl-1",
|
|
1152
|
-
actionMailboxId: "mbox-old",
|
|
1153
|
-
});
|
|
1154
|
-
const newerSemanticFilter = filterItem({
|
|
1155
|
-
filterId: "filter-newer-semantic",
|
|
1156
|
-
ruleChangedAt: 1_000,
|
|
1157
|
-
actionChangedAt: 1_000,
|
|
1158
|
-
actionMailboxId: "mbox-new",
|
|
1159
|
-
hasAnchor: true,
|
|
1160
|
-
});
|
|
1161
|
-
const drifted = staleAnchor("filter-newer-semantic");
|
|
1162
|
-
|
|
1163
|
-
const { messageIds: matched } = await matchAccepted(
|
|
1164
|
-
matchDeps(store),
|
|
1165
|
-
ACCOUNT_CONFIG_ID,
|
|
1166
|
-
p,
|
|
1167
|
-
);
|
|
1168
|
-
const tracked = trackingClient({
|
|
1169
|
-
activeFilters: [newerSemanticFilter],
|
|
1170
|
-
filterAnchorRows: [drifted],
|
|
1171
|
-
threadMessages: { "msg-1": { subject: "Dinner reservation" } },
|
|
1172
|
-
});
|
|
1173
|
-
const mover = trackingMoveService();
|
|
1174
|
-
const result = await applyOrganize(
|
|
1175
|
-
{
|
|
1176
|
-
client: tracked.client,
|
|
1177
|
-
moveService: mover.moveService,
|
|
1178
|
-
match: currentModelDeps(store, [drifted]),
|
|
1179
|
-
},
|
|
1180
|
-
ACCOUNT_CONFIG_ID,
|
|
1181
|
-
matched,
|
|
1182
|
-
p,
|
|
1183
|
-
);
|
|
1184
|
-
|
|
1185
|
-
assert.equal(result.applied, 1);
|
|
1186
|
-
assert.deepEqual(
|
|
1187
|
-
mover.moves,
|
|
1188
|
-
[],
|
|
1189
|
-
"scored against the re-embedded anchor the newer filter contests the move; against the stale one it would silently lose",
|
|
1190
|
-
);
|
|
1191
|
-
assert.deepEqual(
|
|
1192
|
-
tracked.anchorPuts.map((put) => [put.filterId, put.anchorEmbeddingId]),
|
|
1193
|
-
[["filter-newer-semantic", CURRENT_EMBEDDING_ID]],
|
|
1194
|
-
"the repair is written back in place, not left for the next pass",
|
|
1195
|
-
);
|
|
1196
|
-
});
|
|
1197
|
-
|
|
1198
|
-
it("keeps a failed re-embed isolated to its own filter rather than throwing out of the arbitration", async () => {
|
|
1199
|
-
const store = createMemoryVectorStore();
|
|
1200
|
-
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
1201
|
-
const p = predicate({ actionMailboxId: "mbox-target" });
|
|
1202
|
-
// The stale vector would score a match, so scoring against it — which is
|
|
1203
|
-
// exactly what this path did before the repair existed — suppresses the
|
|
1204
|
-
// move. A stamp that cannot be honoured is not a match: the filter is
|
|
1205
|
-
// skipped and loudly logged, and the rest of the pass is unaffected.
|
|
1206
|
-
const drifted = staleAnchor("filter-broken-anchor", ANCHOR_VECTOR);
|
|
1207
|
-
const tracked = trackingClient({
|
|
1208
|
-
activeFilters: [
|
|
1209
|
-
filterItem({
|
|
1210
|
-
filterId: "filter-broken-anchor",
|
|
1211
|
-
ruleChangedAt: 1_000,
|
|
1212
|
-
actionChangedAt: 1_000,
|
|
1213
|
-
actionMailboxId: "mbox-elsewhere",
|
|
1214
|
-
hasAnchor: true,
|
|
1215
|
-
}),
|
|
1216
|
-
],
|
|
1217
|
-
filterAnchorRows: [drifted],
|
|
1218
|
-
threadMessages: { "msg-1": { subject: "Dinner reservation" } },
|
|
1219
|
-
});
|
|
1220
|
-
const mover = trackingMoveService();
|
|
1221
|
-
|
|
1222
|
-
const result = await applyOrganize(
|
|
1223
|
-
{
|
|
1224
|
-
client: tracked.client,
|
|
1225
|
-
moveService: mover.moveService,
|
|
1226
|
-
match: currentModelDeps(store, [drifted], async (text) => {
|
|
1227
|
-
if (text === ANCHOR_SOURCE_TEXT) throw new Error("embedder refused");
|
|
1228
|
-
return ANCHOR_VECTOR;
|
|
1229
|
-
}),
|
|
1230
|
-
},
|
|
1231
|
-
ACCOUNT_CONFIG_ID,
|
|
1232
|
-
["msg-1"],
|
|
1233
|
-
p,
|
|
1234
|
-
);
|
|
1235
|
-
|
|
1236
|
-
assert.equal(result.applied, 1);
|
|
1237
|
-
assert.equal(result.failed, 0);
|
|
1238
|
-
assert.deepEqual(
|
|
1239
|
-
mover.moves.map((move) => move.destinationMailboxId),
|
|
1240
|
-
["mbox-target"],
|
|
1241
|
-
"an un-repairable anchor skips its own filter — it neither decides the arbitration on a stale score nor aborts the pass",
|
|
1242
|
-
);
|
|
1243
|
-
});
|
|
1244
1057
|
});
|
|
1245
1058
|
|
|
1246
1059
|
describe("matchOrganize with ListId and FromDomain clauses", () => {
|
|
@@ -1340,7 +1153,7 @@ describe("matchOrganize with ListId and FromDomain clauses", () => {
|
|
|
1340
1153
|
|
|
1341
1154
|
const tracked = trackingClient();
|
|
1342
1155
|
const result = await applyOrganize(
|
|
1343
|
-
{ client: tracked.client
|
|
1156
|
+
{ client: tracked.client },
|
|
1344
1157
|
ACCOUNT_CONFIG_ID,
|
|
1345
1158
|
applied.messageIds,
|
|
1346
1159
|
p,
|
package/src/service/organize.ts
CHANGED
|
@@ -5,20 +5,15 @@ import type {
|
|
|
5
5
|
IFilterAnchorRepository,
|
|
6
6
|
OrganizeJobRequestItem,
|
|
7
7
|
} from "@remit/data-ports";
|
|
8
|
-
import {
|
|
9
|
-
import { FilterClauseField, FilterState } from "@remit/domain-enums";
|
|
8
|
+
import { FilterClauseField } from "@remit/domain-enums";
|
|
10
9
|
import { logger } from "@remit/logger-lambda";
|
|
11
10
|
import {
|
|
12
|
-
type AnchorEmbedder,
|
|
13
|
-
buildMatchText,
|
|
14
|
-
cosineSimilarity,
|
|
15
11
|
DEFAULT_SEMANTIC_MATCH_THRESHOLD,
|
|
16
12
|
type FilterMessage,
|
|
17
13
|
literalClausesMatch,
|
|
18
14
|
NO_ACTION,
|
|
19
15
|
PlacementMoveService,
|
|
20
16
|
refreshAnchorForEmbedder,
|
|
21
|
-
selectMoveWinner,
|
|
22
17
|
} from "@remit/mailbox-service";
|
|
23
18
|
import {
|
|
24
19
|
type AnchorPayload,
|
|
@@ -96,12 +91,11 @@ export interface OrganizeSemanticDeps {
|
|
|
96
91
|
) => Promise<AnchorPayload | null>;
|
|
97
92
|
vectorStore: Pick<VectorStoreService, "query" | "getByMessage">;
|
|
98
93
|
/**
|
|
99
|
-
* Embed one
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
* kNN read (see `semantic-capability.ts`).
|
|
94
|
+
* Embed one text — the `AnchorEmbedder` half of these deps, used to
|
|
95
|
+
* re-embed a drifted persisted anchor in place ({@link
|
|
96
|
+
* refreshAnchorForEmbedder}). Never called by the anchor widen itself,
|
|
97
|
+
* which only ever runs a vector-store kNN read (see
|
|
98
|
+
* `semantic-capability.ts`).
|
|
105
99
|
*/
|
|
106
100
|
embed: (text: string) => Promise<number[]>;
|
|
107
101
|
/**
|
|
@@ -602,13 +596,6 @@ export const buildOrganizeMoveService = (
|
|
|
602
596
|
export interface ApplyOrganizeDeps {
|
|
603
597
|
client: RemitClient;
|
|
604
598
|
moveService?: PlacementMoveService;
|
|
605
|
-
/**
|
|
606
|
-
* The same matcher deps {@link matchOrganize} uses — reused here only for
|
|
607
|
-
* their vector-store/embedder access, to compare a candidate message
|
|
608
|
-
* against a *different* filter's persisted anchor when checking exclusive-
|
|
609
|
-
* move precedence (RFC 039 Decision 2, reader #350).
|
|
610
|
-
*/
|
|
611
|
-
match: OrganizeMatchDeps;
|
|
612
599
|
}
|
|
613
600
|
|
|
614
601
|
export interface ApplyOrganizeResult {
|
|
@@ -616,181 +603,6 @@ export interface ApplyOrganizeResult {
|
|
|
616
603
|
failed: number;
|
|
617
604
|
}
|
|
618
605
|
|
|
619
|
-
/**
|
|
620
|
-
* The vector-free literal-match projection of one already-stored message,
|
|
621
|
-
* read from its ThreadMessage row — the same fields and the same fidelity
|
|
622
|
-
* tradeoff {@link listAccountFilterMessagesFromClient} accepts for the
|
|
623
|
-
* back-apply predicate's own literal clauses (full-fidelity From/Subject/
|
|
624
|
-
* ListId, empty body text, so a `HasWords` clause on a *different* filter
|
|
625
|
-
* cannot be proven to currently match here). `undefined` when the message no
|
|
626
|
-
* longer exists.
|
|
627
|
-
*/
|
|
628
|
-
const findFilterMessageForPrecedence = async (
|
|
629
|
-
client: Pick<RemitClient, "threadMessage">,
|
|
630
|
-
accountConfigId: string,
|
|
631
|
-
messageId: string,
|
|
632
|
-
): Promise<FilterMessage | undefined> => {
|
|
633
|
-
const row = await client.threadMessage
|
|
634
|
-
.get(accountConfigId, messageId)
|
|
635
|
-
.catch((error: unknown) => {
|
|
636
|
-
if (error instanceof NotFoundError) return undefined;
|
|
637
|
-
throw error;
|
|
638
|
-
});
|
|
639
|
-
if (!row) return undefined;
|
|
640
|
-
return {
|
|
641
|
-
from: row.fromEmail ?? "",
|
|
642
|
-
fromName: row.fromName ?? "",
|
|
643
|
-
subject: row.subject ?? "",
|
|
644
|
-
text: "",
|
|
645
|
-
listId: row.listId ?? "",
|
|
646
|
-
};
|
|
647
|
-
};
|
|
648
|
-
|
|
649
|
-
/**
|
|
650
|
-
* Whether one *other* Active filter with a move action currently matches this
|
|
651
|
-
* message — mirrors `FilterPipeline.filterMatches` (mailbox-service
|
|
652
|
-
* filters/pipeline.ts) exactly: literal clauses first, then, for a filter
|
|
653
|
-
* with a semantic anchor, its own persisted `FilterAnchor` — re-embedded in
|
|
654
|
-
* place through the same {@link refreshAnchorForEmbedder} the pipeline calls
|
|
655
|
-
* when the model has drifted since the anchor was written — compared against
|
|
656
|
-
* the candidate's embedding. A stale/incompatible anchor on the *other*
|
|
657
|
-
* filter is isolated to that filter (skipped, not thrown) — the same
|
|
658
|
-
* resilience `filterMatches` gives index-time matching, so one bad anchor,
|
|
659
|
-
* or one anchor that cannot be re-embedded, never breaks this back-apply's
|
|
660
|
-
* move.
|
|
661
|
-
*/
|
|
662
|
-
const filterCurrentlyMatches = async (
|
|
663
|
-
filterAnchorService: Pick<IFilterAnchorRepository, "get" | "put">,
|
|
664
|
-
embedder: () => AnchorEmbedder,
|
|
665
|
-
accountConfigId: string,
|
|
666
|
-
filter: FilterItem,
|
|
667
|
-
msg: FilterMessage,
|
|
668
|
-
embed: () => Promise<number[] | null>,
|
|
669
|
-
): Promise<boolean> => {
|
|
670
|
-
if (!literalClausesMatch(filter.literalClauses, filter.matchOperator, msg)) {
|
|
671
|
-
return false;
|
|
672
|
-
}
|
|
673
|
-
if (!filter.hasAnchor) {
|
|
674
|
-
return filter.literalClauses.length > 0;
|
|
675
|
-
}
|
|
676
|
-
const stored = await filterAnchorService.get(
|
|
677
|
-
accountConfigId,
|
|
678
|
-
filter.filterId,
|
|
679
|
-
);
|
|
680
|
-
if (!stored) return false;
|
|
681
|
-
const vector = await embed();
|
|
682
|
-
if (!vector) return false;
|
|
683
|
-
const repairAnchor = async (): Promise<FilterAnchorItem> =>
|
|
684
|
-
refreshAnchorForEmbedder(
|
|
685
|
-
{ anchorRepository: filterAnchorService, embedder: embedder() },
|
|
686
|
-
stored,
|
|
687
|
-
);
|
|
688
|
-
return repairAnchor()
|
|
689
|
-
.then(
|
|
690
|
-
(anchor) =>
|
|
691
|
-
cosineSimilarity(vector, anchor.anchorEmbedding) >=
|
|
692
|
-
DEFAULT_SEMANTIC_MATCH_THRESHOLD,
|
|
693
|
-
)
|
|
694
|
-
.catch((error: unknown) => {
|
|
695
|
-
logger.error(
|
|
696
|
-
{
|
|
697
|
-
alert: "filter_anchor_match_failed",
|
|
698
|
-
filterId: filter.filterId,
|
|
699
|
-
accountConfigId,
|
|
700
|
-
errorName: (error as { name?: string })?.name,
|
|
701
|
-
error: inspect(error),
|
|
702
|
-
},
|
|
703
|
-
"Filter anchor comparison failed during back-apply precedence; skipping this filter, the rest still arbitrate (bad/stale anchor vector or a failed repair, non-fatal)",
|
|
704
|
-
);
|
|
705
|
-
return false;
|
|
706
|
-
});
|
|
707
|
-
};
|
|
708
|
-
|
|
709
|
-
/**
|
|
710
|
-
* The filter that currently wins this message's exclusive move, evaluated
|
|
711
|
-
* fresh against every Active filter with a move action — the same
|
|
712
|
-
* cross-filter arbitration index-time matching runs
|
|
713
|
-
* (`FilterPipeline`/`selectMoveWinner`), never the back-apply job's own
|
|
714
|
-
* snapshotted predicate (RFC 039 Decision 2, reader #350). This is what lets
|
|
715
|
-
* back-apply defer to a filter created or edited *after* the job was
|
|
716
|
-
* requested. `movers` is fetched once per {@link applyOrganize} call, not
|
|
717
|
-
* once per message — a back-apply pass runs in one short burst, so the
|
|
718
|
-
* account's Active filter set does not need re-reading per message.
|
|
719
|
-
*
|
|
720
|
-
* Returns `undefined` — meaning "nothing else contests this move" — whenever
|
|
721
|
-
* there are no other Active mover filters, or the message's own projection
|
|
722
|
-
* cannot be built (deleted between match and apply). Both are the common
|
|
723
|
-
* case and the safe default: proceed with the requested move exactly as
|
|
724
|
-
* before this check existed.
|
|
725
|
-
*/
|
|
726
|
-
const findCurrentMoveWinner = async (
|
|
727
|
-
deps: {
|
|
728
|
-
client: Pick<RemitClient, "threadMessage" | "filterAnchor">;
|
|
729
|
-
match: OrganizeMatchDeps;
|
|
730
|
-
},
|
|
731
|
-
movers: readonly FilterItem[],
|
|
732
|
-
accountConfigId: string,
|
|
733
|
-
messageId: string,
|
|
734
|
-
): Promise<FilterItem | undefined> => {
|
|
735
|
-
if (movers.length === 0) return undefined;
|
|
736
|
-
const msg = await findFilterMessageForPrecedence(
|
|
737
|
-
deps.client,
|
|
738
|
-
accountConfigId,
|
|
739
|
-
messageId,
|
|
740
|
-
);
|
|
741
|
-
if (!msg) return undefined;
|
|
742
|
-
|
|
743
|
-
let messageEmbedding: number[] | null | undefined;
|
|
744
|
-
const embed = async (): Promise<number[] | null> => {
|
|
745
|
-
if (messageEmbedding !== undefined) return messageEmbedding;
|
|
746
|
-
try {
|
|
747
|
-
messageEmbedding = await deps.match.semantic().embed(buildMatchText(msg));
|
|
748
|
-
} catch (error) {
|
|
749
|
-
if (!noteSemanticCapabilityAbsence(error)) throw error;
|
|
750
|
-
messageEmbedding = null;
|
|
751
|
-
}
|
|
752
|
-
return messageEmbedding;
|
|
753
|
-
};
|
|
754
|
-
|
|
755
|
-
const matched: FilterItem[] = [];
|
|
756
|
-
for (const filter of movers) {
|
|
757
|
-
const isMatch = await filterCurrentlyMatches(
|
|
758
|
-
deps.client.filterAnchor,
|
|
759
|
-
() => deps.match.semantic(),
|
|
760
|
-
accountConfigId,
|
|
761
|
-
filter,
|
|
762
|
-
msg,
|
|
763
|
-
embed,
|
|
764
|
-
);
|
|
765
|
-
if (isMatch) matched.push(filter);
|
|
766
|
-
}
|
|
767
|
-
return selectMoveWinner(matched);
|
|
768
|
-
};
|
|
769
|
-
|
|
770
|
-
/**
|
|
771
|
-
* Every currently-Active filter with a move action, its lazy Temporary-expiry
|
|
772
|
-
* check already applied — the fixed candidate set {@link findCurrentMoveWinner}
|
|
773
|
-
* arbitrates against for every message in this pass. Empty (and read exactly
|
|
774
|
-
* once) when no move action was requested, so a label-only back-apply never
|
|
775
|
-
* pays for this at all.
|
|
776
|
-
*/
|
|
777
|
-
const listCurrentMovers = async (
|
|
778
|
-
client: Pick<RemitClient, "filter">,
|
|
779
|
-
accountConfigId: string,
|
|
780
|
-
): Promise<FilterItem[]> => {
|
|
781
|
-
const active = await client.filter.listByAccountAndState(
|
|
782
|
-
accountConfigId,
|
|
783
|
-
FilterState.Active,
|
|
784
|
-
);
|
|
785
|
-
const movers: FilterItem[] = [];
|
|
786
|
-
for (const filter of active) {
|
|
787
|
-
if (filter.actionMailboxId === NO_ACTION) continue;
|
|
788
|
-
const usable = await client.filter.refreshExpiry(filter);
|
|
789
|
-
if (usable.state === FilterState.Active) movers.push(usable);
|
|
790
|
-
}
|
|
791
|
-
return movers;
|
|
792
|
-
};
|
|
793
|
-
|
|
794
606
|
/**
|
|
795
607
|
* Apply the back-apply action to every matched message, reusing the index-time
|
|
796
608
|
* apply plumbing: an idempotent MessageLabel upsert (additive) with
|
|
@@ -799,12 +611,13 @@ const listCurrentMovers = async (
|
|
|
799
611
|
* folder move (exclusive). One poisoned message never fails the batch; it is
|
|
800
612
|
* counted as failed and the pass continues.
|
|
801
613
|
*
|
|
802
|
-
*
|
|
803
|
-
*
|
|
804
|
-
*
|
|
805
|
-
*
|
|
806
|
-
*
|
|
807
|
-
*
|
|
614
|
+
* Every requested move happens. A standing filter acts at exactly two moments —
|
|
615
|
+
* when a message is first seen, and when the user presses "run rules now" — and
|
|
616
|
+
* holds no claim on that message afterwards, so it never arbitrates against a
|
|
617
|
+
* later user-initiated apply (reader #497). Cross-filter precedence stays where
|
|
618
|
+
* two filters collide over the same message at the same trigger
|
|
619
|
+
* (`FilterPipeline`/`selectMoveWinner`), and `applied` therefore counts moves
|
|
620
|
+
* that actually happened.
|
|
808
621
|
*/
|
|
809
622
|
export const applyOrganize = async (
|
|
810
623
|
deps: ApplyOrganizeDeps,
|
|
@@ -812,16 +625,12 @@ export const applyOrganize = async (
|
|
|
812
625
|
messageIds: readonly string[],
|
|
813
626
|
predicate: OrganizePredicate,
|
|
814
627
|
): Promise<ApplyOrganizeResult> => {
|
|
815
|
-
const { client, moveService
|
|
628
|
+
const { client, moveService } = deps;
|
|
816
629
|
const applyLabel =
|
|
817
630
|
predicate.actionLabelId !== NO_ACTION && predicate.actionLabelId !== "";
|
|
818
631
|
const applyMove =
|
|
819
632
|
predicate.actionMailboxId !== NO_ACTION && predicate.actionMailboxId !== "";
|
|
820
633
|
|
|
821
|
-
const movers = applyMove
|
|
822
|
-
? await listCurrentMovers(client, accountConfigId)
|
|
823
|
-
: [];
|
|
824
|
-
|
|
825
634
|
const applyToMessage = async (messageId: string): Promise<void> => {
|
|
826
635
|
if (applyLabel) {
|
|
827
636
|
await client.messageLabel.apply({
|
|
@@ -831,18 +640,6 @@ export const applyOrganize = async (
|
|
|
831
640
|
});
|
|
832
641
|
}
|
|
833
642
|
if (applyMove) {
|
|
834
|
-
const winner = await findCurrentMoveWinner(
|
|
835
|
-
{ client, match },
|
|
836
|
-
movers,
|
|
837
|
-
accountConfigId,
|
|
838
|
-
messageId,
|
|
839
|
-
);
|
|
840
|
-
if (winner && winner.actionMailboxId !== predicate.actionMailboxId) {
|
|
841
|
-
// A more-recently-changed filter currently claims this message's
|
|
842
|
-
// move (RFC 034 Decision 3.2) — defer to it. The label above, if
|
|
843
|
-
// requested, has already applied; only the move is suppressed.
|
|
844
|
-
return;
|
|
845
|
-
}
|
|
846
643
|
if (!moveService) {
|
|
847
644
|
// An exclusive move was requested but this caller wired no move
|
|
848
645
|
// service. Never silently pretend it applied — surface it as a
|