@remit/search-index-worker 0.0.14 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/handler.ts +3 -3
- package/src/services.ts +0 -2
- package/src/reindex.ts +0 -63
- package/src/run-reindex.ts +0 -22
package/package.json
CHANGED
package/src/handler.ts
CHANGED
|
@@ -86,9 +86,9 @@ const deleteMessage = async (
|
|
|
86
86
|
// upsert per message means a record S3 Vectors rejects (e.g. a metadata
|
|
87
87
|
// ValidationException) dead-letters on its own — its siblings in the
|
|
88
88
|
// batch still index instead of retrying forever behind a poison record.
|
|
89
|
-
// Exported so the long-running Postgres consumer (`consumer.ts`)
|
|
90
|
-
//
|
|
91
|
-
//
|
|
89
|
+
// Exported so the long-running Postgres consumer (`consumer.ts`) can
|
|
90
|
+
// process one message at a time outside the Lambda batch shape, reusing
|
|
91
|
+
// this exact logic.
|
|
92
92
|
export const upsertMessage = async (
|
|
93
93
|
message: Extract<ParsedQueueMessage, { kind: "upsert" }>,
|
|
94
94
|
services: Services,
|
package/src/services.ts
CHANGED
|
@@ -73,8 +73,6 @@ export const getServices = async (): Promise<Services> => {
|
|
|
73
73
|
return cached;
|
|
74
74
|
};
|
|
75
75
|
|
|
76
|
-
export const createServices = (overrides: Services): Services => overrides;
|
|
77
|
-
|
|
78
76
|
/** Reset the singleton — test use only. */
|
|
79
77
|
export const _resetForTest = (): void => {
|
|
80
78
|
cached = undefined;
|
package/src/reindex.ts
DELETED
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
import type { Logger } from "@remit/logger-lambda";
|
|
2
|
-
import pMap from "p-map";
|
|
3
|
-
import type { Pool } from "pg";
|
|
4
|
-
import { type IndexOutcome, upsertMessage } from "./handler.js";
|
|
5
|
-
import type { Services } from "./services.js";
|
|
6
|
-
|
|
7
|
-
export interface ReindexResult {
|
|
8
|
-
total: number;
|
|
9
|
-
indexed: number;
|
|
10
|
-
skipped: number;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
const NOT_APPLICABLE_ACCOUNT_ID = "reindex-all";
|
|
14
|
-
|
|
15
|
-
/**
|
|
16
|
-
* Re-embed every body-synced Postgres message. Concurrent (pMap) and
|
|
17
|
-
* force-upserting so a model change or a repair repopulates the whole store;
|
|
18
|
-
* keys-only scan keeps the hot path off `describe()`. Postgres-only — the
|
|
19
|
-
* `accountId` on the synthetic upsert message is a placeholder: `services`
|
|
20
|
-
* must carry `resolveAccountId` (true whenever `DATA_BACKEND=postgres`; see
|
|
21
|
-
* `data-ports.ts`), which derives the real one from each message's mailbox.
|
|
22
|
-
*/
|
|
23
|
-
export const reindexAll = async (
|
|
24
|
-
pool: Pool,
|
|
25
|
-
services: Services,
|
|
26
|
-
log: Logger,
|
|
27
|
-
options?: { concurrency?: number },
|
|
28
|
-
): Promise<ReindexResult> => {
|
|
29
|
-
const rows = await pool.query<{ message_id: string }>(
|
|
30
|
-
"SELECT message_id FROM message WHERE body_storage_key IS NOT NULL",
|
|
31
|
-
);
|
|
32
|
-
const messageIds = rows.rows.map((row) => row.message_id);
|
|
33
|
-
|
|
34
|
-
let indexed = 0;
|
|
35
|
-
let skipped = 0;
|
|
36
|
-
await pMap(
|
|
37
|
-
messageIds,
|
|
38
|
-
async (messageId) => {
|
|
39
|
-
let outcome: IndexOutcome | undefined;
|
|
40
|
-
const taskServices: Services = {
|
|
41
|
-
...services,
|
|
42
|
-
onIndexOutcome: (o) => {
|
|
43
|
-
outcome = o;
|
|
44
|
-
},
|
|
45
|
-
};
|
|
46
|
-
await upsertMessage(
|
|
47
|
-
{
|
|
48
|
-
kind: "upsert",
|
|
49
|
-
accountId: NOT_APPLICABLE_ACCOUNT_ID,
|
|
50
|
-
messageId,
|
|
51
|
-
force: true,
|
|
52
|
-
},
|
|
53
|
-
taskServices,
|
|
54
|
-
log,
|
|
55
|
-
);
|
|
56
|
-
if (outcome?.status === "indexed") indexed += 1;
|
|
57
|
-
else skipped += 1;
|
|
58
|
-
},
|
|
59
|
-
{ concurrency: options?.concurrency ?? 8 },
|
|
60
|
-
);
|
|
61
|
-
|
|
62
|
-
return { total: messageIds.length, indexed, skipped };
|
|
63
|
-
};
|
package/src/run-reindex.ts
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { createLogger } from "@remit/logger-lambda";
|
|
2
|
-
import pg from "pg";
|
|
3
|
-
import { reindexAll } from "./reindex.js";
|
|
4
|
-
import { getServices } from "./services.js";
|
|
5
|
-
|
|
6
|
-
const log = createLogger();
|
|
7
|
-
|
|
8
|
-
const main = async (): Promise<void> => {
|
|
9
|
-
const connectionString = process.env.PG_CONNECTION_URL;
|
|
10
|
-
if (!connectionString) throw new Error("PG_CONNECTION_URL is required");
|
|
11
|
-
|
|
12
|
-
const pool = new pg.Pool({ connectionString });
|
|
13
|
-
const services = await getServices();
|
|
14
|
-
|
|
15
|
-
const result = await reindexAll(pool, services, log);
|
|
16
|
-
log.info("reindex complete", { ...result });
|
|
17
|
-
|
|
18
|
-
await pool.end();
|
|
19
|
-
process.exit(0);
|
|
20
|
-
};
|
|
21
|
-
|
|
22
|
-
await main();
|