@open-mercato/core 0.8.1-develop.7295.1.d0e0e33014 → 0.8.1-develop.7296.1.2111d779db

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,9 @@
1
1
  import { sql } from "kysely";
2
+ import { recordIndexerError } from "@open-mercato/shared/lib/indexers/error-log";
3
+ async function searchTokensTableExists(db) {
4
+ const row = await db.selectFrom("information_schema.tables").select(sql`1`.as("one")).where("table_name", "=", "search_tokens").limit(1).executeTakeFirst();
5
+ return !!row;
6
+ }
2
7
  async function purgeOrphans(db, options) {
3
8
  const { entityType, tenantId, partitionIndex, partitionCount, startedAt } = options;
4
9
  let q = db.deleteFrom("entity_indexes").where("entity_type", "=", entityType);
@@ -16,6 +21,43 @@ async function purgeOrphans(db, options) {
16
21
  q = q.where("entity_id", "not in", options.excludeRecordIds);
17
22
  }
18
23
  await q.execute();
24
+ try {
25
+ if (!await searchTokensTableExists(db)) return;
26
+ let tokens = db.deleteFrom("search_tokens").where("entity_type", "=", entityType);
27
+ if (tenantId !== void 0) {
28
+ tokens = tenantId === null ? tokens.where("tenant_id", "is", null) : tokens.where("tenant_id", "=", tenantId);
29
+ }
30
+ if (options.organizationId !== void 0) {
31
+ tokens = options.organizationId === null ? tokens.where("organization_id", "is", null) : tokens.where("organization_id", "=", options.organizationId);
32
+ }
33
+ if (partitionIndex != null && partitionCount != null) {
34
+ tokens = tokens.where(sql`mod(abs(hashtext(entity_id::text)), ${partitionCount}) = ${partitionIndex}`);
35
+ }
36
+ tokens = tokens.where("created_at", "<", startedAt);
37
+ if (options.excludeRecordIds?.length) {
38
+ tokens = tokens.where("entity_id", "not in", options.excludeRecordIds);
39
+ }
40
+ tokens = tokens.where(sql`not exists (
41
+ select 1 from entity_indexes as surviving
42
+ where surviving.entity_type = ${entityType}
43
+ and surviving.entity_id = search_tokens.entity_id
44
+ and surviving.tenant_id is not distinct from search_tokens.tenant_id
45
+ and surviving.organization_id is not distinct from search_tokens.organization_id
46
+ )`);
47
+ await tokens.execute();
48
+ } catch (searchTokenError) {
49
+ await recordIndexerError(
50
+ { db },
51
+ {
52
+ source: "fulltext",
53
+ handler: "query_index:purge-orphans",
54
+ error: searchTokenError,
55
+ entityType,
56
+ tenantId: tenantId ?? null,
57
+ organizationId: options.organizationId ?? null
58
+ }
59
+ ).catch(() => void 0);
60
+ }
19
61
  }
20
62
  export {
21
63
  purgeOrphans
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../../../../src/modules/query_index/lib/stale.ts"],
4
- "sourcesContent": ["import { type Kysely, sql } from 'kysely'\n\ntype PurgeOrphansOptions = {\n entityType: string\n tenantId?: string | null\n organizationId?: string | null\n partitionIndex: number | null\n partitionCount: number | null\n startedAt: Date\n /**\n * Records whose index rows must be preserved even though this run did not refresh\n * them. A row that failed to be written still looks untouched to the `updated_at`\n * predicate below, so without this the purge would delete the very entries the run\n * failed to rebuild.\n */\n excludeRecordIds?: string[]\n}\n\nexport async function purgeOrphans(\n db: Kysely<any>,\n options: PurgeOrphansOptions,\n): Promise<void> {\n const { entityType, tenantId, partitionIndex, partitionCount, startedAt } = options\n let q = db.deleteFrom('entity_indexes' as any).where('entity_type' as any, '=', entityType)\n if (tenantId !== undefined) {\n q = q.where(sql<boolean>`tenant_id is not distinct from ${tenantId ?? null}`)\n }\n if (options.organizationId !== undefined) {\n q = q.where(sql<boolean>`organization_id is not distinct from ${options.organizationId ?? null}`)\n }\n if (partitionIndex != null && partitionCount != null) {\n q = q.where(sql<boolean>`mod(abs(hashtext(entity_id::text)), ${partitionCount}) = ${partitionIndex}`)\n }\n q = q.where('updated_at' as any, '<', startedAt as any)\n if (options.excludeRecordIds?.length) {\n q = q.where('entity_id' as any, 'not in', options.excludeRecordIds)\n }\n await q.execute()\n}\n"],
5
- "mappings": "AAAA,SAAsB,WAAW;AAkBjC,eAAsB,aACpB,IACA,SACe;AACf,QAAM,EAAE,YAAY,UAAU,gBAAgB,gBAAgB,UAAU,IAAI;AAC5E,MAAI,IAAI,GAAG,WAAW,gBAAuB,EAAE,MAAM,eAAsB,KAAK,UAAU;AAC1F,MAAI,aAAa,QAAW;AAC1B,QAAI,EAAE,MAAM,qCAA8C,YAAY,IAAI,EAAE;AAAA,EAC9E;AACA,MAAI,QAAQ,mBAAmB,QAAW;AACxC,QAAI,EAAE,MAAM,2CAAoD,QAAQ,kBAAkB,IAAI,EAAE;AAAA,EAClG;AACA,MAAI,kBAAkB,QAAQ,kBAAkB,MAAM;AACpD,QAAI,EAAE,MAAM,0CAAmD,cAAc,OAAO,cAAc,EAAE;AAAA,EACtG;AACA,MAAI,EAAE,MAAM,cAAqB,KAAK,SAAgB;AACtD,MAAI,QAAQ,kBAAkB,QAAQ;AACpC,QAAI,EAAE,MAAM,aAAoB,UAAU,QAAQ,gBAAgB;AAAA,EACpE;AACA,QAAM,EAAE,QAAQ;AAClB;",
4
+ "sourcesContent": ["import { type Kysely, sql } from 'kysely'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\n\ntype PurgeOrphansOptions = {\n entityType: string\n tenantId?: string | null\n organizationId?: string | null\n partitionIndex: number | null\n partitionCount: number | null\n startedAt: Date\n /**\n * Records whose index rows must be preserved even though this run did not refresh\n * them. A row that failed to be written still looks untouched to the `updated_at`\n * predicate below, so without this the purge would delete the very entries the run\n * failed to rebuild.\n */\n excludeRecordIds?: string[]\n}\n\n/**\n * `entity_indexes` ships in an earlier migration than `search_tokens`, so a deployment\n * running this code against a not-yet-migrated database has the projection table and not\n * the token table. The query engine probes the same way before reading the table\n * (`shared/lib/search/availability.ts`).\n */\nasync function searchTokensTableExists(db: Kysely<any>): Promise<boolean> {\n const row = await db\n .selectFrom('information_schema.tables' as any)\n .select(sql<number>`1`.as('one') as any)\n .where('table_name' as any, '=', 'search_tokens')\n .limit(1)\n .executeTakeFirst()\n return !!row\n}\n\nexport async function purgeOrphans(\n db: Kysely<any>,\n options: PurgeOrphansOptions,\n): Promise<void> {\n const { entityType, tenantId, partitionIndex, partitionCount, startedAt } = options\n\n let q = db.deleteFrom('entity_indexes' as any).where('entity_type' as any, '=', entityType)\n if (tenantId !== undefined) {\n q = q.where(sql<boolean>`tenant_id is not distinct from ${tenantId ?? null}`)\n }\n if (options.organizationId !== undefined) {\n q = q.where(sql<boolean>`organization_id is not distinct from ${options.organizationId ?? null}`)\n }\n if (partitionIndex != null && partitionCount != null) {\n q = q.where(sql<boolean>`mod(abs(hashtext(entity_id::text)), ${partitionCount}) = ${partitionIndex}`)\n }\n q = q.where('updated_at' as any, '<', startedAt as any)\n if (options.excludeRecordIds?.length) {\n q = q.where('entity_id' as any, 'not in', options.excludeRecordIds)\n }\n await q.execute()\n\n // The token sweep runs as its own statement, after the projection delete has committed, and\n // deliberately cannot fail the run. Its anti-join only needs those deletes to be *visible*,\n // and a committed delete is as visible as an uncommitted one inside a shared transaction \u2014\n // so wrapping the pair buys no correctness, only a new fatal path: `reindexer.ts` does not\n // guard this call and marks the whole job failed on a throw, so a statement timeout on a\n // large token delete would discard a rebuild whose projections all wrote successfully. The\n // fallback is the pre-fix steady state: orphan tokens survive, benign, until the next run\n // sweeps them. This matches how the module already treats token-side failures \u2014\n // `batch.ts` records a failed token write via `recordIndexerError` rather than counting it\n // as a write failure, and `indexer.ts` swallows the token delete outside a transaction.\n try {\n if (!(await searchTokensTableExists(db))) return\n\n // Search tokens are written from the same (entity_type, entity_id, tenant_id,\n // organization_id) tuple as the projection, but nothing above ever touched them, so\n // every record this run stopped covering kept a queryable token with no presenter \u2014\n // and token-only orphans left by earlier runs survived every rebuild since.\n let tokens = db.deleteFrom('search_tokens' as any).where('entity_type' as any, '=', entityType)\n // `= / is null` rather than the projection sweep's `is not distinct from`: these are bound\n // constants, so the two spellings are semantically identical here, but only the former can\n // serve as a btree index condition. Without it the delete uses just the `entity_type` prefix\n // of `search_tokens_presence_idx (entity_type, tenant_id, organization_id)` and a\n // tenant-scoped, unpartitioned run reads every tenant's tokens for that entity type \u2014\n // the same planner trap `shared/lib/search/availability.ts` documents for this table (#4723).\n // The anti-join below keeps the null-safe form: it compares columns, not constants.\n if (tenantId !== undefined) {\n tokens = tenantId === null\n ? tokens.where('tenant_id' as any, 'is', null)\n : tokens.where('tenant_id' as any, '=', tenantId)\n }\n if (options.organizationId !== undefined) {\n tokens = options.organizationId === null\n ? tokens.where('organization_id' as any, 'is', null)\n : tokens.where('organization_id' as any, '=', options.organizationId)\n }\n if (partitionIndex != null && partitionCount != null) {\n tokens = tokens.where(sql<boolean>`mod(abs(hashtext(entity_id::text)), ${partitionCount}) = ${partitionIndex}`)\n }\n // Secondary guard only. `replaceSearchTokensForBatch` returns early for a record whose\n // tokens are unchanged, so a perfectly current record keeps tokens older than this run\n // \u2014 a timestamp-driven sweep would make live records unfindable. What this predicate\n // does earn is the concurrent-writer case: a batch inserts tokens before the matching\n // projection, so a token written during the run can outrun its own presenter.\n tokens = tokens.where('created_at' as any, '<', startedAt as any)\n if (options.excludeRecordIds?.length) {\n tokens = tokens.where('entity_id' as any, 'not in', options.excludeRecordIds)\n }\n // The load-bearing predicate: a token survives while any projection for its own record\n // and scope survives, soft-deleted ones included. `search_tokens` has a second writer \u2014\n // `@open-mercato/search`'s token strategy writes rows for the same records under its own\n // `field` spellings \u2014 and this sweep is safe for those rows only because that strategy\n // draws its record set from the query engine, i.e. from the very projections consulted\n // here. Changing where that strategy gets its records would make this delete eat its rows.\n tokens = tokens.where(sql<boolean>`not exists (\n select 1 from entity_indexes as surviving\n where surviving.entity_type = ${entityType}\n and surviving.entity_id = search_tokens.entity_id\n and surviving.tenant_id is not distinct from search_tokens.tenant_id\n and surviving.organization_id is not distinct from search_tokens.organization_id\n )`)\n await tokens.execute()\n } catch (searchTokenError) {\n await recordIndexerError(\n { db },\n {\n source: 'fulltext',\n handler: 'query_index:purge-orphans',\n error: searchTokenError,\n entityType,\n tenantId: tenantId ?? null,\n organizationId: options.organizationId ?? null,\n },\n ).catch(() => undefined)\n }\n}\n"],
5
+ "mappings": "AAAA,SAAsB,WAAW;AACjC,SAAS,0BAA0B;AAwBnC,eAAe,wBAAwB,IAAmC;AACxE,QAAM,MAAM,MAAM,GACf,WAAW,2BAAkC,EAC7C,OAAO,OAAe,GAAG,KAAK,CAAQ,EACtC,MAAM,cAAqB,KAAK,eAAe,EAC/C,MAAM,CAAC,EACP,iBAAiB;AACpB,SAAO,CAAC,CAAC;AACX;AAEA,eAAsB,aACpB,IACA,SACe;AACf,QAAM,EAAE,YAAY,UAAU,gBAAgB,gBAAgB,UAAU,IAAI;AAE5E,MAAI,IAAI,GAAG,WAAW,gBAAuB,EAAE,MAAM,eAAsB,KAAK,UAAU;AAC1F,MAAI,aAAa,QAAW;AAC1B,QAAI,EAAE,MAAM,qCAA8C,YAAY,IAAI,EAAE;AAAA,EAC9E;AACA,MAAI,QAAQ,mBAAmB,QAAW;AACxC,QAAI,EAAE,MAAM,2CAAoD,QAAQ,kBAAkB,IAAI,EAAE;AAAA,EAClG;AACA,MAAI,kBAAkB,QAAQ,kBAAkB,MAAM;AACpD,QAAI,EAAE,MAAM,0CAAmD,cAAc,OAAO,cAAc,EAAE;AAAA,EACtG;AACA,MAAI,EAAE,MAAM,cAAqB,KAAK,SAAgB;AACtD,MAAI,QAAQ,kBAAkB,QAAQ;AACpC,QAAI,EAAE,MAAM,aAAoB,UAAU,QAAQ,gBAAgB;AAAA,EACpE;AACA,QAAM,EAAE,QAAQ;AAYhB,MAAI;AACF,QAAI,CAAE,MAAM,wBAAwB,EAAE,EAAI;AAM1C,QAAI,SAAS,GAAG,WAAW,eAAsB,EAAE,MAAM,eAAsB,KAAK,UAAU;AAQ9F,QAAI,aAAa,QAAW;AAC1B,eAAS,aAAa,OAClB,OAAO,MAAM,aAAoB,MAAM,IAAI,IAC3C,OAAO,MAAM,aAAoB,KAAK,QAAQ;AAAA,IACpD;AACA,QAAI,QAAQ,mBAAmB,QAAW;AACxC,eAAS,QAAQ,mBAAmB,OAChC,OAAO,MAAM,mBAA0B,MAAM,IAAI,IACjD,OAAO,MAAM,mBAA0B,KAAK,QAAQ,cAAc;AAAA,IACxE;AACA,QAAI,kBAAkB,QAAQ,kBAAkB,MAAM;AACpD,eAAS,OAAO,MAAM,0CAAmD,cAAc,OAAO,cAAc,EAAE;AAAA,IAChH;AAMA,aAAS,OAAO,MAAM,cAAqB,KAAK,SAAgB;AAChE,QAAI,QAAQ,kBAAkB,QAAQ;AACpC,eAAS,OAAO,MAAM,aAAoB,UAAU,QAAQ,gBAAgB;AAAA,IAC9E;AAOA,aAAS,OAAO,MAAM;AAAA;AAAA,sCAEY,UAAU;AAAA;AAAA;AAAA;AAAA,MAI1C;AACF,UAAM,OAAO,QAAQ;AAAA,EACvB,SAAS,kBAAkB;AACzB,UAAM;AAAA,MACJ,EAAE,GAAG;AAAA,MACL;AAAA,QACE,QAAQ;AAAA,QACR,SAAS;AAAA,QACT,OAAO;AAAA,QACP;AAAA,QACA,UAAU,YAAY;AAAA,QACtB,gBAAgB,QAAQ,kBAAkB;AAAA,MAC5C;AAAA,IACF,EAAE,MAAM,MAAM,MAAS;AAAA,EACzB;AACF;",
6
6
  "names": []
7
7
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@open-mercato/core",
3
- "version": "0.8.1-develop.7295.1.d0e0e33014",
3
+ "version": "0.8.1-develop.7296.1.2111d779db",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -253,16 +253,16 @@
253
253
  "zod": "^4.4.3"
254
254
  },
255
255
  "peerDependencies": {
256
- "@open-mercato/ai-assistant": "0.8.1-develop.7295.1.d0e0e33014",
257
- "@open-mercato/shared": "0.8.1-develop.7295.1.d0e0e33014",
258
- "@open-mercato/ui": "0.8.1-develop.7295.1.d0e0e33014",
256
+ "@open-mercato/ai-assistant": "0.8.1-develop.7296.1.2111d779db",
257
+ "@open-mercato/shared": "0.8.1-develop.7296.1.2111d779db",
258
+ "@open-mercato/ui": "0.8.1-develop.7296.1.2111d779db",
259
259
  "react": "^19.0.0",
260
260
  "react-dom": "^19.0.0"
261
261
  },
262
262
  "devDependencies": {
263
- "@open-mercato/ai-assistant": "0.8.1-develop.7295.1.d0e0e33014",
264
- "@open-mercato/shared": "0.8.1-develop.7295.1.d0e0e33014",
265
- "@open-mercato/ui": "0.8.1-develop.7295.1.d0e0e33014",
263
+ "@open-mercato/ai-assistant": "0.8.1-develop.7296.1.2111d779db",
264
+ "@open-mercato/shared": "0.8.1-develop.7296.1.2111d779db",
265
+ "@open-mercato/ui": "0.8.1-develop.7296.1.2111d779db",
266
266
  "@testing-library/dom": "^10.4.1",
267
267
  "@testing-library/jest-dom": "^7.0.1",
268
268
  "@testing-library/react": "^16.3.3",
@@ -1,4 +1,5 @@
1
1
  import { type Kysely, sql } from 'kysely'
2
+ import { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'
2
3
 
3
4
  type PurgeOrphansOptions = {
4
5
  entityType: string
@@ -16,11 +17,28 @@ type PurgeOrphansOptions = {
16
17
  excludeRecordIds?: string[]
17
18
  }
18
19
 
20
+ /**
21
+ * `entity_indexes` ships in an earlier migration than `search_tokens`, so a deployment
22
+ * running this code against a not-yet-migrated database has the projection table and not
23
+ * the token table. The query engine probes the same way before reading the table
24
+ * (`shared/lib/search/availability.ts`).
25
+ */
26
+ async function searchTokensTableExists(db: Kysely<any>): Promise<boolean> {
27
+ const row = await db
28
+ .selectFrom('information_schema.tables' as any)
29
+ .select(sql<number>`1`.as('one') as any)
30
+ .where('table_name' as any, '=', 'search_tokens')
31
+ .limit(1)
32
+ .executeTakeFirst()
33
+ return !!row
34
+ }
35
+
19
36
  export async function purgeOrphans(
20
37
  db: Kysely<any>,
21
38
  options: PurgeOrphansOptions,
22
39
  ): Promise<void> {
23
40
  const { entityType, tenantId, partitionIndex, partitionCount, startedAt } = options
41
+
24
42
  let q = db.deleteFrom('entity_indexes' as any).where('entity_type' as any, '=', entityType)
25
43
  if (tenantId !== undefined) {
26
44
  q = q.where(sql<boolean>`tenant_id is not distinct from ${tenantId ?? null}`)
@@ -36,4 +54,79 @@ export async function purgeOrphans(
36
54
  q = q.where('entity_id' as any, 'not in', options.excludeRecordIds)
37
55
  }
38
56
  await q.execute()
57
+
58
+ // The token sweep runs as its own statement, after the projection delete has committed, and
59
+ // deliberately cannot fail the run. Its anti-join only needs those deletes to be *visible*,
60
+ // and a committed delete is as visible as an uncommitted one inside a shared transaction —
61
+ // so wrapping the pair buys no correctness, only a new fatal path: `reindexer.ts` does not
62
+ // guard this call and marks the whole job failed on a throw, so a statement timeout on a
63
+ // large token delete would discard a rebuild whose projections all wrote successfully. The
64
+ // fallback is the pre-fix steady state: orphan tokens survive, benign, until the next run
65
+ // sweeps them. This matches how the module already treats token-side failures —
66
+ // `batch.ts` records a failed token write via `recordIndexerError` rather than counting it
67
+ // as a write failure, and `indexer.ts` swallows the token delete outside a transaction.
68
+ try {
69
+ if (!(await searchTokensTableExists(db))) return
70
+
71
+ // Search tokens are written from the same (entity_type, entity_id, tenant_id,
72
+ // organization_id) tuple as the projection, but nothing above ever touched them, so
73
+ // every record this run stopped covering kept a queryable token with no presenter —
74
+ // and token-only orphans left by earlier runs survived every rebuild since.
75
+ let tokens = db.deleteFrom('search_tokens' as any).where('entity_type' as any, '=', entityType)
76
+ // `= / is null` rather than the projection sweep's `is not distinct from`: these are bound
77
+ // constants, so the two spellings are semantically identical here, but only the former can
78
+ // serve as a btree index condition. Without it the delete uses just the `entity_type` prefix
79
+ // of `search_tokens_presence_idx (entity_type, tenant_id, organization_id)` and a
80
+ // tenant-scoped, unpartitioned run reads every tenant's tokens for that entity type —
81
+ // the same planner trap `shared/lib/search/availability.ts` documents for this table (#4723).
82
+ // The anti-join below keeps the null-safe form: it compares columns, not constants.
83
+ if (tenantId !== undefined) {
84
+ tokens = tenantId === null
85
+ ? tokens.where('tenant_id' as any, 'is', null)
86
+ : tokens.where('tenant_id' as any, '=', tenantId)
87
+ }
88
+ if (options.organizationId !== undefined) {
89
+ tokens = options.organizationId === null
90
+ ? tokens.where('organization_id' as any, 'is', null)
91
+ : tokens.where('organization_id' as any, '=', options.organizationId)
92
+ }
93
+ if (partitionIndex != null && partitionCount != null) {
94
+ tokens = tokens.where(sql<boolean>`mod(abs(hashtext(entity_id::text)), ${partitionCount}) = ${partitionIndex}`)
95
+ }
96
+ // Secondary guard only. `replaceSearchTokensForBatch` returns early for a record whose
97
+ // tokens are unchanged, so a perfectly current record keeps tokens older than this run
98
+ // — a timestamp-driven sweep would make live records unfindable. What this predicate
99
+ // does earn is the concurrent-writer case: a batch inserts tokens before the matching
100
+ // projection, so a token written during the run can outrun its own presenter.
101
+ tokens = tokens.where('created_at' as any, '<', startedAt as any)
102
+ if (options.excludeRecordIds?.length) {
103
+ tokens = tokens.where('entity_id' as any, 'not in', options.excludeRecordIds)
104
+ }
105
+ // The load-bearing predicate: a token survives while any projection for its own record
106
+ // and scope survives, soft-deleted ones included. `search_tokens` has a second writer —
107
+ // `@open-mercato/search`'s token strategy writes rows for the same records under its own
108
+ // `field` spellings — and this sweep is safe for those rows only because that strategy
109
+ // draws its record set from the query engine, i.e. from the very projections consulted
110
+ // here. Changing where that strategy gets its records would make this delete eat its rows.
111
+ tokens = tokens.where(sql<boolean>`not exists (
112
+ select 1 from entity_indexes as surviving
113
+ where surviving.entity_type = ${entityType}
114
+ and surviving.entity_id = search_tokens.entity_id
115
+ and surviving.tenant_id is not distinct from search_tokens.tenant_id
116
+ and surviving.organization_id is not distinct from search_tokens.organization_id
117
+ )`)
118
+ await tokens.execute()
119
+ } catch (searchTokenError) {
120
+ await recordIndexerError(
121
+ { db },
122
+ {
123
+ source: 'fulltext',
124
+ handler: 'query_index:purge-orphans',
125
+ error: searchTokenError,
126
+ entityType,
127
+ tenantId: tenantId ?? null,
128
+ organizationId: options.organizationId ?? null,
129
+ },
130
+ ).catch(() => undefined)
131
+ }
39
132
  }