@open-mercato/core 0.10.1-develop.7776.1.3032ea0741 → 0.10.1-develop.7781.1.079f42c863
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/modules/catalog/components/products/ProductsDataTable.js +6 -5
- package/dist/modules/catalog/components/products/ProductsDataTable.js.map +2 -2
- package/dist/modules/customers/backend/customers/companies/page.js +5 -4
- package/dist/modules/customers/backend/customers/companies/page.js.map +2 -2
- package/dist/modules/customers/backend/customers/deals/page.js +6 -5
- package/dist/modules/customers/backend/customers/deals/page.js.map +2 -2
- package/dist/modules/customers/backend/customers/people/page.js +5 -4
- package/dist/modules/customers/backend/customers/people/page.js.map +2 -2
- package/dist/modules/query_index/cli.js +9 -1
- package/dist/modules/query_index/cli.js.map +2 -2
- package/dist/modules/query_index/lib/coverage.js +13 -2
- package/dist/modules/query_index/lib/coverage.js.map +2 -2
- package/dist/modules/query_index/lib/engine.js +28 -9
- package/dist/modules/query_index/lib/engine.js.map +2 -2
- package/dist/modules/query_index/lib/indexer.js +5 -1
- package/dist/modules/query_index/lib/indexer.js.map +2 -2
- package/dist/modules/query_index/lib/reindexer.js +95 -32
- package/dist/modules/query_index/lib/reindexer.js.map +2 -2
- package/dist/modules/query_index/lib/search-scope.js +101 -0
- package/dist/modules/query_index/lib/search-scope.js.map +7 -0
- package/dist/modules/query_index/lib/stale.js +41 -1
- package/dist/modules/query_index/lib/stale.js.map +2 -2
- package/dist/modules/query_index/lib/subscriber-scope.js +30 -7
- package/dist/modules/query_index/lib/subscriber-scope.js.map +3 -3
- package/dist/modules/query_index/lib/tenant-global.js +41 -0
- package/dist/modules/query_index/lib/tenant-global.js.map +7 -0
- package/dist/modules/query_index/subscribers/reindex.js +9 -2
- package/dist/modules/query_index/subscribers/reindex.js.map +2 -2
- package/dist/modules/sales/components/documents/ItemsSection.js +4 -3
- package/dist/modules/sales/components/documents/ItemsSection.js.map +2 -2
- package/package.json +7 -7
- package/src/modules/catalog/components/products/ProductsDataTable.tsx +6 -5
- package/src/modules/customers/backend/customers/companies/page.tsx +5 -4
- package/src/modules/customers/backend/customers/deals/page.tsx +6 -5
- package/src/modules/customers/backend/customers/people/page.tsx +5 -4
- package/src/modules/query_index/cli.ts +19 -1
- package/src/modules/query_index/lib/coverage.ts +29 -3
- package/src/modules/query_index/lib/engine.ts +34 -9
- package/src/modules/query_index/lib/indexer.ts +6 -1
- package/src/modules/query_index/lib/reindexer.ts +205 -36
- package/src/modules/query_index/lib/search-scope.ts +144 -0
- package/src/modules/query_index/lib/stale.ts +73 -0
- package/src/modules/query_index/lib/subscriber-scope.ts +38 -8
- package/src/modules/query_index/lib/tenant-global.ts +93 -0
- package/src/modules/query_index/subscribers/reindex.ts +24 -1
- package/src/modules/sales/components/documents/ItemsSection.tsx +6 -3
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { resolveQueryIndexSourceTableMetadata } from './subscriber-scope'
|
|
1
2
|
import type { QueryEngine, QueryOptions, QueryResult, QueryResultMeta, EncryptedSortRowCapWarning, ListCountCapWarning, FilterOp, Filter, QueryCustomFieldSource, PartialIndexWarning, QueryExtensionsConfig, Sort } from '@open-mercato/shared/lib/query/types'
|
|
2
3
|
import { SortDir } from '@open-mercato/shared/lib/query/types'
|
|
3
4
|
import type { EntityId } from '@open-mercato/shared/modules/entities'
|
|
4
5
|
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
5
|
-
import { BasicQueryEngine, resolveEntityTableName, resolveRegisteredEntityTableName } from '@open-mercato/shared/lib/query/engine'
|
|
6
|
+
import { BasicQueryEngine, resolveEntityTableName as resolveFallbackEntityTableName, resolveRegisteredEntityTableName } from '@open-mercato/shared/lib/query/engine'
|
|
6
7
|
import { isOrmBackedSystemEntityId } from '@open-mercato/shared/lib/data/engine'
|
|
7
8
|
import {
|
|
8
9
|
buildArrayColumnNoOverlapPredicate,
|
|
@@ -72,6 +73,10 @@ import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
|
72
73
|
import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
|
|
73
74
|
import { createBoundedTtlMemo } from '@open-mercato/shared/lib/query/bounded-ttl-memo'
|
|
74
75
|
|
|
76
|
+
function resolveEntityTableName(em: EntityManager, entity: string): string {
|
|
77
|
+
return resolveQueryIndexSourceTableMetadata(em, entity)?.table ?? resolveFallbackEntityTableName(em, entity)
|
|
78
|
+
}
|
|
79
|
+
|
|
75
80
|
const logger = createLogger('query_index').child({ component: 'engine' })
|
|
76
81
|
|
|
77
82
|
function customDocumentValue(row: Record<string, unknown>, field: string): unknown {
|
|
@@ -488,14 +493,14 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
488
493
|
const baseExists = await profiler.measure('base_table_exists', () => this.tableExists(baseTable))
|
|
489
494
|
if (!baseExists) {
|
|
490
495
|
if (debugEnabled) this.debug('query:fallback:missing-base', { entity, baseTable })
|
|
491
|
-
const fallbackResult = await this.
|
|
496
|
+
const fallbackResult = await this.queryFallback<T>(entity, opts)
|
|
492
497
|
finishProfile({ result: 'fallback', reason: 'missing_base' })
|
|
493
498
|
return await applyAfterExtensions(fallbackResult)
|
|
494
499
|
}
|
|
495
500
|
|
|
496
501
|
if (opts.omitAutomaticTenantOrgScope === true) {
|
|
497
502
|
if (debugEnabled) this.debug('query:fallback:omit-automatic-scope', { entity })
|
|
498
|
-
const fallbackResult = await this.
|
|
503
|
+
const fallbackResult = await this.queryFallback<T>(entity, opts)
|
|
499
504
|
finishProfile({ result: 'fallback', reason: 'omit_automatic_tenant_org_scope' })
|
|
500
505
|
return await applyAfterExtensions(fallbackResult)
|
|
501
506
|
}
|
|
@@ -539,7 +544,7 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
539
544
|
)
|
|
540
545
|
if (!hasIndexRows) {
|
|
541
546
|
if (debugEnabled) this.debug('query:fallback:no-index', { entity })
|
|
542
|
-
const fallbackResult = await this.
|
|
547
|
+
const fallbackResult = await this.queryFallback<T>(entity, opts)
|
|
543
548
|
finishProfile({ result: 'fallback', reason: 'no_index_rows' })
|
|
544
549
|
return await applyAfterExtensions(fallbackResult)
|
|
545
550
|
}
|
|
@@ -568,7 +573,7 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
568
573
|
logger.warn('Partial index coverage detected; falling back to basic engine', { entity })
|
|
569
574
|
if (debugEnabled) this.debug('query:fallback:partial-coverage', { entity })
|
|
570
575
|
}
|
|
571
|
-
const fallbackResult = await this.
|
|
576
|
+
const fallbackResult = await this.queryFallback<T>(entity, opts)
|
|
572
577
|
const resultWithWarning: QueryResult<T> = {
|
|
573
578
|
...fallbackResult,
|
|
574
579
|
meta: {
|
|
@@ -654,6 +659,11 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
654
659
|
}
|
|
655
660
|
}
|
|
656
661
|
|
|
662
|
+
const tenantlessIndexSources = new Set<string>()
|
|
663
|
+
for (const source of preparedCfSources) {
|
|
664
|
+
if (!source.tenantField && !(await this.columnExists(source.table, 'tenant_id'))) tenantlessIndexSources.add(source.indexAlias)
|
|
665
|
+
}
|
|
666
|
+
|
|
657
667
|
const searchSources: SearchTokenSource[] = indexSources
|
|
658
668
|
.map((src) => ({ entity: String(src.entityId), recordIdColumn: src.recordIdColumn }))
|
|
659
669
|
.filter((src) => src.recordIdColumn && src.entity)
|
|
@@ -963,6 +973,8 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
963
973
|
jc = jc
|
|
964
974
|
.onRef('ei.tenant_id', '=', qualify('tenant_id'))
|
|
965
975
|
.on('ei.tenant_id', 'is not', null)
|
|
976
|
+
} else {
|
|
977
|
+
jc = jc.on('ei.tenant_id', 'is', null)
|
|
966
978
|
}
|
|
967
979
|
if (!opts.withDeleted) {
|
|
968
980
|
jc = jc.on('ei.deleted_at', 'is', null)
|
|
@@ -995,7 +1007,9 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
995
1007
|
const tenantRef = source.tenantField
|
|
996
1008
|
? `${source.alias}.${source.tenantField}`
|
|
997
1009
|
: (columns.has('tenant_id') ? qualify('tenant_id') : null)
|
|
998
|
-
if (
|
|
1010
|
+
if (tenantlessIndexSources.has(source.indexAlias)) {
|
|
1011
|
+
jc = jc.on(`${source.indexAlias}.tenant_id`, 'is', null)
|
|
1012
|
+
} else if (tenantRef) {
|
|
999
1013
|
jc = jc
|
|
1000
1014
|
.onRef(`${source.indexAlias}.tenant_id`, '=', tenantRef)
|
|
1001
1015
|
.on(`${source.indexAlias}.tenant_id`, 'is not', null)
|
|
@@ -2572,13 +2586,22 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
2572
2586
|
return result
|
|
2573
2587
|
}
|
|
2574
2588
|
|
|
2589
|
+
private async queryFallback<T = unknown>(entity: EntityId, opts: QueryOptions): Promise<QueryResult<T>> {
|
|
2590
|
+
const source = resolveQueryIndexSourceTableMetadata(this.em, entity)
|
|
2591
|
+
if (source?.schema || source?.table.includes('.')) {
|
|
2592
|
+
throw new Error('[internal] Query fallback cannot preserve a schema-qualified source identity')
|
|
2593
|
+
}
|
|
2594
|
+
return this.fallback.query<T>(entity, opts)
|
|
2595
|
+
}
|
|
2596
|
+
|
|
2575
2597
|
private async tableExists(table: string): Promise<boolean> {
|
|
2576
2598
|
if (baseTableExistsCache.get(table)) return true
|
|
2577
2599
|
const db = this.getDb() as any
|
|
2578
2600
|
const exists = await db
|
|
2579
2601
|
.selectFrom('information_schema.tables')
|
|
2580
2602
|
.select(sql<number>`1`.as('one'))
|
|
2581
|
-
.where('
|
|
2603
|
+
.where(table.includes('.') ? sql<boolean>`table_schema = ${table.split('.')[0]}` : sql<boolean>`table_schema = current_schema()`)
|
|
2604
|
+
.where('table_name', '=', table.split('.').pop())
|
|
2582
2605
|
.executeTakeFirst()
|
|
2583
2606
|
const present = !!exists
|
|
2584
2607
|
if (present) baseTableExistsCache.set(table, true)
|
|
@@ -2949,7 +2972,8 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
2949
2972
|
const exists = await db
|
|
2950
2973
|
.selectFrom('information_schema.columns')
|
|
2951
2974
|
.select(sql<number>`1`.as('one'))
|
|
2952
|
-
.where('
|
|
2975
|
+
.where(table.includes('.') ? sql<boolean>`table_schema = ${table.split('.')[0]}` : sql<boolean>`table_schema = current_schema()`)
|
|
2976
|
+
.where('table_name', '=', table.split('.').pop())
|
|
2953
2977
|
.where('column_name', '=', column)
|
|
2954
2978
|
.executeTakeFirst()
|
|
2955
2979
|
const present = !!exists
|
|
@@ -2964,7 +2988,8 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
2964
2988
|
const rows = await db
|
|
2965
2989
|
.selectFrom('information_schema.columns')
|
|
2966
2990
|
.select(['column_name', 'data_type'])
|
|
2967
|
-
.where('
|
|
2991
|
+
.where(table.includes('.') ? sql<boolean>`table_schema = ${table.split('.')[0]}` : sql<boolean>`table_schema = current_schema()`)
|
|
2992
|
+
.where('table_name', '=', table.split('.').pop())
|
|
2968
2993
|
.execute() as Array<{ column_name: string; data_type: string }>
|
|
2969
2994
|
const map = new Map<string, string>()
|
|
2970
2995
|
for (const r of rows) map.set(r.column_name, r.data_type)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { resolveQueryIndexSourceTableMetadata } from './subscriber-scope'
|
|
1
2
|
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
2
|
-
import { resolveEntityTableName } from '@open-mercato/shared/lib/query/engine'
|
|
3
|
+
import { resolveEntityTableName as resolveFallbackEntityTableName } from '@open-mercato/shared/lib/query/engine'
|
|
3
4
|
import { resolveTenantEncryptionService } from '@open-mercato/shared/lib/encryption/customFieldValues'
|
|
4
5
|
import { decryptIndexDocForSearch, encryptIndexDocForStorage } from '@open-mercato/shared/lib/encryption/indexDoc'
|
|
5
6
|
import {
|
|
@@ -15,6 +16,10 @@ import { replaceSearchTokensForRecord, deleteSearchTokensForRecord } from './sea
|
|
|
15
16
|
import { attachAggregateSearchField, rebuildAggregateSearchField } from './document'
|
|
16
17
|
import { applyIndexDocEnrichers } from './doc-enrichers'
|
|
17
18
|
|
|
19
|
+
function resolveEntityTableName(em: EntityManager, entity: string): string {
|
|
20
|
+
return resolveQueryIndexSourceTableMetadata(em, entity)?.table ?? resolveFallbackEntityTableName(em, entity)
|
|
21
|
+
}
|
|
22
|
+
|
|
18
23
|
const logger = createLogger('query_index').child({ component: 'indexer' })
|
|
19
24
|
|
|
20
25
|
type QueryIndexExecutor = Kysely<any> | Transaction<any>
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
2
2
|
import { type Kysely, sql } from 'kysely'
|
|
3
|
-
import {
|
|
3
|
+
import { resolveQueryIndexSourceTableMetadata } from './subscriber-scope'
|
|
4
4
|
import { resolveTenantEncryptionService } from '@open-mercato/shared/lib/encryption/customFieldValues'
|
|
5
5
|
import { decryptIndexDocForSearch, encryptIndexDocForStorage } from '@open-mercato/shared/lib/encryption/indexDoc'
|
|
6
6
|
import {
|
|
@@ -11,12 +11,19 @@ import {
|
|
|
11
11
|
QueryIndexBatchWriteError,
|
|
12
12
|
type AnyRow,
|
|
13
13
|
} from './batch'
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
refreshCoverageSnapshot,
|
|
16
|
+
writeCoverageCounts,
|
|
17
|
+
applyCoverageAdjustments,
|
|
18
|
+
deleteScopedCoverageForEntity,
|
|
19
|
+
} from './coverage'
|
|
15
20
|
import { prepareJob, updateJobProgress, finalizeJob, type JobScope } from './jobs'
|
|
16
|
-
import { purgeOrphans } from './stale'
|
|
21
|
+
import { purgeOrphans, retireScopedCatalogueRows } from './stale'
|
|
17
22
|
import type { VectorIndexService } from '@open-mercato/search/vector'
|
|
18
23
|
import { isSearchDebugEnabled } from './search-tokens'
|
|
24
|
+
import { isTenantGlobalEntityType } from './tenant-global'
|
|
19
25
|
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
26
|
+
import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
|
|
20
27
|
|
|
21
28
|
const logger = createLogger('query_index').child({ component: 'reindexer' })
|
|
22
29
|
|
|
@@ -37,11 +44,34 @@ export type ReindexJobOptions = {
|
|
|
37
44
|
vectorService?: VectorIndexService | null
|
|
38
45
|
}
|
|
39
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Why a sweep did no work, when it did none on purpose.
|
|
49
|
+
*
|
|
50
|
+
* A refused run is otherwise byte-identical to a successful sweep that found zero rows,
|
|
51
|
+
* and the surfaces an operator reads - `indexer_status_logs`, the indexer status page,
|
|
52
|
+
* the CLI's `0 / 0 (0.00%)` - report the second. The allowlist's safety argument is that
|
|
53
|
+
* a missing `registerTenantGlobalEntityTypes()` declaration "shows up in the log", so the
|
|
54
|
+
* log the product surfaces has to be able to say it.
|
|
55
|
+
*/
|
|
56
|
+
export type ReindexRefusalReason =
|
|
57
|
+
/** The source table has no `tenant_id` column and the type is not a declared catalogue. */
|
|
58
|
+
| 'no-tenant-column'
|
|
59
|
+
/** `information_schema` could not be read, so no column could be proven to exist. */
|
|
60
|
+
| 'column-probe-failed'
|
|
61
|
+
/**
|
|
62
|
+
* The type IS declared tenant-global, but its table carries `organization_id` while
|
|
63
|
+
* lacking `tenant_id`. A global projection means both scope columns absent - the same
|
|
64
|
+
* definition `loadQueryIndexRowScope()` uses - so the declaration cannot be honoured.
|
|
65
|
+
*/
|
|
66
|
+
| 'declared-global-but-org-scoped'
|
|
67
|
+
|
|
40
68
|
export type ReindexJobResult = {
|
|
41
69
|
processed: number
|
|
42
70
|
total: number
|
|
43
71
|
tenantScopes: Array<string | null>
|
|
44
72
|
scopes: Array<{ tenantId: string | null; organizationId: string | null }>
|
|
73
|
+
/** Absent on a sweep that ran. Present, with the reason, on one that deliberately did not. */
|
|
74
|
+
refused?: ReindexRefusalReason
|
|
45
75
|
}
|
|
46
76
|
|
|
47
77
|
export const DEFAULT_REINDEX_PARTITIONS = 5
|
|
@@ -110,17 +140,50 @@ function toNumber(value: unknown): number {
|
|
|
110
140
|
return 0
|
|
111
141
|
}
|
|
112
142
|
|
|
113
|
-
|
|
143
|
+
/**
|
|
144
|
+
* Returns `null` - never an empty set - when the probe itself fails.
|
|
145
|
+
*
|
|
146
|
+
* An empty set is indistinguishable from "this table has no columns we care about", so a
|
|
147
|
+
* transient failure (an aborted transaction on the same connection, a statement timeout, a
|
|
148
|
+
* blip during a partition-parallel sweep) would make every entity type look tenant-less and
|
|
149
|
+
* refuse every tenant-scoped reindex, reported as a successful zero-row run. Failing closed
|
|
150
|
+
* is right; failing closed silently and globally is the hardest state to diagnose.
|
|
151
|
+
*/
|
|
152
|
+
async function getColumnSet(db: Kysely<any>, tableName: string, schema?: string): Promise<Set<string> | null> {
|
|
114
153
|
try {
|
|
115
154
|
const rows = await db
|
|
116
155
|
.selectFrom('information_schema.columns' as any)
|
|
117
156
|
.select(['column_name' as any])
|
|
118
|
-
.where(sql<boolean>`table_schema = current_schema()`)
|
|
157
|
+
.where(schema ? sql<boolean>`table_schema = ${schema}` : sql<boolean>`table_schema = current_schema()`)
|
|
119
158
|
.where('table_name' as any, '=', tableName)
|
|
120
159
|
.execute() as Array<{ column_name: string }>
|
|
121
160
|
return new Set(rows.map((row) => String(row.column_name).toLowerCase()))
|
|
122
|
-
} catch {
|
|
123
|
-
|
|
161
|
+
} catch (error) {
|
|
162
|
+
logger.warn('Failed to read information_schema.columns for a reindex target', {
|
|
163
|
+
table: tableName,
|
|
164
|
+
error: error instanceof Error ? error.message : String(error),
|
|
165
|
+
})
|
|
166
|
+
reportColumnProbeFailure(error, tableName)
|
|
167
|
+
return null
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* The probe's caller refuses the sweep whatever happens here, so a telemetry backend that
|
|
173
|
+
* throws must not turn that refusal into a failed job.
|
|
174
|
+
*/
|
|
175
|
+
function reportColumnProbeFailure(error: unknown, tableName: string): void {
|
|
176
|
+
try {
|
|
177
|
+
getTelemetryRuntime()?.reportError(error, {
|
|
178
|
+
module: 'query_index',
|
|
179
|
+
code: 'query_index.column_probe_failed',
|
|
180
|
+
attributes: { table: tableName },
|
|
181
|
+
})
|
|
182
|
+
} catch (telemetryError) {
|
|
183
|
+
logger.warn('Failed to report a column probe failure', {
|
|
184
|
+
table: tableName,
|
|
185
|
+
error: telemetryError instanceof Error ? telemetryError.message : String(telemetryError),
|
|
186
|
+
})
|
|
124
187
|
}
|
|
125
188
|
}
|
|
126
189
|
|
|
@@ -166,7 +229,8 @@ export async function reindexEntity(
|
|
|
166
229
|
// here: doing so would let a principal with `query_index.reindex` point the
|
|
167
230
|
// reindexer at arbitrary tables (e.g. `auth_users`, `users`) and read their
|
|
168
231
|
// rows into the index, bypassing tenant scoping and entity-level encryption.
|
|
169
|
-
const
|
|
232
|
+
const source = resolveQueryIndexSourceTableMetadata(em, entityType)
|
|
233
|
+
const table = source?.table
|
|
170
234
|
if (!table || entityType === 'query_index:search_token' || table === 'search_tokens') {
|
|
171
235
|
if (!table) {
|
|
172
236
|
logger.warn('Refusing to reindex unregistered entity type', {
|
|
@@ -180,11 +244,103 @@ export async function reindexEntity(
|
|
|
180
244
|
scopes: [],
|
|
181
245
|
}
|
|
182
246
|
}
|
|
183
|
-
const columns = await getColumnSet(db, table)
|
|
247
|
+
const columns = await getColumnSet(db, source?.tableName ?? table, source?.schema)
|
|
248
|
+
// A probe failure proves nothing about the table, so it cannot be read as "no tenant
|
|
249
|
+
// column". Refuse with its own reason rather than letting a database blip masquerade as
|
|
250
|
+
// a permanent classification - and refuse rather than proceed, because proceeding would
|
|
251
|
+
// drop the tenant predicate and stamp the caller's tenant, which is the defect this guard
|
|
252
|
+
// exists for.
|
|
253
|
+
if (columns === null) {
|
|
254
|
+
logger.warn("Refusing reindex: could not read the source table's columns", {
|
|
255
|
+
entityType,
|
|
256
|
+
table,
|
|
257
|
+
tenantId,
|
|
258
|
+
})
|
|
259
|
+
return { processed: 0, total: 0, tenantScopes: [], scopes: [], refused: 'column-probe-failed' }
|
|
260
|
+
}
|
|
184
261
|
const hasOrgCol = columns.has('organization_id')
|
|
185
262
|
const hasTenantCol = columns.has('tenant_id')
|
|
186
263
|
const hasDeletedCol = columns.has('deleted_at')
|
|
187
264
|
|
|
265
|
+
// `applyBaseWhere()` below can only apply the tenant predicate when the source
|
|
266
|
+
// table HAS a `tenant_id` column; when it does not, it silently drops the
|
|
267
|
+
// predicate the caller asked for. `scopeOverrides.tenantId` further down then
|
|
268
|
+
// stamps every swept row with the caller's tenant anyway. A tenant-scoped
|
|
269
|
+
// reindex of a tenant-less table therefore does two wrong things at once: it
|
|
270
|
+
// reads every tenant's rows, and it files them under whichever tenant happened
|
|
271
|
+
// to run it. Nothing downstream corrects that, because both readers ask for an
|
|
272
|
+
// exact tenant match with no NULL branch, so the rows become searchable and
|
|
273
|
+
// readable as that tenant's own.
|
|
274
|
+
//
|
|
275
|
+
// The condition is derived, not enumerated: it is exactly the set of inputs
|
|
276
|
+
// where a predicate is dropped AND an override is stamped. A `tenantId` of
|
|
277
|
+
// `undefined` or `null` sets no override, so those rows land under
|
|
278
|
+
// `tenant_id = NULL` — invisible to both readers, but filed under nobody — and
|
|
279
|
+
// are deliberately left alone here.
|
|
280
|
+
//
|
|
281
|
+
// Indexing every tenant-less table under NULL was considered as the general fix
|
|
282
|
+
// and rejected: an unqualified NULL branch in the readers overloads a value that
|
|
283
|
+
// also means "written by an unscoped reindex", so every mis-scoped row would
|
|
284
|
+
// become globally visible — fail-open, in a fix whose whole value is failing
|
|
285
|
+
// closed. `isTenantGlobalEntityType` is the narrow allowlist for the catalogue
|
|
286
|
+
// case instead; see its comment in ./tenant-global. The readers widen for exactly
|
|
287
|
+
// that declared set and for nothing else, and the projection of a declared type is
|
|
288
|
+
// written under the null tenant just below.
|
|
289
|
+
// A declared platform-wide catalogue has one row per record for every tenant to
|
|
290
|
+
// share, so its projection belongs to no tenant and no organization in particular and
|
|
291
|
+
// is written under `tenant_id = NULL, organization_id = NULL`. That is already where
|
|
292
|
+
// the incremental path files these rows: `resolveQueryIndexRecordScope()` resolves a
|
|
293
|
+
// source table with NEITHER scope column to `kind: 'global'` and requires an explicitly
|
|
294
|
+
// null tenant AND organization. Before this, the sweep disagreed with it and stamped the
|
|
295
|
+
// caller's scope, and since `entity_indexes` is unique on
|
|
296
|
+
// (entity_type, entity_id, organization_id_coalesced) the last caller to reindex simply
|
|
297
|
+
// took the catalogue from the previous one. Both readers in `@open-mercato/search` now
|
|
298
|
+
// carry the matching NULL branch for declared types.
|
|
299
|
+
//
|
|
300
|
+
// The condition matches `loadQueryIndexRowScope()`'s definition of global exactly -
|
|
301
|
+
// NEITHER column, not just no tenant column. Keying on the tenant column alone would let
|
|
302
|
+
// a third-party declaration of a table that HAS `organization_id` take the global branch
|
|
303
|
+
// for a record the event path never classifies as global, so the two writers would
|
|
304
|
+
// disagree again on a different axis.
|
|
305
|
+
//
|
|
306
|
+
// Only the PROJECTION scope moves. `jobScope` keeps the caller's tenant and organization:
|
|
307
|
+
// it records who ran the sweep, so two tenants reindexing the catalogue remain two jobs
|
|
308
|
+
// rather than one that blocks the other behind the active-job guard.
|
|
309
|
+
const writesGlobalProjection =
|
|
310
|
+
columns.has('id') && !hasTenantCol && !hasOrgCol && !source?.tenantColumn && !source?.organizationColumn && isTenantGlobalEntityType(entityType)
|
|
311
|
+
const projectionTenantId = writesGlobalProjection ? null : tenantId
|
|
312
|
+
const projectionOrganizationId = writesGlobalProjection ? null : organizationId
|
|
313
|
+
// Vector work keeps the caller's scope even where the projection moves. Its consumers -
|
|
314
|
+
// the `query_index.vectorize_one` and `query_index.vectorize_purge` subscribers and
|
|
315
|
+
// `removeOrphans` - need a real tenant, and nothing defines a shared vector across
|
|
316
|
+
// dispatch, storage and reads, so a declared catalogue's vectors stay filed per tenant
|
|
317
|
+
// exactly as they were before its projection moved to the null scope.
|
|
318
|
+
const vectorTenantId = writesGlobalProjection ? tenantId : projectionTenantId
|
|
319
|
+
const vectorOrganizationId = writesGlobalProjection ? organizationId : projectionOrganizationId
|
|
320
|
+
|
|
321
|
+
// Refuse anything tenant-less that is not writing a global projection. That is the
|
|
322
|
+
// no-tenant-column case, and also a declaration this module cannot honour: a declared
|
|
323
|
+
// type whose table carries `organization_id` but no `tenant_id`. Letting the second
|
|
324
|
+
// through would silently drop the tenant predicate and stamp the caller's tenant - the
|
|
325
|
+
// exact defect the guard exists for - while the author believes the declaration took
|
|
326
|
+
// effect. Fail closed and name it, in the spirit of the allowlist itself.
|
|
327
|
+
if (!hasTenantCol && tenantId != null && !writesGlobalProjection) {
|
|
328
|
+
const declaredButOrgScoped = isTenantGlobalEntityType(entityType)
|
|
329
|
+
logger.warn(
|
|
330
|
+
declaredButOrgScoped
|
|
331
|
+
? 'Refusing tenant-scoped reindex: entity type is declared tenant-global but its table has an organization_id column'
|
|
332
|
+
: 'Refusing tenant-scoped reindex of a table with no tenant_id column',
|
|
333
|
+
{ entityType, table, tenantId },
|
|
334
|
+
)
|
|
335
|
+
return {
|
|
336
|
+
processed: 0,
|
|
337
|
+
total: 0,
|
|
338
|
+
tenantScopes: [],
|
|
339
|
+
scopes: [],
|
|
340
|
+
refused: declaredButOrgScoped ? 'declared-global-but-org-scoped' : 'no-tenant-column',
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
188
344
|
const jobScope: JobScope = {
|
|
189
345
|
entityType,
|
|
190
346
|
organizationId: organizationId ?? null,
|
|
@@ -268,18 +424,18 @@ export async function reindexEntity(
|
|
|
268
424
|
for (const row of rows) {
|
|
269
425
|
const bucketTenant = groupByTenant
|
|
270
426
|
? ((row as any)?.tenant_id ?? null)
|
|
271
|
-
: (
|
|
427
|
+
: (projectionTenantId === undefined ? null : projectionTenantId ?? null)
|
|
272
428
|
const bucketOrg = groupByOrg
|
|
273
429
|
? ((row as any)?.organization_id ?? null)
|
|
274
|
-
: (
|
|
430
|
+
: (projectionOrganizationId === undefined ? null : projectionOrganizationId ?? null)
|
|
275
431
|
registerBaseCount(bucketTenant, bucketOrg, toNumber((row as any)?.count))
|
|
276
432
|
}
|
|
277
433
|
} else {
|
|
278
434
|
const row = await applyBaseWhere(
|
|
279
435
|
db.selectFrom(`${table} as b` as any).select(sql<number>`count(*)`.as('count')),
|
|
280
436
|
).executeTakeFirst() as { count: unknown } | undefined
|
|
281
|
-
const bucketTenant =
|
|
282
|
-
const bucketOrg =
|
|
437
|
+
const bucketTenant = projectionTenantId === undefined ? null : projectionTenantId ?? null
|
|
438
|
+
const bucketOrg = projectionOrganizationId === undefined ? null : projectionOrganizationId ?? null
|
|
283
439
|
registerBaseCount(bucketTenant, bucketOrg, toNumber(row?.count))
|
|
284
440
|
}
|
|
285
441
|
|
|
@@ -301,11 +457,11 @@ export async function reindexEntity(
|
|
|
301
457
|
: undefined
|
|
302
458
|
|
|
303
459
|
const scopeOverrides: { tenantId?: string; orgId?: string } = {}
|
|
304
|
-
if (
|
|
305
|
-
scopeOverrides.tenantId = String(
|
|
460
|
+
if (projectionTenantId !== undefined && projectionTenantId !== null) {
|
|
461
|
+
scopeOverrides.tenantId = String(projectionTenantId)
|
|
306
462
|
}
|
|
307
|
-
if (
|
|
308
|
-
scopeOverrides.orgId = String(
|
|
463
|
+
if (projectionOrganizationId !== undefined && projectionOrganizationId !== null) {
|
|
464
|
+
scopeOverrides.orgId = String(projectionOrganizationId)
|
|
309
465
|
}
|
|
310
466
|
|
|
311
467
|
const scopeEntries = Array.from(baseCounts.values()).map((entry) => ({
|
|
@@ -334,11 +490,11 @@ export async function reindexEntity(
|
|
|
334
490
|
let purgeQuery = db
|
|
335
491
|
.deleteFrom('entity_indexes' as any)
|
|
336
492
|
.where('entity_type' as any, '=', entityType)
|
|
337
|
-
if (
|
|
338
|
-
purgeQuery = purgeQuery.where(sql<boolean>`tenant_id is not distinct from ${
|
|
493
|
+
if (projectionTenantId !== undefined) {
|
|
494
|
+
purgeQuery = purgeQuery.where(sql<boolean>`tenant_id is not distinct from ${projectionTenantId ?? null}`)
|
|
339
495
|
}
|
|
340
|
-
if (
|
|
341
|
-
purgeQuery = purgeQuery.where(sql<boolean>`organization_id is not distinct from ${
|
|
496
|
+
if (projectionOrganizationId !== undefined) {
|
|
497
|
+
purgeQuery = purgeQuery.where(sql<boolean>`organization_id is not distinct from ${projectionOrganizationId ?? null}`)
|
|
342
498
|
}
|
|
343
499
|
if (usingPartitions && partitionIndex !== null) {
|
|
344
500
|
purgeQuery = purgeQuery.where(
|
|
@@ -363,12 +519,12 @@ export async function reindexEntity(
|
|
|
363
519
|
// leaves it to the caller that owns the fan-out (`api/reindex.ts` queues it once, before
|
|
364
520
|
// dispatching any partition).
|
|
365
521
|
if (force && resetCoverage && !usingPartitions && emitVectorize && eventBus) {
|
|
366
|
-
if (
|
|
522
|
+
if (vectorTenantId !== undefined) {
|
|
367
523
|
const payload: Record<string, unknown> = {
|
|
368
524
|
entityType,
|
|
369
|
-
tenantId:
|
|
525
|
+
tenantId: vectorTenantId ?? null,
|
|
370
526
|
}
|
|
371
|
-
if (
|
|
527
|
+
if (vectorOrganizationId !== undefined) payload.organizationId = vectorOrganizationId ?? null
|
|
372
528
|
try {
|
|
373
529
|
await eventBus.emitEvent('query_index.vectorize_purge', payload)
|
|
374
530
|
} catch (err) {
|
|
@@ -473,11 +629,11 @@ export async function reindexEntity(
|
|
|
473
629
|
|
|
474
630
|
const coverageDeltas = new Map<string, { tenantId: string | null; organizationId: string | null; delta: number }>()
|
|
475
631
|
for (const row of writtenRows) {
|
|
476
|
-
const scopeTenant =
|
|
477
|
-
?
|
|
632
|
+
const scopeTenant = projectionTenantId !== undefined
|
|
633
|
+
? projectionTenantId ?? null
|
|
478
634
|
: (hasTenantCol ? ((row as AnyRow).tenant_id ?? null) : null)
|
|
479
|
-
const scopeOrg =
|
|
480
|
-
?
|
|
635
|
+
const scopeOrg = projectionOrganizationId !== undefined
|
|
636
|
+
? projectionOrganizationId ?? null
|
|
481
637
|
: (hasOrgCol ? ((row as AnyRow).organization_id ?? null) : (deriveOrg ? deriveOrg(row) ?? null : null))
|
|
482
638
|
const key = scopeKey(scopeTenant ?? null, scopeOrg ?? null)
|
|
483
639
|
const existingDelta = coverageDeltas.get(key)
|
|
@@ -505,13 +661,13 @@ export async function reindexEntity(
|
|
|
505
661
|
if (emitVectorize && eventBus) {
|
|
506
662
|
await Promise.all(
|
|
507
663
|
writtenRows.map((row) => {
|
|
508
|
-
const scopeOrg =
|
|
509
|
-
?
|
|
664
|
+
const scopeOrg = vectorOrganizationId !== undefined
|
|
665
|
+
? vectorOrganizationId ?? null
|
|
510
666
|
: hasOrgCol
|
|
511
667
|
? ((row as AnyRow).organization_id ?? null)
|
|
512
668
|
: (deriveOrg ? deriveOrg(row) ?? null : null)
|
|
513
|
-
const scopeTenant =
|
|
514
|
-
?
|
|
669
|
+
const scopeTenant = vectorTenantId !== undefined
|
|
670
|
+
? vectorTenantId ?? null
|
|
515
671
|
: (hasTenantCol ? ((row as AnyRow).tenant_id ?? null) : null)
|
|
516
672
|
return eventBus
|
|
517
673
|
.emitEvent('query_index.vectorize_one', {
|
|
@@ -543,21 +699,34 @@ export async function reindexEntity(
|
|
|
543
699
|
} else {
|
|
544
700
|
await purgeOrphans(db, {
|
|
545
701
|
entityType,
|
|
546
|
-
tenantId,
|
|
547
|
-
organizationId,
|
|
702
|
+
tenantId: projectionTenantId,
|
|
703
|
+
organizationId: projectionOrganizationId,
|
|
548
704
|
partitionIndex: usingPartitions ? partitionIndex : null,
|
|
549
705
|
partitionCount: usingPartitions ? partitionCountRaw : null,
|
|
550
706
|
startedAt: jobStartedAt,
|
|
551
707
|
excludeRecordIds: purgeExclusions,
|
|
552
708
|
})
|
|
709
|
+
// The purge above only sees the canonical null scope. Rows earlier sweeps stamped with
|
|
710
|
+
// a caller's tenant and organization are other keys of the same records, so a sweep of
|
|
711
|
+
// a declared catalogue also retires them - after the canonical rows are written, so the
|
|
712
|
+
// catalogue never goes unserved in between.
|
|
713
|
+
if (writesGlobalProjection) {
|
|
714
|
+
await retireScopedCatalogueRows(db, {
|
|
715
|
+
entityType,
|
|
716
|
+
partitionIndex: usingPartitions ? partitionIndex : null,
|
|
717
|
+
partitionCount: usingPartitions ? partitionCountRaw : null,
|
|
718
|
+
excludeRecordIds: purgeExclusions,
|
|
719
|
+
})
|
|
720
|
+
if (resetCoverage) await deleteScopedCoverageForEntity(db, entityType)
|
|
721
|
+
}
|
|
553
722
|
}
|
|
554
723
|
|
|
555
724
|
if (force && vectorService && (!usingPartitions || partitionIndex === null)) {
|
|
556
725
|
try {
|
|
557
726
|
await vectorService.removeOrphans({
|
|
558
727
|
entityId: entityType,
|
|
559
|
-
tenantId,
|
|
560
|
-
organizationId,
|
|
728
|
+
tenantId: vectorTenantId,
|
|
729
|
+
organizationId: vectorOrganizationId,
|
|
561
730
|
olderThan: jobStartedAt,
|
|
562
731
|
})
|
|
563
732
|
} catch (error) {
|