@open-mercato/search 0.10.1-develop.7787.1.96c7cfc66f → 0.10.1-develop.7793.1.9e0c43fdde
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/di.js +14 -1
- package/dist/di.js.map +2 -2
- package/dist/indexer/search-indexer.js +28 -5
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/lib/debug.js +9 -0
- package/dist/lib/debug.js.map +2 -2
- package/dist/lib/encrypted-custom-field-resolver.js +19 -0
- package/dist/lib/encrypted-custom-field-resolver.js.map +7 -0
- package/dist/lib/plaintext-fields.js +17 -0
- package/dist/lib/plaintext-fields.js.map +7 -0
- package/dist/strategies/fulltext.strategy.js +2 -1
- package/dist/strategies/fulltext.strategy.js.map +2 -2
- package/dist/strategies/vector.strategy.js +2 -1
- package/dist/strategies/vector.strategy.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/di-plaintext-custom-field-policy.test.ts +88 -0
- package/src/__tests__/encrypted-custom-field-resolver.test.ts +43 -0
- package/src/__tests__/plaintext-custom-field-boundaries.test.ts +262 -0
- package/src/di.ts +18 -0
- package/src/indexer/search-indexer.ts +36 -3
- package/src/lib/debug.ts +14 -0
- package/src/lib/encrypted-custom-field-resolver.ts +36 -0
- package/src/lib/plaintext-fields.ts +15 -0
- package/src/strategies/fulltext.strategy.ts +2 -1
- package/src/strategies/vector.strategy.ts +2 -1
|
@@ -13,7 +13,8 @@ import type { QueryEngine } from '@open-mercato/shared/lib/query/types'
|
|
|
13
13
|
import type { Queue } from '@open-mercato/queue'
|
|
14
14
|
import type { FulltextIndexJobPayload } from '../queue/fulltext-indexing'
|
|
15
15
|
import type { VectorIndexJobPayload, VectorBatchRecord } from '../queue/vector-indexing'
|
|
16
|
-
import { searchDebug, searchDebugWarn, searchError } from '../lib/debug'
|
|
16
|
+
import { searchDebug, searchDebugWarn, searchError, reportSearchError } from '../lib/debug'
|
|
17
|
+
import type { EncryptedCustomFieldKeysResolver } from '../lib/encrypted-custom-field-resolver'
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Maximum number of pages to process during reindex to prevent infinite loops.
|
|
@@ -21,6 +22,8 @@ import { searchDebug, searchDebugWarn, searchError } from '../lib/debug'
|
|
|
21
22
|
*/
|
|
22
23
|
const MAX_PAGES = 10000
|
|
23
24
|
|
|
25
|
+
type PlaintextExclusionLookup = (entityId: EntityId) => Promise<IndexableRecord['plaintextExcludedCustomFields']>
|
|
26
|
+
|
|
24
27
|
/**
|
|
25
28
|
* Parameters for indexing a record.
|
|
26
29
|
*/
|
|
@@ -122,6 +125,7 @@ export type SearchIndexerOptions = {
|
|
|
122
125
|
fulltextQueue?: Queue<FulltextIndexJobPayload>
|
|
123
126
|
/** Queue for vector batch indexing */
|
|
124
127
|
vectorQueue?: Queue<VectorIndexJobPayload>
|
|
128
|
+
encryptedCustomFieldKeysResolver?: EncryptedCustomFieldKeysResolver
|
|
125
129
|
}
|
|
126
130
|
|
|
127
131
|
/**
|
|
@@ -133,6 +137,7 @@ export class SearchIndexer {
|
|
|
133
137
|
private readonly queryEngine?: QueryEngine
|
|
134
138
|
private readonly fulltextQueue?: Queue<FulltextIndexJobPayload>
|
|
135
139
|
private readonly vectorQueue?: Queue<VectorIndexJobPayload>
|
|
140
|
+
private readonly encryptedCustomFieldKeysResolver?: EncryptedCustomFieldKeysResolver
|
|
136
141
|
|
|
137
142
|
constructor(
|
|
138
143
|
private readonly searchService: SearchService,
|
|
@@ -143,6 +148,7 @@ export class SearchIndexer {
|
|
|
143
148
|
this.queryEngine = options?.queryEngine
|
|
144
149
|
this.fulltextQueue = options?.fulltextQueue
|
|
145
150
|
this.vectorQueue = options?.vectorQueue
|
|
151
|
+
this.encryptedCustomFieldKeysResolver = options?.encryptedCustomFieldKeysResolver
|
|
146
152
|
for (const moduleConfig of moduleConfigs) {
|
|
147
153
|
for (const entityConfig of moduleConfig.entities) {
|
|
148
154
|
if (entityConfig.enabled !== false) {
|
|
@@ -152,6 +158,23 @@ export class SearchIndexer {
|
|
|
152
158
|
}
|
|
153
159
|
}
|
|
154
160
|
|
|
161
|
+
private createPlaintextExclusionLookup(): PlaintextExclusionLookup {
|
|
162
|
+
const resolver = this.encryptedCustomFieldKeysResolver
|
|
163
|
+
const cache = new Map<EntityId, Promise<IndexableRecord['plaintextExcludedCustomFields']>>()
|
|
164
|
+
return (entityId) => {
|
|
165
|
+
if (!resolver) return Promise.resolve(undefined)
|
|
166
|
+
let pending = cache.get(entityId)
|
|
167
|
+
if (!pending) {
|
|
168
|
+
pending = Promise.resolve().then(() => resolver(entityId)).then((keys) => Array.from(keys)).catch((error): 'all' => {
|
|
169
|
+
reportSearchError('SearchIndexer', error, 'search.encrypted_custom_fields_unresolved', { entityId })
|
|
170
|
+
return 'all'
|
|
171
|
+
})
|
|
172
|
+
cache.set(entityId, pending)
|
|
173
|
+
}
|
|
174
|
+
return pending
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
155
178
|
/**
|
|
156
179
|
* Returns a wrapped QueryEngine that forces skipAutoReindex: true on every query.
|
|
157
180
|
* Used to prevent feedback loops when search indexing callbacks (buildSource, formatResult)
|
|
@@ -197,7 +220,10 @@ export class SearchIndexer {
|
|
|
197
220
|
}
|
|
198
221
|
}
|
|
199
222
|
|
|
200
|
-
private async buildIndexableRecord(
|
|
223
|
+
private async buildIndexableRecord(
|
|
224
|
+
params: IndexRecordParams,
|
|
225
|
+
exclusions = this.createPlaintextExclusionLookup(),
|
|
226
|
+
): Promise<IndexableRecord | null> {
|
|
201
227
|
const config = this.entityConfigMap.get(params.entityId)
|
|
202
228
|
if (!config || config.enabled === false) {
|
|
203
229
|
return null
|
|
@@ -284,6 +310,7 @@ export class SearchIndexer {
|
|
|
284
310
|
tenantId: params.tenantId,
|
|
285
311
|
organizationId: params.organizationId,
|
|
286
312
|
fields: params.record,
|
|
313
|
+
plaintextExcludedCustomFields: await exclusions(params.entityId),
|
|
287
314
|
presenter,
|
|
288
315
|
url,
|
|
289
316
|
links,
|
|
@@ -376,6 +403,7 @@ export class SearchIndexer {
|
|
|
376
403
|
recordIdsByEntity.set(item.entityId, list)
|
|
377
404
|
}
|
|
378
405
|
|
|
406
|
+
const exclusions = this.createPlaintextExclusionLookup()
|
|
379
407
|
const allRecords: IndexableRecord[] = []
|
|
380
408
|
let skipped = 0
|
|
381
409
|
|
|
@@ -409,6 +437,7 @@ export class SearchIndexer {
|
|
|
409
437
|
organizationId ?? null,
|
|
410
438
|
[record],
|
|
411
439
|
config,
|
|
440
|
+
exclusions,
|
|
412
441
|
)
|
|
413
442
|
skipped += dropped
|
|
414
443
|
allRecords.push(...built)
|
|
@@ -593,9 +622,10 @@ export class SearchIndexer {
|
|
|
593
622
|
*/
|
|
594
623
|
async bulkIndexRecords(params: IndexRecordParams[]): Promise<void> {
|
|
595
624
|
const indexableRecords: IndexableRecord[] = []
|
|
625
|
+
const exclusions = this.createPlaintextExclusionLookup()
|
|
596
626
|
|
|
597
627
|
for (const param of params) {
|
|
598
|
-
const indexableRecord = await this.buildIndexableRecord(param)
|
|
628
|
+
const indexableRecord = await this.buildIndexableRecord(param, exclusions)
|
|
599
629
|
if (indexableRecord) {
|
|
600
630
|
indexableRecords.push(indexableRecord)
|
|
601
631
|
}
|
|
@@ -1083,8 +1113,10 @@ export class SearchIndexer {
|
|
|
1083
1113
|
organizationId: string | null,
|
|
1084
1114
|
items: Record<string, unknown>[],
|
|
1085
1115
|
config: SearchEntityConfig,
|
|
1116
|
+
exclusions = this.createPlaintextExclusionLookup(),
|
|
1086
1117
|
): Promise<{ records: IndexableRecord[]; dropped: number }> {
|
|
1087
1118
|
const records: IndexableRecord[] = []
|
|
1119
|
+
const plaintextExcludedCustomFields = await exclusions(entityId)
|
|
1088
1120
|
let dropped = 0
|
|
1089
1121
|
|
|
1090
1122
|
// Debug: log first item to see structure
|
|
@@ -1187,6 +1219,7 @@ export class SearchIndexer {
|
|
|
1187
1219
|
tenantId,
|
|
1188
1220
|
organizationId,
|
|
1189
1221
|
fields: item,
|
|
1222
|
+
plaintextExcludedCustomFields,
|
|
1190
1223
|
presenter,
|
|
1191
1224
|
url,
|
|
1192
1225
|
links,
|
package/src/lib/debug.ts
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
import { createLogger, type Logger } from '@open-mercato/shared/lib/logger'
|
|
12
12
|
import { parseBooleanWithDefault } from '@open-mercato/shared/lib/boolean'
|
|
13
|
+
import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
|
|
13
14
|
|
|
14
15
|
const packageLogger = createLogger('search')
|
|
15
16
|
const componentLoggers = new Map<string, Logger>()
|
|
@@ -59,3 +60,16 @@ export function searchWarn(prefix: string, message: string, data?: Record<string
|
|
|
59
60
|
export function searchError(prefix: string, message: string, data?: Record<string, unknown>): void {
|
|
60
61
|
componentLogger(prefix).error(message, data)
|
|
61
62
|
}
|
|
63
|
+
|
|
64
|
+
export function reportSearchError(
|
|
65
|
+
prefix: string,
|
|
66
|
+
error: unknown,
|
|
67
|
+
code: string,
|
|
68
|
+
attributes?: Record<string, string | number | boolean | undefined>,
|
|
69
|
+
): void {
|
|
70
|
+
try {
|
|
71
|
+
getTelemetryRuntime()?.reportError(error, { module: 'search', code, attributes })
|
|
72
|
+
} catch {
|
|
73
|
+
searchWarn(prefix, 'Telemetry error report failed', { code })
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { Kysely } from 'kysely'
|
|
2
|
+
import type { EntityId } from '@open-mercato/shared/modules/entities'
|
|
3
|
+
import { normalizeDefinitionConfig, normalizeDefinitionKey } from '@open-mercato/shared/lib/crud/custom-field-definition-index'
|
|
4
|
+
|
|
5
|
+
export type CustomFieldDefinitionsDatabase = {
|
|
6
|
+
custom_field_defs: {
|
|
7
|
+
entity_id: string
|
|
8
|
+
key: string
|
|
9
|
+
config_json: unknown
|
|
10
|
+
deleted_at: Date | null
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export type EncryptedCustomFieldKeysResolver = (entityId: EntityId) => Promise<Set<string>>
|
|
15
|
+
|
|
16
|
+
export function createEncryptedCustomFieldKeysResolver(
|
|
17
|
+
db: Kysely<CustomFieldDefinitionsDatabase>,
|
|
18
|
+
): EncryptedCustomFieldKeysResolver {
|
|
19
|
+
return async (entityId) => {
|
|
20
|
+
const definitions = await db
|
|
21
|
+
.selectFrom('custom_field_defs')
|
|
22
|
+
.select(['key', 'config_json'])
|
|
23
|
+
.where('entity_id', '=', entityId)
|
|
24
|
+
.where('deleted_at', 'is', null)
|
|
25
|
+
.execute()
|
|
26
|
+
const keys = new Set<string>()
|
|
27
|
+
for (const definition of definitions) {
|
|
28
|
+
if (!normalizeDefinitionConfig(definition.config_json).encrypted) continue
|
|
29
|
+
const key = normalizeDefinitionKey(definition.key)
|
|
30
|
+
if (!key) continue
|
|
31
|
+
keys.add(key)
|
|
32
|
+
keys.add(key.replace(/[^a-z0-9_]/g, '_'))
|
|
33
|
+
}
|
|
34
|
+
return keys
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { IndexableRecord } from '../types'
|
|
2
|
+
import { normalizeDefinitionKey } from '@open-mercato/shared/lib/crud/custom-field-definition-index'
|
|
3
|
+
|
|
4
|
+
export function plaintextSearchFields(record: IndexableRecord): Record<string, unknown> {
|
|
5
|
+
const exclusion = record.plaintextExcludedCustomFields
|
|
6
|
+
if (!exclusion || exclusion.length === 0) return record.fields
|
|
7
|
+
const keys = exclusion === 'all' ? null : new Set(exclusion.flatMap((key) => {
|
|
8
|
+
const normalized = normalizeDefinitionKey(key)
|
|
9
|
+
return [normalized, normalized.replace(/[^a-z0-9_]/g, '_')]
|
|
10
|
+
}))
|
|
11
|
+
return Object.fromEntries(Object.entries(record.fields).filter(([key]) => {
|
|
12
|
+
if (!key.startsWith('cf:') && !key.startsWith('cf_')) return true
|
|
13
|
+
return keys !== null && !keys.has(normalizeDefinitionKey(key.slice(3)))
|
|
14
|
+
}))
|
|
15
|
+
}
|
|
@@ -6,6 +6,7 @@ import type {
|
|
|
6
6
|
IndexableRecord,
|
|
7
7
|
} from '../types'
|
|
8
8
|
import type { EntityId } from '@open-mercato/shared/modules/entities'
|
|
9
|
+
import { plaintextSearchFields } from '../lib/plaintext-fields'
|
|
9
10
|
import type {
|
|
10
11
|
FullTextSearchDriver,
|
|
11
12
|
FullTextSearchDocument,
|
|
@@ -168,7 +169,7 @@ export class FullTextSearchStrategy implements SearchStrategy {
|
|
|
168
169
|
entityId: record.entityId,
|
|
169
170
|
tenantId: record.tenantId,
|
|
170
171
|
organizationId: record.organizationId,
|
|
171
|
-
fields: record
|
|
172
|
+
fields: plaintextSearchFields(record),
|
|
172
173
|
presenter: record.presenter,
|
|
173
174
|
url: record.url,
|
|
174
175
|
links: record.links,
|
|
@@ -9,6 +9,7 @@ import type {
|
|
|
9
9
|
import type { EntityId } from '@open-mercato/shared/modules/entities'
|
|
10
10
|
import type { VectorDriver, VectorDriverDocument } from '../vector/types'
|
|
11
11
|
import { searchDebugWarn } from '../lib/debug'
|
|
12
|
+
import { plaintextSearchFields } from '../lib/plaintext-fields'
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* Embedding service interface - minimal subset needed by VectorSearchStrategy.
|
|
@@ -192,7 +193,7 @@ export class VectorSearchStrategy implements SearchStrategy {
|
|
|
192
193
|
}
|
|
193
194
|
|
|
194
195
|
// Add string fields from record
|
|
195
|
-
for (const [, value] of Object.entries(record
|
|
196
|
+
for (const [, value] of Object.entries(plaintextSearchFields(record))) {
|
|
196
197
|
if (typeof value === 'string' && value.trim()) {
|
|
197
198
|
parts.push(value)
|
|
198
199
|
}
|