@open-mercato/search 0.10.1-develop.7787.1.96c7cfc66f → 0.10.1-develop.7793.1.9e0c43fdde

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,7 +13,8 @@ import type { QueryEngine } from '@open-mercato/shared/lib/query/types'
13
13
  import type { Queue } from '@open-mercato/queue'
14
14
  import type { FulltextIndexJobPayload } from '../queue/fulltext-indexing'
15
15
  import type { VectorIndexJobPayload, VectorBatchRecord } from '../queue/vector-indexing'
16
- import { searchDebug, searchDebugWarn, searchError } from '../lib/debug'
16
+ import { searchDebug, searchDebugWarn, searchError, reportSearchError } from '../lib/debug'
17
+ import type { EncryptedCustomFieldKeysResolver } from '../lib/encrypted-custom-field-resolver'
17
18
 
18
19
  /**
19
20
  * Maximum number of pages to process during reindex to prevent infinite loops.
@@ -21,6 +22,8 @@ import { searchDebug, searchDebugWarn, searchError } from '../lib/debug'
21
22
  */
22
23
  const MAX_PAGES = 10000
23
24
 
25
+ type PlaintextExclusionLookup = (entityId: EntityId) => Promise<IndexableRecord['plaintextExcludedCustomFields']>
26
+
24
27
  /**
25
28
  * Parameters for indexing a record.
26
29
  */
@@ -122,6 +125,7 @@ export type SearchIndexerOptions = {
122
125
  fulltextQueue?: Queue<FulltextIndexJobPayload>
123
126
  /** Queue for vector batch indexing */
124
127
  vectorQueue?: Queue<VectorIndexJobPayload>
128
+ encryptedCustomFieldKeysResolver?: EncryptedCustomFieldKeysResolver
125
129
  }
126
130
 
127
131
  /**
@@ -133,6 +137,7 @@ export class SearchIndexer {
133
137
  private readonly queryEngine?: QueryEngine
134
138
  private readonly fulltextQueue?: Queue<FulltextIndexJobPayload>
135
139
  private readonly vectorQueue?: Queue<VectorIndexJobPayload>
140
+ private readonly encryptedCustomFieldKeysResolver?: EncryptedCustomFieldKeysResolver
136
141
 
137
142
  constructor(
138
143
  private readonly searchService: SearchService,
@@ -143,6 +148,7 @@ export class SearchIndexer {
143
148
  this.queryEngine = options?.queryEngine
144
149
  this.fulltextQueue = options?.fulltextQueue
145
150
  this.vectorQueue = options?.vectorQueue
151
+ this.encryptedCustomFieldKeysResolver = options?.encryptedCustomFieldKeysResolver
146
152
  for (const moduleConfig of moduleConfigs) {
147
153
  for (const entityConfig of moduleConfig.entities) {
148
154
  if (entityConfig.enabled !== false) {
@@ -152,6 +158,23 @@ export class SearchIndexer {
152
158
  }
153
159
  }
154
160
 
161
+ private createPlaintextExclusionLookup(): PlaintextExclusionLookup {
162
+ const resolver = this.encryptedCustomFieldKeysResolver
163
+ const cache = new Map<EntityId, Promise<IndexableRecord['plaintextExcludedCustomFields']>>()
164
+ return (entityId) => {
165
+ if (!resolver) return Promise.resolve(undefined)
166
+ let pending = cache.get(entityId)
167
+ if (!pending) {
168
+ pending = Promise.resolve().then(() => resolver(entityId)).then((keys) => Array.from(keys)).catch((error): 'all' => {
169
+ reportSearchError('SearchIndexer', error, 'search.encrypted_custom_fields_unresolved', { entityId })
170
+ return 'all'
171
+ })
172
+ cache.set(entityId, pending)
173
+ }
174
+ return pending
175
+ }
176
+ }
177
+
155
178
  /**
156
179
  * Returns a wrapped QueryEngine that forces skipAutoReindex: true on every query.
157
180
  * Used to prevent feedback loops when search indexing callbacks (buildSource, formatResult)
@@ -197,7 +220,10 @@ export class SearchIndexer {
197
220
  }
198
221
  }
199
222
 
200
- private async buildIndexableRecord(params: IndexRecordParams): Promise<IndexableRecord | null> {
223
+ private async buildIndexableRecord(
224
+ params: IndexRecordParams,
225
+ exclusions = this.createPlaintextExclusionLookup(),
226
+ ): Promise<IndexableRecord | null> {
201
227
  const config = this.entityConfigMap.get(params.entityId)
202
228
  if (!config || config.enabled === false) {
203
229
  return null
@@ -284,6 +310,7 @@ export class SearchIndexer {
284
310
  tenantId: params.tenantId,
285
311
  organizationId: params.organizationId,
286
312
  fields: params.record,
313
+ plaintextExcludedCustomFields: await exclusions(params.entityId),
287
314
  presenter,
288
315
  url,
289
316
  links,
@@ -376,6 +403,7 @@ export class SearchIndexer {
376
403
  recordIdsByEntity.set(item.entityId, list)
377
404
  }
378
405
 
406
+ const exclusions = this.createPlaintextExclusionLookup()
379
407
  const allRecords: IndexableRecord[] = []
380
408
  let skipped = 0
381
409
 
@@ -409,6 +437,7 @@ export class SearchIndexer {
409
437
  organizationId ?? null,
410
438
  [record],
411
439
  config,
440
+ exclusions,
412
441
  )
413
442
  skipped += dropped
414
443
  allRecords.push(...built)
@@ -593,9 +622,10 @@ export class SearchIndexer {
593
622
  */
594
623
  async bulkIndexRecords(params: IndexRecordParams[]): Promise<void> {
595
624
  const indexableRecords: IndexableRecord[] = []
625
+ const exclusions = this.createPlaintextExclusionLookup()
596
626
 
597
627
  for (const param of params) {
598
- const indexableRecord = await this.buildIndexableRecord(param)
628
+ const indexableRecord = await this.buildIndexableRecord(param, exclusions)
599
629
  if (indexableRecord) {
600
630
  indexableRecords.push(indexableRecord)
601
631
  }
@@ -1083,8 +1113,10 @@ export class SearchIndexer {
1083
1113
  organizationId: string | null,
1084
1114
  items: Record<string, unknown>[],
1085
1115
  config: SearchEntityConfig,
1116
+ exclusions = this.createPlaintextExclusionLookup(),
1086
1117
  ): Promise<{ records: IndexableRecord[]; dropped: number }> {
1087
1118
  const records: IndexableRecord[] = []
1119
+ const plaintextExcludedCustomFields = await exclusions(entityId)
1088
1120
  let dropped = 0
1089
1121
 
1090
1122
  // Debug: log first item to see structure
@@ -1187,6 +1219,7 @@ export class SearchIndexer {
1187
1219
  tenantId,
1188
1220
  organizationId,
1189
1221
  fields: item,
1222
+ plaintextExcludedCustomFields,
1190
1223
  presenter,
1191
1224
  url,
1192
1225
  links,
package/src/lib/debug.ts CHANGED
@@ -10,6 +10,7 @@
10
10
 
11
11
  import { createLogger, type Logger } from '@open-mercato/shared/lib/logger'
12
12
  import { parseBooleanWithDefault } from '@open-mercato/shared/lib/boolean'
13
+ import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
13
14
 
14
15
  const packageLogger = createLogger('search')
15
16
  const componentLoggers = new Map<string, Logger>()
@@ -59,3 +60,16 @@ export function searchWarn(prefix: string, message: string, data?: Record<string
59
60
  export function searchError(prefix: string, message: string, data?: Record<string, unknown>): void {
60
61
  componentLogger(prefix).error(message, data)
61
62
  }
63
+
64
+ export function reportSearchError(
65
+ prefix: string,
66
+ error: unknown,
67
+ code: string,
68
+ attributes?: Record<string, string | number | boolean | undefined>,
69
+ ): void {
70
+ try {
71
+ getTelemetryRuntime()?.reportError(error, { module: 'search', code, attributes })
72
+ } catch {
73
+ searchWarn(prefix, 'Telemetry error report failed', { code })
74
+ }
75
+ }
@@ -0,0 +1,36 @@
1
+ import type { Kysely } from 'kysely'
2
+ import type { EntityId } from '@open-mercato/shared/modules/entities'
3
+ import { normalizeDefinitionConfig, normalizeDefinitionKey } from '@open-mercato/shared/lib/crud/custom-field-definition-index'
4
+
5
+ export type CustomFieldDefinitionsDatabase = {
6
+ custom_field_defs: {
7
+ entity_id: string
8
+ key: string
9
+ config_json: unknown
10
+ deleted_at: Date | null
11
+ }
12
+ }
13
+
14
+ export type EncryptedCustomFieldKeysResolver = (entityId: EntityId) => Promise<Set<string>>
15
+
16
+ export function createEncryptedCustomFieldKeysResolver(
17
+ db: Kysely<CustomFieldDefinitionsDatabase>,
18
+ ): EncryptedCustomFieldKeysResolver {
19
+ return async (entityId) => {
20
+ const definitions = await db
21
+ .selectFrom('custom_field_defs')
22
+ .select(['key', 'config_json'])
23
+ .where('entity_id', '=', entityId)
24
+ .where('deleted_at', 'is', null)
25
+ .execute()
26
+ const keys = new Set<string>()
27
+ for (const definition of definitions) {
28
+ if (!normalizeDefinitionConfig(definition.config_json).encrypted) continue
29
+ const key = normalizeDefinitionKey(definition.key)
30
+ if (!key) continue
31
+ keys.add(key)
32
+ keys.add(key.replace(/[^a-z0-9_]/g, '_'))
33
+ }
34
+ return keys
35
+ }
36
+ }
@@ -0,0 +1,15 @@
1
+ import type { IndexableRecord } from '../types'
2
+ import { normalizeDefinitionKey } from '@open-mercato/shared/lib/crud/custom-field-definition-index'
3
+
4
+ export function plaintextSearchFields(record: IndexableRecord): Record<string, unknown> {
5
+ const exclusion = record.plaintextExcludedCustomFields
6
+ if (!exclusion || exclusion.length === 0) return record.fields
7
+ const keys = exclusion === 'all' ? null : new Set(exclusion.flatMap((key) => {
8
+ const normalized = normalizeDefinitionKey(key)
9
+ return [normalized, normalized.replace(/[^a-z0-9_]/g, '_')]
10
+ }))
11
+ return Object.fromEntries(Object.entries(record.fields).filter(([key]) => {
12
+ if (!key.startsWith('cf:') && !key.startsWith('cf_')) return true
13
+ return keys !== null && !keys.has(normalizeDefinitionKey(key.slice(3)))
14
+ }))
15
+ }
@@ -6,6 +6,7 @@ import type {
6
6
  IndexableRecord,
7
7
  } from '../types'
8
8
  import type { EntityId } from '@open-mercato/shared/modules/entities'
9
+ import { plaintextSearchFields } from '../lib/plaintext-fields'
9
10
  import type {
10
11
  FullTextSearchDriver,
11
12
  FullTextSearchDocument,
@@ -168,7 +169,7 @@ export class FullTextSearchStrategy implements SearchStrategy {
168
169
  entityId: record.entityId,
169
170
  tenantId: record.tenantId,
170
171
  organizationId: record.organizationId,
171
- fields: record.fields,
172
+ fields: plaintextSearchFields(record),
172
173
  presenter: record.presenter,
173
174
  url: record.url,
174
175
  links: record.links,
@@ -9,6 +9,7 @@ import type {
9
9
  import type { EntityId } from '@open-mercato/shared/modules/entities'
10
10
  import type { VectorDriver, VectorDriverDocument } from '../vector/types'
11
11
  import { searchDebugWarn } from '../lib/debug'
12
+ import { plaintextSearchFields } from '../lib/plaintext-fields'
12
13
 
13
14
  /**
14
15
  * Embedding service interface - minimal subset needed by VectorSearchStrategy.
@@ -192,7 +193,7 @@ export class VectorSearchStrategy implements SearchStrategy {
192
193
  }
193
194
 
194
195
  // Add string fields from record
195
- for (const [, value] of Object.entries(record.fields)) {
196
+ for (const [, value] of Object.entries(plaintextSearchFields(record))) {
196
197
  if (typeof value === 'string' && value.trim()) {
197
198
  parts.push(value)
198
199
  }