@open-mercato/shared 0.8.1-develop.7275.1.f772b944d4 → 0.8.1-develop.7295.1.d0e0e33014
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +2 -1
- package/AGENTS.md +6 -3
- package/build.mjs +4 -1
- package/dist/lib/crud/errors.js +6 -1
- package/dist/lib/crud/errors.js.map +2 -2
- package/dist/lib/crud/factory.js +22 -13
- package/dist/lib/crud/factory.js.map +2 -2
- package/dist/lib/custom-fields/kinds.js +113 -0
- package/dist/lib/custom-fields/kinds.js.map +7 -0
- package/dist/lib/encryption/indexDoc.js +12 -9
- package/dist/lib/encryption/indexDoc.js.map +2 -2
- package/dist/lib/location/countries.generated.js +265 -0
- package/dist/lib/location/countries.generated.js.map +7 -0
- package/dist/lib/location/countries.js +2 -13
- package/dist/lib/location/countries.js.map +2 -2
- package/dist/lib/navigation/pageReload.js +11 -0
- package/dist/lib/navigation/pageReload.js.map +7 -0
- package/dist/lib/query/encrypted-sort.js +4 -1
- package/dist/lib/query/encrypted-sort.js.map +2 -2
- package/dist/lib/query/engine.js +97 -10
- package/dist/lib/query/engine.js.map +3 -3
- package/dist/lib/schedule/interval.js +33 -0
- package/dist/lib/schedule/interval.js.map +7 -0
- package/dist/lib/schedule/invalidScheduleValue.js +19 -0
- package/dist/lib/schedule/invalidScheduleValue.js.map +7 -0
- package/dist/lib/search/config.js.map +2 -2
- package/dist/lib/search/containment.js +32 -0
- package/dist/lib/search/containment.js.map +7 -0
- package/dist/lib/version.js +1 -1
- package/dist/lib/version.js.map +1 -1
- package/package.json +3 -2
- package/scripts/generate-countries.mjs +75 -0
- package/src/lib/crud/__tests__/crud-factory.test.ts +177 -0
- package/src/lib/crud/__tests__/errors.test.ts +31 -1
- package/src/lib/crud/errors.ts +16 -0
- package/src/lib/crud/factory.ts +37 -15
- package/src/lib/custom-fields/__tests__/kinds.test.ts +208 -0
- package/src/lib/custom-fields/kinds.ts +205 -0
- package/src/lib/encryption/__tests__/indexDoc.custom-field-kinds.test.ts +127 -0
- package/src/lib/encryption/__tests__/indexDoc.test.ts +39 -0
- package/src/lib/encryption/indexDoc.ts +16 -7
- package/src/lib/location/__tests__/countries.test.ts +25 -0
- package/src/lib/location/countries.generated.ts +264 -0
- package/src/lib/location/countries.ts +2 -23
- package/src/lib/navigation/__tests__/pageReload.test.ts +58 -0
- package/src/lib/navigation/pageReload.ts +43 -0
- package/src/lib/query/__tests__/engine.test.ts +208 -16
- package/src/lib/query/encrypted-sort.ts +8 -1
- package/src/lib/query/engine.ts +172 -19
- package/src/lib/schedule/__tests__/interval.test.ts +37 -0
- package/src/lib/schedule/interval.ts +45 -0
- package/src/lib/schedule/invalidScheduleValue.ts +33 -0
- package/src/lib/search/__tests__/config.test.ts +27 -0
- package/src/lib/search/__tests__/containment.test.ts +61 -0
- package/src/lib/search/config.ts +28 -6
- package/src/lib/search/containment.ts +66 -0
|
@@ -287,7 +287,13 @@ describe('BasicQueryEngine (Kysely)', () => {
|
|
|
287
287
|
})
|
|
288
288
|
expect(hasTenantFilter).toBe(true)
|
|
289
289
|
const baseCall = fakeDb._calls.find((b: any) => b._ops.table === 'users')
|
|
290
|
-
|
|
290
|
+
// The sort rides its own dedicated `__sort` alias, not the jsonb projection
|
|
291
|
+
// alias `cf_vip` — see the dedicated-alias test above (#5674).
|
|
292
|
+
const hasCfOrder = baseCall._ops.orderBys.some((o: any) => {
|
|
293
|
+
const expr = o[0]
|
|
294
|
+
return typeof expr?.toOperationNode === 'function'
|
|
295
|
+
&& JSON.stringify(expr.toOperationNode()).includes('cf_vip__sort')
|
|
296
|
+
})
|
|
291
297
|
expect(hasCfOrder).toBe(true)
|
|
292
298
|
const hasExtJoin = baseCall._ops.joins.length > 0
|
|
293
299
|
expect(hasExtJoin).toBe(true)
|
|
@@ -303,13 +309,102 @@ describe('BasicQueryEngine (Kysely)', () => {
|
|
|
303
309
|
tenantId: 't1',
|
|
304
310
|
})
|
|
305
311
|
const baseCall = fakeDb._calls.find((b: any) => b._ops.table === 'users')
|
|
306
|
-
|
|
312
|
+
// The sort rides its own dedicated scalar alias, never the jsonb projection
|
|
313
|
+
// alias — ordering by `to_jsonb(...)` compares arrays after every scalar
|
|
314
|
+
// string regardless of contents (#5674). Trailing entry is the stable `id`
|
|
315
|
+
// tiebreak appended when the sort didn't already end on `id`.
|
|
316
|
+
expect(baseCall._ops.orderBys).toHaveLength(2)
|
|
317
|
+
const [sortExpr] = baseCall._ops.orderBys[0]
|
|
318
|
+
expect(JSON.stringify(sortExpr.toOperationNode())).toContain('cf_vip__sort')
|
|
319
|
+
expect(baseCall._ops.orderBys[1]).toEqual(['users.id', 'asc'])
|
|
307
320
|
// Ordering by an alias the query never selected is a Postgres 42703, so the
|
|
308
321
|
// sort has to bring its own projection and joins along (#5521).
|
|
309
|
-
expect(selectAliases(baseCall)).toContain('
|
|
322
|
+
expect(selectAliases(baseCall)).toContain('cf_vip__sort')
|
|
310
323
|
expect(baseCall._ops.joins.length).toBeGreaterThan(0)
|
|
311
324
|
})
|
|
312
325
|
|
|
326
|
+
test('a numeric-kind cf sort casts to numeric instead of ordering as text (#5674)', async () => {
|
|
327
|
+
const fakeDb = createFakeKysely({
|
|
328
|
+
custom_field_defs: [
|
|
329
|
+
{ key: 'rank', entity_id: 'auth:user', is_active: true, config_json: '{}', kind: 'float' },
|
|
330
|
+
],
|
|
331
|
+
})
|
|
332
|
+
const engine = new BasicQueryEngine({} as any, () => fakeDb as any)
|
|
333
|
+
await engine.query('auth:user', {
|
|
334
|
+
fields: ['id', 'email'],
|
|
335
|
+
sort: [{ field: 'cf:rank', dir: SortDir.Asc }],
|
|
336
|
+
organizationId: '1',
|
|
337
|
+
tenantId: 't1',
|
|
338
|
+
})
|
|
339
|
+
const baseCall = fakeDb._calls.find((b: any) => b._ops.table === 'users')
|
|
340
|
+
// The cast lives on the dedicated sort projection, not the ORDER BY clause,
|
|
341
|
+
// which just references that projection's alias by name.
|
|
342
|
+
const sortSelect = baseCall._ops.selects.find((s: any) => String(s?.alias ?? '') === 'cf_rank__sort')
|
|
343
|
+
expect(sortSelect).toBeTruthy()
|
|
344
|
+
const serialized = JSON.stringify(sortSelect.toOperationNode())
|
|
345
|
+
expect(serialized).toContain('::numeric')
|
|
346
|
+
expect(serialized).toContain('value_float')
|
|
347
|
+
expect(baseCall._ops.orderBys).toHaveLength(2)
|
|
348
|
+
const [orderExpr] = baseCall._ops.orderBys[0]
|
|
349
|
+
expect(JSON.stringify(orderExpr.toOperationNode())).toContain('NULLS LAST')
|
|
350
|
+
expect(baseCall._ops.orderBys[1]).toEqual(['users.id', 'asc'])
|
|
351
|
+
})
|
|
352
|
+
|
|
353
|
+
test('an encrypted base sort combined with a cf: sort still orders by the cf value, and the __sort alias never leaks into returned rows (#5674)', async () => {
|
|
354
|
+
const fakeDb = createFakeKysely({
|
|
355
|
+
users: [
|
|
356
|
+
// '1' and '2' decrypt to the same email — only the cf:vip tiebreak can
|
|
357
|
+
// put them in the right relative order. '3' decrypts to a later email
|
|
358
|
+
// so it sorts last regardless of its cf:vip value.
|
|
359
|
+
{ id: '1', tenant_id: 't1', organization_id: 'org1', email: 'cipher-1', cf_vip__sort: 'b' },
|
|
360
|
+
{ id: '2', tenant_id: 't1', organization_id: 'org1', email: 'cipher-2', cf_vip__sort: 'a' },
|
|
361
|
+
{ id: '3', tenant_id: 't1', organization_id: 'org1', email: 'cipher-3', cf_vip__sort: 'z' },
|
|
362
|
+
],
|
|
363
|
+
'information_schema.columns': [
|
|
364
|
+
{ table_name: 'users', column_name: 'id' },
|
|
365
|
+
{ table_name: 'users', column_name: 'tenant_id' },
|
|
366
|
+
{ table_name: 'users', column_name: 'organization_id' },
|
|
367
|
+
{ table_name: 'users', column_name: 'deleted_at' },
|
|
368
|
+
{ table_name: 'users', column_name: 'email' },
|
|
369
|
+
],
|
|
370
|
+
})
|
|
371
|
+
const emailById: Record<string, string> = {
|
|
372
|
+
'1': 'dup@example.com',
|
|
373
|
+
'2': 'dup@example.com',
|
|
374
|
+
'3': 'zzz@example.com',
|
|
375
|
+
}
|
|
376
|
+
const engine = new BasicQueryEngine(
|
|
377
|
+
{} as any,
|
|
378
|
+
() => fakeDb as any,
|
|
379
|
+
() => ({
|
|
380
|
+
isEnabled: () => true,
|
|
381
|
+
getEncryptedFieldNames: async () => ['email'],
|
|
382
|
+
decryptEntityPayload: async (_entityId, payload) => ({
|
|
383
|
+
email: emailById[String(payload.id)],
|
|
384
|
+
}),
|
|
385
|
+
}),
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
const result = await engine.query('auth:user', {
|
|
389
|
+
tenantId: 't1',
|
|
390
|
+
organizationId: 'org1',
|
|
391
|
+
fields: ['id', 'email'],
|
|
392
|
+
sort: [{ field: 'email', dir: SortDir.Asc }, { field: 'cf:vip', dir: SortDir.Asc }],
|
|
393
|
+
page: { page: 1, pageSize: 3 },
|
|
394
|
+
})
|
|
395
|
+
|
|
396
|
+
// Before the fix, `sortRowsInMemory` read `cf:vip` through candidates that
|
|
397
|
+
// never included the dedicated `cf_vip__sort` projection alias, so the value
|
|
398
|
+
// came back `undefined` and the cf: sort silently dropped out of the
|
|
399
|
+
// ordering — '1' and '2' would then only tie-break by `id`.
|
|
400
|
+
expect(result.items.map((item: any) => item.id)).toEqual(['2', '1', '3'])
|
|
401
|
+
// The synthetic sort alias is internal-only — it must never leak into a
|
|
402
|
+
// returned row as a phantom custom field `vip__sort` (#5674 review).
|
|
403
|
+
for (const item of result.items) {
|
|
404
|
+
expect(item).not.toHaveProperty('cf_vip__sort')
|
|
405
|
+
}
|
|
406
|
+
})
|
|
407
|
+
|
|
313
408
|
test('a cf sort that resolves to no definition is dropped, not ordered by', async () => {
|
|
314
409
|
const fakeDb = createFakeKysely()
|
|
315
410
|
const engine = new BasicQueryEngine({} as any, () => fakeDb as any)
|
|
@@ -322,8 +417,9 @@ describe('BasicQueryEngine (Kysely)', () => {
|
|
|
322
417
|
const baseCall = fakeDb._calls.find((b: any) => b._ops.table === 'users')
|
|
323
418
|
// Dropping an unresolvable sort is what the base-column branch already does;
|
|
324
419
|
// the alternative here was an ORDER BY over a column that is never selected.
|
|
420
|
+
// No cf sort survived, so there is nothing to tiebreak either.
|
|
325
421
|
expect(baseCall._ops.orderBys).toEqual([])
|
|
326
|
-
expect(selectAliases(baseCall)).not.toContain('
|
|
422
|
+
expect(selectAliases(baseCall)).not.toContain('cf_no_such_key__sort')
|
|
327
423
|
})
|
|
328
424
|
|
|
329
425
|
test('customFieldSources join additional profiles for custom fields', async () => {
|
|
@@ -1041,7 +1137,12 @@ describe('BasicQueryEngine (Kysely)', () => {
|
|
|
1041
1137
|
})
|
|
1042
1138
|
|
|
1043
1139
|
const baseCall = fakeDb._calls.find((call: any) => call._ops.table === 'customer_entities')
|
|
1044
|
-
|
|
1140
|
+
// A trailing `id` tiebreak is appended when the sort didn't already end on
|
|
1141
|
+
// `id`, so ties/NULLs don't reorder arbitrarily across pages (#5674).
|
|
1142
|
+
expect(baseCall._ops.orderBys).toEqual([
|
|
1143
|
+
['customer_entities.display_name', 'asc'],
|
|
1144
|
+
['customer_entities.id', 'asc'],
|
|
1145
|
+
])
|
|
1045
1146
|
expect(baseCall._ops.limits).toBe(10)
|
|
1046
1147
|
expect(baseCall._ops.offsets).toBe(10)
|
|
1047
1148
|
})
|
|
@@ -1302,9 +1403,9 @@ describe('BasicQueryEngine entity-extension joins', () => {
|
|
|
1302
1403
|
})
|
|
1303
1404
|
|
|
1304
1405
|
describe('BasicQueryEngine like/ilike routing by column encryption', () => {
|
|
1305
|
-
// The gate
|
|
1306
|
-
//
|
|
1307
|
-
// default
|
|
1406
|
+
// The gate stays off by default per #5383, so these cases opt in explicitly to pin the
|
|
1407
|
+
// declared-ILIKE behavior a deployment gets by setting the switch; the last one flips it back
|
|
1408
|
+
// off and pins the legacy rewrite-everything behavior that is the shipped default.
|
|
1308
1409
|
beforeEach(() => {
|
|
1309
1410
|
process.env.OM_SEARCH_USE_ILIKE_FOR_NON_ENCRYPTED_FIELDS = 'true'
|
|
1310
1411
|
})
|
|
@@ -1327,7 +1428,15 @@ describe('BasicQueryEngine like/ilike routing by column encryption', () => {
|
|
|
1327
1428
|
],
|
|
1328
1429
|
})
|
|
1329
1430
|
|
|
1330
|
-
|
|
1431
|
+
const ilikePatternsFor = (fakeDb: any, table: string, column: string): string[] => {
|
|
1432
|
+
const baseCall = fakeDb._calls.find((b: any) => b._ops.table === table)
|
|
1433
|
+
if (!baseCall) return []
|
|
1434
|
+
return baseCall._ops.wheres
|
|
1435
|
+
.filter((w: any) => Array.isArray(w) && String(w[0]).includes(column) && w[1] === 'ilike')
|
|
1436
|
+
.map((w: any) => String(w[2]))
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
test('a plaintext base column runs as SQL ILIKE even when tokens are available', async () => {
|
|
1331
1440
|
const fakeDb = fakeDbWithTokens()
|
|
1332
1441
|
const engine = new BasicQueryEngine(
|
|
1333
1442
|
{} as any,
|
|
@@ -1344,12 +1453,33 @@ describe('BasicQueryEngine like/ilike routing by column encryption', () => {
|
|
|
1344
1453
|
})
|
|
1345
1454
|
|
|
1346
1455
|
expect(applySearchTokensSpy).not.toHaveBeenCalled()
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1456
|
+
// Multi-word terms are ANDed per word rather than matched as one literal substring -- see the
|
|
1457
|
+
// TC-RESO-009 case below for why that distinction is the whole point of the reroute.
|
|
1458
|
+
expect(ilikePatternsFor(fakeDb, 'customer_entities', 'display_name')).toEqual(['%ZK%', '%1/2026%'])
|
|
1459
|
+
})
|
|
1460
|
+
|
|
1461
|
+
// #5803 regression guard, reported by CI on the first shape of this fix. The token subquery
|
|
1462
|
+
// matched a value carrying EVERY token in any order with anything in between, so
|
|
1463
|
+
// `?search=Warehouse <stamp>` matched `Warehouse A <stamp>`; TC-RESO-009 pins that as required
|
|
1464
|
+
// behavior. One verbatim `ILIKE '%Warehouse <stamp>%'` would not match -- the `A ` sits between
|
|
1465
|
+
// the words -- so the reroute has to AND one containment predicate per word to be a fix rather
|
|
1466
|
+
// than a trade of one broken search for another.
|
|
1467
|
+
test('a multi-word term matches words in order-independent positions, as the token path did', async () => {
|
|
1468
|
+
const fakeDb = fakeDbWithTokens()
|
|
1469
|
+
const engine = new BasicQueryEngine(
|
|
1470
|
+
{} as any,
|
|
1471
|
+
() => fakeDb as any,
|
|
1472
|
+
() => ({ getEncryptedFieldNames: async () => [] }) as any,
|
|
1351
1473
|
)
|
|
1352
|
-
|
|
1474
|
+
|
|
1475
|
+
await engine.query('customers:customer_entity', {
|
|
1476
|
+
tenantId: 't1',
|
|
1477
|
+
fields: ['id'],
|
|
1478
|
+
filters: { display_name: { $ilike: '%Warehouse 1757%' } },
|
|
1479
|
+
page: { page: 1, pageSize: 10 },
|
|
1480
|
+
})
|
|
1481
|
+
|
|
1482
|
+
expect(ilikePatternsFor(fakeDb, 'customer_entities', 'display_name')).toEqual(['%Warehouse%', '%1757%'])
|
|
1353
1483
|
})
|
|
1354
1484
|
|
|
1355
1485
|
test('an encrypted base column still routes through search tokens', async () => {
|
|
@@ -1433,8 +1563,70 @@ describe('BasicQueryEngine like/ilike routing by column encryption', () => {
|
|
|
1433
1563
|
expect(applySearchTokensSpy).toHaveBeenCalled()
|
|
1434
1564
|
})
|
|
1435
1565
|
|
|
1436
|
-
|
|
1437
|
-
|
|
1566
|
+
// #5803: the reported regression. `2026-08` and `2026-01` tokenize to the identical set
|
|
1567
|
+
// {202, 2026} because the fragment that tells them apart is shorter than minTokenLength, so the
|
|
1568
|
+
// token rewrite answered a search for one period with the other one. The declared predicate has
|
|
1569
|
+
// to reach SQL verbatim for the exact row to come back at all.
|
|
1570
|
+
test('a search term whose distinguishing fragment is dropped by the tokenizer still reaches SQL verbatim', async () => {
|
|
1571
|
+
const fakeDb = createFakeKysely({
|
|
1572
|
+
accounting_periods: [],
|
|
1573
|
+
search_tokens: [{ one: 1 }],
|
|
1574
|
+
'information_schema.tables': [{ table_name: 'search_tokens' }],
|
|
1575
|
+
'information_schema.columns': [
|
|
1576
|
+
{ table_name: 'accounting_periods', column_name: 'tenant_id' },
|
|
1577
|
+
{ table_name: 'accounting_periods', column_name: 'period_code' },
|
|
1578
|
+
],
|
|
1579
|
+
})
|
|
1580
|
+
const engine = new BasicQueryEngine(
|
|
1581
|
+
{} as any,
|
|
1582
|
+
() => fakeDb as any,
|
|
1583
|
+
() => ({ getEncryptedFieldNames: async () => [] }) as any,
|
|
1584
|
+
)
|
|
1585
|
+
const applySearchTokensSpy = jest.spyOn(engine as any, 'applySearchTokens')
|
|
1586
|
+
|
|
1587
|
+
await engine.query('accounting:accounting_period', {
|
|
1588
|
+
tenantId: 't1',
|
|
1589
|
+
fields: ['id'],
|
|
1590
|
+
filters: { period_code: { $ilike: '%2026-08%' } },
|
|
1591
|
+
page: { page: 1, pageSize: 10 },
|
|
1592
|
+
})
|
|
1593
|
+
|
|
1594
|
+
expect(applySearchTokensSpy).not.toHaveBeenCalled()
|
|
1595
|
+
const baseCall = fakeDb._calls.find((b: any) => b._ops.table === 'accounting_periods')
|
|
1596
|
+
expect(baseCall).toBeTruthy()
|
|
1597
|
+
const ilikeWhere = baseCall._ops.wheres.some(
|
|
1598
|
+
(w: any) => Array.isArray(w) && String(w[0]).includes('period_code') && w[1] === 'ilike' && w[2] === '%2026-08%',
|
|
1599
|
+
)
|
|
1600
|
+
expect(ilikeWhere).toBe(true)
|
|
1601
|
+
})
|
|
1602
|
+
|
|
1603
|
+
// #5803: `08` tokenizes to nothing at all, and the token path answered that with "no predicate"
|
|
1604
|
+
// -- every row in the table. A filter that matches everything is never the honest reading of a
|
|
1605
|
+
// declared containment predicate.
|
|
1606
|
+
test('a term too short to tokenize filters instead of being dropped', async () => {
|
|
1607
|
+
const fakeDb = fakeDbWithTokens()
|
|
1608
|
+
const engine = new BasicQueryEngine(
|
|
1609
|
+
{} as any,
|
|
1610
|
+
() => fakeDb as any,
|
|
1611
|
+
() => ({ getEncryptedFieldNames: async () => [] }) as any,
|
|
1612
|
+
)
|
|
1613
|
+
|
|
1614
|
+
await engine.query('customers:customer_entity', {
|
|
1615
|
+
tenantId: 't1',
|
|
1616
|
+
fields: ['id'],
|
|
1617
|
+
filters: { display_name: { $ilike: '%08%' } },
|
|
1618
|
+
page: { page: 1, pageSize: 10 },
|
|
1619
|
+
})
|
|
1620
|
+
|
|
1621
|
+
const baseCall = fakeDb._calls.find((b: any) => b._ops.table === 'customer_entities')
|
|
1622
|
+
const ilikeWhere = baseCall._ops.wheres.some(
|
|
1623
|
+
(w: any) => Array.isArray(w) && String(w[0]).includes('display_name') && w[1] === 'ilike' && w[2] === '%08%',
|
|
1624
|
+
)
|
|
1625
|
+
expect(ilikeWhere).toBe(true)
|
|
1626
|
+
})
|
|
1627
|
+
|
|
1628
|
+
test('with the flag explicitly off the token rewrite is kept even for plaintext columns', () => {
|
|
1629
|
+
process.env.OM_SEARCH_USE_ILIKE_FOR_NON_ENCRYPTED_FIELDS = 'false'
|
|
1438
1630
|
const fakeDb = fakeDbWithTokens()
|
|
1439
1631
|
const engine = new BasicQueryEngine(
|
|
1440
1632
|
{} as any,
|
|
@@ -20,7 +20,14 @@ export function fieldNameCandidates(field: string): string[] {
|
|
|
20
20
|
const raw = String(field || '').trim()
|
|
21
21
|
if (!raw) return []
|
|
22
22
|
const candidates = [raw, toSnakeCase(raw), toCamelCase(raw)]
|
|
23
|
-
if (raw.startsWith('cf:'))
|
|
23
|
+
if (raw.startsWith('cf:')) {
|
|
24
|
+
const sanitized = raw.replace(/[^a-zA-Z0-9_]/g, '_')
|
|
25
|
+
// The ORM query engine's plaintext-sort candidate scan projects a `cf:`
|
|
26
|
+
// sort key only under its dedicated `<alias>__sort` projection alias, never
|
|
27
|
+
// the plain jsonb-aggregate alias — so the in-memory sort needs that exact
|
|
28
|
+
// name to find the value on the phase-1 candidate rows (#5674 review).
|
|
29
|
+
candidates.push(sanitized, `${sanitized}__sort`)
|
|
30
|
+
}
|
|
24
31
|
return Array.from(new Set(candidates))
|
|
25
32
|
}
|
|
26
33
|
|
package/src/lib/query/engine.ts
CHANGED
|
@@ -20,13 +20,16 @@ import {
|
|
|
20
20
|
type SearchTokenProbeQueryBuilder,
|
|
21
21
|
} from '../search/availability'
|
|
22
22
|
import { tokenizeText } from '../search/tokenize'
|
|
23
|
+
import { buildContainmentPatterns } from '../search/containment'
|
|
23
24
|
import { fieldNameCandidates } from './encrypted-sort'
|
|
24
25
|
import { isTenantDataEncryptionEnabled } from '../encryption/toggles'
|
|
25
26
|
import { runBeforeQueryPipeline, runAfterQueryPipeline, type QueryExtensionContext } from './query-extension-runner'
|
|
26
27
|
import {
|
|
27
28
|
buildCustomFieldDefinitionIndexFromRows,
|
|
29
|
+
normalizeDefinitionKey,
|
|
28
30
|
resolveCfDefIndexOrgCandidates,
|
|
29
31
|
type CustomFieldDefinitionRow,
|
|
32
|
+
type CustomFieldDefinitionSummary,
|
|
30
33
|
type ResolvedCustomFieldDefinitions,
|
|
31
34
|
} from '../crud/custom-field-definition-index'
|
|
32
35
|
import { warnOnCiphertextLikeFallback } from './ciphertext-search-warning'
|
|
@@ -293,6 +296,25 @@ function buildFilterableCustomFieldJoins(
|
|
|
293
296
|
})
|
|
294
297
|
}
|
|
295
298
|
|
|
299
|
+
/** Custom field kinds stored numerically (`custom_field_values.value_int`/`value_float`) — sort numerically, not as text (#5674). */
|
|
300
|
+
const NUMERIC_CF_SORT_KINDS = new Set(['integer', 'float'])
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* Pick the winning `kind` for a `cf:` sort key from an already-resolved
|
|
304
|
+
* definition list — org-scoped beats tenant-wide beats global, same
|
|
305
|
+
* precedence as the dedicated sort-kind lookup (#5674 review).
|
|
306
|
+
*/
|
|
307
|
+
function pickCfSortKind(defs: CustomFieldDefinitionSummary[] | undefined): string | null {
|
|
308
|
+
if (!defs || !defs.length) return null
|
|
309
|
+
let winner: { kind: string; specificity: number } | null = null
|
|
310
|
+
for (const def of defs) {
|
|
311
|
+
if (!def.kind) continue
|
|
312
|
+
const specificity = def.organizationId != null ? 2 : def.tenantId != null ? 1 : 0
|
|
313
|
+
if (!winner || specificity >= winner.specificity) winner = { kind: def.kind, specificity }
|
|
314
|
+
}
|
|
315
|
+
return winner?.kind ?? null
|
|
316
|
+
}
|
|
317
|
+
|
|
296
318
|
function computeCustomFieldScore(cfg: Record<string, unknown>, kind: string, entityIndex: number) {
|
|
297
319
|
const listVisibleScore = cfg.listVisible === false ? 0 : 1
|
|
298
320
|
const formEditableScore = cfg.formEditable === false ? 0 : 1
|
|
@@ -446,9 +468,9 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
446
468
|
? await this.searchAvailability().hasTokens(String(entity), opts.tenantId ?? null, orgScope)
|
|
447
469
|
: false
|
|
448
470
|
const searchActive = searchEnabled && hasSearchTokens
|
|
449
|
-
//
|
|
450
|
-
//
|
|
451
|
-
// through search tokens ONLY for encrypted columns, where
|
|
471
|
+
// Gated on OM_SEARCH_USE_ILIKE_FOR_NON_ENCRYPTED_FIELDS (default false per #5383; set it to
|
|
472
|
+
// true to opt into the #5803 fix ahead of that follow-up). When enabled, base-column
|
|
473
|
+
// like/ilike is rerouted through search tokens ONLY for encrypted columns, where
|
|
452
474
|
// ILIKE against ciphertext cannot match. On a plaintext column SQL ILIKE is exact, and the token
|
|
453
475
|
// rewrite silently changes the result set: tokenization splits on non-alphanumerics and drops
|
|
454
476
|
// tokens shorter than minTokenLength, so a document-number search like "ZK 1/2026" degrades to
|
|
@@ -617,6 +639,27 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
617
639
|
})
|
|
618
640
|
}
|
|
619
641
|
}
|
|
642
|
+
// A PLAINTEXT base column the gate above kept off the token path: apply the declared
|
|
643
|
+
// containment once per word so the token subquery's word-order-independent matching survives
|
|
644
|
+
// the reroute (#5803 / TC-RESO-009). Chained `where`s ARE the AND the token
|
|
645
|
+
// `having count(distinct)` did. `fieldName` is required, so this never fires for a JOINed
|
|
646
|
+
// column — `applyJoinFilters` calls `applyFilterOp` without it (below), and that predicate
|
|
647
|
+
// keeps its pre-existing literal-containment behavior unchanged.
|
|
648
|
+
if (
|
|
649
|
+
(op === 'like' || op === 'ilike') &&
|
|
650
|
+
typeof value === 'string' &&
|
|
651
|
+
searchActive &&
|
|
652
|
+
fieldName &&
|
|
653
|
+
encryptedLikeFields !== null &&
|
|
654
|
+
!isEncryptedLikeField(encryptedLikeFields, fieldName)
|
|
655
|
+
) {
|
|
656
|
+
const patterns = buildContainmentPatterns(value)
|
|
657
|
+
if (patterns.length > 1) {
|
|
658
|
+
let next = builder
|
|
659
|
+
for (const pattern of patterns) next = this.applyColumnOp(next, column, op, pattern)
|
|
660
|
+
return next
|
|
661
|
+
}
|
|
662
|
+
}
|
|
620
663
|
return this.applyColumnOp(builder, column, op, value)
|
|
621
664
|
}
|
|
622
665
|
|
|
@@ -703,6 +746,7 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
703
746
|
hasJoinedAggregates: boolean
|
|
704
747
|
cfJsonAliases: Set<string>
|
|
705
748
|
cfMultiAliasByAlias: Map<string, string>
|
|
749
|
+
cfSortAliases: string[]
|
|
706
750
|
resolvedCustomFieldDefinitions: ResolvedCustomFieldDefinitions | undefined
|
|
707
751
|
}
|
|
708
752
|
|
|
@@ -994,10 +1038,70 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
994
1038
|
}
|
|
995
1039
|
}
|
|
996
1040
|
|
|
1041
|
+
// A `cf:` sort needs to know the field's declared `kind` so the ORDER BY
|
|
1042
|
+
// expression can cast numeric kinds instead of ordering by jsonb text
|
|
1043
|
+
// (#5674). Org-scoped beats tenant-wide beats global for the same key —
|
|
1044
|
+
// a same-key definition from an unrelated organization in the tenant
|
|
1045
|
+
// never gets to decide the kind (#5674 review).
|
|
1046
|
+
const cfSortKeys = Array.from(new Set(
|
|
1047
|
+
resolvedSorts.filter((sort) => sort.field.startsWith('cf:')).map((sort) => sort.field.slice(3))
|
|
1048
|
+
))
|
|
1049
|
+
const cfSortKinds = new Map<string, string>()
|
|
1050
|
+
if (cfSortKeys.length > 0) {
|
|
1051
|
+
// `includeCustomFields === true` already resolved every definition for
|
|
1052
|
+
// these entities into `resolvedCustomFieldDefinitions` above — reuse it
|
|
1053
|
+
// instead of a second `custom_field_defs` round trip (#5674 review).
|
|
1054
|
+
const keysNeedingQuery: string[] = []
|
|
1055
|
+
for (const key of cfSortKeys) {
|
|
1056
|
+
const kind = resolvedCustomFieldDefinitions
|
|
1057
|
+
? pickCfSortKind(resolvedCustomFieldDefinitions.index.get(normalizeDefinitionKey(key)))
|
|
1058
|
+
: null
|
|
1059
|
+
if (kind) cfSortKinds.set(key, kind)
|
|
1060
|
+
else keysNeedingQuery.push(key)
|
|
1061
|
+
}
|
|
1062
|
+
if (keysNeedingQuery.length > 0) {
|
|
1063
|
+
const sortEntityIds = Array.from(new Set(
|
|
1064
|
+
keysNeedingQuery
|
|
1065
|
+
.map((key) => keySource.get(key)?.entityId)
|
|
1066
|
+
.filter((id): id is EntityId => Boolean(id))
|
|
1067
|
+
.map((id) => String(id))
|
|
1068
|
+
))
|
|
1069
|
+
if (sortEntityIds.length > 0) {
|
|
1070
|
+
const cfSortOrgCandidates = resolveCfDefIndexOrgCandidates(opts.organizationIds, opts.organizationId ?? null)
|
|
1071
|
+
const kindRows = await db
|
|
1072
|
+
.selectFrom('custom_field_defs' as any)
|
|
1073
|
+
.select(['key' as any, 'kind' as any, 'organization_id' as any, 'tenant_id' as any])
|
|
1074
|
+
.where('entity_id' as any, 'in', sortEntityIds)
|
|
1075
|
+
.where('key' as any, 'in', keysNeedingQuery)
|
|
1076
|
+
.where('is_active' as any, '=', true)
|
|
1077
|
+
.where((eb: any) => eb.or([
|
|
1078
|
+
eb('tenant_id' as any, '=', tenantId),
|
|
1079
|
+
eb('tenant_id' as any, 'is', null),
|
|
1080
|
+
]))
|
|
1081
|
+
.where((eb: any) => cfSortOrgCandidates.length
|
|
1082
|
+
? eb.or([
|
|
1083
|
+
eb('organization_id' as any, 'is', null),
|
|
1084
|
+
eb('organization_id' as any, 'in', cfSortOrgCandidates),
|
|
1085
|
+
])
|
|
1086
|
+
: eb('organization_id' as any, 'is', null))
|
|
1087
|
+
.execute() as Array<{ key: string; kind: string | null; organization_id: string | null; tenant_id: string | null }>
|
|
1088
|
+
const winners = new Map<string, { kind: string; specificity: number }>()
|
|
1089
|
+
for (const row of kindRows) {
|
|
1090
|
+
if (!row.kind) continue
|
|
1091
|
+
const specificity = row.organization_id != null ? 2 : row.tenant_id != null ? 1 : 0
|
|
1092
|
+
const existing = winners.get(row.key)
|
|
1093
|
+
if (!existing || specificity >= existing.specificity) winners.set(row.key, { kind: row.kind, specificity })
|
|
1094
|
+
}
|
|
1095
|
+
for (const [key, entry] of winners) cfSortKinds.set(key, entry.kind)
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
997
1100
|
const cfValueExprByKey: Record<string, RawBuilder<string | null>> = {}
|
|
998
1101
|
const cfSelectedAliases: string[] = []
|
|
999
1102
|
const cfJsonAliases = new Set<string>()
|
|
1000
1103
|
const cfMultiAliasByAlias = new Map<string, string>()
|
|
1104
|
+
const cfSortAliases: string[] = []
|
|
1001
1105
|
for (const key of cfKeys) {
|
|
1002
1106
|
const source = keySource.get(key)
|
|
1003
1107
|
if (!source) continue
|
|
@@ -1127,6 +1231,12 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
1127
1231
|
// field covered by an encryption map such a leaf therefore compares against
|
|
1128
1232
|
// ciphertext and will not match.
|
|
1129
1233
|
//
|
|
1234
|
+
// This also means the #5803 plaintext-containment split (lib/search/containment) does not
|
|
1235
|
+
// apply here: `buildColumnOpExpression` below keeps a multi-word `like`/`ilike` leaf as one
|
|
1236
|
+
// literal pattern, so an OR-grouped search (e.g. `customers/api/people`'s multi-field
|
|
1237
|
+
// fallback) loses word-order independence on Basic while the Hybrid engine's OR groups
|
|
1238
|
+
// apply the split. Pre-existing divergence between the two engines; not tracked by an issue.
|
|
1239
|
+
//
|
|
1130
1240
|
// The count shape never populates cfValueExprByKey (it joins no cf tables), so
|
|
1131
1241
|
// its applicability test is key resolution itself — the same condition that
|
|
1132
1242
|
// gates the full shape's expression map — and a cf leaf compiles to a
|
|
@@ -1200,30 +1310,57 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
1200
1310
|
}
|
|
1201
1311
|
}
|
|
1202
1312
|
|
|
1203
|
-
// Sorting: base fields and cf:* (
|
|
1313
|
+
// Sorting: base fields and cf:* (a dedicated scalar alias, never the jsonb
|
|
1314
|
+
// projection alias — ordering by `to_jsonb(...)` compares arrays after every
|
|
1315
|
+
// scalar string regardless of contents, and a numeric kind must cast to
|
|
1316
|
+
// numeric instead of ordering the text CASE expression lexicographically (#5674)).
|
|
1317
|
+
let lastEmittedSortField: string | null = null
|
|
1204
1318
|
for (const s of isCountProjection ? [] : resolvedSorts) {
|
|
1205
1319
|
if (s.field.startsWith('cf:')) {
|
|
1206
1320
|
const key = s.field.slice(3)
|
|
1207
|
-
const
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
const
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1321
|
+
const sortAlias = sanitize(`cf:${key}__sort`)
|
|
1322
|
+
if (!cfSortAliases.includes(sortAlias)) {
|
|
1323
|
+
const source = keySource.get(key)
|
|
1324
|
+
const kind = cfSortKinds.get(key)
|
|
1325
|
+
let sortExpr: RawBuilder<unknown> | null = null
|
|
1326
|
+
if (source && kind && NUMERIC_CF_SORT_KINDS.has(kind)) {
|
|
1327
|
+
const sourceAliasSafe = sanitize(source.alias || 'src')
|
|
1328
|
+
const keyAliasSafe = sanitize(key)
|
|
1329
|
+
const valAlias = `cfv_${sourceAliasSafe}_${keyAliasSafe}`
|
|
1330
|
+
const numericColumn = kind === 'integer' ? 'value_int' : 'value_float'
|
|
1331
|
+
sortExpr = sql<string | null>`max((${sql.ref(`${valAlias}.${numericColumn}`)})::numeric)`
|
|
1332
|
+
} else {
|
|
1333
|
+
const expr = cfValueExprByKey[key]
|
|
1334
|
+
if (expr) sortExpr = sql<string | null>`max(${expr})`
|
|
1335
|
+
}
|
|
1336
|
+
if (sortExpr) {
|
|
1337
|
+
q = q.select(sortExpr.as(sortAlias))
|
|
1338
|
+
cfSortAliases.push(sortAlias)
|
|
1214
1339
|
}
|
|
1215
1340
|
}
|
|
1216
|
-
// Only order by an alias the
|
|
1217
|
-
//
|
|
1218
|
-
//
|
|
1219
|
-
//
|
|
1220
|
-
if (!requiresPlaintextSort &&
|
|
1221
|
-
|
|
1341
|
+
// Only order by an alias the query actually selects. A key that resolved
|
|
1342
|
+
// to no expression is dropped rather than emitted as an unselected alias,
|
|
1343
|
+
// which is what the base-column branch below already does when
|
|
1344
|
+
// `resolveBaseColumn` returns null.
|
|
1345
|
+
if (!requiresPlaintextSort && cfSortAliases.includes(sortAlias)) {
|
|
1346
|
+
const direction = sql.raw((s.dir ?? 'asc') === 'desc' ? 'desc' : 'asc')
|
|
1347
|
+
q = q.orderBy(sql`${sql.ref(sortAlias)} ${direction} NULLS LAST`)
|
|
1348
|
+
lastEmittedSortField = s.field
|
|
1222
1349
|
}
|
|
1223
1350
|
} else {
|
|
1224
|
-
if (!requiresPlaintextSort)
|
|
1351
|
+
if (!requiresPlaintextSort) {
|
|
1352
|
+
q = q.orderBy(qualify(s.field), (s.dir ?? 'asc') as any)
|
|
1353
|
+
lastEmittedSortField = s.field
|
|
1354
|
+
}
|
|
1225
1355
|
}
|
|
1226
1356
|
}
|
|
1357
|
+
// Stable tiebreak so ties (or NULLs) don't reorder arbitrarily across pages
|
|
1358
|
+
// (#5674). Only appended when a sort term actually made it into the ORDER
|
|
1359
|
+
// BY — an entirely unresolved cf sort must stay a no-op, not gain an
|
|
1360
|
+
// incidental `id` ordering the caller never asked for.
|
|
1361
|
+
if (!isCountProjection && lastEmittedSortField !== null && lastEmittedSortField !== 'id') {
|
|
1362
|
+
q = q.orderBy(qualify('id'), 'asc' as any)
|
|
1363
|
+
}
|
|
1227
1364
|
|
|
1228
1365
|
// Deduplicate if we joined CFs or extensions by grouping on base id. The count
|
|
1229
1366
|
// shape has neither, and must stay barrier-free for its LIMIT to bind.
|
|
@@ -1235,7 +1372,7 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
1235
1372
|
q = q.groupBy(`${table}.id`)
|
|
1236
1373
|
}
|
|
1237
1374
|
|
|
1238
|
-
return { builder: q, hasJoinedAggregates, cfJsonAliases, cfMultiAliasByAlias, resolvedCustomFieldDefinitions }
|
|
1375
|
+
return { builder: q, hasJoinedAggregates, cfJsonAliases, cfMultiAliasByAlias, cfSortAliases, resolvedCustomFieldDefinitions }
|
|
1239
1376
|
}
|
|
1240
1377
|
|
|
1241
1378
|
// Pagination
|
|
@@ -1247,6 +1384,7 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
1247
1384
|
hasJoinedAggregates,
|
|
1248
1385
|
cfJsonAliases,
|
|
1249
1386
|
cfMultiAliasByAlias,
|
|
1387
|
+
cfSortAliases,
|
|
1250
1388
|
resolvedCustomFieldDefinitions,
|
|
1251
1389
|
} = await buildQuery('full')
|
|
1252
1390
|
|
|
@@ -1330,6 +1468,19 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
1330
1468
|
}
|
|
1331
1469
|
}
|
|
1332
1470
|
|
|
1471
|
+
// The `cf:<key>__sort` alias exists only to give a `cf:` sort's ORDER BY
|
|
1472
|
+
// something to reference (and, for a numeric kind, to host the cast) — it
|
|
1473
|
+
// is never a requested output field. Strip it before rows leave this
|
|
1474
|
+
// function so it doesn't leak into API responses as a phantom custom field
|
|
1475
|
+
// `<key>__sort` (unlike `__is_multi`, which `normalizeCfJsonAliases`
|
|
1476
|
+
// already deletes) (#5674 review).
|
|
1477
|
+
const stripCfSortAliases = (rows: ResultRow[]) => {
|
|
1478
|
+
if (cfSortAliases.length === 0) return
|
|
1479
|
+
for (const row of rows) {
|
|
1480
|
+
for (const alias of cfSortAliases) delete row[alias]
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1333
1484
|
let pagedItems: ResultRow[]
|
|
1334
1485
|
let encryptedSortRowCapWarning: EncryptedSortRowCapWarning | undefined
|
|
1335
1486
|
|
|
@@ -1372,6 +1523,7 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
1372
1523
|
} else {
|
|
1373
1524
|
const pageRows = await qFull.where(qualify('id'), 'in', pageIds).execute() as ResultRow[]
|
|
1374
1525
|
normalizeCfJsonAliases(pageRows)
|
|
1526
|
+
stripCfSortAliases(pageRows)
|
|
1375
1527
|
const decryptedPageRows = decryptPayload
|
|
1376
1528
|
? await mapWithConcurrency(pageRows, DECRYPT_CONCURRENCY, decryptRow)
|
|
1377
1529
|
: pageRows
|
|
@@ -1384,6 +1536,7 @@ export class BasicQueryEngine implements QueryEngine {
|
|
|
1384
1536
|
const dataQuery = qFull.limit(pageSize).offset((page - 1) * pageSize)
|
|
1385
1537
|
const items = await dataQuery.execute() as ResultRow[]
|
|
1386
1538
|
normalizeCfJsonAliases(items)
|
|
1539
|
+
stripCfSortAliases(items)
|
|
1387
1540
|
pagedItems = decryptPayload
|
|
1388
1541
|
? await mapWithConcurrency(items, DECRYPT_CONCURRENCY, decryptRow)
|
|
1389
1542
|
: items
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import {
|
|
2
|
+
MIN_SCHEDULE_INTERVAL_MS,
|
|
3
|
+
isValidScheduleInterval,
|
|
4
|
+
parseScheduleInterval,
|
|
5
|
+
} from '../interval'
|
|
6
|
+
|
|
7
|
+
describe('parseScheduleInterval', () => {
|
|
8
|
+
it('converts each documented unit to milliseconds', () => {
|
|
9
|
+
expect(parseScheduleInterval('90s')).toBe(90 * 1000)
|
|
10
|
+
expect(parseScheduleInterval('15m')).toBe(15 * 60 * 1000)
|
|
11
|
+
expect(parseScheduleInterval('6h')).toBe(6 * 60 * 60 * 1000)
|
|
12
|
+
expect(parseScheduleInterval('1d')).toBe(24 * 60 * 60 * 1000)
|
|
13
|
+
})
|
|
14
|
+
|
|
15
|
+
it('rejects a value that carries no unit', () => {
|
|
16
|
+
expect(() => parseScheduleInterval('3600')).toThrow('Invalid interval format: 3600')
|
|
17
|
+
})
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
describe('isValidScheduleInterval', () => {
|
|
21
|
+
it.each(['1h', '6h', '24h', '15m', '60s', '1d'])('accepts the documented format %s', (interval) => {
|
|
22
|
+
expect(isValidScheduleInterval(interval)).toBe(true)
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
it.each(['3600', '', ' 1h', '1h ', '1.5h', '-1h', 'hourly', '1w', '1H'])(
|
|
26
|
+
'rejects %p, which the scheduler could never run',
|
|
27
|
+
(interval) => {
|
|
28
|
+
expect(isValidScheduleInterval(interval)).toBe(false)
|
|
29
|
+
},
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
it('rejects an interval shorter than the enforced minimum', () => {
|
|
33
|
+
expect(MIN_SCHEDULE_INTERVAL_MS).toBe(60 * 1000)
|
|
34
|
+
expect(isValidScheduleInterval('30s')).toBe(false)
|
|
35
|
+
expect(isValidScheduleInterval('0m')).toBe(false)
|
|
36
|
+
})
|
|
37
|
+
})
|