@open-mercato/shared 0.6.5-develop.5048.1.fd82f4ae17 → 0.6.5-develop.5099.1.85c0aff78c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/auth/apiKeyAuthCache.js +17 -6
- package/dist/lib/auth/apiKeyAuthCache.js.map +2 -2
- package/dist/lib/commands/scope.js +51 -37
- package/dist/lib/commands/scope.js.map +2 -2
- package/dist/lib/encryption/kms.js +43 -15
- package/dist/lib/encryption/kms.js.map +2 -2
- package/dist/lib/encryption/tenantDataEncryptionService.js +31 -4
- package/dist/lib/encryption/tenantDataEncryptionService.js.map +2 -2
- package/dist/lib/query/engine.js +41 -31
- package/dist/lib/query/engine.js.map +2 -2
- package/dist/lib/version.js +1 -1
- package/dist/lib/version.js.map +1 -1
- package/package.json +2 -2
- package/src/lib/auth/__tests__/apiKeyAuthCache.test.ts +35 -0
- package/src/lib/auth/apiKeyAuthCache.ts +20 -6
- package/src/lib/commands/__tests__/scope.test.ts +48 -0
- package/src/lib/commands/scope.ts +70 -55
- package/src/lib/encryption/__tests__/dek-lifecycle.test.ts +194 -0
- package/src/lib/encryption/__tests__/kms.test.ts +11 -5
- package/src/lib/encryption/kms.ts +60 -15
- package/src/lib/encryption/tenantDataEncryptionService.ts +37 -4
- package/src/lib/query/__tests__/resolve-registered-entity-table.test.ts +83 -0
- package/src/lib/query/engine.ts +59 -30
- package/src/modules/vector.ts +7 -0
|
@@ -1,44 +1,69 @@
|
|
|
1
1
|
import { CrudHttpError } from '@open-mercato/shared/lib/crud/errors'
|
|
2
2
|
import type { CommandRuntimeContext } from '@open-mercato/shared/lib/commands'
|
|
3
3
|
import { isOrganizationAccessAllowed } from '@open-mercato/shared/lib/auth/organizationAccess'
|
|
4
|
+
import { parseBooleanWithDefault } from '@open-mercato/shared/lib/boolean'
|
|
4
5
|
import { env } from 'process'
|
|
5
6
|
|
|
7
|
+
function buildScopeLogContext(ctx: CommandRuntimeContext) {
|
|
8
|
+
const requestInfo =
|
|
9
|
+
ctx.request && typeof ctx.request === 'object'
|
|
10
|
+
? {
|
|
11
|
+
method: (ctx.request as Request).method ?? undefined,
|
|
12
|
+
url: (ctx.request as Request).url ?? undefined,
|
|
13
|
+
}
|
|
14
|
+
: null
|
|
15
|
+
const scope = ctx.organizationScope
|
|
16
|
+
? {
|
|
17
|
+
selectedId: ctx.organizationScope.selectedId ?? null,
|
|
18
|
+
tenantId: ctx.organizationScope.tenantId ?? null,
|
|
19
|
+
allowedIdsCount: Array.isArray(ctx.organizationScope.allowedIds)
|
|
20
|
+
? ctx.organizationScope.allowedIds.length
|
|
21
|
+
: null,
|
|
22
|
+
filterIdsCount: Array.isArray(ctx.organizationScope.filterIds)
|
|
23
|
+
? ctx.organizationScope.filterIds.length
|
|
24
|
+
: null,
|
|
25
|
+
}
|
|
26
|
+
: null
|
|
27
|
+
return {
|
|
28
|
+
userId: ctx.auth?.sub ?? null,
|
|
29
|
+
actorTenantId: ctx.auth?.tenantId ?? null,
|
|
30
|
+
actorOrganizationId: ctx.auth?.orgId ?? null,
|
|
31
|
+
selectedOrganizationId: ctx.selectedOrganizationId ?? null,
|
|
32
|
+
organizationIdsCount: Array.isArray(ctx.organizationIds) ? ctx.organizationIds.length : null,
|
|
33
|
+
scope,
|
|
34
|
+
request: requestInfo,
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function isStrictOrganizationScopeEnforced(): boolean {
|
|
39
|
+
return parseBooleanWithDefault(env.OM_ENFORCE_ORG_SCOPE_STRICT, false)
|
|
40
|
+
}
|
|
41
|
+
|
|
6
42
|
function logScopeViolation(
|
|
7
43
|
ctx: CommandRuntimeContext,
|
|
8
44
|
expected: string,
|
|
9
45
|
actual: string | null
|
|
10
46
|
): void {
|
|
11
47
|
try {
|
|
12
|
-
const requestInfo =
|
|
13
|
-
ctx.request && typeof ctx.request === 'object'
|
|
14
|
-
? {
|
|
15
|
-
method: (ctx.request as Request).method ?? undefined,
|
|
16
|
-
url: (ctx.request as Request).url ?? undefined,
|
|
17
|
-
}
|
|
18
|
-
: null
|
|
19
|
-
const scope = ctx.organizationScope
|
|
20
|
-
? {
|
|
21
|
-
selectedId: ctx.organizationScope.selectedId ?? null,
|
|
22
|
-
tenantId: ctx.organizationScope.tenantId ?? null,
|
|
23
|
-
allowedIdsCount: Array.isArray(ctx.organizationScope.allowedIds)
|
|
24
|
-
? ctx.organizationScope.allowedIds.length
|
|
25
|
-
: null,
|
|
26
|
-
filterIdsCount: Array.isArray(ctx.organizationScope.filterIds)
|
|
27
|
-
? ctx.organizationScope.filterIds.length
|
|
28
|
-
: null,
|
|
29
|
-
}
|
|
30
|
-
: null
|
|
31
48
|
if (env.NODE_ENV !== 'test') {
|
|
32
49
|
console.warn('[scope] Forbidden organization scope mismatch detected', {
|
|
33
50
|
expectedId: expected,
|
|
34
51
|
actualId: actual,
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
52
|
+
...buildScopeLogContext(ctx),
|
|
53
|
+
})
|
|
54
|
+
}
|
|
55
|
+
} catch {
|
|
56
|
+
// best-effort logging
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function logUnscopedOrganizationAccess(ctx: CommandRuntimeContext, organizationId: string): void {
|
|
61
|
+
try {
|
|
62
|
+
if (env.NODE_ENV !== 'test') {
|
|
63
|
+
console.warn('[scope] Unscoped organization command executed without organization context', {
|
|
64
|
+
targetOrganizationId: organizationId,
|
|
65
|
+
strictEnforcement: isStrictOrganizationScopeEnforced(),
|
|
66
|
+
...buildScopeLogContext(ctx),
|
|
42
67
|
})
|
|
43
68
|
}
|
|
44
69
|
} catch {
|
|
@@ -52,36 +77,11 @@ function logTenantScopeViolation(
|
|
|
52
77
|
actualTenantId: string | null
|
|
53
78
|
): void {
|
|
54
79
|
try {
|
|
55
|
-
const requestInfo =
|
|
56
|
-
ctx.request && typeof ctx.request === 'object'
|
|
57
|
-
? {
|
|
58
|
-
method: (ctx.request as Request).method ?? undefined,
|
|
59
|
-
url: (ctx.request as Request).url ?? undefined,
|
|
60
|
-
}
|
|
61
|
-
: null
|
|
62
|
-
const scope = ctx.organizationScope
|
|
63
|
-
? {
|
|
64
|
-
selectedId: ctx.organizationScope.selectedId ?? null,
|
|
65
|
-
tenantId: ctx.organizationScope.tenantId ?? null,
|
|
66
|
-
allowedIdsCount: Array.isArray(ctx.organizationScope.allowedIds)
|
|
67
|
-
? ctx.organizationScope.allowedIds.length
|
|
68
|
-
: null,
|
|
69
|
-
filterIdsCount: Array.isArray(ctx.organizationScope.filterIds)
|
|
70
|
-
? ctx.organizationScope.filterIds.length
|
|
71
|
-
: null,
|
|
72
|
-
}
|
|
73
|
-
: null
|
|
74
80
|
if (env.NODE_ENV !== 'test') {
|
|
75
81
|
console.warn('[scope] Forbidden tenant scope mismatch detected', {
|
|
76
82
|
expectedTenantId,
|
|
77
83
|
actualTenantId,
|
|
78
|
-
|
|
79
|
-
actorTenantId: ctx.auth?.tenantId ?? null,
|
|
80
|
-
actorOrganizationId: ctx.auth?.orgId ?? null,
|
|
81
|
-
selectedOrganizationId: ctx.selectedOrganizationId ?? null,
|
|
82
|
-
organizationIdsCount: Array.isArray(ctx.organizationIds) ? ctx.organizationIds.length : null,
|
|
83
|
-
scope,
|
|
84
|
-
request: requestInfo,
|
|
84
|
+
...buildScopeLogContext(ctx),
|
|
85
85
|
})
|
|
86
86
|
}
|
|
87
87
|
} catch {
|
|
@@ -100,9 +100,24 @@ export function ensureOrganizationScope(ctx: CommandRuntimeContext, organization
|
|
|
100
100
|
if (!scope) {
|
|
101
101
|
if (isSuperAdmin) return
|
|
102
102
|
const currentOrg = ctx.selectedOrganizationId ?? ctx.auth?.orgId ?? null
|
|
103
|
-
if (currentOrg
|
|
104
|
-
|
|
105
|
-
|
|
103
|
+
if (currentOrg) {
|
|
104
|
+
if (currentOrg !== organizationId) {
|
|
105
|
+
logScopeViolation(ctx, organizationId, currentOrg)
|
|
106
|
+
throw new CrudHttpError(403, { error: 'Forbidden' })
|
|
107
|
+
}
|
|
108
|
+
return
|
|
109
|
+
}
|
|
110
|
+
// No current org could be resolved either. This branch previously returned
|
|
111
|
+
// with no validation and no signal — a fail-open-by-omission shape (#2441):
|
|
112
|
+
// a new command path reaching here with `organizationScope: null` would act
|
|
113
|
+
// on an arbitrary target org silently. Preserve the legacy allow behavior by
|
|
114
|
+
// default (the path is load-bearing) but make the unscoped access observable,
|
|
115
|
+
// and let operators harden it into a deny via OM_ENFORCE_ORG_SCOPE_STRICT.
|
|
116
|
+
if (organizationId) {
|
|
117
|
+
logUnscopedOrganizationAccess(ctx, organizationId)
|
|
118
|
+
if (isStrictOrganizationScopeEnforced()) {
|
|
119
|
+
throw new CrudHttpError(403, { error: 'Forbidden' })
|
|
120
|
+
}
|
|
106
121
|
}
|
|
107
122
|
return
|
|
108
123
|
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { decryptWithAesGcm, generateDek } from '../aes'
|
|
2
|
+
import { HashicorpVaultKmsService, type KmsService, type TenantDek } from '../kms'
|
|
3
|
+
import { TenantDataEncryptionService } from '../tenantDataEncryptionService'
|
|
4
|
+
|
|
5
|
+
const originalEnv = { ...process.env }
|
|
6
|
+
|
|
7
|
+
type DuckResponse = { ok: boolean; status: number; json: () => Promise<unknown> }
|
|
8
|
+
|
|
9
|
+
function jsonResponse(status: number, body: Record<string, unknown>): DuckResponse {
|
|
10
|
+
return { ok: status >= 200 && status < 300, status, json: async () => body }
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// Minimal in-memory Vault KV v2 simulator: GET reads the stored key, POST writes
|
|
14
|
+
// it unless a `cas: 0` write loses to an existing version (concurrent create).
|
|
15
|
+
function makeVaultSim() {
|
|
16
|
+
const store = new Map<string, string>()
|
|
17
|
+
const counts = { reads: 0, posts: 0, persisted: 0 }
|
|
18
|
+
const fetchMock = jest.fn(async (input: unknown, init?: RequestInit): Promise<DuckResponse> => {
|
|
19
|
+
const url = String(input)
|
|
20
|
+
const path = url.slice(url.indexOf('/v1/') + '/v1/'.length)
|
|
21
|
+
const method = (init?.method || 'GET').toUpperCase()
|
|
22
|
+
if (method === 'GET') {
|
|
23
|
+
counts.reads++
|
|
24
|
+
const key = store.get(path)
|
|
25
|
+
if (!key) return jsonResponse(404, { errors: [] })
|
|
26
|
+
return jsonResponse(200, { data: { data: { key } } })
|
|
27
|
+
}
|
|
28
|
+
counts.posts++
|
|
29
|
+
const body = init?.body ? (JSON.parse(String(init.body)) as { data?: { key?: string }; options?: { cas?: number } }) : {}
|
|
30
|
+
const cas = body.options?.cas
|
|
31
|
+
if (cas === 0 && store.has(path)) {
|
|
32
|
+
return jsonResponse(400, { errors: ['check-and-set parameter did not match the current version'] })
|
|
33
|
+
}
|
|
34
|
+
store.set(path, body.data?.key ?? '')
|
|
35
|
+
counts.persisted++
|
|
36
|
+
return jsonResponse(200, { data: { version: 1 } })
|
|
37
|
+
})
|
|
38
|
+
return { store, counts, fetchMock }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function installFetch(fetchMock: jest.Mock) {
|
|
42
|
+
;(globalThis as { fetch?: typeof fetch }).fetch = fetchMock as unknown as typeof fetch
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const fixedKey = (fill: number) => Buffer.alloc(32, fill).toString('base64')
|
|
46
|
+
|
|
47
|
+
describe('HashicorpVaultKmsService DEK creation race (issue #2746)', () => {
|
|
48
|
+
afterEach(() => {
|
|
49
|
+
process.env = { ...originalEnv }
|
|
50
|
+
jest.restoreAllMocks()
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
it('reuses an existing Vault DEK instead of overwriting it (read-before-write)', async () => {
|
|
54
|
+
const { store, counts, fetchMock } = makeVaultSim()
|
|
55
|
+
installFetch(fetchMock)
|
|
56
|
+
const service = new HashicorpVaultKmsService({ vaultAddr: 'http://vault.test', vaultToken: 'token' })
|
|
57
|
+
const existingKey = fixedKey(7)
|
|
58
|
+
store.set('secret/data/tenant_key_tenant-existing', existingKey)
|
|
59
|
+
|
|
60
|
+
const dek = await service.createTenantDek('tenant-existing')
|
|
61
|
+
|
|
62
|
+
expect(dek?.key).toBe(existingKey)
|
|
63
|
+
expect(counts.persisted).toBe(0) // never overwrote the active key
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
it('converges concurrent createTenantDek calls on a single key via CAS', async () => {
|
|
67
|
+
const { counts, fetchMock } = makeVaultSim()
|
|
68
|
+
installFetch(fetchMock)
|
|
69
|
+
const service = new HashicorpVaultKmsService({ vaultAddr: 'http://vault.test', vaultToken: 'token' })
|
|
70
|
+
|
|
71
|
+
const [a, b] = await Promise.all([
|
|
72
|
+
service.createTenantDek('tenant-cas'),
|
|
73
|
+
service.createTenantDek('tenant-cas'),
|
|
74
|
+
])
|
|
75
|
+
|
|
76
|
+
expect(a?.key).toBeTruthy()
|
|
77
|
+
expect(a?.key).toBe(b?.key) // both adopt the same winning key — no orphaned rows
|
|
78
|
+
expect(counts.persisted).toBe(1) // exactly one key persisted, last-write-wins eliminated
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
it('invalidateDek forces a re-read from Vault', async () => {
|
|
82
|
+
const { store, counts, fetchMock } = makeVaultSim()
|
|
83
|
+
installFetch(fetchMock)
|
|
84
|
+
const service = new HashicorpVaultKmsService({ vaultAddr: 'http://vault.test', vaultToken: 'token' })
|
|
85
|
+
const path = 'secret/data/tenant_key_tenant-inv'
|
|
86
|
+
store.set(path, fixedKey(3))
|
|
87
|
+
|
|
88
|
+
const first = await service.getTenantDek('tenant-inv')
|
|
89
|
+
await service.getTenantDek('tenant-inv') // served from the KMS TTL cache
|
|
90
|
+
expect(counts.reads).toBe(1)
|
|
91
|
+
|
|
92
|
+
store.set(path, fixedKey(9)) // operator rotates the key in Vault
|
|
93
|
+
service.invalidateDek('tenant-inv')
|
|
94
|
+
const rotated = await service.getTenantDek('tenant-inv')
|
|
95
|
+
|
|
96
|
+
expect(counts.reads).toBe(2)
|
|
97
|
+
expect(rotated?.key).not.toBe(first?.key)
|
|
98
|
+
})
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
describe('TenantDataEncryptionService DEK lifecycle (issue #2746)', () => {
|
|
102
|
+
const entityId = 'customers:person'
|
|
103
|
+
let tenantSeq = 0
|
|
104
|
+
const uniqueTenant = (label: string) => `tenant-${label}-2746-${tenantSeq++}`
|
|
105
|
+
|
|
106
|
+
afterEach(() => {
|
|
107
|
+
jest.restoreAllMocks()
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
function makeCreatingKms() {
|
|
111
|
+
const created: string[] = []
|
|
112
|
+
const kms: KmsService = {
|
|
113
|
+
isHealthy: () => true,
|
|
114
|
+
getTenantDek: jest.fn(async (): Promise<TenantDek | null> => null),
|
|
115
|
+
createTenantDek: jest.fn(async (tenantId: string): Promise<TenantDek | null> => {
|
|
116
|
+
const key = generateDek()
|
|
117
|
+
created.push(key)
|
|
118
|
+
return { tenantId, key, fetchedAt: Date.now() }
|
|
119
|
+
}),
|
|
120
|
+
}
|
|
121
|
+
return { kms, created }
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function makeFetchingKms() {
|
|
125
|
+
const kms: KmsService = {
|
|
126
|
+
isHealthy: () => true,
|
|
127
|
+
getTenantDek: jest.fn(async (tenantId: string): Promise<TenantDek | null> => ({
|
|
128
|
+
tenantId,
|
|
129
|
+
key: generateDek(),
|
|
130
|
+
fetchedAt: Date.now(),
|
|
131
|
+
})),
|
|
132
|
+
createTenantDek: jest.fn(async (): Promise<TenantDek | null> => null),
|
|
133
|
+
}
|
|
134
|
+
return { kms }
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
it('dedupes concurrent first-time DEK creation so no row is orphaned', async () => {
|
|
138
|
+
const { kms, created } = makeCreatingKms()
|
|
139
|
+
const service = new TenantDataEncryptionService({} as never, { kms })
|
|
140
|
+
jest.spyOn(service, 'isEnabled').mockReturnValue(true)
|
|
141
|
+
;(service as unknown as { getMap: () => Promise<{ entityId: string; fields: { field: string }[] }> }).getMap =
|
|
142
|
+
jest.fn(async () => ({ entityId, fields: [{ field: 'secret' }] }))
|
|
143
|
+
const tenantId = uniqueTenant('race')
|
|
144
|
+
|
|
145
|
+
const rows = await Promise.all(
|
|
146
|
+
Array.from({ length: 12 }, (_, i) =>
|
|
147
|
+
service.encryptEntityPayload(entityId, { secret: `value-${i}` }, tenantId),
|
|
148
|
+
),
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
expect((kms.createTenantDek as jest.Mock)).toHaveBeenCalledTimes(1)
|
|
152
|
+
expect(new Set(created).size).toBe(1)
|
|
153
|
+
|
|
154
|
+
const dek = await service.getDek(tenantId)
|
|
155
|
+
expect(dek).not.toBeNull()
|
|
156
|
+
rows.forEach((row, i) => {
|
|
157
|
+
expect(decryptWithAesGcm(String(row.secret), (dek as TenantDek).key)).toBe(`value-${i}`)
|
|
158
|
+
})
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
it('re-fetches a cached DEK after the TTL elapses (no stale key after rotation)', async () => {
|
|
162
|
+
const { kms } = makeFetchingKms()
|
|
163
|
+
const service = new TenantDataEncryptionService({} as never, { kms })
|
|
164
|
+
const tenantId = uniqueTenant('ttl')
|
|
165
|
+
let now = 1_700_000_000_000
|
|
166
|
+
jest.spyOn(Date, 'now').mockImplementation(() => now)
|
|
167
|
+
|
|
168
|
+
await service.getDek(tenantId)
|
|
169
|
+
await service.getDek(tenantId)
|
|
170
|
+
expect((kms.getTenantDek as jest.Mock)).toHaveBeenCalledTimes(1) // cached within TTL
|
|
171
|
+
|
|
172
|
+
now += 15 * 60 * 1000 + 1
|
|
173
|
+
await service.getDek(tenantId)
|
|
174
|
+
expect((kms.getTenantDek as jest.Mock)).toHaveBeenCalledTimes(2) // expired → re-fetch
|
|
175
|
+
})
|
|
176
|
+
|
|
177
|
+
it('invalidateDek clears both the service cache and the KMS cache', async () => {
|
|
178
|
+
const { kms } = makeFetchingKms()
|
|
179
|
+
const kmsInvalidate = jest.fn()
|
|
180
|
+
;(kms as KmsService).invalidateDek = kmsInvalidate
|
|
181
|
+
const service = new TenantDataEncryptionService({} as never, { kms })
|
|
182
|
+
const tenantId = uniqueTenant('invalidate')
|
|
183
|
+
|
|
184
|
+
await service.getDek(tenantId)
|
|
185
|
+
await service.getDek(tenantId)
|
|
186
|
+
expect((kms.getTenantDek as jest.Mock)).toHaveBeenCalledTimes(1)
|
|
187
|
+
|
|
188
|
+
service.invalidateDek(tenantId)
|
|
189
|
+
expect(kmsInvalidate).toHaveBeenCalledWith(tenantId)
|
|
190
|
+
|
|
191
|
+
await service.getDek(tenantId)
|
|
192
|
+
expect((kms.getTenantDek as jest.Mock)).toHaveBeenCalledTimes(2)
|
|
193
|
+
})
|
|
194
|
+
})
|
|
@@ -10,11 +10,17 @@ describe('kms timeout handling', () => {
|
|
|
10
10
|
})
|
|
11
11
|
|
|
12
12
|
it('marks Vault unhealthy after a timed out write', async () => {
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
// createTenantDek now reads-before-write (#2746): the read probe answers fast
|
|
14
|
+
// with no existing key so the flow reaches the write, which then times out.
|
|
15
|
+
const fetchMock = jest.fn((_url: string, init?: RequestInit) => {
|
|
16
|
+
const method = (init?.method || 'GET').toUpperCase()
|
|
17
|
+
if (method === 'GET') {
|
|
18
|
+
return Promise.resolve({ ok: true, status: 200, json: async () => ({ data: { data: {} } }) })
|
|
19
|
+
}
|
|
20
|
+
return new Promise((_resolve, reject) => {
|
|
15
21
|
init?.signal?.addEventListener('abort', () => reject(new Error('aborted')))
|
|
16
|
-
})
|
|
17
|
-
)
|
|
22
|
+
})
|
|
23
|
+
})
|
|
18
24
|
;(globalThis as { fetch?: typeof fetch }).fetch = fetchMock as typeof fetch
|
|
19
25
|
|
|
20
26
|
const service = new HashicorpVaultKmsService({
|
|
@@ -25,7 +31,7 @@ describe('kms timeout handling', () => {
|
|
|
25
31
|
|
|
26
32
|
await expect(service.createTenantDek('tenant-1')).resolves.toBeNull()
|
|
27
33
|
expect(service.isHealthy()).toBe(false)
|
|
28
|
-
expect(fetchMock).toHaveBeenCalledTimes(
|
|
34
|
+
expect(fetchMock).toHaveBeenCalledTimes(2) // read probe + the timed-out write
|
|
29
35
|
})
|
|
30
36
|
|
|
31
37
|
it('falls back to derived keys after the primary Vault call times out', async () => {
|
|
@@ -22,6 +22,7 @@ export interface KmsService {
|
|
|
22
22
|
getTenantDek(tenantId: string): Promise<TenantDek | null>
|
|
23
23
|
createTenantDek(tenantId: string): Promise<TenantDek | null>
|
|
24
24
|
isHealthy(): boolean
|
|
25
|
+
invalidateDek?(tenantId: string): void
|
|
25
26
|
}
|
|
26
27
|
|
|
27
28
|
class FallbackKmsService implements KmsService {
|
|
@@ -76,6 +77,11 @@ class FallbackKmsService implements KmsService {
|
|
|
76
77
|
}
|
|
77
78
|
return null
|
|
78
79
|
}
|
|
80
|
+
|
|
81
|
+
invalidateDek(tenantId: string): void {
|
|
82
|
+
this.primary.invalidateDek?.(tenantId)
|
|
83
|
+
this.fallback?.invalidateDek?.(tenantId)
|
|
84
|
+
}
|
|
79
85
|
}
|
|
80
86
|
|
|
81
87
|
type VaultClientOpts = {
|
|
@@ -90,6 +96,10 @@ type VaultReadResponse = {
|
|
|
90
96
|
data?: { data?: { key?: string; version?: number }; metadata?: Record<string, unknown> }
|
|
91
97
|
}
|
|
92
98
|
|
|
99
|
+
// 'conflict' = a check-and-set write lost to a concurrent writer (normal race
|
|
100
|
+
// outcome, Vault still healthy); 'error' = the write genuinely failed.
|
|
101
|
+
type VaultWriteOutcome = 'ok' | 'conflict' | 'error'
|
|
102
|
+
|
|
93
103
|
function normalizeEnv(value: string | undefined): string {
|
|
94
104
|
if (!value) return ''
|
|
95
105
|
return value.trim().replace(/(?:^['"]|['"]$)/g, '')
|
|
@@ -230,11 +240,13 @@ export class HashicorpVaultKmsService implements KmsService {
|
|
|
230
240
|
}
|
|
231
241
|
}
|
|
232
242
|
|
|
233
|
-
private async writeVault(path: string, key: string): Promise<
|
|
243
|
+
private async writeVault(path: string, key: string, opts?: { cas?: number }): Promise<VaultWriteOutcome> {
|
|
234
244
|
if (!this.vaultAddr || !this.vaultToken) {
|
|
235
245
|
this.healthy = false
|
|
236
|
-
return
|
|
246
|
+
return 'error'
|
|
237
247
|
}
|
|
248
|
+
const body: { data: { key: string }; options?: { cas: number } } = { data: { key } }
|
|
249
|
+
if (typeof opts?.cas === 'number') body.options = { cas: opts.cas }
|
|
238
250
|
try {
|
|
239
251
|
const res = await fetchWithTimeout(`${this.vaultAddr}/v1/${path}`, {
|
|
240
252
|
method: 'POST',
|
|
@@ -242,14 +254,23 @@ export class HashicorpVaultKmsService implements KmsService {
|
|
|
242
254
|
'X-Vault-Token': this.vaultToken,
|
|
243
255
|
'Content-Type': 'application/json',
|
|
244
256
|
},
|
|
245
|
-
body: JSON.stringify(
|
|
257
|
+
body: JSON.stringify(body),
|
|
246
258
|
timeoutMs: this.requestTimeoutMs,
|
|
247
259
|
})
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
260
|
+
if (res.ok) {
|
|
261
|
+
this.healthy = true
|
|
262
|
+
return 'ok'
|
|
251
263
|
}
|
|
252
|
-
|
|
264
|
+
// KV v2 returns 400 when a check-and-set write loses to a concurrent
|
|
265
|
+
// writer (path already at a newer version). That is a normal race outcome,
|
|
266
|
+
// not an unhealthy Vault — don't flip `healthy`.
|
|
267
|
+
if (typeof opts?.cas === 'number' && res.status === 400) {
|
|
268
|
+
console.warn('⚠️ [encryption][kms] Vault write CAS conflict (concurrent DEK create)', { path, status: res.status })
|
|
269
|
+
return 'conflict'
|
|
270
|
+
}
|
|
271
|
+
this.healthy = false
|
|
272
|
+
console.warn('⚠️ [encryption][kms] Vault write failed', { path, status: res.status })
|
|
273
|
+
return 'error'
|
|
253
274
|
} catch (err) {
|
|
254
275
|
this.healthy = false
|
|
255
276
|
console.warn('⚠️ [encryption][kms] Vault write error', {
|
|
@@ -257,7 +278,7 @@ export class HashicorpVaultKmsService implements KmsService {
|
|
|
257
278
|
error: (err as Error)?.message || String(err),
|
|
258
279
|
timeoutMs: this.requestTimeoutMs,
|
|
259
280
|
})
|
|
260
|
-
return
|
|
281
|
+
return 'error'
|
|
261
282
|
}
|
|
262
283
|
}
|
|
263
284
|
|
|
@@ -287,16 +308,40 @@ export class HashicorpVaultKmsService implements KmsService {
|
|
|
287
308
|
}
|
|
288
309
|
|
|
289
310
|
async createTenantDek(tenantId: string): Promise<TenantDek | null> {
|
|
290
|
-
const key = generateDek()
|
|
291
311
|
const path = this.buildKeyPath(tenantId)
|
|
292
|
-
|
|
293
|
-
|
|
312
|
+
// Read-before-write: if a DEK already exists for this tenant (another request
|
|
313
|
+
// or process created it first), adopt it instead of overwriting the active
|
|
314
|
+
// key — overwriting orphans every row already encrypted under it (#2746).
|
|
315
|
+
const existing = await this.readVault(path)
|
|
316
|
+
const existingKey = existing?.data?.data?.key
|
|
317
|
+
if (existingKey) {
|
|
318
|
+
return this.remember({ tenantId, key: existingKey, fetchedAt: this.now() })
|
|
319
|
+
}
|
|
320
|
+
// A read failure (timeout / 5xx) flips `healthy` off; don't blind-write a new
|
|
321
|
+
// key over a possibly-existing one we just couldn't read — let the caller fall back.
|
|
322
|
+
if (!this.healthy) return null
|
|
323
|
+
const key = generateDek()
|
|
324
|
+
const outcome = await this.writeVault(path, key, { cas: 0 })
|
|
325
|
+
if (outcome === 'ok') {
|
|
294
326
|
console.info('🔑 [encryption][kms] Stored tenant DEK in Vault', { tenantId, path })
|
|
295
|
-
|
|
296
|
-
|
|
327
|
+
return this.remember({ tenantId, key, fetchedAt: this.now() })
|
|
328
|
+
}
|
|
329
|
+
if (outcome === 'conflict') {
|
|
330
|
+
// A concurrent create won the CAS race — adopt the winner's key so both
|
|
331
|
+
// callers encrypt under the same DEK.
|
|
332
|
+
const winner = await this.readVault(path)
|
|
333
|
+
const winnerKey = winner?.data?.data?.key
|
|
334
|
+
if (winnerKey) {
|
|
335
|
+
console.info('🔑 [encryption][kms] Adopted concurrently-created tenant DEK', { tenantId, path })
|
|
336
|
+
return this.remember({ tenantId, key: winnerKey, fetchedAt: this.now() })
|
|
337
|
+
}
|
|
297
338
|
}
|
|
298
|
-
|
|
299
|
-
return
|
|
339
|
+
console.warn('⚠️ [encryption][kms] Failed to store tenant DEK in Vault', { tenantId, path })
|
|
340
|
+
return null
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
invalidateDek(tenantId: string): void {
|
|
344
|
+
this.cache.delete(tenantId)
|
|
300
345
|
}
|
|
301
346
|
}
|
|
302
347
|
|
|
@@ -22,6 +22,10 @@ type MapCacheKey = {
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
const MAP_MISS_TTL_MS = 5 * 60 * 1000
|
|
25
|
+
// Mirror the Vault KMS default DEK TTL so a rotated/revoked tenant key is picked
|
|
26
|
+
// up by long-lived processes without a restart (#2746). The service-level cache
|
|
27
|
+
// previously had no TTL and shadowed the KMS's own 15-minute expiry.
|
|
28
|
+
const DEK_CACHE_TTL_MS = 15 * 60 * 1000
|
|
25
29
|
|
|
26
30
|
function cacheKey(key: MapCacheKey): string {
|
|
27
31
|
return [
|
|
@@ -106,11 +110,13 @@ export class TenantDataEncryptionService {
|
|
|
106
110
|
private static globalMemoryCache = new Map<string, EncryptionMapRecord>()
|
|
107
111
|
private static globalInflightMaps = new Map<string, Promise<EncryptionMapRecord | null>>()
|
|
108
112
|
private static globalDekCache = new Map<string, TenantDek>()
|
|
113
|
+
private static globalInflightDeks = new Map<string, Promise<TenantDek | null>>()
|
|
109
114
|
private static globalMissCache = new Map<string, number>()
|
|
110
115
|
private readonly kms: KmsService
|
|
111
116
|
private readonly cache?: CacheStrategy
|
|
112
117
|
private readonly memoryCache = TenantDataEncryptionService.globalMemoryCache
|
|
113
118
|
private readonly dekCache = TenantDataEncryptionService.globalDekCache
|
|
119
|
+
private readonly inflightDeks = TenantDataEncryptionService.globalInflightDeks
|
|
114
120
|
private readonly inflightMaps = TenantDataEncryptionService.globalInflightMaps
|
|
115
121
|
private readonly missCache = TenantDataEncryptionService.globalMissCache
|
|
116
122
|
|
|
@@ -126,10 +132,15 @@ export class TenantDataEncryptionService {
|
|
|
126
132
|
return isTenantDataEncryptionEnabled() && this.kms.isHealthy()
|
|
127
133
|
}
|
|
128
134
|
|
|
135
|
+
private isDekExpired(dek: TenantDek): boolean {
|
|
136
|
+
return Date.now() - dek.fetchedAt > DEK_CACHE_TTL_MS
|
|
137
|
+
}
|
|
138
|
+
|
|
129
139
|
async getDek(tenantId: string | null | undefined): Promise<TenantDek | null> {
|
|
130
140
|
if (!tenantId) return null
|
|
131
141
|
const cached = this.dekCache.get(tenantId)
|
|
132
|
-
if (cached) return cached
|
|
142
|
+
if (cached && !this.isDekExpired(cached)) return cached
|
|
143
|
+
if (cached) this.dekCache.delete(tenantId)
|
|
133
144
|
const dek = await this.kms.getTenantDek(tenantId)
|
|
134
145
|
if (!dek) {
|
|
135
146
|
debug('🔎 dek.miss', { tenantId })
|
|
@@ -144,9 +155,22 @@ export class TenantDataEncryptionService {
|
|
|
144
155
|
const existing = await this.getDek(tenantId)
|
|
145
156
|
if (existing || !tenantId) return existing ?? null
|
|
146
157
|
if (typeof this.kms.createTenantDek !== 'function') return existing ?? null
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
158
|
+
// Dedupe concurrent first-time creation within this process so two callers
|
|
159
|
+
// can't each generate a distinct DEK and overwrite one another (#2746).
|
|
160
|
+
// Mirrors the encryption-map inflight dedupe (`globalInflightMaps`).
|
|
161
|
+
const pending = this.inflightDeks.get(tenantId)
|
|
162
|
+
if (pending) return pending
|
|
163
|
+
const creation = (async () => {
|
|
164
|
+
const created = await this.kms.createTenantDek(tenantId)
|
|
165
|
+
if (created) this.dekCache.set(tenantId, created)
|
|
166
|
+
return created ?? null
|
|
167
|
+
})()
|
|
168
|
+
this.inflightDeks.set(tenantId, creation)
|
|
169
|
+
try {
|
|
170
|
+
return await creation
|
|
171
|
+
} finally {
|
|
172
|
+
this.inflightDeks.delete(tenantId)
|
|
173
|
+
}
|
|
150
174
|
}
|
|
151
175
|
|
|
152
176
|
async createDek(tenantId: string): Promise<TenantDek | null> {
|
|
@@ -246,6 +270,15 @@ export class TenantDataEncryptionService {
|
|
|
246
270
|
}
|
|
247
271
|
}
|
|
248
272
|
|
|
273
|
+
// Force a flush of a tenant's cached DEK across the service-level cache and the
|
|
274
|
+
// underlying KMS cache so an operator can pick up a rotated/revoked key without
|
|
275
|
+
// a process restart (#2746).
|
|
276
|
+
invalidateDek(tenantId: string): void {
|
|
277
|
+
this.dekCache.delete(tenantId)
|
|
278
|
+
this.inflightDeks.delete(tenantId)
|
|
279
|
+
this.kms.invalidateDek?.(tenantId)
|
|
280
|
+
}
|
|
281
|
+
|
|
249
282
|
async getEncryptedFieldNames(
|
|
250
283
|
entityId: string,
|
|
251
284
|
tenantId: string | null | undefined,
|