@open-mercato/shared 0.6.7-develop.6788.1.c2a1520a30 → 0.6.7-develop.6814.1.0627c7e9f1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/AGENTS.md +1 -0
- package/dist/lib/auth/organizationScope.js +34 -0
- package/dist/lib/auth/organizationScope.js.map +7 -0
- package/dist/lib/di/container.js +33 -3
- package/dist/lib/di/container.js.map +2 -2
- package/dist/lib/email/send.js +56 -0
- package/dist/lib/email/send.js.map +2 -2
- package/dist/lib/query/engine.js +25 -42
- package/dist/lib/query/engine.js.map +2 -2
- package/dist/lib/query/types.js.map +1 -1
- package/dist/lib/search/availability.js +107 -0
- package/dist/lib/search/availability.js.map +7 -0
- package/dist/lib/search/config.js +24 -2
- package/dist/lib/search/config.js.map +2 -2
- package/dist/lib/search/tokenize.js +23 -11
- package/dist/lib/search/tokenize.js.map +2 -2
- package/dist/lib/version.js +1 -1
- package/dist/lib/version.js.map +1 -1
- package/dist/lib/webhooks/body.js +57 -0
- package/dist/lib/webhooks/body.js.map +7 -0
- package/dist/lib/webhooks/index.js +13 -1
- package/dist/lib/webhooks/index.js.map +2 -2
- package/dist/lib/webhooks/verify.js +12 -7
- package/dist/lib/webhooks/verify.js.map +2 -2
- package/dist/modules/payment_gateways/types.js +8 -1
- package/dist/modules/payment_gateways/types.js.map +2 -2
- package/dist/modules/widgets/extension-points.js.map +1 -1
- package/package.json +8 -8
- package/src/lib/auth/__tests__/organizationScope.test.ts +107 -0
- package/src/lib/auth/organizationScope.ts +65 -0
- package/src/lib/di/__tests__/container-app-di-absent.test.ts +139 -0
- package/src/lib/di/__tests__/container-app-di.test.ts +173 -0
- package/src/lib/di/container.ts +45 -4
- package/src/lib/email/__tests__/send.test.ts +37 -1
- package/src/lib/email/send.ts +79 -0
- package/src/lib/i18n/__tests__/server-dictionary-cache.test.ts +30 -0
- package/src/lib/query/__tests__/engine.test.ts +65 -13
- package/src/lib/query/engine.ts +36 -60
- package/src/lib/query/types.ts +4 -5
- package/src/lib/search/__tests__/availability.test.ts +211 -0
- package/src/lib/search/__tests__/config.test.ts +59 -0
- package/src/lib/search/__tests__/tokenize.test.ts +49 -0
- package/src/lib/search/availability.ts +232 -0
- package/src/lib/search/config.ts +28 -0
- package/src/lib/search/tokenize.ts +33 -11
- package/src/lib/webhooks/__tests__/body.test.ts +95 -0
- package/src/lib/webhooks/__tests__/verify.test.ts +20 -1
- package/src/lib/webhooks/body.ts +70 -0
- package/src/lib/webhooks/index.ts +7 -1
- package/src/lib/webhooks/verify.ts +17 -9
- package/src/modules/payment_gateways/__tests__/types.test.ts +37 -0
- package/src/modules/payment_gateways/types.ts +15 -0
- package/src/modules/widgets/extension-points.ts +104 -1
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { sql } from 'kysely'
|
|
2
|
+
import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
|
|
3
|
+
|
|
4
|
+
export type OrganizationScope = { ids: string[]; includeNull: boolean }
|
|
5
|
+
|
|
6
|
+
export type SearchTokenSourceRef = { entity: string; recordIdColumn?: string }
|
|
7
|
+
|
|
8
|
+
type ProbeExpression = object | string | readonly string[] | null
|
|
9
|
+
|
|
10
|
+
export type SearchTokenProbeQueryBuilder = {
|
|
11
|
+
select: (selection: ProbeExpression) => SearchTokenProbeQueryBuilder
|
|
12
|
+
where: (column: ProbeExpression, operator?: string, value?: ProbeExpression) => SearchTokenProbeQueryBuilder
|
|
13
|
+
limit: (count: number) => SearchTokenProbeQueryBuilder
|
|
14
|
+
executeTakeFirst: () => Promise<object | undefined>
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export type SearchTokenProbeDb = { selectFrom: (table: string) => SearchTokenProbeQueryBuilder }
|
|
18
|
+
|
|
19
|
+
export type SearchTokenAvailabilityDebugPayload = {
|
|
20
|
+
entity: string
|
|
21
|
+
tenantId: string | null
|
|
22
|
+
organizationScope?: OrganizationScope | null
|
|
23
|
+
recordIdColumn?: string
|
|
24
|
+
hasTokens?: boolean
|
|
25
|
+
error?: string
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export type SearchTokenAvailabilityDeps = {
|
|
29
|
+
getDb: () => SearchTokenProbeDb
|
|
30
|
+
getConfig: () => { enabled: boolean }
|
|
31
|
+
applyOrganizationScope: (
|
|
32
|
+
query: SearchTokenProbeQueryBuilder,
|
|
33
|
+
column: string,
|
|
34
|
+
scope: OrganizationScope,
|
|
35
|
+
) => SearchTokenProbeQueryBuilder
|
|
36
|
+
logDebug: (event: string, payload: SearchTokenAvailabilityDebugPayload) => void
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type SearchTokenAvailability = {
|
|
40
|
+
/**
|
|
41
|
+
* The cheap, statically-known half of the decision: search is configured on
|
|
42
|
+
* AND the `search_tokens` table exists. Safe to resolve eagerly (consumers
|
|
43
|
+
* like custom-field source attachment need it before any filter is
|
|
44
|
+
* inspected); the table probe is memoized per instance.
|
|
45
|
+
*/
|
|
46
|
+
staticEnabled: () => Promise<boolean>
|
|
47
|
+
/**
|
|
48
|
+
* The expensive half: does `search_tokens` hold any row for this
|
|
49
|
+
* (entity, tenant, organization scope)? Historically the `LIMIT 1` probe
|
|
50
|
+
* could degenerate into a seq scan on a large table (#4723), so callers MUST
|
|
51
|
+
* only ask when the query actually carries a like/ilike filter — use
|
|
52
|
+
* `hasSearchFilter` for the gate. Three layers keep it cheap for the end
|
|
53
|
+
* user: index-usable predicates (no `IS NOT DISTINCT FROM`) served by the
|
|
54
|
+
* dedicated `search_tokens_presence_idx (entity_type, tenant_id,
|
|
55
|
+
* organization_id)` prefix — which makes the MISS as cheap as the hit — a
|
|
56
|
+
* per-request instance memo, and a process-level TTL cache
|
|
57
|
+
* (`OM_SEARCH_TOKEN_PRESENCE_CACHE_MS`, default 30s) that amortizes the
|
|
58
|
+
* probe across requests. Probe errors log `search:has-tokens-error`,
|
|
59
|
+
* resolve to `false`, and are never TTL-cached.
|
|
60
|
+
*/
|
|
61
|
+
hasTokens: (entity: string, tenantId: string | null, orgScope?: OrganizationScope | null) => Promise<boolean>
|
|
62
|
+
/** First-hit sweep over token sources; logs `search:source-has-tokens` per probed source. */
|
|
63
|
+
anySourceHasTokens: (
|
|
64
|
+
sources: SearchTokenSourceRef[],
|
|
65
|
+
tenantId: string | null,
|
|
66
|
+
orgScope?: OrganizationScope | null,
|
|
67
|
+
) => Promise<boolean>
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function isSearchFilterOp(op: string | null | undefined): boolean {
|
|
71
|
+
return op === 'like' || op === 'ilike'
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The single definition of "this query actually searches". Every consumer of
|
|
76
|
+
* the token-availability answer sits behind a like/ilike guard, so when this
|
|
77
|
+
* returns `false` the `hasTokens` probe's answer would never be read — gate
|
|
78
|
+
* the probe on it.
|
|
79
|
+
*/
|
|
80
|
+
export function hasSearchFilter(filters: ReadonlyArray<{ op?: string | null }>): boolean {
|
|
81
|
+
return filters.some((filter) => isSearchFilterOp(filter.op))
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function orgScopeKey(scope: OrganizationScope | null | undefined): string {
|
|
85
|
+
if (!scope) return 'none'
|
|
86
|
+
return `${scope.includeNull ? '1' : '0'}:${[...scope.ids].sort((left, right) => left.localeCompare(right)).join(',')}`
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const PRESENCE_CACHE_DEFAULT_TTL_MS = 30_000
|
|
90
|
+
const PRESENCE_CACHE_MAX_ENTRIES = 10_000
|
|
91
|
+
|
|
92
|
+
// Process-level TTL cache for the token-presence answer. Module-level (process-global) on
|
|
93
|
+
// purpose: `createRequestContainer` builds fresh engines — and with them fresh resolver
|
|
94
|
+
// instances — per request, so an instance-scoped memo alone re-pays the probe on every
|
|
95
|
+
// request. On a large `search_tokens` one probe can be pathologically expensive (#4723),
|
|
96
|
+
// so the answer is amortized across requests here and only re-checked once per TTL.
|
|
97
|
+
// Staleness contract: a stale `false` keeps like/ilike on the plain-column fallback for up
|
|
98
|
+
// to the TTL after an entity's first tokens are written; a stale `true` routes search
|
|
99
|
+
// through an emptied token set for up to the TTL after a purge. Both converge within the
|
|
100
|
+
// TTL; set OM_SEARCH_TOKEN_PRESENCE_CACHE_MS=0 to disable and probe per request again.
|
|
101
|
+
const presenceCache = new Map<string, { value: boolean; expiresAt: number }>()
|
|
102
|
+
|
|
103
|
+
function resolvePresenceCacheTtlMs(): number {
|
|
104
|
+
return parseNumberWithDefault(process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS, PRESENCE_CACHE_DEFAULT_TTL_MS, { integer: true, min: 0 })
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function storePresence(key: string, value: boolean, ttlMs: number): void {
|
|
108
|
+
if (presenceCache.size >= PRESENCE_CACHE_MAX_ENTRIES) {
|
|
109
|
+
const now = Date.now()
|
|
110
|
+
for (const [entryKey, entry] of presenceCache) {
|
|
111
|
+
if (entry.expiresAt <= now) presenceCache.delete(entryKey)
|
|
112
|
+
}
|
|
113
|
+
if (presenceCache.size >= PRESENCE_CACHE_MAX_ENTRIES) presenceCache.clear()
|
|
114
|
+
}
|
|
115
|
+
presenceCache.set(key, { value, expiresAt: Date.now() + ttlMs })
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export function clearSearchTokenPresenceCache(): void {
|
|
119
|
+
presenceCache.clear()
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* One place that answers "is token search usable here?" for both query
|
|
124
|
+
* engines, instead of each hand-assembling config + table-existence +
|
|
125
|
+
* token-presence probes with private duplicate helpers and ad-hoc memo maps.
|
|
126
|
+
*
|
|
127
|
+
* Memoization is per instance; engines are constructed per request
|
|
128
|
+
* (`createRequestContainer`), so entries never outlive a request — the same
|
|
129
|
+
* staleness contract the engines' previous per-query join maps had. A
|
|
130
|
+
* rejected table probe is evicted so the next call retries instead of
|
|
131
|
+
* observing a poisoned cache entry.
|
|
132
|
+
*/
|
|
133
|
+
export function createSearchTokenAvailability(deps: SearchTokenAvailabilityDeps): SearchTokenAvailability {
|
|
134
|
+
const tablePresence = new Map<string, Promise<boolean>>()
|
|
135
|
+
const tokenPresence = new Map<string, Promise<boolean>>()
|
|
136
|
+
|
|
137
|
+
const tableExists = (table: string): Promise<boolean> => {
|
|
138
|
+
const cached = tablePresence.get(table)
|
|
139
|
+
if (cached) return cached
|
|
140
|
+
const probe = (async () => {
|
|
141
|
+
const row = await deps.getDb()
|
|
142
|
+
.selectFrom('information_schema.tables')
|
|
143
|
+
.select(sql<number>`1`.as('one'))
|
|
144
|
+
.where('table_name', '=', table)
|
|
145
|
+
.limit(1)
|
|
146
|
+
.executeTakeFirst()
|
|
147
|
+
return !!row
|
|
148
|
+
})()
|
|
149
|
+
tablePresence.set(table, probe)
|
|
150
|
+
probe.catch(() => tablePresence.delete(table))
|
|
151
|
+
return probe
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const probeTokens = async (
|
|
155
|
+
entity: string,
|
|
156
|
+
tenantId: string | null,
|
|
157
|
+
orgScope?: OrganizationScope | null,
|
|
158
|
+
): Promise<boolean> => {
|
|
159
|
+
let query = deps.getDb()
|
|
160
|
+
.selectFrom('search_tokens')
|
|
161
|
+
.select(sql<number>`1`.as('one'))
|
|
162
|
+
.where('entity_type', '=', entity)
|
|
163
|
+
// Deliberately `= / IS NULL` instead of `IS NOT DISTINCT FROM` (identical semantics
|
|
164
|
+
// for a string|null tenant): the latter cannot serve as an index condition, which is
|
|
165
|
+
// part of why the planner degraded this probe to a seq scan on large tables (#4723).
|
|
166
|
+
// With plain predicates the probe is a pure prefix seek on
|
|
167
|
+
// `search_tokens_presence_idx (entity_type, tenant_id, organization_id)`, making the
|
|
168
|
+
// miss as cheap as the hit.
|
|
169
|
+
query = tenantId == null
|
|
170
|
+
? query.where('tenant_id', 'is', null)
|
|
171
|
+
: query.where('tenant_id', '=', tenantId)
|
|
172
|
+
if (orgScope) {
|
|
173
|
+
query = deps.applyOrganizationScope(query, 'search_tokens.organization_id', orgScope)
|
|
174
|
+
}
|
|
175
|
+
const row = await query.limit(1).executeTakeFirst()
|
|
176
|
+
return !!row
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const hasTokens = (
|
|
180
|
+
entity: string,
|
|
181
|
+
tenantId: string | null,
|
|
182
|
+
orgScope?: OrganizationScope | null,
|
|
183
|
+
): Promise<boolean> => {
|
|
184
|
+
const key = `${entity}|${tenantId ?? '__null__'}|${orgScopeKey(orgScope)}`
|
|
185
|
+
const ttlMs = resolvePresenceCacheTtlMs()
|
|
186
|
+
if (ttlMs > 0) {
|
|
187
|
+
const entry = presenceCache.get(key)
|
|
188
|
+
if (entry && entry.expiresAt > Date.now()) return Promise.resolve(entry.value)
|
|
189
|
+
}
|
|
190
|
+
const cached = tokenPresence.get(key)
|
|
191
|
+
if (cached) return cached
|
|
192
|
+
const probe = (async () => {
|
|
193
|
+
try {
|
|
194
|
+
const value = await probeTokens(entity, tenantId, orgScope)
|
|
195
|
+
// Only genuine probe results enter the process-level cache — caching an
|
|
196
|
+
// error-driven `false` would pin degraded search for a full TTL after a
|
|
197
|
+
// transient DB failure.
|
|
198
|
+
if (ttlMs > 0) storePresence(key, value, ttlMs)
|
|
199
|
+
return value
|
|
200
|
+
} catch (err) {
|
|
201
|
+
deps.logDebug('search:has-tokens-error', {
|
|
202
|
+
entity,
|
|
203
|
+
tenantId,
|
|
204
|
+
organizationScope: orgScope,
|
|
205
|
+
error: err instanceof Error ? err.message : String(err),
|
|
206
|
+
})
|
|
207
|
+
return false
|
|
208
|
+
}
|
|
209
|
+
})()
|
|
210
|
+
tokenPresence.set(key, probe)
|
|
211
|
+
return probe
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
return {
|
|
215
|
+
staticEnabled: async () => deps.getConfig().enabled && await tableExists('search_tokens'),
|
|
216
|
+
hasTokens,
|
|
217
|
+
anySourceHasTokens: async (sources, tenantId, orgScope) => {
|
|
218
|
+
for (const source of sources) {
|
|
219
|
+
const ok = await hasTokens(source.entity, tenantId, orgScope)
|
|
220
|
+
deps.logDebug('search:source-has-tokens', {
|
|
221
|
+
entity: source.entity,
|
|
222
|
+
recordIdColumn: source.recordIdColumn,
|
|
223
|
+
tenantId,
|
|
224
|
+
organizationScope: orgScope,
|
|
225
|
+
hasTokens: ok,
|
|
226
|
+
})
|
|
227
|
+
if (ok) return true
|
|
228
|
+
}
|
|
229
|
+
return false
|
|
230
|
+
},
|
|
231
|
+
}
|
|
232
|
+
}
|
package/src/lib/search/config.ts
CHANGED
|
@@ -10,9 +10,21 @@ export type SearchConfig = {
|
|
|
10
10
|
storeRawTokens: boolean
|
|
11
11
|
blocklistedFields: string[]
|
|
12
12
|
entityBlocklistedFields?: Record<string, string[]>
|
|
13
|
+
maxFieldChars?: number
|
|
14
|
+
maxTokensPerField?: number
|
|
15
|
+
maxTokensPerRecord?: number
|
|
13
16
|
}
|
|
14
17
|
|
|
15
18
|
export const DEFAULT_SEARCH_MIN_TOKEN_LENGTH = 3
|
|
19
|
+
export const DEFAULT_SEARCH_MAX_FIELD_CHARS = 20_000
|
|
20
|
+
export const DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD = 5_000
|
|
21
|
+
export const DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD = 20_000
|
|
22
|
+
|
|
23
|
+
export type SearchTokenLimits = {
|
|
24
|
+
maxFieldChars: number
|
|
25
|
+
maxTokensPerField: number
|
|
26
|
+
maxTokensPerRecord: number
|
|
27
|
+
}
|
|
16
28
|
|
|
17
29
|
const DEFAULT_BLOCKLIST = ['password', 'token', 'secret', 'hash']
|
|
18
30
|
|
|
@@ -26,6 +38,19 @@ function parseNumber(raw: string | undefined, fallback: number, min = 1): number
|
|
|
26
38
|
return parseNumberWithDefault(raw, fallback, { integer: true, min })
|
|
27
39
|
}
|
|
28
40
|
|
|
41
|
+
export function resolveSearchTokenLimits(config: SearchConfig): SearchTokenLimits {
|
|
42
|
+
const resolveLimit = (value: number | undefined, fallback: number): number => {
|
|
43
|
+
if (value === undefined) return fallback
|
|
44
|
+
if (!Number.isFinite(value) || value < 0) return fallback
|
|
45
|
+
return Math.trunc(value)
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
maxFieldChars: resolveLimit(config.maxFieldChars, DEFAULT_SEARCH_MAX_FIELD_CHARS),
|
|
49
|
+
maxTokensPerField: resolveLimit(config.maxTokensPerField, DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD),
|
|
50
|
+
maxTokensPerRecord: resolveLimit(config.maxTokensPerRecord, DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD),
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
29
54
|
function parseHashAlgorithm(raw: string | undefined): 'sha256' | 'sha1' | 'md5' {
|
|
30
55
|
const value = (raw ?? '').trim().toLowerCase()
|
|
31
56
|
if (value === 'sha1') return 'sha1'
|
|
@@ -89,6 +114,9 @@ export function resolveSearchConfig(): SearchConfig {
|
|
|
89
114
|
storeRawTokens: parseBoolean(process.env.OM_SEARCH_STORE_RAW_TOKENS, false),
|
|
90
115
|
blocklistedFields: blocklist.global,
|
|
91
116
|
entityBlocklistedFields: blocklist.byEntity,
|
|
117
|
+
maxFieldChars: parseNumber(process.env.OM_SEARCH_MAX_FIELD_CHARS, DEFAULT_SEARCH_MAX_FIELD_CHARS, 0),
|
|
118
|
+
maxTokensPerField: parseNumber(process.env.OM_SEARCH_MAX_TOKENS_PER_FIELD, DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD, 0),
|
|
119
|
+
maxTokensPerRecord: parseNumber(process.env.OM_SEARCH_MAX_TOKENS_PER_RECORD, DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD, 0),
|
|
92
120
|
}
|
|
93
121
|
}
|
|
94
122
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import crypto from 'crypto'
|
|
2
|
-
import { resolveSearchConfig, type SearchConfig } from './config'
|
|
2
|
+
import { resolveSearchConfig, resolveSearchTokenLimits, type SearchConfig } from './config'
|
|
3
3
|
|
|
4
4
|
export type TokenizationResult = {
|
|
5
5
|
tokens: string[]
|
|
@@ -20,13 +20,28 @@ function splitTokens(text: string, minLength: number): string[] {
|
|
|
20
20
|
.filter((token) => token.length >= minLength)
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
function
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
23
|
+
function appendExpandedToken(
|
|
24
|
+
token: string,
|
|
25
|
+
config: SearchConfig,
|
|
26
|
+
seen: Set<string>,
|
|
27
|
+
tokens: string[],
|
|
28
|
+
limit: number,
|
|
29
|
+
): void {
|
|
30
|
+
const append = (candidate: string): boolean => {
|
|
31
|
+
if (seen.has(candidate)) return tokens.length < limit
|
|
32
|
+
seen.add(candidate)
|
|
33
|
+
tokens.push(candidate)
|
|
34
|
+
return tokens.length < limit
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
if (!config.enablePartials) {
|
|
38
|
+
append(token)
|
|
39
|
+
return
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
for (let length = config.minTokenLength; length <= token.length; length += 1) {
|
|
43
|
+
if (!append(token.slice(0, length))) return
|
|
28
44
|
}
|
|
29
|
-
return results
|
|
30
45
|
}
|
|
31
46
|
|
|
32
47
|
export function hashToken(token: string, config?: SearchConfig): string {
|
|
@@ -36,10 +51,17 @@ export function hashToken(token: string, config?: SearchConfig): string {
|
|
|
36
51
|
|
|
37
52
|
export function tokenizeText(text: string, config?: SearchConfig): TokenizationResult {
|
|
38
53
|
const cfg = config ?? resolveSearchConfig()
|
|
39
|
-
const
|
|
40
|
-
const
|
|
41
|
-
const
|
|
42
|
-
const
|
|
54
|
+
const limits = resolveSearchTokenLimits(cfg)
|
|
55
|
+
const boundedText = limits.maxFieldChars > 0 ? text.slice(0, limits.maxFieldChars) : text
|
|
56
|
+
const tokenLimit = limits.maxTokensPerField > 0 ? limits.maxTokensPerField : Number.POSITIVE_INFINITY
|
|
57
|
+
const seen = new Set<string>()
|
|
58
|
+
const tokens: string[] = []
|
|
59
|
+
|
|
60
|
+
for (const token of splitTokens(boundedText, cfg.minTokenLength)) {
|
|
61
|
+
if (tokens.length >= tokenLimit) break
|
|
62
|
+
appendExpandedToken(token, cfg, seen, tokens, tokenLimit)
|
|
63
|
+
}
|
|
64
|
+
|
|
43
65
|
const hashes = tokens.map((token) => hashToken(token, cfg))
|
|
44
66
|
return { tokens, hashes }
|
|
45
67
|
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/** @jest-environment node */
|
|
2
|
+
import {
|
|
3
|
+
DEFAULT_WEBHOOK_BODY_LIMIT_BYTES,
|
|
4
|
+
readBoundedRequestBody,
|
|
5
|
+
resolveWebhookBodyLimitBytes,
|
|
6
|
+
WebhookBodyTooLargeError,
|
|
7
|
+
} from '../body'
|
|
8
|
+
|
|
9
|
+
function makeStreamingRequest(chunks: Uint8Array[], headers?: HeadersInit) {
|
|
10
|
+
let index = 0
|
|
11
|
+
const cancel = jest.fn()
|
|
12
|
+
const body = new ReadableStream<Uint8Array>({
|
|
13
|
+
pull(controller) {
|
|
14
|
+
const chunk = chunks[index]
|
|
15
|
+
index += 1
|
|
16
|
+
if (chunk) controller.enqueue(chunk)
|
|
17
|
+
else controller.close()
|
|
18
|
+
},
|
|
19
|
+
cancel,
|
|
20
|
+
})
|
|
21
|
+
const request = new Request('http://localhost/webhook', {
|
|
22
|
+
method: 'POST',
|
|
23
|
+
headers,
|
|
24
|
+
body,
|
|
25
|
+
duplex: 'half',
|
|
26
|
+
} as RequestInit & { duplex: 'half' })
|
|
27
|
+
return { request, cancel }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
describe('readBoundedRequestBody', () => {
|
|
31
|
+
const encoder = new TextEncoder()
|
|
32
|
+
|
|
33
|
+
it('rejects an oversized declared length before reading the body', async () => {
|
|
34
|
+
const request = new Request('http://localhost/webhook', {
|
|
35
|
+
method: 'POST',
|
|
36
|
+
headers: { 'content-length': '6' },
|
|
37
|
+
body: 'ok',
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
await expect(readBoundedRequestBody(request, { maxBytes: 5 })).rejects.toEqual(
|
|
41
|
+
expect.objectContaining<WebhookBodyTooLargeError>({ limitBytes: 5 }),
|
|
42
|
+
)
|
|
43
|
+
})
|
|
44
|
+
|
|
45
|
+
it('stops a chunked body when its actual byte count crosses the limit', async () => {
|
|
46
|
+
const { request, cancel } = makeStreamingRequest([
|
|
47
|
+
encoder.encode('abc'),
|
|
48
|
+
encoder.encode('def'),
|
|
49
|
+
encoder.encode('never-read'),
|
|
50
|
+
])
|
|
51
|
+
|
|
52
|
+
await expect(readBoundedRequestBody(request, { maxBytes: 5 })).rejects.toBeInstanceOf(
|
|
53
|
+
WebhookBodyTooLargeError,
|
|
54
|
+
)
|
|
55
|
+
expect(cancel).toHaveBeenCalled()
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
it.each([
|
|
59
|
+
['invalid', 'not-a-number'],
|
|
60
|
+
['lying', '3'],
|
|
61
|
+
])('does not let a %s Content-Length bypass the streamed cap', async (_case, contentLength) => {
|
|
62
|
+
const { request } = makeStreamingRequest(
|
|
63
|
+
[encoder.encode('abc'), encoder.encode('def')],
|
|
64
|
+
{ 'content-length': contentLength },
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
await expect(readBoundedRequestBody(request, { maxBytes: 5 })).rejects.toBeInstanceOf(
|
|
68
|
+
WebhookBodyTooLargeError,
|
|
69
|
+
)
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
it('preserves the exact decoded body at the byte boundary', async () => {
|
|
73
|
+
const bytes = encoder.encode('a€')
|
|
74
|
+
const { request } = makeStreamingRequest([bytes.slice(0, 2), bytes.slice(2)])
|
|
75
|
+
|
|
76
|
+
await expect(readBoundedRequestBody(request, { maxBytes: bytes.byteLength })).resolves.toBe('a€')
|
|
77
|
+
})
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
describe('resolveWebhookBodyLimitBytes', () => {
|
|
81
|
+
it('uses the documented default for missing or invalid configuration', () => {
|
|
82
|
+
expect(resolveWebhookBodyLimitBytes(undefined)).toBe(DEFAULT_WEBHOOK_BODY_LIMIT_BYTES)
|
|
83
|
+
expect(resolveWebhookBodyLimitBytes('invalid')).toBe(DEFAULT_WEBHOOK_BODY_LIMIT_BYTES)
|
|
84
|
+
expect(resolveWebhookBodyLimitBytes('0')).toBe(DEFAULT_WEBHOOK_BODY_LIMIT_BYTES)
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
it('accepts a positive safe integer override', () => {
|
|
88
|
+
expect(resolveWebhookBodyLimitBytes('2097152')).toBe(2 * 1024 * 1024)
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
it('supports a source-specific fallback without weakening validation', () => {
|
|
92
|
+
expect(resolveWebhookBodyLimitBytes(undefined, 2 * 1024 * 1024)).toBe(2 * 1024 * 1024)
|
|
93
|
+
expect(resolveWebhookBodyLimitBytes('invalid', 2 * 1024 * 1024)).toBe(2 * 1024 * 1024)
|
|
94
|
+
})
|
|
95
|
+
})
|
|
@@ -1,7 +1,26 @@
|
|
|
1
1
|
import { signWebhookPayload } from '../sign'
|
|
2
|
-
import { verifyWebhookSignature } from '../verify'
|
|
2
|
+
import { isWebhookTimestampWithinTolerance, verifyWebhookSignature } from '../verify'
|
|
3
3
|
import { generateWebhookSecret } from '../secrets'
|
|
4
4
|
|
|
5
|
+
describe('isWebhookTimestampWithinTolerance', () => {
|
|
6
|
+
it('accepts timestamps inside the configured tolerance', () => {
|
|
7
|
+
expect(isWebhookTimestampWithinTolerance('1700000000', 300, 1700000299)).toBe(true)
|
|
8
|
+
})
|
|
9
|
+
|
|
10
|
+
it('rejects stale timestamps outside the configured tolerance', () => {
|
|
11
|
+
expect(isWebhookTimestampWithinTolerance('1700000000', 300, 1700000301)).toBe(false)
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
it('rejects timestamps too far in the future', () => {
|
|
15
|
+
expect(isWebhookTimestampWithinTolerance('1700000601', 300, 1700000000)).toBe(false)
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
it('rejects malformed timestamps', () => {
|
|
19
|
+
expect(isWebhookTimestampWithinTolerance('not-a-timestamp', 300, 1700000000)).toBe(false)
|
|
20
|
+
expect(isWebhookTimestampWithinTolerance('1700000000abc', 300, 1700000000)).toBe(false)
|
|
21
|
+
})
|
|
22
|
+
})
|
|
23
|
+
|
|
5
24
|
describe('verifyWebhookSignature', () => {
|
|
6
25
|
const msgId = 'msg_test123'
|
|
7
26
|
const body = '{"type":"test.event","data":{}}'
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { parseNumberWithDefault } from '../number'
|
|
2
|
+
|
|
3
|
+
export const DEFAULT_WEBHOOK_BODY_LIMIT_BYTES = 1024 * 1024
|
|
4
|
+
|
|
5
|
+
export class WebhookBodyTooLargeError extends Error {
|
|
6
|
+
readonly limitBytes: number
|
|
7
|
+
|
|
8
|
+
constructor(limitBytes: number) {
|
|
9
|
+
super(`Webhook body exceeds the ${limitBytes}-byte limit`)
|
|
10
|
+
this.name = 'WebhookBodyTooLargeError'
|
|
11
|
+
this.limitBytes = limitBytes
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function resolveWebhookBodyLimitBytes(
|
|
16
|
+
raw = typeof process === 'undefined' ? undefined : process.env.OM_WEBHOOK_MAX_BODY_BYTES,
|
|
17
|
+
fallbackBytes = DEFAULT_WEBHOOK_BODY_LIMIT_BYTES,
|
|
18
|
+
): number {
|
|
19
|
+
const fallback = Number.isSafeInteger(fallbackBytes) && fallbackBytes > 0
|
|
20
|
+
? fallbackBytes
|
|
21
|
+
: DEFAULT_WEBHOOK_BODY_LIMIT_BYTES
|
|
22
|
+
const parsed = parseNumberWithDefault(raw, fallback, { min: 1 })
|
|
23
|
+
return Number.isSafeInteger(parsed) ? parsed : fallback
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function readBoundedRequestBody(
|
|
27
|
+
request: Request,
|
|
28
|
+
options?: { maxBytes?: number },
|
|
29
|
+
): Promise<string> {
|
|
30
|
+
const configuredLimit = options?.maxBytes ?? resolveWebhookBodyLimitBytes()
|
|
31
|
+
const maxBytes = Number.isSafeInteger(configuredLimit) && configuredLimit > 0
|
|
32
|
+
? configuredLimit
|
|
33
|
+
: DEFAULT_WEBHOOK_BODY_LIMIT_BYTES
|
|
34
|
+
const declaredLength = request.headers.get('content-length')?.trim()
|
|
35
|
+
if (declaredLength && /^\d+$/.test(declaredLength)) {
|
|
36
|
+
const declaredBytes = Number(declaredLength)
|
|
37
|
+
if (!Number.isSafeInteger(declaredBytes) || declaredBytes > maxBytes) {
|
|
38
|
+
throw new WebhookBodyTooLargeError(maxBytes)
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
if (!request.body) return ''
|
|
43
|
+
|
|
44
|
+
const reader = request.body.getReader()
|
|
45
|
+
const chunks: Uint8Array[] = []
|
|
46
|
+
let totalBytes = 0
|
|
47
|
+
|
|
48
|
+
try {
|
|
49
|
+
while (true) {
|
|
50
|
+
const { done, value } = await reader.read()
|
|
51
|
+
if (done) break
|
|
52
|
+
totalBytes += value.byteLength
|
|
53
|
+
if (totalBytes > maxBytes) {
|
|
54
|
+
await reader.cancel().catch(() => undefined)
|
|
55
|
+
throw new WebhookBodyTooLargeError(maxBytes)
|
|
56
|
+
}
|
|
57
|
+
chunks.push(value)
|
|
58
|
+
}
|
|
59
|
+
} finally {
|
|
60
|
+
reader.releaseLock()
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const body = new Uint8Array(totalBytes)
|
|
64
|
+
let offset = 0
|
|
65
|
+
for (const chunk of chunks) {
|
|
66
|
+
body.set(chunk, offset)
|
|
67
|
+
offset += chunk.byteLength
|
|
68
|
+
}
|
|
69
|
+
return new TextDecoder().decode(body)
|
|
70
|
+
}
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
export { signWebhookPayload, buildWebhookHeaders, generateMessageId } from './sign'
|
|
2
|
-
export { verifyWebhookSignature } from './verify'
|
|
2
|
+
export { WEBHOOK_SIGNATURE_TOLERANCE_SECONDS, isWebhookTimestampWithinTolerance, verifyWebhookSignature } from './verify'
|
|
3
3
|
export { generateWebhookSecret, parseWebhookSecret, isValidWebhookSecret } from './secrets'
|
|
4
|
+
export {
|
|
5
|
+
DEFAULT_WEBHOOK_BODY_LIMIT_BYTES,
|
|
6
|
+
readBoundedRequestBody,
|
|
7
|
+
resolveWebhookBodyLimitBytes,
|
|
8
|
+
WebhookBodyTooLargeError,
|
|
9
|
+
} from './body'
|
|
4
10
|
export type { StandardWebhookHeaders, WebhookSigningKey, WebhookVerificationResult, StandardWebhookPayload } from './types'
|
|
5
11
|
export type {
|
|
6
12
|
InboundWebhookRequest,
|
|
@@ -2,7 +2,21 @@ import { timingSafeEqual, createHmac } from 'node:crypto'
|
|
|
2
2
|
import { parseWebhookSecret } from './secrets'
|
|
3
3
|
import type { WebhookVerificationResult } from './types'
|
|
4
4
|
|
|
5
|
-
const
|
|
5
|
+
export const WEBHOOK_SIGNATURE_TOLERANCE_SECONDS = 5 * 60 // 5 minutes
|
|
6
|
+
|
|
7
|
+
export function isWebhookTimestampWithinTolerance(
|
|
8
|
+
timestamp: string,
|
|
9
|
+
toleranceSeconds: number = WEBHOOK_SIGNATURE_TOLERANCE_SECONDS,
|
|
10
|
+
nowSeconds: number = Math.floor(Date.now() / 1000),
|
|
11
|
+
): boolean {
|
|
12
|
+
const normalizedTimestamp = timestamp.trim()
|
|
13
|
+
if (!/^\d+$/.test(normalizedTimestamp)) {
|
|
14
|
+
return false
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const timestampNum = Number.parseInt(normalizedTimestamp, 10)
|
|
18
|
+
return Math.abs(nowSeconds - timestampNum) <= toleranceSeconds
|
|
19
|
+
}
|
|
6
20
|
|
|
7
21
|
/**
|
|
8
22
|
* Verify a Standard Webhooks signature against one or more secrets.
|
|
@@ -14,15 +28,9 @@ export function verifyWebhookSignature(
|
|
|
14
28
|
body: string,
|
|
15
29
|
signatureHeader: string,
|
|
16
30
|
secrets: string[],
|
|
17
|
-
toleranceSeconds: number =
|
|
31
|
+
toleranceSeconds: number = WEBHOOK_SIGNATURE_TOLERANCE_SECONDS,
|
|
18
32
|
): WebhookVerificationResult {
|
|
19
|
-
|
|
20
|
-
if (isNaN(timestampNum)) {
|
|
21
|
-
return { valid: false }
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
const now = Math.floor(Date.now() / 1000)
|
|
25
|
-
if (Math.abs(now - timestampNum) > toleranceSeconds) {
|
|
33
|
+
if (!isWebhookTimestampWithinTolerance(timestamp, toleranceSeconds)) {
|
|
26
34
|
return { valid: false }
|
|
27
35
|
}
|
|
28
36
|
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import {
|
|
2
|
+
clearWebhookHandlers,
|
|
3
|
+
getWebhookHandler,
|
|
4
|
+
registerWebhookHandler,
|
|
5
|
+
type VerifyWebhookInput,
|
|
6
|
+
type WebhookEvent,
|
|
7
|
+
} from '../types'
|
|
8
|
+
|
|
9
|
+
const handler = async (_input: VerifyWebhookInput): Promise<WebhookEvent> => ({
|
|
10
|
+
eventType: 'payment.updated',
|
|
11
|
+
eventId: 'evt-1',
|
|
12
|
+
data: {},
|
|
13
|
+
idempotencyKey: 'evt-1',
|
|
14
|
+
timestamp: new Date(0),
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
describe('payment gateway webhook registration', () => {
|
|
18
|
+
beforeEach(() => {
|
|
19
|
+
clearWebhookHandlers()
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
it('preserves an optional valid body limit on the registration', () => {
|
|
23
|
+
registerWebhookHandler('limited', handler, { maxBodyBytes: 64 * 1024 })
|
|
24
|
+
|
|
25
|
+
expect(getWebhookHandler('limited')?.maxBodyBytes).toBe(64 * 1024)
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
it.each([0, -1, 1.5, Number.MAX_SAFE_INTEGER])(
|
|
29
|
+
'rejects an invalid body limit: %s',
|
|
30
|
+
(maxBodyBytes) => {
|
|
31
|
+
expect(() => registerWebhookHandler('invalid', handler, { maxBodyBytes })).toThrow(
|
|
32
|
+
'[internal] Payment gateway webhook maxBodyBytes must be a positive safe integer',
|
|
33
|
+
)
|
|
34
|
+
expect(getWebhookHandler('invalid')).toBeUndefined()
|
|
35
|
+
},
|
|
36
|
+
)
|
|
37
|
+
})
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { DEFAULT_WEBHOOK_BODY_LIMIT_BYTES } from '../../lib/webhooks/body'
|
|
2
|
+
|
|
1
3
|
// ── Unified Payment Status ──────────────────────────────────────────────────
|
|
2
4
|
|
|
3
5
|
export type UnifiedPaymentStatus =
|
|
@@ -251,6 +253,7 @@ export interface WebhookHandlerRegistration {
|
|
|
251
253
|
handler: (input: VerifyWebhookInput) => Promise<WebhookEvent>
|
|
252
254
|
queue?: string
|
|
253
255
|
readSessionIdHint?: (payload: Record<string, unknown> | null) => string | null
|
|
256
|
+
maxBodyBytes?: number
|
|
254
257
|
}
|
|
255
258
|
|
|
256
259
|
// ── Adapter Registry Options ────────────────────────────────────────────────
|
|
@@ -345,13 +348,25 @@ export function registerWebhookHandler(
|
|
|
345
348
|
options?: {
|
|
346
349
|
queue?: string
|
|
347
350
|
readSessionIdHint?: (payload: Record<string, unknown> | null) => string | null
|
|
351
|
+
maxBodyBytes?: number
|
|
348
352
|
},
|
|
349
353
|
): () => void {
|
|
354
|
+
if (
|
|
355
|
+
options?.maxBodyBytes !== undefined
|
|
356
|
+
&& (!Number.isSafeInteger(options.maxBodyBytes)
|
|
357
|
+
|| options.maxBodyBytes <= 0
|
|
358
|
+
|| options.maxBodyBytes > DEFAULT_WEBHOOK_BODY_LIMIT_BYTES)
|
|
359
|
+
) {
|
|
360
|
+
throw new Error(
|
|
361
|
+
`[internal] Payment gateway webhook maxBodyBytes must be a positive safe integer no greater than ${DEFAULT_WEBHOOK_BODY_LIMIT_BYTES}`,
|
|
362
|
+
)
|
|
363
|
+
}
|
|
350
364
|
const webhookHandlerRegistry = getWebhookHandlerRegistry()
|
|
351
365
|
webhookHandlerRegistry.set(providerKey, {
|
|
352
366
|
handler,
|
|
353
367
|
queue: options?.queue,
|
|
354
368
|
readSessionIdHint: options?.readSessionIdHint,
|
|
369
|
+
maxBodyBytes: options?.maxBodyBytes,
|
|
355
370
|
})
|
|
356
371
|
return () => {
|
|
357
372
|
webhookHandlerRegistry.delete(providerKey)
|