@agentlensai/server 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cloud/auth/api-key-middleware.d.ts +66 -0
- package/dist/cloud/auth/api-key-middleware.d.ts.map +1 -0
- package/dist/cloud/auth/api-key-middleware.js +147 -0
- package/dist/cloud/auth/api-key-middleware.js.map +1 -0
- package/dist/cloud/auth/api-keys.d.ts +90 -0
- package/dist/cloud/auth/api-keys.d.ts.map +1 -0
- package/dist/cloud/auth/api-keys.js +162 -0
- package/dist/cloud/auth/api-keys.js.map +1 -0
- package/dist/cloud/auth/audit-log.d.ts +66 -0
- package/dist/cloud/auth/audit-log.d.ts.map +1 -0
- package/dist/cloud/auth/audit-log.js +92 -0
- package/dist/cloud/auth/audit-log.js.map +1 -0
- package/dist/cloud/auth/auth-service.d.ts +77 -0
- package/dist/cloud/auth/auth-service.d.ts.map +1 -0
- package/dist/cloud/auth/auth-service.js +229 -0
- package/dist/cloud/auth/auth-service.js.map +1 -0
- package/dist/cloud/auth/brute-force.d.ts +36 -0
- package/dist/cloud/auth/brute-force.d.ts.map +1 -0
- package/dist/cloud/auth/brute-force.js +67 -0
- package/dist/cloud/auth/brute-force.js.map +1 -0
- package/dist/cloud/auth/index.d.ts +11 -0
- package/dist/cloud/auth/index.d.ts.map +1 -0
- package/dist/cloud/auth/index.js +11 -0
- package/dist/cloud/auth/index.js.map +1 -0
- package/dist/cloud/auth/jwt.d.ts +34 -0
- package/dist/cloud/auth/jwt.d.ts.map +1 -0
- package/dist/cloud/auth/jwt.js +68 -0
- package/dist/cloud/auth/jwt.js.map +1 -0
- package/dist/cloud/auth/oauth.d.ts +37 -0
- package/dist/cloud/auth/oauth.d.ts.map +1 -0
- package/dist/cloud/auth/oauth.js +120 -0
- package/dist/cloud/auth/oauth.js.map +1 -0
- package/dist/cloud/auth/passwords.d.ts +25 -0
- package/dist/cloud/auth/passwords.d.ts.map +1 -0
- package/dist/cloud/auth/passwords.js +50 -0
- package/dist/cloud/auth/passwords.js.map +1 -0
- package/dist/cloud/auth/rbac.d.ts +51 -0
- package/dist/cloud/auth/rbac.d.ts.map +1 -0
- package/dist/cloud/auth/rbac.js +89 -0
- package/dist/cloud/auth/rbac.js.map +1 -0
- package/dist/cloud/auth/tokens.d.ts +18 -0
- package/dist/cloud/auth/tokens.d.ts.map +1 -0
- package/dist/cloud/auth/tokens.js +29 -0
- package/dist/cloud/auth/tokens.js.map +1 -0
- package/dist/cloud/billing/billing-service.d.ts +44 -0
- package/dist/cloud/billing/billing-service.d.ts.map +1 -0
- package/dist/cloud/billing/billing-service.js +153 -0
- package/dist/cloud/billing/billing-service.js.map +1 -0
- package/dist/cloud/billing/index.d.ts +11 -0
- package/dist/cloud/billing/index.d.ts.map +1 -0
- package/dist/cloud/billing/index.js +11 -0
- package/dist/cloud/billing/index.js.map +1 -0
- package/dist/cloud/billing/invoice-service.d.ts +57 -0
- package/dist/cloud/billing/invoice-service.d.ts.map +1 -0
- package/dist/cloud/billing/invoice-service.js +123 -0
- package/dist/cloud/billing/invoice-service.js.map +1 -0
- package/dist/cloud/billing/plan-management.d.ts +46 -0
- package/dist/cloud/billing/plan-management.d.ts.map +1 -0
- package/dist/cloud/billing/plan-management.js +157 -0
- package/dist/cloud/billing/plan-management.js.map +1 -0
- package/dist/cloud/billing/quota-enforcement.d.ts +53 -0
- package/dist/cloud/billing/quota-enforcement.d.ts.map +1 -0
- package/dist/cloud/billing/quota-enforcement.js +143 -0
- package/dist/cloud/billing/quota-enforcement.js.map +1 -0
- package/dist/cloud/billing/stripe-client.d.ts +142 -0
- package/dist/cloud/billing/stripe-client.d.ts.map +1 -0
- package/dist/cloud/billing/stripe-client.js +169 -0
- package/dist/cloud/billing/stripe-client.js.map +1 -0
- package/dist/cloud/billing/trial-service.d.ts +47 -0
- package/dist/cloud/billing/trial-service.d.ts.map +1 -0
- package/dist/cloud/billing/trial-service.js +104 -0
- package/dist/cloud/billing/trial-service.js.map +1 -0
- package/dist/cloud/billing/usage-metering.d.ts +83 -0
- package/dist/cloud/billing/usage-metering.d.ts.map +1 -0
- package/dist/cloud/billing/usage-metering.js +174 -0
- package/dist/cloud/billing/usage-metering.js.map +1 -0
- package/dist/cloud/ingestion/backpressure.d.ts +107 -0
- package/dist/cloud/ingestion/backpressure.d.ts.map +1 -0
- package/dist/cloud/ingestion/backpressure.js +134 -0
- package/dist/cloud/ingestion/backpressure.js.map +1 -0
- package/dist/cloud/ingestion/batch-writer.d.ts +115 -0
- package/dist/cloud/ingestion/batch-writer.d.ts.map +1 -0
- package/dist/cloud/ingestion/batch-writer.js +319 -0
- package/dist/cloud/ingestion/batch-writer.js.map +1 -0
- package/dist/cloud/ingestion/dlq-manager.d.ts +116 -0
- package/dist/cloud/ingestion/dlq-manager.d.ts.map +1 -0
- package/dist/cloud/ingestion/dlq-manager.js +244 -0
- package/dist/cloud/ingestion/dlq-manager.js.map +1 -0
- package/dist/cloud/ingestion/event-queue.d.ts +105 -0
- package/dist/cloud/ingestion/event-queue.d.ts.map +1 -0
- package/dist/cloud/ingestion/event-queue.js +185 -0
- package/dist/cloud/ingestion/event-queue.js.map +1 -0
- package/dist/cloud/ingestion/gateway.d.ts +68 -0
- package/dist/cloud/ingestion/gateway.d.ts.map +1 -0
- package/dist/cloud/ingestion/gateway.js +198 -0
- package/dist/cloud/ingestion/gateway.js.map +1 -0
- package/dist/cloud/ingestion/index.d.ts +7 -0
- package/dist/cloud/ingestion/index.d.ts.map +1 -0
- package/dist/cloud/ingestion/index.js +7 -0
- package/dist/cloud/ingestion/index.js.map +1 -0
- package/dist/cloud/ingestion/rate-limiter.d.ts +73 -0
- package/dist/cloud/ingestion/rate-limiter.d.ts.map +1 -0
- package/dist/cloud/ingestion/rate-limiter.js +153 -0
- package/dist/cloud/ingestion/rate-limiter.js.map +1 -0
- package/dist/cloud/migrate.d.ts +45 -0
- package/dist/cloud/migrate.d.ts.map +1 -0
- package/dist/cloud/migrate.js +147 -0
- package/dist/cloud/migrate.js.map +1 -0
- package/dist/cloud/migration/export-import.d.ts +56 -0
- package/dist/cloud/migration/export-import.d.ts.map +1 -0
- package/dist/cloud/migration/export-import.js +289 -0
- package/dist/cloud/migration/export-import.js.map +1 -0
- package/dist/cloud/migration/index.d.ts +5 -0
- package/dist/cloud/migration/index.d.ts.map +1 -0
- package/dist/cloud/migration/index.js +5 -0
- package/dist/cloud/migration/index.js.map +1 -0
- package/dist/cloud/org-service.d.ts +68 -0
- package/dist/cloud/org-service.d.ts.map +1 -0
- package/dist/cloud/org-service.js +169 -0
- package/dist/cloud/org-service.js.map +1 -0
- package/dist/cloud/partition-maintenance.d.ts +29 -0
- package/dist/cloud/partition-maintenance.d.ts.map +1 -0
- package/dist/cloud/partition-maintenance.js +96 -0
- package/dist/cloud/partition-maintenance.js.map +1 -0
- package/dist/cloud/retention/index.d.ts +7 -0
- package/dist/cloud/retention/index.d.ts.map +1 -0
- package/dist/cloud/retention/index.js +7 -0
- package/dist/cloud/retention/index.js.map +1 -0
- package/dist/cloud/retention/partition-management.d.ts +61 -0
- package/dist/cloud/retention/partition-management.d.ts.map +1 -0
- package/dist/cloud/retention/partition-management.js +167 -0
- package/dist/cloud/retention/partition-management.js.map +1 -0
- package/dist/cloud/retention/retention-job.d.ts +70 -0
- package/dist/cloud/retention/retention-job.d.ts.map +1 -0
- package/dist/cloud/retention/retention-job.js +160 -0
- package/dist/cloud/retention/retention-job.js.map +1 -0
- package/dist/cloud/retention/retention-policy.d.ts +27 -0
- package/dist/cloud/retention/retention-policy.d.ts.map +1 -0
- package/dist/cloud/retention/retention-policy.js +36 -0
- package/dist/cloud/retention/retention-policy.js.map +1 -0
- package/dist/cloud/routes/api-key-routes.d.ts +38 -0
- package/dist/cloud/routes/api-key-routes.d.ts.map +1 -0
- package/dist/cloud/routes/api-key-routes.js +84 -0
- package/dist/cloud/routes/api-key-routes.js.map +1 -0
- package/dist/cloud/routes/audit-routes.d.ts +36 -0
- package/dist/cloud/routes/audit-routes.d.ts.map +1 -0
- package/dist/cloud/routes/audit-routes.js +47 -0
- package/dist/cloud/routes/audit-routes.js.map +1 -0
- package/dist/cloud/routes/billing-routes.d.ts +51 -0
- package/dist/cloud/routes/billing-routes.d.ts.map +1 -0
- package/dist/cloud/routes/billing-routes.js +114 -0
- package/dist/cloud/routes/billing-routes.js.map +1 -0
- package/dist/cloud/routes/onboarding-routes.d.ts +34 -0
- package/dist/cloud/routes/onboarding-routes.d.ts.map +1 -0
- package/dist/cloud/routes/onboarding-routes.js +58 -0
- package/dist/cloud/routes/onboarding-routes.js.map +1 -0
- package/dist/cloud/routes/org-routes.d.ts +80 -0
- package/dist/cloud/routes/org-routes.d.ts.map +1 -0
- package/dist/cloud/routes/org-routes.js +153 -0
- package/dist/cloud/routes/org-routes.js.map +1 -0
- package/dist/cloud/routes/usage-routes.d.ts +18 -0
- package/dist/cloud/routes/usage-routes.d.ts.map +1 -0
- package/dist/cloud/routes/usage-routes.js +66 -0
- package/dist/cloud/routes/usage-routes.js.map +1 -0
- package/dist/cloud/storage/adapter.d.ts +102 -0
- package/dist/cloud/storage/adapter.d.ts.map +1 -0
- package/dist/cloud/storage/adapter.js +21 -0
- package/dist/cloud/storage/adapter.js.map +1 -0
- package/dist/cloud/storage/index.d.ts +8 -0
- package/dist/cloud/storage/index.d.ts.map +1 -0
- package/dist/cloud/storage/index.js +7 -0
- package/dist/cloud/storage/index.js.map +1 -0
- package/dist/cloud/storage/postgres-adapter.d.ts +34 -0
- package/dist/cloud/storage/postgres-adapter.d.ts.map +1 -0
- package/dist/cloud/storage/postgres-adapter.js +544 -0
- package/dist/cloud/storage/postgres-adapter.js.map +1 -0
- package/dist/cloud/storage/sqlite-adapter.d.ts +29 -0
- package/dist/cloud/storage/sqlite-adapter.d.ts.map +1 -0
- package/dist/cloud/storage/sqlite-adapter.js +176 -0
- package/dist/cloud/storage/sqlite-adapter.js.map +1 -0
- package/dist/cloud/tenant-pool.d.ts +49 -0
- package/dist/cloud/tenant-pool.d.ts.map +1 -0
- package/dist/cloud/tenant-pool.js +61 -0
- package/dist/cloud/tenant-pool.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* API Gateway Service (S-3.2)
|
|
3
|
+
*
|
|
4
|
+
* Ingestion endpoints:
|
|
5
|
+
* POST /v1/events — single event
|
|
6
|
+
* POST /v1/events/batch — up to 100 events
|
|
7
|
+
*
|
|
8
|
+
* Flow: Auth → Validate → Enrich → Publish to queue → 202 Accepted
|
|
9
|
+
*/
|
|
10
|
+
import { randomUUID } from 'crypto';
|
|
11
|
+
import { BACKPRESSURE_THRESHOLD } from './event-queue.js';
|
|
12
|
+
// ═══════════════════════════════════════════
|
|
13
|
+
// Known event types
|
|
14
|
+
// ═══════════════════════════════════════════
|
|
15
|
+
const KNOWN_EVENT_TYPES = new Set([
|
|
16
|
+
'llm_call',
|
|
17
|
+
'tool_use',
|
|
18
|
+
'agent_action',
|
|
19
|
+
'error',
|
|
20
|
+
'session_start',
|
|
21
|
+
'session_end',
|
|
22
|
+
'guardrail',
|
|
23
|
+
'benchmark',
|
|
24
|
+
'custom',
|
|
25
|
+
'health_check',
|
|
26
|
+
'lesson',
|
|
27
|
+
'embedding',
|
|
28
|
+
]);
|
|
29
|
+
// ═══════════════════════════════════════════
|
|
30
|
+
// Validation
|
|
31
|
+
// ═══════════════════════════════════════════
|
|
32
|
+
export function validateEvent(event, index) {
|
|
33
|
+
if (!event || typeof event !== 'object') {
|
|
34
|
+
return { index, error: 'event must be an object' };
|
|
35
|
+
}
|
|
36
|
+
const e = event;
|
|
37
|
+
if (!e.type || typeof e.type !== 'string') {
|
|
38
|
+
return { index, error: 'missing required field: type' };
|
|
39
|
+
}
|
|
40
|
+
if (!KNOWN_EVENT_TYPES.has(e.type)) {
|
|
41
|
+
return { index, error: `unknown event type: ${e.type}` };
|
|
42
|
+
}
|
|
43
|
+
if (!e.session_id || typeof e.session_id !== 'string') {
|
|
44
|
+
return { index, error: 'missing required field: session_id' };
|
|
45
|
+
}
|
|
46
|
+
if (e.timestamp !== undefined) {
|
|
47
|
+
const ts = new Date(e.timestamp);
|
|
48
|
+
if (isNaN(ts.getTime())) {
|
|
49
|
+
return { index, error: 'invalid timestamp format' };
|
|
50
|
+
}
|
|
51
|
+
// Reject timestamps more than 5 minutes in the future
|
|
52
|
+
if (ts.getTime() > Date.now() + 5 * 60 * 1000) {
|
|
53
|
+
return { index, error: 'timestamp in future' };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (e.data !== undefined && (typeof e.data !== 'object' || e.data === null || Array.isArray(e.data))) {
|
|
57
|
+
return { index, error: 'data must be an object' };
|
|
58
|
+
}
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
// ═══════════════════════════════════════════
|
|
62
|
+
// Enrichment
|
|
63
|
+
// ═══════════════════════════════════════════
|
|
64
|
+
function enrichEvent(event, auth, requestId) {
|
|
65
|
+
return {
|
|
66
|
+
id: event.id ?? randomUUID(),
|
|
67
|
+
type: event.type,
|
|
68
|
+
timestamp: event.timestamp ?? new Date().toISOString(),
|
|
69
|
+
session_id: event.session_id,
|
|
70
|
+
data: event.data ?? {},
|
|
71
|
+
org_id: auth.orgId,
|
|
72
|
+
api_key_id: auth.keyId,
|
|
73
|
+
received_at: new Date().toISOString(),
|
|
74
|
+
request_id: requestId,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
// ═══════════════════════════════════════════
|
|
78
|
+
// Gateway Service
|
|
79
|
+
// ═══════════════════════════════════════════
|
|
80
|
+
export class IngestionGateway {
|
|
81
|
+
queue;
|
|
82
|
+
constructor(queue) {
|
|
83
|
+
this.queue = queue;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* POST /v1/events — ingest a single event
|
|
87
|
+
*/
|
|
88
|
+
async ingestSingle(event, auth) {
|
|
89
|
+
const requestId = randomUUID();
|
|
90
|
+
// Check backpressure
|
|
91
|
+
const streamLen = await this.queue.getStreamLength();
|
|
92
|
+
if (streamLen >= BACKPRESSURE_THRESHOLD) {
|
|
93
|
+
return {
|
|
94
|
+
status: 503,
|
|
95
|
+
body: { error: 'Service temporarily unavailable. Retry later.' },
|
|
96
|
+
requestId,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
// Validate
|
|
100
|
+
const error = validateEvent(event, 0);
|
|
101
|
+
if (error) {
|
|
102
|
+
return {
|
|
103
|
+
status: 400,
|
|
104
|
+
body: { error: error.error },
|
|
105
|
+
requestId,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
// Check ingest scope
|
|
109
|
+
if (!auth.scopes.includes('ingest')) {
|
|
110
|
+
return {
|
|
111
|
+
status: 403,
|
|
112
|
+
body: { error: 'API key does not have ingest scope' },
|
|
113
|
+
requestId,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
// Enrich & publish
|
|
117
|
+
const enriched = enrichEvent(event, auth, requestId);
|
|
118
|
+
await this.queue.publish(enriched);
|
|
119
|
+
return {
|
|
120
|
+
status: 202,
|
|
121
|
+
body: { accepted: true, request_id: requestId },
|
|
122
|
+
requestId,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* POST /v1/events/batch — ingest up to 100 events
|
|
127
|
+
*/
|
|
128
|
+
async ingestBatch(events, auth) {
|
|
129
|
+
const requestId = randomUUID();
|
|
130
|
+
// Must be array
|
|
131
|
+
if (!Array.isArray(events)) {
|
|
132
|
+
return {
|
|
133
|
+
status: 400,
|
|
134
|
+
body: { error: 'events must be an array' },
|
|
135
|
+
requestId,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
// Max 100
|
|
139
|
+
if (events.length > 100) {
|
|
140
|
+
return {
|
|
141
|
+
status: 400,
|
|
142
|
+
body: { error: 'batch size exceeds maximum of 100 events' },
|
|
143
|
+
requestId,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
if (events.length === 0) {
|
|
147
|
+
return {
|
|
148
|
+
status: 400,
|
|
149
|
+
body: { error: 'events array must not be empty' },
|
|
150
|
+
requestId,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
// Check ingest scope
|
|
154
|
+
if (!auth.scopes.includes('ingest')) {
|
|
155
|
+
return {
|
|
156
|
+
status: 403,
|
|
157
|
+
body: { error: 'API key does not have ingest scope' },
|
|
158
|
+
requestId,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
// Check backpressure
|
|
162
|
+
const streamLen = await this.queue.getStreamLength();
|
|
163
|
+
if (streamLen >= BACKPRESSURE_THRESHOLD) {
|
|
164
|
+
return {
|
|
165
|
+
status: 503,
|
|
166
|
+
body: { error: 'Service temporarily unavailable. Retry later.' },
|
|
167
|
+
requestId,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
// Validate each event
|
|
171
|
+
const errors = [];
|
|
172
|
+
const valid = [];
|
|
173
|
+
for (let i = 0; i < events.length; i++) {
|
|
174
|
+
const err = validateEvent(events[i], i);
|
|
175
|
+
if (err) {
|
|
176
|
+
errors.push(err);
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
valid.push(enrichEvent(events[i], auth, requestId));
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
// Publish valid events
|
|
183
|
+
if (valid.length > 0) {
|
|
184
|
+
await this.queue.publishBatch(valid);
|
|
185
|
+
}
|
|
186
|
+
return {
|
|
187
|
+
status: 202,
|
|
188
|
+
body: {
|
|
189
|
+
accepted: valid.length,
|
|
190
|
+
rejected: errors.length,
|
|
191
|
+
errors,
|
|
192
|
+
request_id: requestId,
|
|
193
|
+
},
|
|
194
|
+
requestId,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
//# sourceMappingURL=gateway.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gateway.js","sourceRoot":"","sources":["../../../src/cloud/ingestion/gateway.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAyC1D,8CAA8C;AAC9C,oBAAoB;AACpB,8CAA8C;AAE9C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,UAAU;IACV,UAAU;IACV,cAAc;IACd,OAAO;IACP,eAAe;IACf,aAAa;IACb,WAAW;IACX,WAAW;IACX,QAAQ;IACR,cAAc;IACd,QAAQ;IACR,WAAW;CACZ,CAAC,CAAC;AAEH,8CAA8C;AAC9C,aAAa;AACb,8CAA8C;AAE9C,MAAM,UAAU,aAAa,CAAC,KAAc,EAAE,KAAa;IACzD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC;IACrD,CAAC;IAED,MAAM,CAAC,GAAG,KAAgC,CAAC;IAE3C,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,8BAA8B,EAAE,CAAC;IAC1D,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,UAAU,IAAI,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QACtD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,oCAAoC,EAAE,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAmB,CAAC,CAAC;QAC3C,IAAI,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC;QACtD,CAAC;QACD,sDAAsD;QACtD,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;YAC9C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;QACjD,CAAC;IACH,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACrG,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;IACpD,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8CAA8C;AAC9C,aAAa;AACb,8CAA8C;AAE9C,SAAS,WAAW,CAClB,KAAoB,EACpB,IAAuB,EACvB,SAAiB;IAEjB,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,EAAE,IAAI,UAAU,EAAE;QAC5B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACtD,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE;QACtB,MAAM,EAAE,IAAI,CAAC,KAAK;QAClB,UAAU,EAAE,IAAI,CAAC,KAAK;QACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,UAAU,EAAE,SAAS;KACtB,CAAC;AACJ,CAAC;AAED,8CAA8C;AAC9C,kBAAkB;AAClB,8CAA8C;AAE9C,MAAM,OAAO,gBAAgB;IACP;IAApB,YAAoB,KAAiB;QAAjB,UAAK,GAAL,KAAK,CAAY;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,YAAY,CAChB,KAAc,EACd,IAAuB;QAEvB,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC;QAE/B,qBAAqB;QACrB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACrD,IAAI,SAAS,IAAI,sBAAsB,EAAE,CAAC;YACxC,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,EAAE,KAAK,EAAE,+CAA+C,EAAuB;gBACrF,SAAS;aACV,CAAC;QACJ,CAAC;QAED,WAAW;QACX,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACtC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAuB;gBACjD,SAAS;aACV,CAAC;QACJ,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,EAAE,KAAK,EAAE,oCAAoC,EAAuB;gBAC1E,SAAS;aACV,CAAC;QACJ,CAAC;QAED,mBAAmB;QACnB,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAsB,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QACtE,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEnC,OAAO;YACL,MAAM,EAAE,GAAG;YACX,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE;YAC/C,SAAS;SACV,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACf,MAAe,EACf,IAAuB;QAEvB,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC;QAE/B,gBAAgB;QAChB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,EAAE,KAAK,EAAE,yBAAyB,EAAuB;gBAC/D,SAAS;aACV,CAAC;QACJ,CAAC;QAED,UAAU;QACV,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACxB,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,EAAE,KAAK,EAAE,0CAA0C,EAAuB;gBAChF,SAAS;aACV,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,EAAE,KAAK,EAAE,gCAAgC,EAAuB;gBACtE,SAAS;aACV,CAAC;QACJ,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,EAAE,KAAK,EAAE,oCAAoC,EAAuB;gBAC1E,SAAS;aACV,CAAC;QACJ,CAAC;QAED,qBAAqB;QACrB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACrD,IAAI,SAAS,IAAI,sBAAsB,EAAE,CAAC;YACxC,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,EAAE,KAAK,EAAE,+CAA+C,EAAuB;gBACrF,SAAS;aACV,CAAC;QACJ,CAAC;QAED,sBAAsB;QACtB,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAkB,EAAE,CAAC;QAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACxC,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAkB,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;QAED,OAAO;YACL,MAAM,EAAE,GAAG;YACX,IAAI,EAAE;gBACJ,QAAQ,EAAE,KAAK,CAAC,MAAM;gBACtB,QAAQ,EAAE,MAAM,CAAC,MAAM;gBACvB,MAAM;gBACN,UAAU,EAAE,SAAS;aACtB;YACD,SAAS;SACV,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { type EventQueue, type QueuedEvent, type QueueHealth, type RedisClient, type RedisPipeline, InMemoryEventQueue, RedisEventQueue, createEventQueue, STREAM_NAME, DLQ_STREAM_NAME, CONSUMER_GROUP, BACKPRESSURE_THRESHOLD, } from './event-queue.js';
|
|
2
|
+
export { IngestionGateway, validateEvent, type IncomingEvent, type SingleEventRequest, type BatchEventRequest, type SingleEventResponse, type BatchEventResponse, type ValidationError, } from './gateway.js';
|
|
3
|
+
export { BatchWriter, InMemoryBatchWriter, calculateCost, computeHash, type StreamMessage, type BatchWriterConfig, type WriterStats, type InMemoryBatchWriterDeps, type ConsumerRedisClient, } from './batch-writer.js';
|
|
4
|
+
export { RedisRateLimiter, InMemoryRateLimiter, TIER_LIMITS, type Tier, type RateLimitResult, type RateLimitCheckParams, type RateLimitConfig, type RateLimitRedisClient, } from './rate-limiter.js';
|
|
5
|
+
export { BackpressureMonitor, generateCloudWatchAlarmConfig, generateAutoScalingPolicy, type BackpressureStatus, type BackpressureConfig, type CloudWatchAlarmConfig, type AutoScalingPolicy, } from './backpressure.js';
|
|
6
|
+
export { DlqManager, InMemoryDlqManager, type DlqEntry, type DlqStats, type DlqHealthInfo, type DlqRedisClient, } from './dlq-manager.js';
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/cloud/ingestion/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,kBAAkB,EAClB,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,eAAe,EACf,cAAc,EACd,sBAAsB,GACvB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,eAAe,GACrB,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,WAAW,EACX,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,uBAAuB,EAC5B,KAAK,mBAAmB,GACzB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,WAAW,EACX,KAAK,IAAI,EACT,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,mBAAmB,EACnB,6BAA6B,EAC7B,yBAAyB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,GACvB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,UAAU,EACV,kBAAkB,EAClB,KAAK,QAAQ,EACb,KAAK,QAAQ,EACb,KAAK,aAAa,EAClB,KAAK,cAAc,GACpB,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { InMemoryEventQueue, RedisEventQueue, createEventQueue, STREAM_NAME, DLQ_STREAM_NAME, CONSUMER_GROUP, BACKPRESSURE_THRESHOLD, } from './event-queue.js';
|
|
2
|
+
export { IngestionGateway, validateEvent, } from './gateway.js';
|
|
3
|
+
export { BatchWriter, InMemoryBatchWriter, calculateCost, computeHash, } from './batch-writer.js';
|
|
4
|
+
export { RedisRateLimiter, InMemoryRateLimiter, TIER_LIMITS, } from './rate-limiter.js';
|
|
5
|
+
export { BackpressureMonitor, generateCloudWatchAlarmConfig, generateAutoScalingPolicy, } from './backpressure.js';
|
|
6
|
+
export { DlqManager, InMemoryDlqManager, } from './dlq-manager.js';
|
|
7
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/cloud/ingestion/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAML,kBAAkB,EAClB,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,eAAe,EACf,cAAc,EACd,sBAAsB,GACvB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,gBAAgB,EAChB,aAAa,GAOd,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,WAAW,GAMZ,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,WAAW,GAMZ,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,mBAAmB,EACnB,6BAA6B,EAC7B,yBAAyB,GAK1B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,UAAU,EACV,kBAAkB,GAKnB,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rate Limiter (S-3.4)
|
|
3
|
+
*
|
|
4
|
+
* Sliding window rate limiting. Per-key and per-org limits.
|
|
5
|
+
* Tier defaults: Free=100/min, Pro=5K/min, Team=50K/min.
|
|
6
|
+
* 429 with Retry-After header. Per-key override supported.
|
|
7
|
+
*/
|
|
8
|
+
export type Tier = 'free' | 'pro' | 'team' | 'enterprise';
|
|
9
|
+
export interface RateLimitResult {
|
|
10
|
+
allowed: boolean;
|
|
11
|
+
/** Current count in window */
|
|
12
|
+
current: number;
|
|
13
|
+
/** Limit that was applied */
|
|
14
|
+
limit: number;
|
|
15
|
+
/** Seconds until window resets (for Retry-After header) */
|
|
16
|
+
retryAfterSeconds: number;
|
|
17
|
+
/** Which limit was hit: 'key' | 'org' | null */
|
|
18
|
+
limitedBy: 'key' | 'org' | null;
|
|
19
|
+
}
|
|
20
|
+
export interface RateLimitConfig {
|
|
21
|
+
/** Window size in seconds (default: 60) */
|
|
22
|
+
windowSeconds?: number;
|
|
23
|
+
}
|
|
24
|
+
/** Per-tier default limits (events per minute) */
|
|
25
|
+
export declare const TIER_LIMITS: Record<Tier, {
|
|
26
|
+
perKey: number;
|
|
27
|
+
perOrg: number;
|
|
28
|
+
}>;
|
|
29
|
+
export interface RateLimitCheckParams {
|
|
30
|
+
orgId: string;
|
|
31
|
+
keyId: string;
|
|
32
|
+
tier: Tier;
|
|
33
|
+
/** Per-key override limit (from api_keys.rate_limit_override), null = use tier default */
|
|
34
|
+
keyOverride: number | null;
|
|
35
|
+
/** Number of events in this request (default: 1) */
|
|
36
|
+
count?: number;
|
|
37
|
+
}
|
|
38
|
+
export interface RateLimitRedisClient {
|
|
39
|
+
multi(): RateLimitRedisMulti;
|
|
40
|
+
}
|
|
41
|
+
export interface RateLimitRedisMulti {
|
|
42
|
+
zremrangebyscore(key: string, min: number | string, max: number | string): RateLimitRedisMulti;
|
|
43
|
+
zadd(key: string, score: number, member: string): RateLimitRedisMulti;
|
|
44
|
+
zcard(key: string): RateLimitRedisMulti;
|
|
45
|
+
expire(key: string, seconds: number): RateLimitRedisMulti;
|
|
46
|
+
exec(): Promise<Array<[Error | null, unknown]> | null>;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Redis-backed sliding window rate limiter.
|
|
50
|
+
*
|
|
51
|
+
* Uses sorted sets with timestamp scores. Each request adds a member,
|
|
52
|
+
* old entries outside the window are pruned.
|
|
53
|
+
*/
|
|
54
|
+
export declare class RedisRateLimiter {
|
|
55
|
+
private redis;
|
|
56
|
+
private windowSeconds;
|
|
57
|
+
constructor(redis: RateLimitRedisClient, config?: RateLimitConfig);
|
|
58
|
+
check(params: RateLimitCheckParams): Promise<RateLimitResult>;
|
|
59
|
+
private slidingWindowCount;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* In-memory sliding window rate limiter for testing.
|
|
63
|
+
*/
|
|
64
|
+
export declare class InMemoryRateLimiter {
|
|
65
|
+
private windows;
|
|
66
|
+
private windowSeconds;
|
|
67
|
+
constructor(config?: RateLimitConfig);
|
|
68
|
+
check(params: RateLimitCheckParams): Promise<RateLimitResult>;
|
|
69
|
+
/** Reset all state */
|
|
70
|
+
reset(): void;
|
|
71
|
+
private addAndCount;
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=rate-limiter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rate-limiter.d.ts","sourceRoot":"","sources":["../../../src/cloud/ingestion/rate-limiter.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,MAAM,MAAM,IAAI,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,YAAY,CAAC;AAE1D,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gDAAgD;IAChD,SAAS,EAAE,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,2CAA2C;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,kDAAkD;AAClD,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAKxE,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,IAAI,CAAC;IACX,0FAA0F;IAC1F,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAMD,MAAM,WAAW,oBAAoB;IACnC,KAAK,IAAI,mBAAmB,CAAC;CAC9B;AAED,MAAM,WAAW,mBAAmB;IAClC,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,mBAAmB,CAAC;IAC/F,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACtE,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACxC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC1D,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;CACxD;AAED;;;;;GAKG;AACH,qBAAa,gBAAgB;IAIzB,OAAO,CAAC,KAAK;IAHf,OAAO,CAAC,aAAa,CAAS;gBAGpB,KAAK,EAAE,oBAAoB,EACnC,MAAM,CAAC,EAAE,eAAe;IAKpB,KAAK,CAAC,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,CAAC;YA0CrD,kBAAkB;CA8BjC;AAMD;;GAEG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,aAAa,CAAS;gBAElB,MAAM,CAAC,EAAE,eAAe;IAI9B,KAAK,CAAC,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,CAAC;IA4CnE,sBAAsB;IACtB,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,WAAW;CAWpB"}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rate Limiter (S-3.4)
|
|
3
|
+
*
|
|
4
|
+
* Sliding window rate limiting. Per-key and per-org limits.
|
|
5
|
+
* Tier defaults: Free=100/min, Pro=5K/min, Team=50K/min.
|
|
6
|
+
* 429 with Retry-After header. Per-key override supported.
|
|
7
|
+
*/
|
|
8
|
+
/** Per-tier default limits (events per minute) */
|
|
9
|
+
export const TIER_LIMITS = {
|
|
10
|
+
free: { perKey: 100, perOrg: 200 },
|
|
11
|
+
pro: { perKey: 5_000, perOrg: 10_000 },
|
|
12
|
+
team: { perKey: 50_000, perOrg: 100_000 },
|
|
13
|
+
enterprise: { perKey: 100_000, perOrg: 500_000 },
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Redis-backed sliding window rate limiter.
|
|
17
|
+
*
|
|
18
|
+
* Uses sorted sets with timestamp scores. Each request adds a member,
|
|
19
|
+
* old entries outside the window are pruned.
|
|
20
|
+
*/
|
|
21
|
+
export class RedisRateLimiter {
|
|
22
|
+
redis;
|
|
23
|
+
windowSeconds;
|
|
24
|
+
constructor(redis, config) {
|
|
25
|
+
this.redis = redis;
|
|
26
|
+
this.windowSeconds = config?.windowSeconds ?? 60;
|
|
27
|
+
}
|
|
28
|
+
async check(params) {
|
|
29
|
+
const { orgId, keyId, tier, keyOverride, count = 1 } = params;
|
|
30
|
+
const tierLimits = TIER_LIMITS[tier];
|
|
31
|
+
const keyLimit = keyOverride ?? tierLimits.perKey;
|
|
32
|
+
const orgLimit = tierLimits.perOrg;
|
|
33
|
+
const now = Date.now();
|
|
34
|
+
const windowStart = now - this.windowSeconds * 1000;
|
|
35
|
+
// Check per-key limit
|
|
36
|
+
const keyCount = await this.slidingWindowCount(`rate:key:${keyId}`, now, windowStart, count);
|
|
37
|
+
if (keyCount > keyLimit) {
|
|
38
|
+
return {
|
|
39
|
+
allowed: false,
|
|
40
|
+
current: keyCount,
|
|
41
|
+
limit: keyLimit,
|
|
42
|
+
retryAfterSeconds: this.windowSeconds,
|
|
43
|
+
limitedBy: 'key',
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
// Check per-org limit
|
|
47
|
+
const orgCount = await this.slidingWindowCount(`rate:org:${orgId}`, now, windowStart, count);
|
|
48
|
+
if (orgCount > orgLimit) {
|
|
49
|
+
return {
|
|
50
|
+
allowed: false,
|
|
51
|
+
current: orgCount,
|
|
52
|
+
limit: orgLimit,
|
|
53
|
+
retryAfterSeconds: this.windowSeconds,
|
|
54
|
+
limitedBy: 'org',
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
allowed: true,
|
|
59
|
+
current: keyCount,
|
|
60
|
+
limit: keyLimit,
|
|
61
|
+
retryAfterSeconds: 0,
|
|
62
|
+
limitedBy: null,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
async slidingWindowCount(key, now, windowStart, count) {
|
|
66
|
+
const multi = this.redis.multi();
|
|
67
|
+
// Remove entries outside window
|
|
68
|
+
multi.zremrangebyscore(key, 0, windowStart);
|
|
69
|
+
// Add new entries
|
|
70
|
+
for (let i = 0; i < count; i++) {
|
|
71
|
+
multi.zadd(key, now, `${now}:${Math.random().toString(36).slice(2, 8)}:${i}`);
|
|
72
|
+
}
|
|
73
|
+
// Count entries in window
|
|
74
|
+
multi.zcard(key);
|
|
75
|
+
// Set expiry to auto-cleanup
|
|
76
|
+
multi.expire(key, this.windowSeconds * 2);
|
|
77
|
+
const results = await multi.exec();
|
|
78
|
+
if (!results)
|
|
79
|
+
return 0;
|
|
80
|
+
// zcard result is at index 2 + count (after zremrangebyscore + count zadds)
|
|
81
|
+
const zcardIdx = 1 + count;
|
|
82
|
+
const zcardResult = results[zcardIdx];
|
|
83
|
+
return zcardResult?.[1] ?? 0;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
// ═══════════════════════════════════════════
|
|
87
|
+
// In-Memory Rate Limiter (testing / no Redis)
|
|
88
|
+
// ═══════════════════════════════════════════
|
|
89
|
+
/**
|
|
90
|
+
* In-memory sliding window rate limiter for testing.
|
|
91
|
+
*/
|
|
92
|
+
export class InMemoryRateLimiter {
|
|
93
|
+
windows = new Map();
|
|
94
|
+
windowSeconds;
|
|
95
|
+
constructor(config) {
|
|
96
|
+
this.windowSeconds = config?.windowSeconds ?? 60;
|
|
97
|
+
}
|
|
98
|
+
async check(params) {
|
|
99
|
+
const { orgId, keyId, tier, keyOverride, count = 1 } = params;
|
|
100
|
+
const tierLimits = TIER_LIMITS[tier];
|
|
101
|
+
const keyLimit = keyOverride ?? tierLimits.perKey;
|
|
102
|
+
const orgLimit = tierLimits.perOrg;
|
|
103
|
+
const now = Date.now();
|
|
104
|
+
const windowStart = now - this.windowSeconds * 1000;
|
|
105
|
+
// Check per-key
|
|
106
|
+
const keyKey = `rate:key:${keyId}`;
|
|
107
|
+
const keyCount = this.addAndCount(keyKey, now, windowStart, count);
|
|
108
|
+
if (keyCount > keyLimit) {
|
|
109
|
+
return {
|
|
110
|
+
allowed: false,
|
|
111
|
+
current: keyCount,
|
|
112
|
+
limit: keyLimit,
|
|
113
|
+
retryAfterSeconds: this.windowSeconds,
|
|
114
|
+
limitedBy: 'key',
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
// Check per-org
|
|
118
|
+
const orgKey = `rate:org:${orgId}`;
|
|
119
|
+
const orgCount = this.addAndCount(orgKey, now, windowStart, count);
|
|
120
|
+
if (orgCount > orgLimit) {
|
|
121
|
+
return {
|
|
122
|
+
allowed: false,
|
|
123
|
+
current: orgCount,
|
|
124
|
+
limit: orgLimit,
|
|
125
|
+
retryAfterSeconds: this.windowSeconds,
|
|
126
|
+
limitedBy: 'org',
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
return {
|
|
130
|
+
allowed: true,
|
|
131
|
+
current: keyCount,
|
|
132
|
+
limit: keyLimit,
|
|
133
|
+
retryAfterSeconds: 0,
|
|
134
|
+
limitedBy: null,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
/** Reset all state */
|
|
138
|
+
reset() {
|
|
139
|
+
this.windows.clear();
|
|
140
|
+
}
|
|
141
|
+
addAndCount(key, now, windowStart, count) {
|
|
142
|
+
let entries = this.windows.get(key) ?? [];
|
|
143
|
+
// Prune old entries
|
|
144
|
+
entries = entries.filter((ts) => ts > windowStart);
|
|
145
|
+
// Add new
|
|
146
|
+
for (let i = 0; i < count; i++) {
|
|
147
|
+
entries.push(now);
|
|
148
|
+
}
|
|
149
|
+
this.windows.set(key, entries);
|
|
150
|
+
return entries.length;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
//# sourceMappingURL=rate-limiter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rate-limiter.js","sourceRoot":"","sources":["../../../src/cloud/ingestion/rate-limiter.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAyBH,kDAAkD;AAClD,MAAM,CAAC,MAAM,WAAW,GAAqD;IAC3E,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;IAClC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;IACtC,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE;IACzC,UAAU,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;CACjD,CAAC;AA4BF;;;;;GAKG;AACH,MAAM,OAAO,gBAAgB;IAIjB;IAHF,aAAa,CAAS;IAE9B,YACU,KAA2B,EACnC,MAAwB;QADhB,UAAK,GAAL,KAAK,CAAsB;QAGnC,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA4B;QACtC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,MAAM,CAAC;QAC9D,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,WAAW,IAAI,UAAU,CAAC,MAAM,CAAC;QAClD,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC;QAEnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,GAAG,GAAG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAEpD,sBAAsB;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAC7F,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,QAAQ;gBACjB,KAAK,EAAE,QAAQ;gBACf,iBAAiB,EAAE,IAAI,CAAC,aAAa;gBACrC,SAAS,EAAE,KAAK;aACjB,CAAC;QACJ,CAAC;QAED,sBAAsB;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAC7F,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,QAAQ;gBACjB,KAAK,EAAE,QAAQ;gBACf,iBAAiB,EAAE,IAAI,CAAC,aAAa;gBACrC,SAAS,EAAE,KAAK;aACjB,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,QAAQ;YACjB,KAAK,EAAE,QAAQ;YACf,iBAAiB,EAAE,CAAC;YACpB,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,GAAW,EACX,GAAW,EACX,WAAmB,EACnB,KAAa;QAEb,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAEjC,gCAAgC;QAChC,KAAK,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;QAE5C,kBAAkB;QAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,0BAA0B;QAC1B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjB,6BAA6B;QAC7B,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;QAE1C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,OAAO;YAAE,OAAO,CAAC,CAAC;QAEvB,4EAA4E;QAC5E,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QACtC,OAAQ,WAAW,EAAE,CAAC,CAAC,CAAY,IAAI,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,8CAA8C;AAC9C,8CAA8C;AAC9C,8CAA8C;AAE9C;;GAEG;AACH,MAAM,OAAO,mBAAmB;IACtB,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IACtC,aAAa,CAAS;IAE9B,YAAY,MAAwB;QAClC,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA4B;QACtC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,MAAM,CAAC;QAC9D,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,WAAW,IAAI,UAAU,CAAC,MAAM,CAAC;QAClD,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC;QAEnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,GAAG,GAAG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAEpD,gBAAgB;QAChB,MAAM,MAAM,GAAG,YAAY,KAAK,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QACnE,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,QAAQ;gBACjB,KAAK,EAAE,QAAQ;gBACf,iBAAiB,EAAE,IAAI,CAAC,aAAa;gBACrC,SAAS,EAAE,KAAK;aACjB,CAAC;QACJ,CAAC;QAED,gBAAgB;QAChB,MAAM,MAAM,GAAG,YAAY,KAAK,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QACnE,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,QAAQ;gBACjB,KAAK,EAAE,QAAQ;gBACf,iBAAiB,EAAE,IAAI,CAAC,aAAa;gBACrC,SAAS,EAAE,KAAK;aACjB,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,QAAQ;YACjB,KAAK,EAAE,QAAQ;YACf,iBAAiB,EAAE,CAAC;YACpB,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,sBAAsB;IACtB,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAEO,WAAW,CAAC,GAAW,EAAE,GAAW,EAAE,WAAmB,EAAE,KAAa;QAC9E,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC1C,oBAAoB;QACpB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC;QACnD,UAAU;QACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/B,OAAO,OAAO,CAAC,MAAM,CAAC;IACxB,CAAC;CACF"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cloud Migration Runner
|
|
3
|
+
*
|
|
4
|
+
* Executes numbered SQL migration files against PostgreSQL.
|
|
5
|
+
* Tracks applied migrations in a `_cloud_migrations` table.
|
|
6
|
+
* Idempotent: skips already-applied migrations.
|
|
7
|
+
*/
|
|
8
|
+
export interface MigrationClient {
|
|
9
|
+
query(sql: string, params?: unknown[]): Promise<{
|
|
10
|
+
rows: unknown[];
|
|
11
|
+
}>;
|
|
12
|
+
}
|
|
13
|
+
export interface MigrationResult {
|
|
14
|
+
applied: string[];
|
|
15
|
+
skipped: string[];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Get ordered migration files from the migrations directory.
|
|
19
|
+
*/
|
|
20
|
+
export declare function getMigrationFiles(dir?: string): string[];
|
|
21
|
+
/**
|
|
22
|
+
* Read the SQL content of a migration file.
|
|
23
|
+
*/
|
|
24
|
+
export declare function readMigration(filename: string, dir?: string): string;
|
|
25
|
+
/**
|
|
26
|
+
* Run all pending migrations in order.
|
|
27
|
+
*/
|
|
28
|
+
export declare function runMigrations(client: MigrationClient, dir?: string): Promise<MigrationResult>;
|
|
29
|
+
/**
|
|
30
|
+
* Validate migration SQL files without executing them.
|
|
31
|
+
* Returns parsing issues (basic checks).
|
|
32
|
+
*/
|
|
33
|
+
export declare function validateMigrations(dir?: string): {
|
|
34
|
+
valid: boolean;
|
|
35
|
+
files: string[];
|
|
36
|
+
errors: {
|
|
37
|
+
file: string;
|
|
38
|
+
error: string;
|
|
39
|
+
}[];
|
|
40
|
+
};
|
|
41
|
+
export declare const PARTITIONED_TABLES: readonly ["events", "audit_log", "usage_records"];
|
|
42
|
+
export declare const TENANT_SCOPED_TABLES: readonly ["events", "sessions", "agents", "alert_rules", "alert_history", "lessons", "embeddings", "session_summaries", "sharing_config", "agent_sharing_config", "deny_list_rules", "sharing_audit_log", "sharing_review_queue", "anonymous_id_map", "capability_registry", "discovery_config", "delegation_log", "api_keys", "usage_records", "invoices", "audit_log", "org_members", "org_invitations"];
|
|
43
|
+
export declare const CLOUD_TABLES: readonly ["orgs", "users", "org_members", "org_invitations", "api_keys", "usage_records", "invoices", "audit_log"];
|
|
44
|
+
export declare const EXISTING_TABLES: readonly ["events", "sessions", "agents", "alert_rules", "alert_history", "lessons", "embeddings", "session_summaries", "sharing_config", "agent_sharing_config", "deny_list_rules", "sharing_audit_log", "sharing_review_queue", "anonymous_id_map", "capability_registry", "discovery_config", "delegation_log"];
|
|
45
|
+
//# sourceMappingURL=migrate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrate.d.ts","sourceRoot":"","sources":["../../src/cloud/migrate.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,OAAO,EAAE,CAAA;KAAE,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAID;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,GAAE,MAAuB,GAAG,MAAM,EAAE,CAIxE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,GAAE,MAAuB,GAAG,MAAM,CAEpF;AAsBD;;GAEG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,eAAe,EACvB,GAAG,GAAE,MAAuB,GAC3B,OAAO,CAAC,eAAe,CAAC,CAoB1B;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,GAAE,MAAuB,GAAG;IAChE,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC3C,CAqBA;AAGD,eAAO,MAAM,kBAAkB,mDAAoD,CAAC;AAGpF,eAAO,MAAM,oBAAoB,4YAwBvB,CAAC;AAGX,eAAO,MAAM,YAAY,oHASf,CAAC;AAGX,eAAO,MAAM,eAAe,oTAkBlB,CAAC"}
|