@agentlensai/server 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/dist/cloud/auth/api-key-middleware.d.ts +66 -0
  2. package/dist/cloud/auth/api-key-middleware.d.ts.map +1 -0
  3. package/dist/cloud/auth/api-key-middleware.js +147 -0
  4. package/dist/cloud/auth/api-key-middleware.js.map +1 -0
  5. package/dist/cloud/auth/api-keys.d.ts +90 -0
  6. package/dist/cloud/auth/api-keys.d.ts.map +1 -0
  7. package/dist/cloud/auth/api-keys.js +162 -0
  8. package/dist/cloud/auth/api-keys.js.map +1 -0
  9. package/dist/cloud/auth/audit-log.d.ts +66 -0
  10. package/dist/cloud/auth/audit-log.d.ts.map +1 -0
  11. package/dist/cloud/auth/audit-log.js +92 -0
  12. package/dist/cloud/auth/audit-log.js.map +1 -0
  13. package/dist/cloud/auth/auth-service.d.ts +77 -0
  14. package/dist/cloud/auth/auth-service.d.ts.map +1 -0
  15. package/dist/cloud/auth/auth-service.js +229 -0
  16. package/dist/cloud/auth/auth-service.js.map +1 -0
  17. package/dist/cloud/auth/brute-force.d.ts +36 -0
  18. package/dist/cloud/auth/brute-force.d.ts.map +1 -0
  19. package/dist/cloud/auth/brute-force.js +67 -0
  20. package/dist/cloud/auth/brute-force.js.map +1 -0
  21. package/dist/cloud/auth/index.d.ts +11 -0
  22. package/dist/cloud/auth/index.d.ts.map +1 -0
  23. package/dist/cloud/auth/index.js +11 -0
  24. package/dist/cloud/auth/index.js.map +1 -0
  25. package/dist/cloud/auth/jwt.d.ts +34 -0
  26. package/dist/cloud/auth/jwt.d.ts.map +1 -0
  27. package/dist/cloud/auth/jwt.js +68 -0
  28. package/dist/cloud/auth/jwt.js.map +1 -0
  29. package/dist/cloud/auth/oauth.d.ts +37 -0
  30. package/dist/cloud/auth/oauth.d.ts.map +1 -0
  31. package/dist/cloud/auth/oauth.js +120 -0
  32. package/dist/cloud/auth/oauth.js.map +1 -0
  33. package/dist/cloud/auth/passwords.d.ts +25 -0
  34. package/dist/cloud/auth/passwords.d.ts.map +1 -0
  35. package/dist/cloud/auth/passwords.js +50 -0
  36. package/dist/cloud/auth/passwords.js.map +1 -0
  37. package/dist/cloud/auth/rbac.d.ts +51 -0
  38. package/dist/cloud/auth/rbac.d.ts.map +1 -0
  39. package/dist/cloud/auth/rbac.js +89 -0
  40. package/dist/cloud/auth/rbac.js.map +1 -0
  41. package/dist/cloud/auth/tokens.d.ts +18 -0
  42. package/dist/cloud/auth/tokens.d.ts.map +1 -0
  43. package/dist/cloud/auth/tokens.js +29 -0
  44. package/dist/cloud/auth/tokens.js.map +1 -0
  45. package/dist/cloud/billing/billing-service.d.ts +44 -0
  46. package/dist/cloud/billing/billing-service.d.ts.map +1 -0
  47. package/dist/cloud/billing/billing-service.js +153 -0
  48. package/dist/cloud/billing/billing-service.js.map +1 -0
  49. package/dist/cloud/billing/index.d.ts +11 -0
  50. package/dist/cloud/billing/index.d.ts.map +1 -0
  51. package/dist/cloud/billing/index.js +11 -0
  52. package/dist/cloud/billing/index.js.map +1 -0
  53. package/dist/cloud/billing/invoice-service.d.ts +57 -0
  54. package/dist/cloud/billing/invoice-service.d.ts.map +1 -0
  55. package/dist/cloud/billing/invoice-service.js +123 -0
  56. package/dist/cloud/billing/invoice-service.js.map +1 -0
  57. package/dist/cloud/billing/plan-management.d.ts +46 -0
  58. package/dist/cloud/billing/plan-management.d.ts.map +1 -0
  59. package/dist/cloud/billing/plan-management.js +157 -0
  60. package/dist/cloud/billing/plan-management.js.map +1 -0
  61. package/dist/cloud/billing/quota-enforcement.d.ts +53 -0
  62. package/dist/cloud/billing/quota-enforcement.d.ts.map +1 -0
  63. package/dist/cloud/billing/quota-enforcement.js +143 -0
  64. package/dist/cloud/billing/quota-enforcement.js.map +1 -0
  65. package/dist/cloud/billing/stripe-client.d.ts +142 -0
  66. package/dist/cloud/billing/stripe-client.d.ts.map +1 -0
  67. package/dist/cloud/billing/stripe-client.js +169 -0
  68. package/dist/cloud/billing/stripe-client.js.map +1 -0
  69. package/dist/cloud/billing/trial-service.d.ts +47 -0
  70. package/dist/cloud/billing/trial-service.d.ts.map +1 -0
  71. package/dist/cloud/billing/trial-service.js +104 -0
  72. package/dist/cloud/billing/trial-service.js.map +1 -0
  73. package/dist/cloud/billing/usage-metering.d.ts +83 -0
  74. package/dist/cloud/billing/usage-metering.d.ts.map +1 -0
  75. package/dist/cloud/billing/usage-metering.js +174 -0
  76. package/dist/cloud/billing/usage-metering.js.map +1 -0
  77. package/dist/cloud/ingestion/backpressure.d.ts +107 -0
  78. package/dist/cloud/ingestion/backpressure.d.ts.map +1 -0
  79. package/dist/cloud/ingestion/backpressure.js +134 -0
  80. package/dist/cloud/ingestion/backpressure.js.map +1 -0
  81. package/dist/cloud/ingestion/batch-writer.d.ts +115 -0
  82. package/dist/cloud/ingestion/batch-writer.d.ts.map +1 -0
  83. package/dist/cloud/ingestion/batch-writer.js +319 -0
  84. package/dist/cloud/ingestion/batch-writer.js.map +1 -0
  85. package/dist/cloud/ingestion/dlq-manager.d.ts +116 -0
  86. package/dist/cloud/ingestion/dlq-manager.d.ts.map +1 -0
  87. package/dist/cloud/ingestion/dlq-manager.js +244 -0
  88. package/dist/cloud/ingestion/dlq-manager.js.map +1 -0
  89. package/dist/cloud/ingestion/event-queue.d.ts +105 -0
  90. package/dist/cloud/ingestion/event-queue.d.ts.map +1 -0
  91. package/dist/cloud/ingestion/event-queue.js +185 -0
  92. package/dist/cloud/ingestion/event-queue.js.map +1 -0
  93. package/dist/cloud/ingestion/gateway.d.ts +68 -0
  94. package/dist/cloud/ingestion/gateway.d.ts.map +1 -0
  95. package/dist/cloud/ingestion/gateway.js +198 -0
  96. package/dist/cloud/ingestion/gateway.js.map +1 -0
  97. package/dist/cloud/ingestion/index.d.ts +7 -0
  98. package/dist/cloud/ingestion/index.d.ts.map +1 -0
  99. package/dist/cloud/ingestion/index.js +7 -0
  100. package/dist/cloud/ingestion/index.js.map +1 -0
  101. package/dist/cloud/ingestion/rate-limiter.d.ts +73 -0
  102. package/dist/cloud/ingestion/rate-limiter.d.ts.map +1 -0
  103. package/dist/cloud/ingestion/rate-limiter.js +153 -0
  104. package/dist/cloud/ingestion/rate-limiter.js.map +1 -0
  105. package/dist/cloud/migrate.d.ts +45 -0
  106. package/dist/cloud/migrate.d.ts.map +1 -0
  107. package/dist/cloud/migrate.js +147 -0
  108. package/dist/cloud/migrate.js.map +1 -0
  109. package/dist/cloud/migration/export-import.d.ts +56 -0
  110. package/dist/cloud/migration/export-import.d.ts.map +1 -0
  111. package/dist/cloud/migration/export-import.js +289 -0
  112. package/dist/cloud/migration/export-import.js.map +1 -0
  113. package/dist/cloud/migration/index.d.ts +5 -0
  114. package/dist/cloud/migration/index.d.ts.map +1 -0
  115. package/dist/cloud/migration/index.js +5 -0
  116. package/dist/cloud/migration/index.js.map +1 -0
  117. package/dist/cloud/org-service.d.ts +68 -0
  118. package/dist/cloud/org-service.d.ts.map +1 -0
  119. package/dist/cloud/org-service.js +169 -0
  120. package/dist/cloud/org-service.js.map +1 -0
  121. package/dist/cloud/partition-maintenance.d.ts +29 -0
  122. package/dist/cloud/partition-maintenance.d.ts.map +1 -0
  123. package/dist/cloud/partition-maintenance.js +96 -0
  124. package/dist/cloud/partition-maintenance.js.map +1 -0
  125. package/dist/cloud/retention/index.d.ts +7 -0
  126. package/dist/cloud/retention/index.d.ts.map +1 -0
  127. package/dist/cloud/retention/index.js +7 -0
  128. package/dist/cloud/retention/index.js.map +1 -0
  129. package/dist/cloud/retention/partition-management.d.ts +61 -0
  130. package/dist/cloud/retention/partition-management.d.ts.map +1 -0
  131. package/dist/cloud/retention/partition-management.js +167 -0
  132. package/dist/cloud/retention/partition-management.js.map +1 -0
  133. package/dist/cloud/retention/retention-job.d.ts +70 -0
  134. package/dist/cloud/retention/retention-job.d.ts.map +1 -0
  135. package/dist/cloud/retention/retention-job.js +160 -0
  136. package/dist/cloud/retention/retention-job.js.map +1 -0
  137. package/dist/cloud/retention/retention-policy.d.ts +27 -0
  138. package/dist/cloud/retention/retention-policy.d.ts.map +1 -0
  139. package/dist/cloud/retention/retention-policy.js +36 -0
  140. package/dist/cloud/retention/retention-policy.js.map +1 -0
  141. package/dist/cloud/routes/api-key-routes.d.ts +38 -0
  142. package/dist/cloud/routes/api-key-routes.d.ts.map +1 -0
  143. package/dist/cloud/routes/api-key-routes.js +84 -0
  144. package/dist/cloud/routes/api-key-routes.js.map +1 -0
  145. package/dist/cloud/routes/audit-routes.d.ts +36 -0
  146. package/dist/cloud/routes/audit-routes.d.ts.map +1 -0
  147. package/dist/cloud/routes/audit-routes.js +47 -0
  148. package/dist/cloud/routes/audit-routes.js.map +1 -0
  149. package/dist/cloud/routes/billing-routes.d.ts +51 -0
  150. package/dist/cloud/routes/billing-routes.d.ts.map +1 -0
  151. package/dist/cloud/routes/billing-routes.js +114 -0
  152. package/dist/cloud/routes/billing-routes.js.map +1 -0
  153. package/dist/cloud/routes/onboarding-routes.d.ts +34 -0
  154. package/dist/cloud/routes/onboarding-routes.d.ts.map +1 -0
  155. package/dist/cloud/routes/onboarding-routes.js +58 -0
  156. package/dist/cloud/routes/onboarding-routes.js.map +1 -0
  157. package/dist/cloud/routes/org-routes.d.ts +80 -0
  158. package/dist/cloud/routes/org-routes.d.ts.map +1 -0
  159. package/dist/cloud/routes/org-routes.js +153 -0
  160. package/dist/cloud/routes/org-routes.js.map +1 -0
  161. package/dist/cloud/routes/usage-routes.d.ts +18 -0
  162. package/dist/cloud/routes/usage-routes.d.ts.map +1 -0
  163. package/dist/cloud/routes/usage-routes.js +66 -0
  164. package/dist/cloud/routes/usage-routes.js.map +1 -0
  165. package/dist/cloud/storage/adapter.d.ts +102 -0
  166. package/dist/cloud/storage/adapter.d.ts.map +1 -0
  167. package/dist/cloud/storage/adapter.js +21 -0
  168. package/dist/cloud/storage/adapter.js.map +1 -0
  169. package/dist/cloud/storage/index.d.ts +8 -0
  170. package/dist/cloud/storage/index.d.ts.map +1 -0
  171. package/dist/cloud/storage/index.js +7 -0
  172. package/dist/cloud/storage/index.js.map +1 -0
  173. package/dist/cloud/storage/postgres-adapter.d.ts +34 -0
  174. package/dist/cloud/storage/postgres-adapter.d.ts.map +1 -0
  175. package/dist/cloud/storage/postgres-adapter.js +544 -0
  176. package/dist/cloud/storage/postgres-adapter.js.map +1 -0
  177. package/dist/cloud/storage/sqlite-adapter.d.ts +29 -0
  178. package/dist/cloud/storage/sqlite-adapter.d.ts.map +1 -0
  179. package/dist/cloud/storage/sqlite-adapter.js +176 -0
  180. package/dist/cloud/storage/sqlite-adapter.js.map +1 -0
  181. package/dist/cloud/tenant-pool.d.ts +49 -0
  182. package/dist/cloud/tenant-pool.d.ts.map +1 -0
  183. package/dist/cloud/tenant-pool.js +61 -0
  184. package/dist/cloud/tenant-pool.js.map +1 -0
  185. package/package.json +1 -1
@@ -0,0 +1,174 @@
1
+ /**
2
+ * Usage Metering Pipeline (S-6.2)
3
+ *
4
+ * Tracks event counts per org per month. Provides:
5
+ * - In-memory batch accumulator (flushes every 10s or 100 events)
6
+ * - Monthly usage query for billing
7
+ * - Stripe usage reporting for metered billing
8
+ * - Redis counter sync for fast quota checks
9
+ */
10
+ import { TIER_CONFIG } from './stripe-client.js';
11
+ /**
12
+ * In-memory accumulator that batches usage increments
13
+ * before flushing to the database.
14
+ */
15
+ export class UsageAccumulator {
16
+ deps;
17
+ redisStore;
18
+ buffer = new Map(); // key: orgId:YYYY-MM, value: count
19
+ lastFlush = Date.now();
20
+ totalBuffered = 0;
21
+ flushIntervalMs;
22
+ flushThreshold;
23
+ constructor(deps, redisStore, config) {
24
+ this.deps = deps;
25
+ this.redisStore = redisStore;
26
+ this.flushIntervalMs = config?.flushIntervalMs ?? 10_000;
27
+ this.flushThreshold = config?.flushThreshold ?? 100;
28
+ }
29
+ /**
30
+ * Record events for an org. Accumulates in memory,
31
+ * flushes when threshold or interval is reached.
32
+ */
33
+ async recordEvents(orgId, count, timestamp) {
34
+ const date = timestamp ?? new Date();
35
+ // Truncate to hour to match batch-writer granularity and avoid double-counting
36
+ const hourDate = new Date(date);
37
+ hourDate.setUTCMinutes(0, 0, 0);
38
+ const hourKey = hourDate.toISOString();
39
+ const month = `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, '0')}`;
40
+ const key = `${orgId}:${hourKey}`;
41
+ this.buffer.set(key, (this.buffer.get(key) ?? 0) + count);
42
+ this.totalBuffered += count;
43
+ // Update Redis counter immediately for fast quota checks
44
+ if (this.redisStore) {
45
+ const redisKey = `usage:${orgId}:${month}`;
46
+ await this.redisStore.incrby(redisKey, count);
47
+ // Expire at end of month + 7 days
48
+ await this.redisStore.expire(redisKey, 40 * 86400);
49
+ }
50
+ // Check if we should flush
51
+ if (this.totalBuffered >= this.flushThreshold || Date.now() - this.lastFlush >= this.flushIntervalMs) {
52
+ await this.flush();
53
+ }
54
+ }
55
+ /**
56
+ * Flush accumulated counts to the database.
57
+ */
58
+ async flush() {
59
+ if (this.buffer.size === 0)
60
+ return 0;
61
+ const entries = Array.from(this.buffer.entries());
62
+ this.buffer.clear();
63
+ const flushed = this.totalBuffered;
64
+ this.totalBuffered = 0;
65
+ this.lastFlush = Date.now();
66
+ for (const [key, count] of entries) {
67
+ const [orgId, ...rest] = key.split(':');
68
+ const hour = rest.join(':'); // hour ISO string stored as key
69
+ await this.deps.db.query(`INSERT INTO usage_records (org_id, hour, event_count, api_key_id)
70
+ VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000')
71
+ ON CONFLICT (org_id, hour, api_key_id)
72
+ DO UPDATE SET event_count = usage_records.event_count + $3`, [orgId, hour, count]);
73
+ }
74
+ return flushed;
75
+ }
76
+ /** Get current buffer size (for testing) */
77
+ getBufferSize() {
78
+ return this.totalBuffered;
79
+ }
80
+ }
81
+ /**
82
+ * Query usage data for an org.
83
+ */
84
+ export class UsageQuery {
85
+ db;
86
+ constructor(db) {
87
+ this.db = db;
88
+ }
89
+ /**
90
+ * Get current month usage summary for an org.
91
+ */
92
+ async getCurrentMonthUsage(orgId) {
93
+ const now = new Date();
94
+ const month = `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}`;
95
+ const monthStart = `${month}-01T00:00:00Z`;
96
+ const nextMonth = new Date(now.getUTCFullYear(), now.getUTCMonth() + 1, 1);
97
+ const monthEnd = nextMonth.toISOString();
98
+ // Get total events this month
99
+ const usageResult = await this.db.query(`SELECT COALESCE(SUM(event_count), 0)::int as total
100
+ FROM usage_records
101
+ WHERE org_id = $1 AND hour >= $2 AND hour < $3`, [orgId, monthStart, monthEnd]);
102
+ const totalEvents = usageResult.rows[0]?.total ?? 0;
103
+ // Get org plan and quota
104
+ const orgResult = await this.db.query(`SELECT plan, event_quota FROM orgs WHERE id = $1`, [orgId]);
105
+ const org = orgResult.rows[0];
106
+ const plan = (org?.plan ?? 'free');
107
+ const quota = org?.event_quota ?? TIER_CONFIG.free.event_quota;
108
+ const overageEvents = Math.max(0, totalEvents - quota);
109
+ const usagePct = quota > 0 ? (totalEvents / quota) * 100 : 0;
110
+ return {
111
+ org_id: orgId,
112
+ month,
113
+ total_events: totalEvents,
114
+ quota,
115
+ plan,
116
+ overage_events: overageEvents,
117
+ usage_pct: Math.round(usagePct * 100) / 100,
118
+ };
119
+ }
120
+ /**
121
+ * Get usage history (monthly summaries).
122
+ */
123
+ async getUsageHistory(orgId, months = 6) {
124
+ const result = await this.db.query(`SELECT
125
+ to_char(date_trunc('month', hour), 'YYYY-MM') as month,
126
+ COALESCE(SUM(event_count), 0)::int as total_events
127
+ FROM usage_records
128
+ WHERE org_id = $1 AND hour >= (now() - make_interval(months => $2))
129
+ GROUP BY date_trunc('month', hour)
130
+ ORDER BY month DESC`, [orgId, months]);
131
+ return result.rows;
132
+ }
133
+ }
134
+ /**
135
+ * Report overage usage to Stripe for metered billing.
136
+ * Called by an hourly cron job.
137
+ */
138
+ export async function reportOverageToStripe(db, stripe) {
139
+ // Find orgs on paid plans with overage this month
140
+ const now = new Date();
141
+ const month = `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}`;
142
+ const monthStart = `${month}-01T00:00:00Z`;
143
+ const orgsResult = await db.query(`SELECT o.id, o.plan, o.event_quota, o.stripe_subscription_id,
144
+ COALESCE(SUM(u.event_count), 0)::int as total_events
145
+ FROM orgs o
146
+ LEFT JOIN usage_records u ON u.org_id = o.id AND u.hour >= $1
147
+ WHERE o.plan IN ('pro', 'team') AND o.stripe_subscription_id IS NOT NULL
148
+ GROUP BY o.id
149
+ HAVING COALESCE(SUM(u.event_count), 0) > o.event_quota`, [monthStart]);
150
+ let reported = 0;
151
+ for (const org of orgsResult.rows) {
152
+ const overageEvents = org.total_events - org.event_quota;
153
+ if (overageEvents <= 0)
154
+ continue;
155
+ // Find the metered subscription item
156
+ const sub = await stripe.getSubscription(org.stripe_subscription_id);
157
+ if (!sub)
158
+ continue;
159
+ const meteredItem = sub.items.data.find((item) => item.price.recurring?.usage_type === 'metered');
160
+ if (!meteredItem)
161
+ continue;
162
+ // Report overage in units of 1K events
163
+ const overageUnits = Math.ceil(overageEvents / 1000);
164
+ await stripe.reportUsage({
165
+ subscription_item: meteredItem.id,
166
+ quantity: overageUnits,
167
+ timestamp: Math.floor(Date.now() / 1000),
168
+ action: 'set',
169
+ });
170
+ reported++;
171
+ }
172
+ return reported;
173
+ }
174
+ //# sourceMappingURL=usage-metering.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"usage-metering.js","sourceRoot":"","sources":["../../../src/cloud/billing/usage-metering.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,EAAE,WAAW,EAAiB,MAAM,oBAAoB,CAAC;AAwBhE;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IAQjB;IACA;IARF,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,mCAAmC;IACvE,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,aAAa,GAAG,CAAC,CAAC;IAClB,eAAe,CAAS;IACxB,cAAc,CAAS;IAE/B,YACU,IAAuB,EACvB,UAA4B,EACpC,MAA8D;QAFtD,SAAI,GAAJ,IAAI,CAAmB;QACvB,eAAU,GAAV,UAAU,CAAkB;QAGpC,IAAI,CAAC,eAAe,GAAG,MAAM,EAAE,eAAe,IAAI,MAAM,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,GAAG,CAAC;IACtD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,KAAa,EAAE,SAAgB;QAC/D,MAAM,IAAI,GAAG,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;QACrC,+EAA+E;QAC/E,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,cAAc,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAC5F,MAAM,GAAG,GAAG,GAAG,KAAK,IAAI,OAAO,EAAE,CAAC;QAElC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;QAC1D,IAAI,CAAC,aAAa,IAAI,KAAK,CAAC;QAE5B,yDAAyD;QACzD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;YAC3C,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC9C,kCAAkC;YAClC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,CAAC;QACrD,CAAC;QAED,2BAA2B;QAC3B,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACrG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAErC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE5B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YACnC,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,gCAAgC;YAE7D,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CACtB;;;oEAG4D,EAC5D,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CACrB,CAAC;QACJ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4CAA4C;IAC5C,aAAa;QACX,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,UAAU;IACD;IAApB,YAAoB,EAAmB;QAAnB,OAAE,GAAF,EAAE,CAAiB;IAAG,CAAC;IAE3C;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,KAAa;QACtC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,cAAc,EAAE,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAC1F,MAAM,UAAU,GAAG,GAAG,KAAK,eAAe,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAEzC,8BAA8B;QAC9B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACrC;;sDAEgD,EAChD,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAC,CAC9B,CAAC;QACF,MAAM,WAAW,GAAI,WAAW,CAAC,IAAc,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QAE/D,yBAAyB;QACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACnC,kDAAkD,EAClD,CAAC,KAAK,CAAC,CACR,CAAC;QACF,MAAM,GAAG,GAAI,SAAS,CAAC,IAAc,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,MAAM,CAAa,CAAC;QAC/C,MAAM,KAAK,GAAG,GAAG,EAAE,WAAW,IAAI,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC;QAE/D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,KAAK,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE7D,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK;YACL,YAAY,EAAE,WAAW;YACzB,KAAK;YACL,IAAI;YACJ,cAAc,EAAE,aAAa;YAC7B,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,GAAG;SAC5C,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,SAAiB,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAChC;;;;;;2BAMqB,EACrB,CAAC,KAAK,EAAE,MAAM,CAAC,CAChB,CAAC;QACF,OAAO,MAAM,CAAC,IAAsD,CAAC;IACvE,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,EAAmB,EACnB,MAAqB;IAErB,kDAAkD;IAClD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,cAAc,EAAE,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAC1F,MAAM,UAAU,GAAG,GAAG,KAAK,eAAe,CAAC;IAE3C,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,KAAK,CAC/B;;;;;;4DAMwD,EACxD,CAAC,UAAU,CAAC,CACb,CAAC;IAEF,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,IAAa,EAAE,CAAC;QAC3C,MAAM,aAAa,GAAG,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,WAAW,CAAC;QACzD,IAAI,aAAa,IAAI,CAAC;YAAE,SAAS;QAEjC,qCAAqC;QACrC,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACrE,IAAI,CAAC,GAAG;YAAE,SAAS;QAEnB,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CACrC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,UAAU,KAAK,SAAS,CACzD,CAAC;QACF,IAAI,CAAC,WAAW;YAAE,SAAS;QAE3B,uCAAuC;QACvC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;QACrD,MAAM,MAAM,CAAC,WAAW,CAAC;YACvB,iBAAiB,EAAE,WAAW,CAAC,EAAE;YACjC,QAAQ,EAAE,YAAY;YACtB,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;YACxC,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QACH,QAAQ,EAAE,CAAC;IACb,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
@@ -0,0 +1,107 @@
1
+ /**
2
+ * Backpressure Mechanism (S-3.5)
3
+ *
4
+ * Monitors Redis Stream length. When pending messages exceed a configurable
5
+ * threshold (default 100K, env: BACKPRESSURE_THRESHOLD), the API gateway
6
+ * returns 503 with Retry-After header.
7
+ *
8
+ * Includes CloudWatch alarm configuration and auto-scaling policy definitions.
9
+ */
10
+ import type { EventQueue } from './event-queue.js';
11
+ export interface BackpressureStatus {
12
+ /** Whether the system is under backpressure */
13
+ underPressure: boolean;
14
+ /** Current stream depth */
15
+ streamLength: number;
16
+ /** Configured threshold */
17
+ threshold: number;
18
+ /** Suggested Retry-After in seconds (0 if not under pressure) */
19
+ retryAfterSeconds: number;
20
+ }
21
+ export interface BackpressureConfig {
22
+ /** Pending message threshold (default: from env or 100K) */
23
+ threshold?: number;
24
+ /** Retry-After header value in seconds when under pressure (default: 5) */
25
+ retryAfterSeconds?: number;
26
+ /** Check interval for cached status in ms (default: 1000) */
27
+ cacheMs?: number;
28
+ }
29
+ export interface CloudWatchAlarmConfig {
30
+ alarmName: string;
31
+ metricName: string;
32
+ namespace: string;
33
+ threshold: number;
34
+ evaluationPeriods: number;
35
+ period: number;
36
+ statistic: string;
37
+ comparisonOperator: string;
38
+ alarmActions: string[];
39
+ dimensions: Array<{
40
+ Name: string;
41
+ Value: string;
42
+ }>;
43
+ }
44
+ export interface AutoScalingPolicy {
45
+ policyName: string;
46
+ serviceNamespace: string;
47
+ resourceId: string;
48
+ scalableDimension: string;
49
+ stepAdjustments: Array<{
50
+ metricIntervalLowerBound: number;
51
+ metricIntervalUpperBound?: number;
52
+ scalingAdjustment: number;
53
+ }>;
54
+ cooldownSeconds: number;
55
+ }
56
+ /**
57
+ * Monitors queue depth and reports backpressure status.
58
+ * Caches the stream length check to avoid hitting Redis on every request.
59
+ */
60
+ export declare class BackpressureMonitor {
61
+ private queue;
62
+ private threshold;
63
+ private retryAfterSeconds;
64
+ private cacheMs;
65
+ private cachedStatus;
66
+ private lastCheckTime;
67
+ constructor(queue: EventQueue, config?: BackpressureConfig);
68
+ /** Get the configured threshold */
69
+ getThreshold(): number;
70
+ /**
71
+ * Check whether the system is under backpressure.
72
+ * Uses a cached value if checked within the cache interval.
73
+ */
74
+ check(): Promise<BackpressureStatus>;
75
+ /** Force-clear the cache (useful after scaling or for tests) */
76
+ clearCache(): void;
77
+ /**
78
+ * Generate a 503 response body for backpressure.
79
+ */
80
+ static make503Response(status: BackpressureStatus): {
81
+ status: 503;
82
+ headers: Record<string, string>;
83
+ body: {
84
+ error: string;
85
+ retry_after: number;
86
+ stream_depth: number;
87
+ };
88
+ };
89
+ }
90
+ /**
91
+ * Generate CloudWatch alarm configuration for stream depth monitoring.
92
+ */
93
+ export declare function generateCloudWatchAlarmConfig(opts?: {
94
+ threshold?: number;
95
+ snsTopicArn?: string;
96
+ streamName?: string;
97
+ }): CloudWatchAlarmConfig;
98
+ /**
99
+ * Generate auto-scaling step policy for worker tasks.
100
+ * When stream depth exceeds threshold, add workers in steps.
101
+ */
102
+ export declare function generateAutoScalingPolicy(opts?: {
103
+ ecsCluster?: string;
104
+ ecsService?: string;
105
+ cooldownSeconds?: number;
106
+ }): AutoScalingPolicy;
107
+ //# sourceMappingURL=backpressure.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"backpressure.d.ts","sourceRoot":"","sources":["../../../src/cloud/ingestion/backpressure.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAOnD,MAAM,WAAW,kBAAkB;IACjC,+CAA+C;IAC/C,aAAa,EAAE,OAAO,CAAC;IACvB,2BAA2B;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACpD;AAED,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,KAAK,CAAC;QACrB,wBAAwB,EAAE,MAAM,CAAC;QACjC,wBAAwB,CAAC,EAAE,MAAM,CAAC;QAClC,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC,CAAC;IACH,eAAe,EAAE,MAAM,CAAC;CACzB;AAMD;;;GAGG;AACH,qBAAa,mBAAmB;IAQ5B,OAAO,CAAC,KAAK;IAPf,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAmC;IACvD,OAAO,CAAC,aAAa,CAAK;gBAGhB,KAAK,EAAE,UAAU,EACzB,MAAM,CAAC,EAAE,kBAAkB;IAY7B,mCAAmC;IACnC,YAAY,IAAI,MAAM;IAItB;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAoB1C,gEAAgE;IAChE,UAAU,IAAI,IAAI;IAKlB;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,kBAAkB,GAAG;QAClD,MAAM,EAAE,GAAG,CAAC;QACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAA;SAAE,CAAC;KACpE;CAaF;AAMD;;GAEG;AACH,wBAAgB,6BAA6B,CAAC,IAAI,CAAC,EAAE;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,qBAAqB,CAkBxB;AAMD;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,CAAC,EAAE;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,iBAAiB,CAsBpB"}
@@ -0,0 +1,134 @@
1
+ /**
2
+ * Backpressure Mechanism (S-3.5)
3
+ *
4
+ * Monitors Redis Stream length. When pending messages exceed a configurable
5
+ * threshold (default 100K, env: BACKPRESSURE_THRESHOLD), the API gateway
6
+ * returns 503 with Retry-After header.
7
+ *
8
+ * Includes CloudWatch alarm configuration and auto-scaling policy definitions.
9
+ */
10
+ import { BACKPRESSURE_THRESHOLD } from './event-queue.js';
11
+ // ═══════════════════════════════════════════
12
+ // BackpressureMonitor
13
+ // ═══════════════════════════════════════════
14
+ /**
15
+ * Monitors queue depth and reports backpressure status.
16
+ * Caches the stream length check to avoid hitting Redis on every request.
17
+ */
18
+ export class BackpressureMonitor {
19
+ queue;
20
+ threshold;
21
+ retryAfterSeconds;
22
+ cacheMs;
23
+ cachedStatus = null;
24
+ lastCheckTime = 0;
25
+ constructor(queue, config) {
26
+ this.queue = queue;
27
+ // Allow env override
28
+ const envThreshold = typeof process !== 'undefined'
29
+ ? parseInt(process.env.BACKPRESSURE_THRESHOLD ?? '', 10)
30
+ : NaN;
31
+ this.threshold = config?.threshold ?? (isNaN(envThreshold) ? BACKPRESSURE_THRESHOLD : envThreshold);
32
+ this.retryAfterSeconds = config?.retryAfterSeconds ?? 5;
33
+ this.cacheMs = config?.cacheMs ?? 1000;
34
+ }
35
+ /** Get the configured threshold */
36
+ getThreshold() {
37
+ return this.threshold;
38
+ }
39
+ /**
40
+ * Check whether the system is under backpressure.
41
+ * Uses a cached value if checked within the cache interval.
42
+ */
43
+ async check() {
44
+ const now = Date.now();
45
+ if (this.cachedStatus && (now - this.lastCheckTime) < this.cacheMs) {
46
+ return this.cachedStatus;
47
+ }
48
+ const streamLength = await this.queue.getStreamLength();
49
+ const underPressure = streamLength >= this.threshold;
50
+ this.cachedStatus = {
51
+ underPressure,
52
+ streamLength,
53
+ threshold: this.threshold,
54
+ retryAfterSeconds: underPressure ? this.retryAfterSeconds : 0,
55
+ };
56
+ this.lastCheckTime = now;
57
+ return this.cachedStatus;
58
+ }
59
+ /** Force-clear the cache (useful after scaling or for tests) */
60
+ clearCache() {
61
+ this.cachedStatus = null;
62
+ this.lastCheckTime = 0;
63
+ }
64
+ /**
65
+ * Generate a 503 response body for backpressure.
66
+ */
67
+ static make503Response(status) {
68
+ return {
69
+ status: 503,
70
+ headers: {
71
+ 'Retry-After': String(status.retryAfterSeconds),
72
+ },
73
+ body: {
74
+ error: 'Service temporarily unavailable due to high load. Please retry.',
75
+ retry_after: status.retryAfterSeconds,
76
+ stream_depth: status.streamLength,
77
+ },
78
+ };
79
+ }
80
+ }
81
+ // ═══════════════════════════════════════════
82
+ // CloudWatch Alarm Configuration
83
+ // ═══════════════════════════════════════════
84
+ /**
85
+ * Generate CloudWatch alarm configuration for stream depth monitoring.
86
+ */
87
+ export function generateCloudWatchAlarmConfig(opts) {
88
+ const threshold = opts?.threshold ?? BACKPRESSURE_THRESHOLD;
89
+ const snsArn = opts?.snsTopicArn ?? 'arn:aws:sns:us-east-1:ACCOUNT_ID:agentlens-alerts';
90
+ return {
91
+ alarmName: 'AgentLens-IngestionStreamDepth-High',
92
+ metricName: 'StreamPendingMessages',
93
+ namespace: 'AgentLens/Ingestion',
94
+ threshold,
95
+ evaluationPeriods: 2,
96
+ period: 60,
97
+ statistic: 'Average',
98
+ comparisonOperator: 'GreaterThanOrEqualToThreshold',
99
+ alarmActions: [snsArn],
100
+ dimensions: [
101
+ { Name: 'StreamName', Value: opts?.streamName ?? 'event_ingestion' },
102
+ ],
103
+ };
104
+ }
105
+ // ═══════════════════════════════════════════
106
+ // Auto-Scaling Policy Configuration
107
+ // ═══════════════════════════════════════════
108
+ /**
109
+ * Generate auto-scaling step policy for worker tasks.
110
+ * When stream depth exceeds threshold, add workers in steps.
111
+ */
112
+ export function generateAutoScalingPolicy(opts) {
113
+ const cluster = opts?.ecsCluster ?? 'agentlens-cloud';
114
+ const service = opts?.ecsService ?? 'ingestion-workers';
115
+ return {
116
+ policyName: 'AgentLens-IngestionWorker-ScaleUp',
117
+ serviceNamespace: 'ecs',
118
+ resourceId: `service/${cluster}/${service}`,
119
+ scalableDimension: 'ecs:service:DesiredCount',
120
+ stepAdjustments: [
121
+ {
122
+ metricIntervalLowerBound: 0,
123
+ metricIntervalUpperBound: 50_000,
124
+ scalingAdjustment: 2,
125
+ },
126
+ {
127
+ metricIntervalLowerBound: 50_000,
128
+ scalingAdjustment: 5,
129
+ },
130
+ ],
131
+ cooldownSeconds: opts?.cooldownSeconds ?? 120,
132
+ };
133
+ }
134
+ //# sourceMappingURL=backpressure.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"backpressure.js","sourceRoot":"","sources":["../../../src/cloud/ingestion/backpressure.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAoD1D,8CAA8C;AAC9C,sBAAsB;AACtB,8CAA8C;AAE9C;;;GAGG;AACH,MAAM,OAAO,mBAAmB;IAQpB;IAPF,SAAS,CAAS;IAClB,iBAAiB,CAAS;IAC1B,OAAO,CAAS;IAChB,YAAY,GAA8B,IAAI,CAAC;IAC/C,aAAa,GAAG,CAAC,CAAC;IAE1B,YACU,KAAiB,EACzB,MAA2B;QADnB,UAAK,GAAL,KAAK,CAAY;QAGzB,qBAAqB;QACrB,MAAM,YAAY,GAAG,OAAO,OAAO,KAAK,WAAW;YACjD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,EAAE,EAAE,EAAE,CAAC;YACxD,CAAC,CAAC,GAAG,CAAC;QAER,IAAI,CAAC,SAAS,GAAG,MAAM,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACpG,IAAI,CAAC,iBAAiB,GAAG,MAAM,EAAE,iBAAiB,IAAI,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC;IACzC,CAAC;IAED,mCAAmC;IACnC,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YACnE,OAAO,IAAI,CAAC,YAAY,CAAC;QAC3B,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACxD,MAAM,aAAa,GAAG,YAAY,IAAI,IAAI,CAAC,SAAS,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG;YAClB,aAAa;YACb,YAAY;YACZ,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,iBAAiB,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;SAC9D,CAAC;QACF,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;QAEzB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,gEAAgE;IAChE,UAAU;QACR,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,MAA0B;QAK/C,OAAO;YACL,MAAM,EAAE,GAAG;YACX,OAAO,EAAE;gBACP,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC;aAChD;YACD,IAAI,EAAE;gBACJ,KAAK,EAAE,iEAAiE;gBACxE,WAAW,EAAE,MAAM,CAAC,iBAAiB;gBACrC,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC;SACF,CAAC;IACJ,CAAC;CACF;AAED,8CAA8C;AAC9C,iCAAiC;AACjC,8CAA8C;AAE9C;;GAEG;AACH,MAAM,UAAU,6BAA6B,CAAC,IAI7C;IACC,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,sBAAsB,CAAC;IAC5D,MAAM,MAAM,GAAG,IAAI,EAAE,WAAW,IAAI,mDAAmD,CAAC;IAExF,OAAO;QACL,SAAS,EAAE,qCAAqC;QAChD,UAAU,EAAE,uBAAuB;QACnC,SAAS,EAAE,qBAAqB;QAChC,SAAS;QACT,iBAAiB,EAAE,CAAC;QACpB,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,SAAS;QACpB,kBAAkB,EAAE,+BAA+B;QACnD,YAAY,EAAE,CAAC,MAAM,CAAC;QACtB,UAAU,EAAE;YACV,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,IAAI,iBAAiB,EAAE;SACrE;KACF,CAAC;AACJ,CAAC;AAED,8CAA8C;AAC9C,oCAAoC;AACpC,8CAA8C;AAE9C;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CAAC,IAIzC;IACC,MAAM,OAAO,GAAG,IAAI,EAAE,UAAU,IAAI,iBAAiB,CAAC;IACtD,MAAM,OAAO,GAAG,IAAI,EAAE,UAAU,IAAI,mBAAmB,CAAC;IAExD,OAAO;QACL,UAAU,EAAE,mCAAmC;QAC/C,gBAAgB,EAAE,KAAK;QACvB,UAAU,EAAE,WAAW,OAAO,IAAI,OAAO,EAAE;QAC3C,iBAAiB,EAAE,0BAA0B;QAC7C,eAAe,EAAE;YACf;gBACE,wBAAwB,EAAE,CAAC;gBAC3B,wBAAwB,EAAE,MAAM;gBAChC,iBAAiB,EAAE,CAAC;aACrB;YACD;gBACE,wBAAwB,EAAE,MAAM;gBAChC,iBAAiB,EAAE,CAAC;aACrB;SACF;QACD,eAAe,EAAE,IAAI,EAAE,eAAe,IAAI,GAAG;KAC9C,CAAC;AACJ,CAAC"}
@@ -0,0 +1,115 @@
1
+ /**
2
+ * Batch Writer — Queue Worker (S-3.3)
3
+ *
4
+ * Consumer group workers: XREADGROUP up to 50 messages from the
5
+ * `event_ingestion` stream, enrich (cost calculation, hash chain),
6
+ * batch INSERT into Postgres, XACK on success. After 3 failures → DLQ.
7
+ */
8
+ import type { QueuedEvent, RedisClient } from './event-queue.js';
9
+ import type { Pool } from '../tenant-pool.js';
10
+ export interface StreamMessage {
11
+ /** Redis stream message ID (e.g. "1234567890-0") */
12
+ streamId: string;
13
+ /** Parsed event payload */
14
+ event: QueuedEvent;
15
+ }
16
+ export interface BatchWriterConfig {
17
+ /** Consumer name (unique per worker instance) */
18
+ consumerName: string;
19
+ /** Max messages per XREADGROUP call (default: 50) */
20
+ batchSize?: number;
21
+ /** Block timeout in ms for XREADGROUP (default: 5000) */
22
+ blockMs?: number;
23
+ /** Max retries before DLQ (default: 3) */
24
+ maxRetries?: number;
25
+ }
26
+ export interface WriterStats {
27
+ processed: number;
28
+ failed: number;
29
+ dlqd: number;
30
+ }
31
+ /** Extended Redis client interface for consumer operations */
32
+ export interface ConsumerRedisClient extends RedisClient {
33
+ xreadgroup(...args: unknown[]): Promise<Array<[string, Array<[string, string[]]>]> | null>;
34
+ xack(...args: unknown[]): Promise<number>;
35
+ xadd(...args: unknown[]): Promise<string>;
36
+ }
37
+ /**
38
+ * Calculate cost for an LLM call event.
39
+ * Returns cost in USD or null if not calculable.
40
+ */
41
+ export declare function calculateCost(event: QueuedEvent): number | null;
42
+ /**
43
+ * Compute hash for event in the hash chain.
44
+ * hash = SHA-256(previousHash + event.id + event.type + event.timestamp)
45
+ */
46
+ export declare function computeHash(event: QueuedEvent, previousHash: string): string;
47
+ export declare class BatchWriter {
48
+ private redis;
49
+ private pool;
50
+ private config;
51
+ private running;
52
+ private stats;
53
+ /** Retry counts by stream message ID */
54
+ private retryCounts;
55
+ /** Last hash per org for hash chain continuity */
56
+ private lastHashByOrg;
57
+ constructor(redis: ConsumerRedisClient, pool: Pool, config: BatchWriterConfig);
58
+ /** Start the worker loop */
59
+ start(): Promise<void>;
60
+ /** Stop the worker loop */
61
+ stop(): void;
62
+ /** Get current stats */
63
+ getStats(): WriterStats;
64
+ /**
65
+ * Process a single batch: XREADGROUP → enrich → INSERT → XACK.
66
+ * Exposed for testing.
67
+ */
68
+ processBatch(): Promise<number>;
69
+ private readMessages;
70
+ private enrichMessages;
71
+ private batchInsert;
72
+ private ackMessages;
73
+ private moveToDlq;
74
+ }
75
+ export interface InMemoryBatchWriterDeps {
76
+ /** Events available for consumption */
77
+ pendingEvents: Array<{
78
+ streamId: string;
79
+ event: QueuedEvent;
80
+ }>;
81
+ /** Acknowledged stream IDs */
82
+ ackedIds: string[];
83
+ /** DLQ entries */
84
+ dlqEvents: QueuedEvent[];
85
+ /** Inserted events (grouped by org) */
86
+ insertedEvents: Map<string, QueuedEvent[]>;
87
+ /** Usage increments */
88
+ usageIncrements: Array<{
89
+ orgId: string;
90
+ count: number;
91
+ }>;
92
+ /** Simulate DB failure for these org IDs */
93
+ failingOrgs?: Set<string>;
94
+ }
95
+ /**
96
+ * In-memory batch writer for testing without Redis/Postgres.
97
+ */
98
+ export declare class InMemoryBatchWriter {
99
+ private deps;
100
+ private stats;
101
+ private retryCounts;
102
+ private lastHashByOrg;
103
+ private maxRetries;
104
+ private batchSize;
105
+ constructor(deps: InMemoryBatchWriterDeps, config?: {
106
+ maxRetries?: number;
107
+ batchSize?: number;
108
+ });
109
+ getStats(): WriterStats;
110
+ /**
111
+ * Process one batch from pendingEvents.
112
+ */
113
+ processBatch(): Promise<number>;
114
+ }
115
+ //# sourceMappingURL=batch-writer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batch-writer.d.ts","sourceRoot":"","sources":["../../../src/cloud/ingestion/batch-writer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEjE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAM9C,MAAM,WAAW,aAAa;IAC5B,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,KAAK,EAAE,WAAW,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,8DAA8D;AAC9D,MAAM,WAAW,mBAAoB,SAAQ,WAAW;IACtD,UAAU,CACR,GAAG,IAAI,EAAE,OAAO,EAAE,GACjB,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC9D,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC3C;AAqBD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI,CAqB/D;AAMD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAG5E;AAMD,qBAAa,WAAW;IAUpB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,IAAI;IAVd,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,KAAK,CAAqD;IAClE,wCAAwC;IACxC,OAAO,CAAC,WAAW,CAA6B;IAChD,kDAAkD;IAClD,OAAO,CAAC,aAAa,CAA6B;gBAGxC,KAAK,EAAE,mBAAmB,EAC1B,IAAI,EAAE,IAAI,EAClB,MAAM,EAAE,iBAAiB;IAU3B,4BAA4B;IACtB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B,2BAA2B;IAC3B,IAAI,IAAI,IAAI;IAIZ,wBAAwB;IACxB,QAAQ,IAAI,WAAW;IAIvB;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;YAqDvB,YAAY;IA8B1B,OAAO,CAAC,cAAc;YAiBR,WAAW;YA2DX,WAAW;YAKX,SAAS;CASxB;AAMD,MAAM,WAAW,uBAAuB;IACtC,uCAAuC;IACvC,aAAa,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,WAAW,CAAA;KAAE,CAAC,CAAC;IAC/D,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,kBAAkB;IAClB,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,uCAAuC;IACvC,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,uBAAuB;IACvB,eAAe,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACzD,4CAA4C;IAC5C,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CAC3B;AAED;;GAEG;AACH,qBAAa,mBAAmB;IAQ5B,OAAO,CAAC,IAAI;IAPd,OAAO,CAAC,KAAK,CAAqD;IAClE,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,aAAa,CAA6B;IAClD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGhB,IAAI,EAAE,uBAAuB,EACrC,MAAM,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE;IAMtD,QAAQ,IAAI,WAAW;IAIvB;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;CAiEtC"}