@rce-mcp/data-plane 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts ADDED
@@ -0,0 +1,3393 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { mkdirSync } from "node:fs";
3
+ import { dirname, resolve as resolvePath } from "node:path";
4
+ import { Pool, type PoolClient, type QueryResultRow } from "pg";
5
+ import { Redis, type Redis as RedisClient } from "ioredis";
6
+ import type { SearchContextOutput } from "@rce-mcp/contracts";
7
+ import { openSqliteDatabase, sqliteRuntimeDriverName, type SqliteDatabase } from "./sqlite-runtime.js";
8
+
9
+ export { sqliteRuntimeDriverName } from "./sqlite-runtime.js";
10
+
11
+ export type RuntimeMode = "cloud" | "local" | "hybrid";
12
+
13
+ export type LocalSqliteRuntime = ReturnType<typeof sqliteRuntimeDriverName>;
14
+
15
+ const RUNTIME_MODES = new Set<RuntimeMode>(["cloud", "local", "hybrid"]);
16
+ export const CLOUD_REQUIRED_ENV_VARS = [
17
+ "DATABASE_URL",
18
+ "REDIS_URL",
19
+ "S3_BUCKET",
20
+ "S3_REGION",
21
+ "S3_ACCESS_KEY_ID",
22
+ "S3_SECRET_ACCESS_KEY"
23
+ ] as const;
24
+
25
+ export interface RuntimeModeResolution {
26
+ requested_mode: RuntimeMode;
27
+ effective_mode: "cloud" | "local";
28
+ cloud_configured: boolean;
29
+ missing_cloud_vars: string[];
30
+ }
31
+
32
+ export function parseRuntimeMode(value: string | undefined): RuntimeMode {
33
+ const normalized = (value ?? "hybrid").trim().toLowerCase();
34
+ if (RUNTIME_MODES.has(normalized as RuntimeMode)) {
35
+ return normalized as RuntimeMode;
36
+ }
37
+ throw new Error(`invalid RCE_RUNTIME_MODE: ${value ?? ""}. Expected cloud|local|hybrid.`);
38
+ }
39
+
40
+ export function missingCloudRuntimeEnvVars(env: NodeJS.ProcessEnv): string[] {
41
+ return CLOUD_REQUIRED_ENV_VARS.filter((key) => {
42
+ const value = env[key];
43
+ return typeof value !== "string" || value.trim() === "";
44
+ });
45
+ }
46
+
47
+ export function isCloudRuntimeConfigured(env: NodeJS.ProcessEnv): boolean {
48
+ return missingCloudRuntimeEnvVars(env).length === 0;
49
+ }
50
+
51
+ export function resolveRuntimeMode(env: NodeJS.ProcessEnv): RuntimeModeResolution {
52
+ const requested_mode = parseRuntimeMode(env.RCE_RUNTIME_MODE);
53
+ const missing_cloud_vars = missingCloudRuntimeEnvVars(env);
54
+ const cloud_configured = missing_cloud_vars.length === 0;
55
+
56
+ if (requested_mode === "cloud") {
57
+ if (!cloud_configured) {
58
+ throw new Error(`RCE_RUNTIME_MODE=cloud requires: ${missing_cloud_vars.join(", ")}`);
59
+ }
60
+ return {
61
+ requested_mode,
62
+ effective_mode: "cloud",
63
+ cloud_configured,
64
+ missing_cloud_vars
65
+ };
66
+ }
67
+
68
+ if (requested_mode === "local") {
69
+ return {
70
+ requested_mode,
71
+ effective_mode: "local",
72
+ cloud_configured,
73
+ missing_cloud_vars
74
+ };
75
+ }
76
+
77
+ return {
78
+ requested_mode,
79
+ effective_mode: cloud_configured ? "cloud" : "local",
80
+ cloud_configured,
81
+ missing_cloud_vars
82
+ };
83
+ }
84
+
85
+ export interface CandidateScoreWeights {
86
+ lexical_weight: number;
87
+ vector_weight: number;
88
+ path_match_boost: number;
89
+ recency_boost: number;
90
+ generated_penalty: number;
91
+ }
92
+
93
+ const DEFAULT_CANDIDATE_SCORE_WEIGHTS: CandidateScoreWeights = {
94
+ lexical_weight: 0.6,
95
+ vector_weight: 0.4,
96
+ path_match_boost: 0.2,
97
+ recency_boost: 0.1,
98
+ generated_penalty: 0.2
99
+ };
100
+
101
+ export interface WorkspaceRecord {
102
+ workspace_id: string;
103
+ tenant_id: string;
104
+ project_root_path: string;
105
+ name: string;
106
+ }
107
+
108
+ export interface ReadyIndexRecord {
109
+ index_id: string;
110
+ workspace_id: string;
111
+ tenant_id: string;
112
+ index_version: string;
113
+ status: "indexing" | "ready" | "failed";
114
+ created_at: string;
115
+ updated_at: string;
116
+ }
117
+
118
+ export interface IndexMetadataRecord {
119
+ embedding_provider: string;
120
+ embedding_model?: string;
121
+ embedding_dimensions: number;
122
+ embedding_version?: string;
123
+ chunking_strategy: "language_aware" | "sliding";
124
+ chunking_fallback_strategy: "sliding";
125
+ created_at: string;
126
+ }
127
+
128
+ export interface RankChunksInput {
129
+ tenant_id: string;
130
+ index_id: string;
131
+ query: string;
132
+ query_embedding: number[];
133
+ query_tokens: string[];
134
+ top_k: number;
135
+ candidate_weights?: CandidateScoreWeights;
136
+ filters?: {
137
+ language?: string;
138
+ path_prefix?: string;
139
+ glob?: string;
140
+ };
141
+ }
142
+
143
+ export interface RankedChunkCandidate {
144
+ chunk_id: string;
145
+ file_id: string;
146
+ path: string;
147
+ start_line: number;
148
+ end_line: number;
149
+ snippet: string;
150
+ language?: string;
151
+ generated?: boolean;
152
+ updated_at: string;
153
+ score: number;
154
+ lexical_score: number;
155
+ vector_score: number;
156
+ path_match: boolean;
157
+ recency_boosted: boolean;
158
+ }
159
+
160
+ export interface PersistedChunk {
161
+ chunk_id: string;
162
+ file_id: string;
163
+ path: string;
164
+ start_line: number;
165
+ end_line: number;
166
+ snippet: string;
167
+ language?: string;
168
+ generated?: boolean;
169
+ updated_at: string;
170
+ embedding: number[];
171
+ }
172
+
173
+ export interface PersistedFileRecord {
174
+ file_id: string;
175
+ repo_path: string;
176
+ content_hash: string;
177
+ language?: string;
178
+ }
179
+
180
+ export interface UpsertFileInput {
181
+ tenant_id: string;
182
+ index_id: string;
183
+ repo_path: string;
184
+ content_hash: string;
185
+ size_bytes: number;
186
+ language?: string;
187
+ warning_metadata?: Record<string, unknown>;
188
+ updated_at?: string;
189
+ }
190
+
191
+ export interface UpsertChunkInput {
192
+ start_line: number;
193
+ end_line: number;
194
+ snippet: string;
195
+ embedding: number[];
196
+ generated?: boolean;
197
+ updated_at?: string;
198
+ }
199
+
200
+ export interface IndexRepository {
201
+ migrate(): Promise<void>;
202
+ upsertWorkspace(input: WorkspaceRecord): Promise<void>;
203
+ resolveWorkspaceByProjectRoot(tenant_id: string, project_root_path: string): Promise<WorkspaceRecord | undefined>;
204
+ resolveWorkspaceByWorkspaceId(tenant_id: string, workspace_id: string): Promise<WorkspaceRecord | undefined>;
205
+ createIndexVersion(input: {
206
+ tenant_id: string;
207
+ workspace_id: string;
208
+ index_version: string;
209
+ status?: "indexing" | "ready" | "failed";
210
+ }): Promise<ReadyIndexRecord>;
211
+ markIndexStatus(input: {
212
+ tenant_id: string;
213
+ workspace_id: string;
214
+ index_id: string;
215
+ status: "indexing" | "ready" | "failed";
216
+ }): Promise<void>;
217
+ getIndexByVersion(input: {
218
+ tenant_id: string;
219
+ workspace_id: string;
220
+ index_version: string;
221
+ }): Promise<ReadyIndexRecord | undefined>;
222
+ resetIndexContent(input: {
223
+ tenant_id: string;
224
+ index_id: string;
225
+ }): Promise<void>;
226
+ getLatestReadyIndex(input: { tenant_id: string; workspace_id: string }): Promise<ReadyIndexRecord | undefined>;
227
+ getFilesByIndex(input: { tenant_id: string; index_id: string }): Promise<PersistedFileRecord[]>;
228
+ copyFileFromIndex(input: {
229
+ tenant_id: string;
230
+ source_index_id: string;
231
+ target_index_id: string;
232
+ repo_path: string;
233
+ }): Promise<void>;
234
+ upsertFile(input: UpsertFileInput): Promise<{ file_id: string }>;
235
+ replaceFileChunks(input: {
236
+ tenant_id: string;
237
+ file_id: string;
238
+ repo_path: string;
239
+ chunks: UpsertChunkInput[];
240
+ }): Promise<void>;
241
+ saveManifest(input: {
242
+ index_id: string;
243
+ object_key: string;
244
+ checksum: string;
245
+ }): Promise<void>;
246
+ saveIndexMetadata?(input: {
247
+ tenant_id: string;
248
+ index_id: string;
249
+ embedding_provider: string;
250
+ embedding_model?: string;
251
+ embedding_dimensions: number;
252
+ embedding_version?: string;
253
+ chunking_strategy: "language_aware" | "sliding";
254
+ chunking_fallback_strategy: "sliding";
255
+ }): Promise<void>;
256
+ getIndexMetadata?(input: {
257
+ tenant_id: string;
258
+ index_id: string;
259
+ }): Promise<IndexMetadataRecord | undefined>;
260
+ listChunksByIndex(input: {
261
+ tenant_id: string;
262
+ index_id: string;
263
+ filters?: {
264
+ language?: string;
265
+ path_prefix?: string;
266
+ glob?: string;
267
+ };
268
+ }): Promise<PersistedChunk[]>;
269
+ rankChunksByIndex?(input: RankChunksInput): Promise<RankedChunkCandidate[]>;
270
+ }
271
+
272
+ function sha256(value: string): string {
273
+ return createHash("sha256").update(value).digest("hex");
274
+ }
275
+
276
+ function parseEmbedding(raw: unknown): number[] {
277
+ if (Array.isArray(raw)) {
278
+ return raw.map((value) => Number(value));
279
+ }
280
+
281
+ if (typeof raw === "string") {
282
+ const trimmed = raw.trim();
283
+ const body = trimmed.startsWith("[") && trimmed.endsWith("]") ? trimmed.slice(1, -1) : trimmed;
284
+ if (body.length === 0) {
285
+ return [];
286
+ }
287
+ return body.split(",").map((value) => Number.parseFloat(value.trim()));
288
+ }
289
+
290
+ return [];
291
+ }
292
+
293
+ function toVectorLiteral(embedding: number[]): string {
294
+ return `[${embedding.join(",")}]`;
295
+ }
296
+
297
+ function toIsoString(value: unknown): string {
298
+ if (value instanceof Date) {
299
+ return value.toISOString();
300
+ }
301
+ if (typeof value === "string") {
302
+ return value;
303
+ }
304
+ return new Date(String(value)).toISOString();
305
+ }
306
+
307
+ interface Queryable {
308
+ query<T extends QueryResultRow = QueryResultRow>(text: string, values?: readonly unknown[]): Promise<{ rows: T[] }>;
309
+ }
310
+
311
+ async function runTx<T>(pool: Pool, fn: (client: PoolClient) => Promise<T>): Promise<T> {
312
+ const client = await pool.connect();
313
+ try {
314
+ await client.query("BEGIN");
315
+ const output = await fn(client);
316
+ await client.query("COMMIT");
317
+ return output;
318
+ } catch (error) {
319
+ await client.query("ROLLBACK");
320
+ throw error;
321
+ } finally {
322
+ client.release();
323
+ }
324
+ }
325
+
326
+ function ensureSqliteParent(dbPath: string): void {
327
+ if (dbPath === ":memory:") {
328
+ return;
329
+ }
330
+ mkdirSync(dirname(resolvePath(dbPath)), { recursive: true });
331
+ }
332
+
333
+ function compileGlob(glob: string): RegExp {
334
+ const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&");
335
+ return new RegExp(`^${escaped.replace(/\*/g, ".*").replace(/\?/g, ".")}$`);
336
+ }
337
+
338
+ function sqliteBool(value: unknown): boolean {
339
+ return Number(value) === 1;
340
+ }
341
+
342
+ function nowIso(): string {
343
+ return new Date().toISOString();
344
+ }
345
+
346
+ function percentile(values: number[], p: number): number {
347
+ if (values.length === 0) {
348
+ return 0;
349
+ }
350
+ const sorted = [...values].sort((a, b) => a - b);
351
+ const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * p) - 1));
352
+ return sorted[idx] ?? 0;
353
+ }
354
+
355
+ function tokenizeForRanking(text: string): string[] {
356
+ const coarseTokens = text
357
+ .toLowerCase()
358
+ .split(/[^a-z0-9_./-]+/)
359
+ .map((token) => token.trim())
360
+ .filter(Boolean);
361
+
362
+ const expandedTokens: string[] = [];
363
+ for (const token of coarseTokens) {
364
+ expandedTokens.push(token);
365
+ for (const part of token.split(/[./_-]+/).filter(Boolean)) {
366
+ expandedTokens.push(part);
367
+ }
368
+ }
369
+
370
+ return [...new Set(expandedTokens)];
371
+ }
372
+
373
+ function lexicalScoreForRanking(queryTokens: string[], haystack: string): number {
374
+ if (queryTokens.length === 0) {
375
+ return 0;
376
+ }
377
+ const haystackTokens = new Set(tokenizeForRanking(haystack));
378
+ let overlap = 0;
379
+ for (const token of queryTokens) {
380
+ if (haystackTokens.has(token)) {
381
+ overlap += 1;
382
+ }
383
+ }
384
+ return overlap / queryTokens.length;
385
+ }
386
+
387
+ function cosineSimilarity(a: number[], b: number[]): number {
388
+ if (a.length === 0 || b.length === 0) {
389
+ return 0;
390
+ }
391
+ const max = Math.min(a.length, b.length);
392
+ let dot = 0;
393
+ let normA = 0;
394
+ let normB = 0;
395
+ for (let i = 0; i < max; i += 1) {
396
+ dot += (a[i] ?? 0) * (b[i] ?? 0);
397
+ normA += (a[i] ?? 0) * (a[i] ?? 0);
398
+ normB += (b[i] ?? 0) * (b[i] ?? 0);
399
+ }
400
+ if (normA === 0 || normB === 0) {
401
+ return 0;
402
+ }
403
+ return dot / (Math.sqrt(normA) * Math.sqrt(normB));
404
+ }
405
+
406
+ export class SqliteIndexRepository implements IndexRepository {
407
+ private readonly db: SqliteDatabase;
408
+
409
+ constructor(private readonly dbPath: string) {
410
+ ensureSqliteParent(dbPath);
411
+ this.db = openSqliteDatabase(dbPath);
412
+ this.db.exec("PRAGMA journal_mode = WAL;");
413
+ this.db.exec("PRAGMA foreign_keys = ON;");
414
+ }
415
+
416
+ close(): void {
417
+ this.db.close();
418
+ }
419
+
420
+ async migrate(): Promise<void> {
421
+ this.db.exec(`
422
+ CREATE TABLE IF NOT EXISTS workspaces (
423
+ id TEXT PRIMARY KEY,
424
+ tenant_id TEXT NOT NULL,
425
+ name TEXT NOT NULL,
426
+ project_root_path TEXT NOT NULL,
427
+ created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
428
+ UNIQUE (tenant_id, project_root_path)
429
+ );
430
+
431
+ CREATE TABLE IF NOT EXISTS indexes (
432
+ id TEXT PRIMARY KEY,
433
+ tenant_id TEXT NOT NULL,
434
+ workspace_id TEXT NOT NULL,
435
+ version TEXT NOT NULL,
436
+ status TEXT NOT NULL,
437
+ created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
438
+ updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
439
+ UNIQUE (workspace_id, version)
440
+ );
441
+
442
+ CREATE TABLE IF NOT EXISTS manifests (
443
+ id TEXT PRIMARY KEY,
444
+ index_id TEXT NOT NULL,
445
+ object_key TEXT NOT NULL,
446
+ checksum TEXT NOT NULL,
447
+ created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
448
+ );
449
+
450
+ CREATE TABLE IF NOT EXISTS index_metadata (
451
+ index_id TEXT PRIMARY KEY,
452
+ tenant_id TEXT NOT NULL,
453
+ embedding_provider TEXT NOT NULL,
454
+ embedding_model TEXT,
455
+ embedding_dimensions INTEGER NOT NULL,
456
+ embedding_version TEXT,
457
+ chunking_strategy TEXT NOT NULL,
458
+ chunking_fallback_strategy TEXT NOT NULL,
459
+ created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
460
+ );
461
+
462
+ CREATE TABLE IF NOT EXISTS files (
463
+ id TEXT PRIMARY KEY,
464
+ tenant_id TEXT NOT NULL,
465
+ index_id TEXT NOT NULL,
466
+ repo_path TEXT NOT NULL,
467
+ content_hash TEXT NOT NULL,
468
+ size_bytes INTEGER NOT NULL,
469
+ language TEXT,
470
+ warning_metadata TEXT,
471
+ updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
472
+ UNIQUE (index_id, repo_path)
473
+ );
474
+
475
+ CREATE TABLE IF NOT EXISTS chunks (
476
+ id TEXT PRIMARY KEY,
477
+ tenant_id TEXT NOT NULL,
478
+ file_id TEXT NOT NULL,
479
+ repo_path TEXT NOT NULL,
480
+ start_line INTEGER NOT NULL,
481
+ end_line INTEGER NOT NULL,
482
+ text TEXT NOT NULL,
483
+ embedding TEXT NOT NULL,
484
+ generated INTEGER NOT NULL DEFAULT 0,
485
+ lexical_doc TEXT,
486
+ updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
487
+ );
488
+ `);
489
+
490
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_sqlite_workspaces_tenant_path ON workspaces(tenant_id, project_root_path)");
491
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_sqlite_indexes_workspace_status ON indexes(workspace_id, status, created_at DESC)");
492
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_sqlite_index_metadata_tenant_index ON index_metadata(tenant_id, index_id)");
493
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_sqlite_files_index_path ON files(index_id, repo_path)");
494
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_sqlite_chunks_file_id ON chunks(file_id)");
495
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_sqlite_chunks_repo_path ON chunks(repo_path)");
496
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_sqlite_chunks_lexical_doc ON chunks(lexical_doc)");
497
+ }
498
+
499
+ async upsertWorkspace(input: WorkspaceRecord): Promise<void> {
500
+ this.db
501
+ .prepare(
502
+ `
503
+ INSERT INTO workspaces (id, tenant_id, name, project_root_path)
504
+ VALUES (?, ?, ?, ?)
505
+ ON CONFLICT (id)
506
+ DO UPDATE SET tenant_id = excluded.tenant_id, name = excluded.name, project_root_path = excluded.project_root_path
507
+ `
508
+ )
509
+ .run(input.workspace_id, input.tenant_id, input.name, input.project_root_path);
510
+ }
511
+
512
+ async resolveWorkspaceByProjectRoot(tenant_id: string, project_root_path: string): Promise<WorkspaceRecord | undefined> {
513
+ const row = this.db
514
+ .prepare(
515
+ `
516
+ SELECT id AS workspace_id, tenant_id, project_root_path, name
517
+ FROM workspaces
518
+ WHERE tenant_id = ? AND project_root_path = ?
519
+ LIMIT 1
520
+ `
521
+ )
522
+ .get(tenant_id, project_root_path) as
523
+ | {
524
+ workspace_id: string;
525
+ tenant_id: string;
526
+ project_root_path: string;
527
+ name: string;
528
+ }
529
+ | undefined;
530
+
531
+ if (!row) {
532
+ return undefined;
533
+ }
534
+
535
+ return {
536
+ workspace_id: row.workspace_id,
537
+ tenant_id: row.tenant_id,
538
+ project_root_path: row.project_root_path,
539
+ name: row.name
540
+ };
541
+ }
542
+
543
+ async resolveWorkspaceByWorkspaceId(tenant_id: string, workspace_id: string): Promise<WorkspaceRecord | undefined> {
544
+ const row = this.db
545
+ .prepare(
546
+ `
547
+ SELECT id AS workspace_id, tenant_id, project_root_path, name
548
+ FROM workspaces
549
+ WHERE tenant_id = ? AND id = ?
550
+ LIMIT 1
551
+ `
552
+ )
553
+ .get(tenant_id, workspace_id) as
554
+ | {
555
+ workspace_id: string;
556
+ tenant_id: string;
557
+ project_root_path: string;
558
+ name: string;
559
+ }
560
+ | undefined;
561
+
562
+ if (!row) {
563
+ return undefined;
564
+ }
565
+
566
+ return {
567
+ workspace_id: row.workspace_id,
568
+ tenant_id: row.tenant_id,
569
+ project_root_path: row.project_root_path,
570
+ name: row.name
571
+ };
572
+ }
573
+
574
+ async createIndexVersion(input: {
575
+ tenant_id: string;
576
+ workspace_id: string;
577
+ index_version: string;
578
+ status?: "indexing" | "ready" | "failed";
579
+ }): Promise<ReadyIndexRecord> {
580
+ const index_id = `idx_${randomUUID()}`;
581
+ const status = input.status ?? "indexing";
582
+ const now = nowIso();
583
+
584
+ this.db
585
+ .prepare(
586
+ `
587
+ INSERT INTO indexes (id, tenant_id, workspace_id, version, status, created_at, updated_at)
588
+ VALUES (?, ?, ?, ?, ?, ?, ?)
589
+ `
590
+ )
591
+ .run(index_id, input.tenant_id, input.workspace_id, input.index_version, status, now, now);
592
+
593
+ return {
594
+ index_id,
595
+ workspace_id: input.workspace_id,
596
+ tenant_id: input.tenant_id,
597
+ index_version: input.index_version,
598
+ status,
599
+ created_at: now,
600
+ updated_at: now
601
+ };
602
+ }
603
+
604
+ async markIndexStatus(input: {
605
+ tenant_id: string;
606
+ workspace_id: string;
607
+ index_id: string;
608
+ status: "indexing" | "ready" | "failed";
609
+ }): Promise<void> {
610
+ this.db
611
+ .prepare(
612
+ `
613
+ UPDATE indexes
614
+ SET status = ?, updated_at = ?
615
+ WHERE id = ? AND tenant_id = ? AND workspace_id = ?
616
+ `
617
+ )
618
+ .run(input.status, nowIso(), input.index_id, input.tenant_id, input.workspace_id);
619
+ }
620
+
621
+ async getIndexByVersion(input: {
622
+ tenant_id: string;
623
+ workspace_id: string;
624
+ index_version: string;
625
+ }): Promise<ReadyIndexRecord | undefined> {
626
+ const row = this.db
627
+ .prepare(
628
+ `
629
+ SELECT id AS index_id, workspace_id, tenant_id, version AS index_version, status, created_at, updated_at
630
+ FROM indexes
631
+ WHERE tenant_id = ? AND workspace_id = ? AND version = ?
632
+ LIMIT 1
633
+ `
634
+ )
635
+ .get(input.tenant_id, input.workspace_id, input.index_version) as
636
+ | {
637
+ index_id: string;
638
+ workspace_id: string;
639
+ tenant_id: string;
640
+ index_version: string;
641
+ status: "indexing" | "ready" | "failed";
642
+ created_at: string;
643
+ updated_at: string;
644
+ }
645
+ | undefined;
646
+
647
+ if (!row) {
648
+ return undefined;
649
+ }
650
+
651
+ return {
652
+ ...row,
653
+ created_at: toIsoString(row.created_at),
654
+ updated_at: toIsoString(row.updated_at)
655
+ };
656
+ }
657
+
658
+ async resetIndexContent(input: {
659
+ tenant_id: string;
660
+ index_id: string;
661
+ }): Promise<void> {
662
+ this.db.exec("BEGIN");
663
+ try {
664
+ this.db
665
+ .prepare(
666
+ `
667
+ DELETE FROM chunks
668
+ WHERE tenant_id = ? AND file_id IN (
669
+ SELECT id FROM files WHERE tenant_id = ? AND index_id = ?
670
+ )
671
+ `
672
+ )
673
+ .run(input.tenant_id, input.tenant_id, input.index_id);
674
+ this.db.prepare("DELETE FROM files WHERE tenant_id = ? AND index_id = ?").run(input.tenant_id, input.index_id);
675
+ this.db
676
+ .prepare(
677
+ `
678
+ DELETE FROM manifests
679
+ WHERE index_id = ?
680
+ `
681
+ )
682
+ .run(input.index_id);
683
+ this.db.exec("COMMIT");
684
+ } catch (error) {
685
+ this.db.exec("ROLLBACK");
686
+ throw error;
687
+ }
688
+ }
689
+
690
+ async getLatestReadyIndex(input: { tenant_id: string; workspace_id: string }): Promise<ReadyIndexRecord | undefined> {
691
+ const row = this.db
692
+ .prepare(
693
+ `
694
+ SELECT id AS index_id, workspace_id, tenant_id, version AS index_version, status, created_at, updated_at
695
+ FROM indexes
696
+ WHERE tenant_id = ? AND workspace_id = ? AND status = 'ready'
697
+ ORDER BY created_at DESC
698
+ LIMIT 1
699
+ `
700
+ )
701
+ .get(input.tenant_id, input.workspace_id) as
702
+ | {
703
+ index_id: string;
704
+ workspace_id: string;
705
+ tenant_id: string;
706
+ index_version: string;
707
+ status: "indexing" | "ready" | "failed";
708
+ created_at: string;
709
+ updated_at: string;
710
+ }
711
+ | undefined;
712
+
713
+ if (!row) {
714
+ return undefined;
715
+ }
716
+
717
+ return {
718
+ ...row,
719
+ created_at: toIsoString(row.created_at),
720
+ updated_at: toIsoString(row.updated_at)
721
+ };
722
+ }
723
+
724
+ async getFilesByIndex(input: { tenant_id: string; index_id: string }): Promise<PersistedFileRecord[]> {
725
+ const rows = this.db
726
+ .prepare(
727
+ `
728
+ SELECT id AS file_id, repo_path, content_hash, language
729
+ FROM files
730
+ WHERE tenant_id = ? AND index_id = ?
731
+ `
732
+ )
733
+ .all(input.tenant_id, input.index_id) as Array<{
734
+ file_id: string;
735
+ repo_path: string;
736
+ content_hash: string;
737
+ language: string | null;
738
+ }>;
739
+
740
+ return rows.map((row) => ({
741
+ file_id: row.file_id,
742
+ repo_path: row.repo_path,
743
+ content_hash: row.content_hash,
744
+ ...(row.language ? { language: row.language } : {})
745
+ }));
746
+ }
747
+
748
+ async copyFileFromIndex(input: {
749
+ tenant_id: string;
750
+ source_index_id: string;
751
+ target_index_id: string;
752
+ repo_path: string;
753
+ }): Promise<void> {
754
+ this.db.exec("BEGIN");
755
+ try {
756
+ const sourceFile = this.db
757
+ .prepare(
758
+ `
759
+ SELECT id AS file_id, repo_path, content_hash, size_bytes, language, warning_metadata, updated_at
760
+ FROM files
761
+ WHERE tenant_id = ? AND index_id = ? AND repo_path = ?
762
+ LIMIT 1
763
+ `
764
+ )
765
+ .get(input.tenant_id, input.source_index_id, input.repo_path) as
766
+ | {
767
+ file_id: string;
768
+ repo_path: string;
769
+ content_hash: string;
770
+ size_bytes: number;
771
+ language: string | null;
772
+ warning_metadata: string | null;
773
+ updated_at: string;
774
+ }
775
+ | undefined;
776
+
777
+ if (!sourceFile) {
778
+ this.db.exec("COMMIT");
779
+ return;
780
+ }
781
+
782
+ const targetFileId = `fil_${randomUUID()}`;
783
+ this.db
784
+ .prepare(
785
+ `
786
+ INSERT INTO files (id, tenant_id, index_id, repo_path, content_hash, size_bytes, language, warning_metadata, updated_at)
787
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
788
+ `
789
+ )
790
+ .run(
791
+ targetFileId,
792
+ input.tenant_id,
793
+ input.target_index_id,
794
+ sourceFile.repo_path,
795
+ sourceFile.content_hash,
796
+ sourceFile.size_bytes,
797
+ sourceFile.language,
798
+ sourceFile.warning_metadata,
799
+ toIsoString(sourceFile.updated_at)
800
+ );
801
+
802
+ const chunks = this.db
803
+ .prepare(
804
+ `
805
+ SELECT repo_path, start_line, end_line, text, embedding, generated, updated_at
806
+ FROM chunks
807
+ WHERE tenant_id = ? AND file_id = ?
808
+ `
809
+ )
810
+ .all(input.tenant_id, sourceFile.file_id) as Array<{
811
+ repo_path: string;
812
+ start_line: number;
813
+ end_line: number;
814
+ text: string;
815
+ embedding: string;
816
+ generated: number;
817
+ updated_at: string;
818
+ }>;
819
+
820
+ const insertChunk = this.db.prepare(
821
+ `
822
+ INSERT INTO chunks (id, tenant_id, file_id, repo_path, start_line, end_line, text, embedding, generated, lexical_doc, updated_at)
823
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
824
+ `
825
+ );
826
+ for (const chunk of chunks) {
827
+ insertChunk.run(
828
+ `chk_${randomUUID()}`,
829
+ input.tenant_id,
830
+ targetFileId,
831
+ chunk.repo_path,
832
+ chunk.start_line,
833
+ chunk.end_line,
834
+ chunk.text,
835
+ chunk.embedding,
836
+ chunk.generated,
837
+ chunk.text,
838
+ toIsoString(chunk.updated_at)
839
+ );
840
+ }
841
+
842
+ this.db.exec("COMMIT");
843
+ } catch (error) {
844
+ this.db.exec("ROLLBACK");
845
+ throw error;
846
+ }
847
+ }
848
+
849
+ async upsertFile(input: UpsertFileInput): Promise<{ file_id: string }> {
850
+ const existing = this.db
851
+ .prepare(
852
+ `
853
+ SELECT id AS file_id
854
+ FROM files
855
+ WHERE index_id = ? AND repo_path = ?
856
+ LIMIT 1
857
+ `
858
+ )
859
+ .get(input.index_id, input.repo_path) as { file_id: string } | undefined;
860
+
861
+ const warningMetadata = JSON.stringify(input.warning_metadata ?? null);
862
+ if (existing) {
863
+ this.db
864
+ .prepare(
865
+ `
866
+ UPDATE files
867
+ SET content_hash = ?, size_bytes = ?, language = ?, warning_metadata = ?, updated_at = ?
868
+ WHERE id = ?
869
+ `
870
+ )
871
+ .run(
872
+ input.content_hash,
873
+ input.size_bytes,
874
+ input.language ?? null,
875
+ warningMetadata,
876
+ input.updated_at ?? nowIso(),
877
+ existing.file_id
878
+ );
879
+ return { file_id: existing.file_id };
880
+ }
881
+
882
+ const file_id = `fil_${randomUUID()}`;
883
+ this.db
884
+ .prepare(
885
+ `
886
+ INSERT INTO files (id, tenant_id, index_id, repo_path, content_hash, size_bytes, language, warning_metadata, updated_at)
887
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
888
+ `
889
+ )
890
+ .run(
891
+ file_id,
892
+ input.tenant_id,
893
+ input.index_id,
894
+ input.repo_path,
895
+ input.content_hash,
896
+ input.size_bytes,
897
+ input.language ?? null,
898
+ warningMetadata,
899
+ input.updated_at ?? nowIso()
900
+ );
901
+
902
+ return { file_id };
903
+ }
904
+
905
+ async replaceFileChunks(input: {
906
+ tenant_id: string;
907
+ file_id: string;
908
+ repo_path: string;
909
+ chunks: UpsertChunkInput[];
910
+ }): Promise<void> {
911
+ this.db.exec("BEGIN");
912
+ try {
913
+ this.db.prepare("DELETE FROM chunks WHERE tenant_id = ? AND file_id = ?").run(input.tenant_id, input.file_id);
914
+ const insert = this.db.prepare(
915
+ `
916
+ INSERT INTO chunks (id, tenant_id, file_id, repo_path, start_line, end_line, text, embedding, generated, lexical_doc, updated_at)
917
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
918
+ `
919
+ );
920
+ for (const chunk of input.chunks) {
921
+ insert.run(
922
+ `chk_${randomUUID()}`,
923
+ input.tenant_id,
924
+ input.file_id,
925
+ input.repo_path,
926
+ chunk.start_line,
927
+ chunk.end_line,
928
+ chunk.snippet,
929
+ JSON.stringify(chunk.embedding),
930
+ chunk.generated ? 1 : 0,
931
+ chunk.snippet,
932
+ chunk.updated_at ?? nowIso()
933
+ );
934
+ }
935
+ this.db.exec("COMMIT");
936
+ } catch (error) {
937
+ this.db.exec("ROLLBACK");
938
+ throw error;
939
+ }
940
+ }
941
+
942
+ async saveManifest(input: {
943
+ index_id: string;
944
+ object_key: string;
945
+ checksum: string;
946
+ }): Promise<void> {
947
+ this.db
948
+ .prepare(
949
+ `
950
+ INSERT INTO manifests (id, index_id, object_key, checksum, created_at)
951
+ VALUES (?, ?, ?, ?, ?)
952
+ `
953
+ )
954
+ .run(`mft_${randomUUID()}`, input.index_id, input.object_key, input.checksum, nowIso());
955
+ }
956
+
957
+ async saveIndexMetadata(input: {
958
+ tenant_id: string;
959
+ index_id: string;
960
+ embedding_provider: string;
961
+ embedding_model?: string;
962
+ embedding_dimensions: number;
963
+ embedding_version?: string;
964
+ chunking_strategy: "language_aware" | "sliding";
965
+ chunking_fallback_strategy: "sliding";
966
+ }): Promise<void> {
967
+ this.db
968
+ .prepare(
969
+ `
970
+ INSERT INTO index_metadata (
971
+ index_id,
972
+ tenant_id,
973
+ embedding_provider,
974
+ embedding_model,
975
+ embedding_dimensions,
976
+ embedding_version,
977
+ chunking_strategy,
978
+ chunking_fallback_strategy,
979
+ created_at
980
+ )
981
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
982
+ ON CONFLICT (index_id)
983
+ DO UPDATE SET
984
+ tenant_id = excluded.tenant_id,
985
+ embedding_provider = excluded.embedding_provider,
986
+ embedding_model = excluded.embedding_model,
987
+ embedding_dimensions = excluded.embedding_dimensions,
988
+ embedding_version = excluded.embedding_version,
989
+ chunking_strategy = excluded.chunking_strategy,
990
+ chunking_fallback_strategy = excluded.chunking_fallback_strategy
991
+ `
992
+ )
993
+ .run(
994
+ input.index_id,
995
+ input.tenant_id,
996
+ input.embedding_provider,
997
+ input.embedding_model ?? null,
998
+ input.embedding_dimensions,
999
+ input.embedding_version ?? null,
1000
+ input.chunking_strategy,
1001
+ input.chunking_fallback_strategy,
1002
+ nowIso()
1003
+ );
1004
+ }
1005
+
1006
+ async getIndexMetadata(input: { tenant_id: string; index_id: string }): Promise<IndexMetadataRecord | undefined> {
1007
+ const row = this.db
1008
+ .prepare(
1009
+ `
1010
+ SELECT
1011
+ embedding_provider,
1012
+ embedding_model,
1013
+ embedding_dimensions,
1014
+ embedding_version,
1015
+ chunking_strategy,
1016
+ chunking_fallback_strategy,
1017
+ created_at
1018
+ FROM index_metadata
1019
+ WHERE tenant_id = ? AND index_id = ?
1020
+ LIMIT 1
1021
+ `
1022
+ )
1023
+ .get(input.tenant_id, input.index_id) as
1024
+ | {
1025
+ embedding_provider: string;
1026
+ embedding_model: string | null;
1027
+ embedding_dimensions: number;
1028
+ embedding_version: string | null;
1029
+ chunking_strategy: "language_aware" | "sliding";
1030
+ chunking_fallback_strategy: "sliding";
1031
+ created_at: string;
1032
+ }
1033
+ | undefined;
1034
+ if (!row) {
1035
+ return undefined;
1036
+ }
1037
+ return {
1038
+ embedding_provider: row.embedding_provider,
1039
+ ...(row.embedding_model ? { embedding_model: row.embedding_model } : {}),
1040
+ embedding_dimensions: row.embedding_dimensions,
1041
+ ...(row.embedding_version ? { embedding_version: row.embedding_version } : {}),
1042
+ chunking_strategy: row.chunking_strategy,
1043
+ chunking_fallback_strategy: row.chunking_fallback_strategy,
1044
+ created_at: toIsoString(row.created_at)
1045
+ };
1046
+ }
1047
+
1048
+ async listChunksByIndex(input: {
1049
+ tenant_id: string;
1050
+ index_id: string;
1051
+ filters?: {
1052
+ language?: string;
1053
+ path_prefix?: string;
1054
+ glob?: string;
1055
+ };
1056
+ }): Promise<PersistedChunk[]> {
1057
+ const where: string[] = ["f.tenant_id = ?", "f.index_id = ?"];
1058
+ const params: Array<string | number> = [input.tenant_id, input.index_id];
1059
+
1060
+ if (input.filters?.language) {
1061
+ where.push("f.language = ?");
1062
+ params.push(input.filters.language);
1063
+ }
1064
+
1065
+ if (input.filters?.path_prefix) {
1066
+ where.push("c.repo_path LIKE ?");
1067
+ params.push(`${input.filters.path_prefix}%`);
1068
+ }
1069
+
1070
+ const rows = this.db
1071
+ .prepare(
1072
+ `
1073
+ SELECT
1074
+ c.id AS chunk_id,
1075
+ c.file_id,
1076
+ c.repo_path AS path,
1077
+ c.start_line,
1078
+ c.end_line,
1079
+ c.text AS snippet,
1080
+ f.language,
1081
+ c.generated,
1082
+ c.updated_at,
1083
+ c.embedding
1084
+ FROM chunks c
1085
+ INNER JOIN files f ON f.id = c.file_id
1086
+ WHERE ${where.join(" AND ")}
1087
+ `
1088
+ )
1089
+ .all(...params) as Array<{
1090
+ chunk_id: string;
1091
+ file_id: string;
1092
+ path: string;
1093
+ start_line: number;
1094
+ end_line: number;
1095
+ snippet: string;
1096
+ language: string | null;
1097
+ generated: number;
1098
+ updated_at: string;
1099
+ embedding: string;
1100
+ }>;
1101
+
1102
+ const globRegex = input.filters?.glob ? compileGlob(input.filters.glob) : undefined;
1103
+ return rows
1104
+ .filter((row) => (globRegex ? globRegex.test(row.path) : true))
1105
+ .map((row) => ({
1106
+ chunk_id: row.chunk_id,
1107
+ file_id: row.file_id,
1108
+ path: row.path,
1109
+ start_line: row.start_line,
1110
+ end_line: row.end_line,
1111
+ snippet: row.snippet,
1112
+ ...(row.language ? { language: row.language } : {}),
1113
+ ...(sqliteBool(row.generated) ? { generated: true } : {}),
1114
+ updated_at: toIsoString(row.updated_at),
1115
+ embedding: parseEmbedding(row.embedding)
1116
+ }));
1117
+ }
1118
+
1119
+ async rankChunksByIndex(input: RankChunksInput): Promise<RankedChunkCandidate[]> {
1120
+ const weights = input.candidate_weights ?? DEFAULT_CANDIDATE_SCORE_WEIGHTS;
1121
+ const where: string[] = ["f.tenant_id = ?", "f.index_id = ?"];
1122
+ const params: Array<string | number> = [input.tenant_id, input.index_id];
1123
+
1124
+ if (input.filters?.language) {
1125
+ where.push("f.language = ?");
1126
+ params.push(input.filters.language);
1127
+ }
1128
+
1129
+ if (input.filters?.path_prefix) {
1130
+ where.push("c.repo_path LIKE ?");
1131
+ params.push(`${input.filters.path_prefix}%`);
1132
+ }
1133
+
1134
+ const rows = this.db
1135
+ .prepare(
1136
+ `
1137
+ SELECT
1138
+ c.id AS chunk_id,
1139
+ c.file_id,
1140
+ c.repo_path AS path,
1141
+ c.start_line,
1142
+ c.end_line,
1143
+ c.text AS snippet,
1144
+ f.language,
1145
+ c.generated,
1146
+ c.updated_at,
1147
+ c.embedding
1148
+ FROM chunks c
1149
+ INNER JOIN files f ON f.id = c.file_id
1150
+ WHERE ${where.join(" AND ")}
1151
+ `
1152
+ )
1153
+ .all(...params) as Array<{
1154
+ chunk_id: string;
1155
+ file_id: string;
1156
+ path: string;
1157
+ start_line: number;
1158
+ end_line: number;
1159
+ snippet: string;
1160
+ language: string | null;
1161
+ generated: number;
1162
+ updated_at: string;
1163
+ embedding: string;
1164
+ }>;
1165
+
1166
+ const globRegex = input.filters?.glob ? compileGlob(input.filters.glob) : undefined;
1167
+ const ranked = rows
1168
+ .filter((row) => (globRegex ? globRegex.test(row.path) : true))
1169
+ .map((row) => {
1170
+ const lexical = lexicalScoreForRanking(input.query_tokens, `${row.path}\n${row.snippet}`);
1171
+ const vector = cosineSimilarity(input.query_embedding, parseEmbedding(row.embedding));
1172
+ const pathMatch = input.query_tokens.some((token) => row.path.toLowerCase().includes(token));
1173
+ const recencyBoost = Date.now() - new Date(toIsoString(row.updated_at)).getTime() < 14 * 24 * 3600 * 1_000;
1174
+
1175
+ let score = lexical * weights.lexical_weight + vector * weights.vector_weight;
1176
+ if (pathMatch) {
1177
+ score += weights.path_match_boost;
1178
+ }
1179
+ if (recencyBoost) {
1180
+ score += weights.recency_boost;
1181
+ }
1182
+ if (sqliteBool(row.generated)) {
1183
+ score -= weights.generated_penalty;
1184
+ }
1185
+
1186
+ return {
1187
+ chunk_id: row.chunk_id,
1188
+ file_id: row.file_id,
1189
+ path: row.path,
1190
+ start_line: row.start_line,
1191
+ end_line: row.end_line,
1192
+ snippet: row.snippet,
1193
+ ...(row.language ? { language: row.language } : {}),
1194
+ ...(sqliteBool(row.generated) ? { generated: true } : {}),
1195
+ updated_at: toIsoString(row.updated_at),
1196
+ score,
1197
+ lexical_score: lexical,
1198
+ vector_score: vector,
1199
+ path_match: pathMatch,
1200
+ recency_boosted: recencyBoost
1201
+ };
1202
+ })
1203
+ .sort((a, b) => b.score - a.score)
1204
+ .slice(0, Math.max(input.top_k * 4, input.top_k));
1205
+
1206
+ return ranked;
1207
+ }
1208
+ }
1209
+
1210
+ export class PostgresIndexRepository implements IndexRepository {
1211
+ private embeddingStorage: "vector" | "array" = "vector";
1212
+
1213
+ constructor(
1214
+ private readonly pool: Pool,
1215
+ private readonly options: {
1216
+ chunkEmbeddingDimensions?: number;
1217
+ preferPgVector?: boolean;
1218
+ } = {}
1219
+ ) {}
1220
+
1221
+ async migrate(): Promise<void> {
1222
+ const dimensions = this.options.chunkEmbeddingDimensions ?? 24;
1223
+ const preferPgVector = this.options.preferPgVector ?? true;
1224
+
1225
+ this.embeddingStorage = "array";
1226
+ if (preferPgVector) {
1227
+ try {
1228
+ await this.pool.query("CREATE EXTENSION IF NOT EXISTS vector");
1229
+ await this.pool.query("SELECT '[1,2]'::vector");
1230
+ this.embeddingStorage = "vector";
1231
+ } catch {
1232
+ this.embeddingStorage = "array";
1233
+ }
1234
+ }
1235
+
1236
+ await this.pool.query(`
1237
+ CREATE TABLE IF NOT EXISTS tenants (
1238
+ id TEXT PRIMARY KEY,
1239
+ name TEXT NOT NULL,
1240
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
1241
+ );
1242
+ `);
1243
+
1244
+ await this.pool.query(`
1245
+ CREATE TABLE IF NOT EXISTS workspaces (
1246
+ id TEXT PRIMARY KEY,
1247
+ tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
1248
+ name TEXT NOT NULL,
1249
+ project_root_path TEXT NOT NULL,
1250
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
1251
+ UNIQUE (tenant_id, project_root_path)
1252
+ );
1253
+ `);
1254
+
1255
+ await this.pool.query(`
1256
+ CREATE TABLE IF NOT EXISTS indexes (
1257
+ id TEXT PRIMARY KEY,
1258
+ tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
1259
+ workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
1260
+ version TEXT NOT NULL,
1261
+ status TEXT NOT NULL,
1262
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
1263
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
1264
+ UNIQUE (workspace_id, version)
1265
+ );
1266
+ `);
1267
+
1268
+ await this.pool.query(`
1269
+ CREATE TABLE IF NOT EXISTS manifests (
1270
+ id TEXT PRIMARY KEY,
1271
+ index_id TEXT NOT NULL REFERENCES indexes(id) ON DELETE CASCADE,
1272
+ object_key TEXT NOT NULL,
1273
+ checksum TEXT NOT NULL,
1274
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
1275
+ );
1276
+ `);
1277
+
1278
+ await this.pool.query(`
1279
+ CREATE TABLE IF NOT EXISTS index_metadata (
1280
+ index_id TEXT PRIMARY KEY REFERENCES indexes(id) ON DELETE CASCADE,
1281
+ tenant_id TEXT NOT NULL,
1282
+ embedding_provider TEXT NOT NULL,
1283
+ embedding_model TEXT,
1284
+ embedding_dimensions INTEGER NOT NULL,
1285
+ embedding_version TEXT,
1286
+ chunking_strategy TEXT NOT NULL,
1287
+ chunking_fallback_strategy TEXT NOT NULL,
1288
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
1289
+ );
1290
+ `);
1291
+
1292
+ await this.pool.query(`
1293
+ CREATE TABLE IF NOT EXISTS files (
1294
+ id TEXT PRIMARY KEY,
1295
+ tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
1296
+ index_id TEXT NOT NULL REFERENCES indexes(id) ON DELETE CASCADE,
1297
+ repo_path TEXT NOT NULL,
1298
+ content_hash TEXT NOT NULL,
1299
+ size_bytes INTEGER NOT NULL,
1300
+ language TEXT,
1301
+ warning_metadata JSONB,
1302
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
1303
+ UNIQUE (index_id, repo_path)
1304
+ );
1305
+ `);
1306
+
1307
+ const embeddingType = this.embeddingStorage === "vector" ? `vector(${dimensions})` : "DOUBLE PRECISION[]";
1308
+ await this.pool.query(`
1309
+ CREATE TABLE IF NOT EXISTS chunks (
1310
+ id TEXT PRIMARY KEY,
1311
+ tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
1312
+ file_id TEXT NOT NULL REFERENCES files(id) ON DELETE CASCADE,
1313
+ repo_path TEXT NOT NULL,
1314
+ start_line INTEGER NOT NULL,
1315
+ end_line INTEGER NOT NULL,
1316
+ text TEXT NOT NULL,
1317
+ embedding ${embeddingType} NOT NULL,
1318
+ generated BOOLEAN NOT NULL DEFAULT FALSE,
1319
+ lexical_doc TEXT,
1320
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
1321
+ );
1322
+ `);
1323
+
1324
+ await this.pool.query("CREATE INDEX IF NOT EXISTS idx_workspaces_tenant_path ON workspaces(tenant_id, project_root_path)");
1325
+ await this.pool.query("CREATE INDEX IF NOT EXISTS idx_indexes_workspace_status ON indexes(workspace_id, status, created_at DESC)");
1326
+ await this.pool.query("CREATE INDEX IF NOT EXISTS idx_index_metadata_tenant_index ON index_metadata(tenant_id, index_id)");
1327
+ await this.pool.query("CREATE INDEX IF NOT EXISTS idx_files_index_path ON files(index_id, repo_path)");
1328
+ await this.pool.query("CREATE INDEX IF NOT EXISTS idx_chunks_file_id ON chunks(file_id)");
1329
+ await this.pool.query("CREATE INDEX IF NOT EXISTS idx_chunks_repo_path ON chunks(repo_path)");
1330
+ await this.pool.query("CREATE INDEX IF NOT EXISTS idx_chunks_lexical_doc ON chunks(lexical_doc)");
1331
+
1332
+ if (this.embeddingStorage === "vector") {
1333
+ try {
1334
+ await this.pool.query(
1335
+ "CREATE INDEX IF NOT EXISTS idx_chunks_embedding_vector ON chunks USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100)"
1336
+ );
1337
+ } catch {
1338
+ // Some test DBs cannot create ivfflat indexes.
1339
+ }
1340
+ }
1341
+ }
1342
+
1343
+ async upsertWorkspace(input: WorkspaceRecord): Promise<void> {
1344
+ await this.pool.query(
1345
+ "INSERT INTO tenants (id, name) VALUES ($1, $2) ON CONFLICT (id) DO NOTHING",
1346
+ [input.tenant_id, input.tenant_id]
1347
+ );
1348
+
1349
+ await this.pool.query(
1350
+ `
1351
+ INSERT INTO workspaces (id, tenant_id, name, project_root_path)
1352
+ VALUES ($1, $2, $3, $4)
1353
+ ON CONFLICT (id)
1354
+ DO UPDATE SET tenant_id = EXCLUDED.tenant_id, name = EXCLUDED.name, project_root_path = EXCLUDED.project_root_path
1355
+ `,
1356
+ [input.workspace_id, input.tenant_id, input.name, input.project_root_path]
1357
+ );
1358
+ }
1359
+
1360
+ async resolveWorkspaceByProjectRoot(tenant_id: string, project_root_path: string): Promise<WorkspaceRecord | undefined> {
1361
+ const result = await this.pool.query<{
1362
+ workspace_id: string;
1363
+ tenant_id: string;
1364
+ project_root_path: string;
1365
+ name: string;
1366
+ }>(
1367
+ `
1368
+ SELECT id AS workspace_id, tenant_id, project_root_path, name
1369
+ FROM workspaces
1370
+ WHERE tenant_id = $1 AND project_root_path = $2
1371
+ LIMIT 1
1372
+ `,
1373
+ [tenant_id, project_root_path]
1374
+ );
1375
+
1376
+ const row = result.rows[0];
1377
+ if (!row) {
1378
+ return undefined;
1379
+ }
1380
+
1381
+ return {
1382
+ workspace_id: row.workspace_id,
1383
+ tenant_id: row.tenant_id,
1384
+ project_root_path: row.project_root_path,
1385
+ name: row.name
1386
+ };
1387
+ }
1388
+
1389
+ async resolveWorkspaceByWorkspaceId(tenant_id: string, workspace_id: string): Promise<WorkspaceRecord | undefined> {
1390
+ const result = await this.pool.query<{
1391
+ workspace_id: string;
1392
+ tenant_id: string;
1393
+ project_root_path: string;
1394
+ name: string;
1395
+ }>(
1396
+ `
1397
+ SELECT id AS workspace_id, tenant_id, project_root_path, name
1398
+ FROM workspaces
1399
+ WHERE tenant_id = $1 AND id = $2
1400
+ LIMIT 1
1401
+ `,
1402
+ [tenant_id, workspace_id]
1403
+ );
1404
+
1405
+ const row = result.rows[0];
1406
+ if (!row) {
1407
+ return undefined;
1408
+ }
1409
+
1410
+ return {
1411
+ workspace_id: row.workspace_id,
1412
+ tenant_id: row.tenant_id,
1413
+ project_root_path: row.project_root_path,
1414
+ name: row.name
1415
+ };
1416
+ }
1417
+
1418
+ async createIndexVersion(input: {
1419
+ tenant_id: string;
1420
+ workspace_id: string;
1421
+ index_version: string;
1422
+ status?: "indexing" | "ready" | "failed";
1423
+ }): Promise<ReadyIndexRecord> {
1424
+ const status = input.status ?? "indexing";
1425
+ const indexId = `idx_${randomUUID()}`;
1426
+ const result = await this.pool.query<{
1427
+ index_id: string;
1428
+ workspace_id: string;
1429
+ tenant_id: string;
1430
+ index_version: string;
1431
+ status: "indexing" | "ready" | "failed";
1432
+ created_at: unknown;
1433
+ updated_at: unknown;
1434
+ }>(
1435
+ `
1436
+ INSERT INTO indexes (id, tenant_id, workspace_id, version, status)
1437
+ VALUES ($1, $2, $3, $4, $5)
1438
+ RETURNING id AS index_id, workspace_id, tenant_id, version AS index_version, status, created_at, updated_at
1439
+ `,
1440
+ [indexId, input.tenant_id, input.workspace_id, input.index_version, status]
1441
+ );
1442
+
1443
+ const row = result.rows[0]!;
1444
+ return {
1445
+ ...row,
1446
+ created_at: toIsoString(row.created_at),
1447
+ updated_at: toIsoString(row.updated_at)
1448
+ };
1449
+ }
1450
+
1451
+ async markIndexStatus(input: {
1452
+ tenant_id: string;
1453
+ workspace_id: string;
1454
+ index_id: string;
1455
+ status: "indexing" | "ready" | "failed";
1456
+ }): Promise<void> {
1457
+ await this.pool.query(
1458
+ `
1459
+ UPDATE indexes
1460
+ SET status = $4, updated_at = NOW()
1461
+ WHERE id = $1 AND tenant_id = $2 AND workspace_id = $3
1462
+ `,
1463
+ [input.index_id, input.tenant_id, input.workspace_id, input.status]
1464
+ );
1465
+ }
1466
+
1467
+ async getIndexByVersion(input: {
1468
+ tenant_id: string;
1469
+ workspace_id: string;
1470
+ index_version: string;
1471
+ }): Promise<ReadyIndexRecord | undefined> {
1472
+ const result = await this.pool.query<{
1473
+ index_id: string;
1474
+ workspace_id: string;
1475
+ tenant_id: string;
1476
+ index_version: string;
1477
+ status: "indexing" | "ready" | "failed";
1478
+ created_at: unknown;
1479
+ updated_at: unknown;
1480
+ }>(
1481
+ `
1482
+ SELECT id AS index_id, workspace_id, tenant_id, version AS index_version, status, created_at, updated_at
1483
+ FROM indexes
1484
+ WHERE tenant_id = $1 AND workspace_id = $2 AND version = $3
1485
+ LIMIT 1
1486
+ `,
1487
+ [input.tenant_id, input.workspace_id, input.index_version]
1488
+ );
1489
+
1490
+ const row = result.rows[0];
1491
+ if (!row) {
1492
+ return undefined;
1493
+ }
1494
+
1495
+ return {
1496
+ ...row,
1497
+ created_at: toIsoString(row.created_at),
1498
+ updated_at: toIsoString(row.updated_at)
1499
+ };
1500
+ }
1501
+
1502
+ async resetIndexContent(input: {
1503
+ tenant_id: string;
1504
+ index_id: string;
1505
+ }): Promise<void> {
1506
+ await runTx(this.pool, async (client) => {
1507
+ await client.query(
1508
+ `
1509
+ DELETE FROM chunks
1510
+ WHERE tenant_id = $1 AND file_id IN (
1511
+ SELECT id FROM files WHERE tenant_id = $1 AND index_id = $2
1512
+ )
1513
+ `,
1514
+ [input.tenant_id, input.index_id]
1515
+ );
1516
+ await client.query("DELETE FROM files WHERE tenant_id = $1 AND index_id = $2", [input.tenant_id, input.index_id]);
1517
+ await client.query("DELETE FROM manifests WHERE index_id = $1", [input.index_id]);
1518
+ });
1519
+ }
1520
+
1521
+ async getLatestReadyIndex(input: { tenant_id: string; workspace_id: string }): Promise<ReadyIndexRecord | undefined> {
1522
+ const result = await this.pool.query<{
1523
+ index_id: string;
1524
+ workspace_id: string;
1525
+ tenant_id: string;
1526
+ index_version: string;
1527
+ status: "indexing" | "ready" | "failed";
1528
+ created_at: unknown;
1529
+ updated_at: unknown;
1530
+ }>(
1531
+ `
1532
+ SELECT id AS index_id, workspace_id, tenant_id, version AS index_version, status, created_at, updated_at
1533
+ FROM indexes
1534
+ WHERE tenant_id = $1 AND workspace_id = $2 AND status = 'ready'
1535
+ ORDER BY created_at DESC
1536
+ LIMIT 1
1537
+ `,
1538
+ [input.tenant_id, input.workspace_id]
1539
+ );
1540
+
1541
+ const row = result.rows[0];
1542
+ if (!row) {
1543
+ return undefined;
1544
+ }
1545
+ return {
1546
+ ...row,
1547
+ created_at: toIsoString(row.created_at),
1548
+ updated_at: toIsoString(row.updated_at)
1549
+ };
1550
+ }
1551
+
1552
+ async getFilesByIndex(input: { tenant_id: string; index_id: string }): Promise<PersistedFileRecord[]> {
1553
+ const result = await this.pool.query<{
1554
+ file_id: string;
1555
+ repo_path: string;
1556
+ content_hash: string;
1557
+ language: string | null;
1558
+ }>(
1559
+ `
1560
+ SELECT id AS file_id, repo_path, content_hash, language
1561
+ FROM files
1562
+ WHERE tenant_id = $1 AND index_id = $2
1563
+ `,
1564
+ [input.tenant_id, input.index_id]
1565
+ );
1566
+
1567
+ return result.rows.map((row) => ({
1568
+ file_id: row.file_id,
1569
+ repo_path: row.repo_path,
1570
+ content_hash: row.content_hash,
1571
+ ...(row.language ? { language: row.language } : {})
1572
+ }));
1573
+ }
1574
+
1575
+ async copyFileFromIndex(input: {
1576
+ tenant_id: string;
1577
+ source_index_id: string;
1578
+ target_index_id: string;
1579
+ repo_path: string;
1580
+ }): Promise<void> {
1581
+ await runTx(this.pool, async (client) => {
1582
+ const sourceFileResult = await client.query<{
1583
+ file_id: string;
1584
+ repo_path: string;
1585
+ content_hash: string;
1586
+ size_bytes: number;
1587
+ language: string | null;
1588
+ warning_metadata: Record<string, unknown> | null;
1589
+ updated_at: unknown;
1590
+ }>(
1591
+ `
1592
+ SELECT id AS file_id, repo_path, content_hash, size_bytes, language, warning_metadata, updated_at
1593
+ FROM files
1594
+ WHERE tenant_id = $1 AND index_id = $2 AND repo_path = $3
1595
+ LIMIT 1
1596
+ `,
1597
+ [input.tenant_id, input.source_index_id, input.repo_path]
1598
+ );
1599
+
1600
+ const sourceFile = sourceFileResult.rows[0];
1601
+ if (!sourceFile) {
1602
+ return;
1603
+ }
1604
+
1605
+ const targetFileId = `fil_${randomUUID()}`;
1606
+ await client.query(
1607
+ `
1608
+ INSERT INTO files (id, tenant_id, index_id, repo_path, content_hash, size_bytes, language, warning_metadata, updated_at)
1609
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9::timestamptz)
1610
+ `,
1611
+ [
1612
+ targetFileId,
1613
+ input.tenant_id,
1614
+ input.target_index_id,
1615
+ sourceFile.repo_path,
1616
+ sourceFile.content_hash,
1617
+ sourceFile.size_bytes,
1618
+ sourceFile.language,
1619
+ JSON.stringify(sourceFile.warning_metadata ?? null),
1620
+ toIsoString(sourceFile.updated_at)
1621
+ ]
1622
+ );
1623
+
1624
+ const chunkRows = await client.query<{
1625
+ repo_path: string;
1626
+ start_line: number;
1627
+ end_line: number;
1628
+ text: string;
1629
+ embedding: unknown;
1630
+ generated: boolean;
1631
+ updated_at: unknown;
1632
+ }>(
1633
+ `
1634
+ SELECT repo_path, start_line, end_line, text, embedding, generated, updated_at
1635
+ FROM chunks
1636
+ WHERE tenant_id = $1 AND file_id = $2
1637
+ `,
1638
+ [input.tenant_id, sourceFile.file_id]
1639
+ );
1640
+
1641
+ for (const chunk of chunkRows.rows) {
1642
+ const embedding = parseEmbedding(chunk.embedding);
1643
+ await this.insertChunk(client, {
1644
+ tenant_id: input.tenant_id,
1645
+ file_id: targetFileId,
1646
+ repo_path: chunk.repo_path,
1647
+ start_line: chunk.start_line,
1648
+ end_line: chunk.end_line,
1649
+ snippet: chunk.text,
1650
+ embedding,
1651
+ generated: chunk.generated,
1652
+ updated_at: toIsoString(chunk.updated_at)
1653
+ });
1654
+ }
1655
+ });
1656
+ }
1657
+
1658
+ async upsertFile(input: UpsertFileInput): Promise<{ file_id: string }> {
1659
+ const fileId = `fil_${randomUUID()}`;
1660
+ const result = await this.pool.query<{ file_id: string }>(
1661
+ `
1662
+ INSERT INTO files (id, tenant_id, index_id, repo_path, content_hash, size_bytes, language, warning_metadata, updated_at)
1663
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9::timestamptz)
1664
+ ON CONFLICT (index_id, repo_path)
1665
+ DO UPDATE SET
1666
+ content_hash = EXCLUDED.content_hash,
1667
+ size_bytes = EXCLUDED.size_bytes,
1668
+ language = EXCLUDED.language,
1669
+ warning_metadata = EXCLUDED.warning_metadata,
1670
+ updated_at = EXCLUDED.updated_at
1671
+ RETURNING id AS file_id
1672
+ `,
1673
+ [
1674
+ fileId,
1675
+ input.tenant_id,
1676
+ input.index_id,
1677
+ input.repo_path,
1678
+ input.content_hash,
1679
+ input.size_bytes,
1680
+ input.language ?? null,
1681
+ JSON.stringify(input.warning_metadata ?? null),
1682
+ input.updated_at ?? new Date().toISOString()
1683
+ ]
1684
+ );
1685
+
1686
+ return result.rows[0]!;
1687
+ }
1688
+
1689
+ async replaceFileChunks(input: {
1690
+ tenant_id: string;
1691
+ file_id: string;
1692
+ repo_path: string;
1693
+ chunks: UpsertChunkInput[];
1694
+ }): Promise<void> {
1695
+ await runTx(this.pool, async (client) => {
1696
+ await client.query("DELETE FROM chunks WHERE tenant_id = $1 AND file_id = $2", [input.tenant_id, input.file_id]);
1697
+
1698
+ for (const chunk of input.chunks) {
1699
+ await this.insertChunk(client, {
1700
+ tenant_id: input.tenant_id,
1701
+ file_id: input.file_id,
1702
+ repo_path: input.repo_path,
1703
+ start_line: chunk.start_line,
1704
+ end_line: chunk.end_line,
1705
+ snippet: chunk.snippet,
1706
+ embedding: chunk.embedding,
1707
+ generated: chunk.generated,
1708
+ updated_at: chunk.updated_at ?? new Date().toISOString()
1709
+ });
1710
+ }
1711
+ });
1712
+ }
1713
+
1714
+ async saveManifest(input: {
1715
+ index_id: string;
1716
+ object_key: string;
1717
+ checksum: string;
1718
+ }): Promise<void> {
1719
+ await this.pool.query(
1720
+ `
1721
+ INSERT INTO manifests (id, index_id, object_key, checksum)
1722
+ VALUES ($1, $2, $3, $4)
1723
+ `,
1724
+ [`mft_${randomUUID()}`, input.index_id, input.object_key, input.checksum]
1725
+ );
1726
+ }
1727
+
1728
+ async saveIndexMetadata(input: {
1729
+ tenant_id: string;
1730
+ index_id: string;
1731
+ embedding_provider: string;
1732
+ embedding_model?: string;
1733
+ embedding_dimensions: number;
1734
+ embedding_version?: string;
1735
+ chunking_strategy: "language_aware" | "sliding";
1736
+ chunking_fallback_strategy: "sliding";
1737
+ }): Promise<void> {
1738
+ await this.pool.query(
1739
+ `
1740
+ INSERT INTO index_metadata (
1741
+ index_id,
1742
+ tenant_id,
1743
+ embedding_provider,
1744
+ embedding_model,
1745
+ embedding_dimensions,
1746
+ embedding_version,
1747
+ chunking_strategy,
1748
+ chunking_fallback_strategy
1749
+ )
1750
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
1751
+ ON CONFLICT (index_id)
1752
+ DO UPDATE SET
1753
+ tenant_id = EXCLUDED.tenant_id,
1754
+ embedding_provider = EXCLUDED.embedding_provider,
1755
+ embedding_model = EXCLUDED.embedding_model,
1756
+ embedding_dimensions = EXCLUDED.embedding_dimensions,
1757
+ embedding_version = EXCLUDED.embedding_version,
1758
+ chunking_strategy = EXCLUDED.chunking_strategy,
1759
+ chunking_fallback_strategy = EXCLUDED.chunking_fallback_strategy
1760
+ `,
1761
+ [
1762
+ input.index_id,
1763
+ input.tenant_id,
1764
+ input.embedding_provider,
1765
+ input.embedding_model ?? null,
1766
+ input.embedding_dimensions,
1767
+ input.embedding_version ?? null,
1768
+ input.chunking_strategy,
1769
+ input.chunking_fallback_strategy
1770
+ ]
1771
+ );
1772
+ }
1773
+
1774
+ async getIndexMetadata(input: { tenant_id: string; index_id: string }): Promise<IndexMetadataRecord | undefined> {
1775
+ const result = await this.pool.query<{
1776
+ embedding_provider: string;
1777
+ embedding_model: string | null;
1778
+ embedding_dimensions: number;
1779
+ embedding_version: string | null;
1780
+ chunking_strategy: "language_aware" | "sliding";
1781
+ chunking_fallback_strategy: "sliding";
1782
+ created_at: unknown;
1783
+ }>(
1784
+ `
1785
+ SELECT
1786
+ embedding_provider,
1787
+ embedding_model,
1788
+ embedding_dimensions,
1789
+ embedding_version,
1790
+ chunking_strategy,
1791
+ chunking_fallback_strategy,
1792
+ created_at
1793
+ FROM index_metadata
1794
+ WHERE tenant_id = $1 AND index_id = $2
1795
+ LIMIT 1
1796
+ `,
1797
+ [input.tenant_id, input.index_id]
1798
+ );
1799
+ const row = result.rows[0];
1800
+ if (!row) {
1801
+ return undefined;
1802
+ }
1803
+ return {
1804
+ embedding_provider: row.embedding_provider,
1805
+ ...(row.embedding_model ? { embedding_model: row.embedding_model } : {}),
1806
+ embedding_dimensions: row.embedding_dimensions,
1807
+ ...(row.embedding_version ? { embedding_version: row.embedding_version } : {}),
1808
+ chunking_strategy: row.chunking_strategy,
1809
+ chunking_fallback_strategy: row.chunking_fallback_strategy,
1810
+ created_at: toIsoString(row.created_at)
1811
+ };
1812
+ }
1813
+
1814
+ async listChunksByIndex(input: {
1815
+ tenant_id: string;
1816
+ index_id: string;
1817
+ filters?: {
1818
+ language?: string;
1819
+ path_prefix?: string;
1820
+ glob?: string;
1821
+ };
1822
+ }): Promise<PersistedChunk[]> {
1823
+ const params: unknown[] = [input.tenant_id, input.index_id];
1824
+ const where: string[] = ["f.tenant_id = $1", "f.index_id = $2"];
1825
+
1826
+ if (input.filters?.language) {
1827
+ params.push(input.filters.language);
1828
+ where.push(`f.language = $${params.length}`);
1829
+ }
1830
+
1831
+ if (input.filters?.path_prefix) {
1832
+ params.push(`${input.filters.path_prefix}%`);
1833
+ where.push(`c.repo_path LIKE $${params.length}`);
1834
+ }
1835
+
1836
+ if (input.filters?.glob) {
1837
+ const regex = this.globToPostgresRegex(input.filters.glob);
1838
+ params.push(regex);
1839
+ where.push(`c.repo_path ~ $${params.length}`);
1840
+ }
1841
+
1842
+ const result = await this.pool.query<{
1843
+ chunk_id: string;
1844
+ file_id: string;
1845
+ path: string;
1846
+ start_line: number;
1847
+ end_line: number;
1848
+ snippet: string;
1849
+ language: string | null;
1850
+ generated: boolean;
1851
+ updated_at: unknown;
1852
+ embedding: unknown;
1853
+ }>(
1854
+ `
1855
+ SELECT
1856
+ c.id AS chunk_id,
1857
+ c.file_id,
1858
+ c.repo_path AS path,
1859
+ c.start_line,
1860
+ c.end_line,
1861
+ c.text AS snippet,
1862
+ f.language,
1863
+ c.generated,
1864
+ c.updated_at,
1865
+ c.embedding
1866
+ FROM chunks c
1867
+ INNER JOIN files f ON f.id = c.file_id
1868
+ WHERE ${where.join(" AND ")}
1869
+ `,
1870
+ params
1871
+ );
1872
+
1873
+ return result.rows.map((row) => ({
1874
+ chunk_id: row.chunk_id,
1875
+ file_id: row.file_id,
1876
+ path: row.path,
1877
+ start_line: row.start_line,
1878
+ end_line: row.end_line,
1879
+ snippet: row.snippet,
1880
+ ...(row.language ? { language: row.language } : {}),
1881
+ ...(row.generated ? { generated: row.generated } : {}),
1882
+ updated_at: toIsoString(row.updated_at),
1883
+ embedding: parseEmbedding(row.embedding)
1884
+ }));
1885
+ }
1886
+
1887
+ async rankChunksByIndex(input: RankChunksInput): Promise<RankedChunkCandidate[]> {
1888
+ const weights = input.candidate_weights ?? DEFAULT_CANDIDATE_SCORE_WEIGHTS;
1889
+ const params: unknown[] = [input.tenant_id, input.index_id];
1890
+ const where: string[] = ["f.tenant_id = $1", "f.index_id = $2"];
1891
+
1892
+ if (input.filters?.language) {
1893
+ params.push(input.filters.language);
1894
+ where.push(`f.language = $${params.length}`);
1895
+ }
1896
+
1897
+ if (input.filters?.path_prefix) {
1898
+ params.push(`${input.filters.path_prefix}%`);
1899
+ where.push(`c.repo_path LIKE $${params.length}`);
1900
+ }
1901
+
1902
+ if (input.filters?.glob) {
1903
+ const regex = this.globToPostgresRegex(input.filters.glob);
1904
+ params.push(regex);
1905
+ where.push(`c.repo_path ~ $${params.length}`);
1906
+ }
1907
+
1908
+ if (this.embeddingStorage === "vector") {
1909
+ const normalizedTokens = input.query_tokens.map((token) => token.toLowerCase()).filter((token) => token.length > 0);
1910
+ params.push(normalizedTokens.length > 0 ? normalizedTokens : [""]);
1911
+ const tokenArrayIndex = params.length;
1912
+ params.push(toVectorLiteral(input.query_embedding));
1913
+ const queryVectorIndex = params.length;
1914
+ params.push(Math.max(input.top_k * 4, input.top_k));
1915
+ const limitIndex = params.length;
1916
+ params.push(weights.lexical_weight);
1917
+ const lexicalWeightIndex = params.length;
1918
+ params.push(weights.vector_weight);
1919
+ const vectorWeightIndex = params.length;
1920
+ params.push(weights.path_match_boost);
1921
+ const pathMatchBoostIndex = params.length;
1922
+ params.push(weights.recency_boost);
1923
+ const recencyBoostIndex = params.length;
1924
+ params.push(weights.generated_penalty);
1925
+ const generatedPenaltyIndex = params.length;
1926
+
1927
+ const result = await this.pool.query<{
1928
+ chunk_id: string;
1929
+ file_id: string;
1930
+ path: string;
1931
+ start_line: number;
1932
+ end_line: number;
1933
+ snippet: string;
1934
+ language: string | null;
1935
+ generated: boolean;
1936
+ updated_at: unknown;
1937
+ score: number;
1938
+ lexical_score: number;
1939
+ vector_score: number;
1940
+ path_match: boolean;
1941
+ recency_boosted: boolean;
1942
+ }>(
1943
+ `
1944
+ WITH scored AS (
1945
+ SELECT
1946
+ c.id AS chunk_id,
1947
+ c.file_id,
1948
+ c.repo_path AS path,
1949
+ c.start_line,
1950
+ c.end_line,
1951
+ c.text AS snippet,
1952
+ f.language,
1953
+ c.generated,
1954
+ c.updated_at,
1955
+ (
1956
+ SELECT COALESCE(COUNT(*), 0)
1957
+ FROM unnest($${tokenArrayIndex}::text[]) AS token
1958
+ WHERE token <> '' AND (
1959
+ position(token in lower(c.repo_path)) > 0
1960
+ OR position(token in lower(c.lexical_doc)) > 0
1961
+ )
1962
+ )::double precision / GREATEST(array_length($${tokenArrayIndex}::text[], 1), 1)::double precision AS lexical_score,
1963
+ (1 - (c.embedding <=> $${queryVectorIndex}::vector))::double precision AS vector_score,
1964
+ EXISTS (
1965
+ SELECT 1
1966
+ FROM unnest($${tokenArrayIndex}::text[]) AS token
1967
+ WHERE token <> '' AND position(token in lower(c.repo_path)) > 0
1968
+ ) AS path_match,
1969
+ (c.updated_at >= NOW() - INTERVAL '14 days') AS recency_boosted
1970
+ FROM chunks c
1971
+ INNER JOIN files f ON f.id = c.file_id
1972
+ WHERE ${where.join(" AND ")}
1973
+ )
1974
+ SELECT
1975
+ chunk_id,
1976
+ file_id,
1977
+ path,
1978
+ start_line,
1979
+ end_line,
1980
+ snippet,
1981
+ language,
1982
+ generated,
1983
+ updated_at,
1984
+ (lexical_score * $${lexicalWeightIndex})
1985
+ + (vector_score * $${vectorWeightIndex})
1986
+ + (CASE WHEN path_match THEN $${pathMatchBoostIndex} ELSE 0 END)
1987
+ + (CASE WHEN recency_boosted THEN $${recencyBoostIndex} ELSE 0 END)
1988
+ + (CASE WHEN generated THEN -$${generatedPenaltyIndex} ELSE 0 END) AS score,
1989
+ lexical_score,
1990
+ vector_score,
1991
+ path_match,
1992
+ recency_boosted
1993
+ FROM scored
1994
+ ORDER BY score DESC
1995
+ LIMIT $${limitIndex}
1996
+ `,
1997
+ params
1998
+ );
1999
+
2000
+ return result.rows.map((row) => ({
2001
+ chunk_id: row.chunk_id,
2002
+ file_id: row.file_id,
2003
+ path: row.path,
2004
+ start_line: row.start_line,
2005
+ end_line: row.end_line,
2006
+ snippet: row.snippet,
2007
+ ...(row.language ? { language: row.language } : {}),
2008
+ ...(row.generated ? { generated: row.generated } : {}),
2009
+ updated_at: toIsoString(row.updated_at),
2010
+ score: Number(row.score),
2011
+ lexical_score: Number(row.lexical_score),
2012
+ vector_score: Number(row.vector_score),
2013
+ path_match: row.path_match,
2014
+ recency_boosted: row.recency_boosted
2015
+ }));
2016
+ }
2017
+
2018
+ const rows = await this.listChunksByIndex({
2019
+ tenant_id: input.tenant_id,
2020
+ index_id: input.index_id,
2021
+ filters: input.filters
2022
+ });
2023
+
2024
+ return rows
2025
+ .map((row) => {
2026
+ const lexical = lexicalScoreForRanking(input.query_tokens, `${row.path}\n${row.snippet}`);
2027
+ const vector = cosineSimilarity(input.query_embedding, row.embedding);
2028
+ const pathMatch = input.query_tokens.some((token) => row.path.toLowerCase().includes(token));
2029
+ const recencyBoost = Date.now() - new Date(row.updated_at).getTime() < 14 * 24 * 3600 * 1_000;
2030
+
2031
+ let score = lexical * weights.lexical_weight + vector * weights.vector_weight;
2032
+ if (pathMatch) {
2033
+ score += weights.path_match_boost;
2034
+ }
2035
+ if (recencyBoost) {
2036
+ score += weights.recency_boost;
2037
+ }
2038
+ if (row.generated) {
2039
+ score -= weights.generated_penalty;
2040
+ }
2041
+
2042
+ return {
2043
+ chunk_id: row.chunk_id,
2044
+ file_id: row.file_id,
2045
+ path: row.path,
2046
+ start_line: row.start_line,
2047
+ end_line: row.end_line,
2048
+ snippet: row.snippet,
2049
+ ...(row.language ? { language: row.language } : {}),
2050
+ ...(row.generated ? { generated: true } : {}),
2051
+ updated_at: row.updated_at,
2052
+ score,
2053
+ lexical_score: lexical,
2054
+ vector_score: vector,
2055
+ path_match: pathMatch,
2056
+ recency_boosted: recencyBoost
2057
+ };
2058
+ })
2059
+ .sort((a, b) => b.score - a.score)
2060
+ .slice(0, Math.max(input.top_k * 4, input.top_k));
2061
+ }
2062
+
2063
+ private async insertChunk(
2064
+ client: Queryable,
2065
+ input: {
2066
+ tenant_id: string;
2067
+ file_id: string;
2068
+ repo_path: string;
2069
+ start_line: number;
2070
+ end_line: number;
2071
+ snippet: string;
2072
+ embedding: number[];
2073
+ generated?: boolean;
2074
+ updated_at: string;
2075
+ }
2076
+ ): Promise<void> {
2077
+ const embedding = this.embeddingStorage === "vector" ? toVectorLiteral(input.embedding) : input.embedding;
2078
+ const cast = this.embeddingStorage === "vector" ? "::vector" : "::double precision[]";
2079
+
2080
+ await client.query(
2081
+ `
2082
+ INSERT INTO chunks (id, tenant_id, file_id, repo_path, start_line, end_line, text, embedding, generated, lexical_doc, updated_at)
2083
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8${cast}, $9, $7, $10::timestamptz)
2084
+ `,
2085
+ [
2086
+ `chk_${randomUUID()}`,
2087
+ input.tenant_id,
2088
+ input.file_id,
2089
+ input.repo_path,
2090
+ input.start_line,
2091
+ input.end_line,
2092
+ input.snippet,
2093
+ embedding,
2094
+ input.generated ?? false,
2095
+ input.updated_at
2096
+ ]
2097
+ );
2098
+ }
2099
+
2100
+ private globToPostgresRegex(glob: string): string {
2101
+ const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&");
2102
+ return `^${escaped.replace(/\*/g, ".*").replace(/\?/g, ".")}$`;
2103
+ }
2104
+ }
2105
+
2106
+ export function createPostgresPool(databaseUrl: string): Pool {
2107
+ return new Pool({ connectionString: databaseUrl });
2108
+ }
2109
+
2110
+ export interface QueryCache {
2111
+ get(cacheKey: string): Promise<SearchContextOutput | undefined>;
2112
+ set(cacheKey: string, value: SearchContextOutput, ttlSeconds?: number): Promise<void>;
2113
+ invalidateWorkspace(workspace_id: string): Promise<void>;
2114
+ }
2115
+
2116
+ export interface RedisLike {
2117
+ get(key: string): Promise<string | null>;
2118
+ set(key: string, value: string, mode: "EX", ttlSeconds: number): Promise<unknown>;
2119
+ keys(pattern: string): Promise<string[]>;
2120
+ del(...keys: string[]): Promise<number>;
2121
+ rpush(key: string, ...values: string[]): Promise<number>;
2122
+ lrem(key: string, count: number, value: string): Promise<number>;
2123
+ brpoplpush(source: string, destination: string, timeout: number): Promise<string | null>;
2124
+ lrange(key: string, start: number, stop: number): Promise<string[]>;
2125
+ llen(key: string): Promise<number>;
2126
+ connect?(): Promise<unknown>;
2127
+ status?: string;
2128
+ }
2129
+
2130
+ export class RedisQueryCache implements QueryCache {
2131
+ private readonly keyPrefix: string;
2132
+
2133
+ constructor(
2134
+ private readonly redis: RedisLike,
2135
+ options?: {
2136
+ keyPrefix?: string;
2137
+ }
2138
+ ) {
2139
+ this.keyPrefix = options?.keyPrefix ?? "rce:query_cache";
2140
+ }
2141
+
2142
+ async get(cacheKey: string): Promise<SearchContextOutput | undefined> {
2143
+ const raw = await this.redis.get(this.wrapKey(cacheKey));
2144
+ if (!raw) {
2145
+ return undefined;
2146
+ }
2147
+
2148
+ return JSON.parse(raw) as SearchContextOutput;
2149
+ }
2150
+
2151
+ async set(cacheKey: string, value: SearchContextOutput, ttlSeconds = 60): Promise<void> {
2152
+ await this.redis.set(this.wrapKey(cacheKey), JSON.stringify(value), "EX", ttlSeconds);
2153
+ }
2154
+
2155
+ async invalidateWorkspace(workspace_id: string): Promise<void> {
2156
+ const keys = await this.redis.keys(`${this.keyPrefix}:${workspace_id}:*`);
2157
+ if (keys.length > 0) {
2158
+ await this.redis.del(...keys);
2159
+ }
2160
+ }
2161
+
2162
+ private wrapKey(cacheKey: string): string {
2163
+ return `${this.keyPrefix}:${cacheKey}`;
2164
+ }
2165
+ }
2166
+
2167
+ export class InMemoryQueryCache implements QueryCache {
2168
+ private readonly cache = new Map<string, { expires_at: number; value: SearchContextOutput }>();
2169
+
2170
+ async get(cacheKey: string): Promise<SearchContextOutput | undefined> {
2171
+ const hit = this.cache.get(cacheKey);
2172
+ if (!hit || hit.expires_at <= Date.now()) {
2173
+ return undefined;
2174
+ }
2175
+ return hit.value;
2176
+ }
2177
+
2178
+ async set(cacheKey: string, value: SearchContextOutput, ttlSeconds = 60): Promise<void> {
2179
+ this.cache.set(cacheKey, {
2180
+ value,
2181
+ expires_at: Date.now() + ttlSeconds * 1_000
2182
+ });
2183
+ }
2184
+
2185
+ async invalidateWorkspace(workspace_id: string): Promise<void> {
2186
+ for (const key of this.cache.keys()) {
2187
+ if (key.startsWith(`${workspace_id}:`)) {
2188
+ this.cache.delete(key);
2189
+ }
2190
+ }
2191
+ }
2192
+ }
2193
+
2194
+ function workspaceIdFromCacheKey(cacheKey: string): string {
2195
+ const split = cacheKey.split(":", 1)[0];
2196
+ return split && split.length > 0 ? split : "unknown";
2197
+ }
2198
+
2199
+ export class SqliteQueryCache implements QueryCache {
2200
+ private readonly db: SqliteDatabase;
2201
+
2202
+ constructor(private readonly dbPath: string) {
2203
+ ensureSqliteParent(dbPath);
2204
+ this.db = openSqliteDatabase(dbPath);
2205
+ this.db.exec("PRAGMA journal_mode = WAL;");
2206
+ this.db.exec(`
2207
+ CREATE TABLE IF NOT EXISTS query_cache (
2208
+ cache_key TEXT PRIMARY KEY,
2209
+ workspace_id TEXT NOT NULL,
2210
+ value TEXT NOT NULL,
2211
+ expires_at INTEGER NOT NULL
2212
+ );
2213
+ CREATE INDEX IF NOT EXISTS idx_sqlite_query_cache_workspace ON query_cache(workspace_id);
2214
+ CREATE INDEX IF NOT EXISTS idx_sqlite_query_cache_expires ON query_cache(expires_at);
2215
+ `);
2216
+ }
2217
+
2218
+ close(): void {
2219
+ this.db.close();
2220
+ }
2221
+
2222
+ async get(cacheKey: string): Promise<SearchContextOutput | undefined> {
2223
+ const row = this.db
2224
+ .prepare("SELECT value, expires_at FROM query_cache WHERE cache_key = ? LIMIT 1")
2225
+ .get(cacheKey) as { value: string; expires_at: number } | undefined;
2226
+ if (!row) {
2227
+ return undefined;
2228
+ }
2229
+
2230
+ if (row.expires_at <= Date.now()) {
2231
+ this.db.prepare("DELETE FROM query_cache WHERE cache_key = ?").run(cacheKey);
2232
+ return undefined;
2233
+ }
2234
+
2235
+ return JSON.parse(row.value) as SearchContextOutput;
2236
+ }
2237
+
2238
+ async set(cacheKey: string, value: SearchContextOutput, ttlSeconds = 60): Promise<void> {
2239
+ const expires_at = Date.now() + ttlSeconds * 1_000;
2240
+ this.db
2241
+ .prepare(
2242
+ `
2243
+ INSERT INTO query_cache (cache_key, workspace_id, value, expires_at)
2244
+ VALUES (?, ?, ?, ?)
2245
+ ON CONFLICT (cache_key)
2246
+ DO UPDATE SET workspace_id = excluded.workspace_id, value = excluded.value, expires_at = excluded.expires_at
2247
+ `
2248
+ )
2249
+ .run(cacheKey, workspaceIdFromCacheKey(cacheKey), JSON.stringify(value), expires_at);
2250
+ }
2251
+
2252
+ async invalidateWorkspace(workspace_id: string): Promise<void> {
2253
+ this.db.prepare("DELETE FROM query_cache WHERE workspace_id = ?").run(workspace_id);
2254
+ }
2255
+ }
2256
+
2257
+ export interface IndexJobPayload {
2258
+ job_id: string;
2259
+ tenant_id: string;
2260
+ workspace_id: string;
2261
+ index_version: string;
2262
+ manifest_key: string;
2263
+ attempts: number;
2264
+ enqueued_at: string;
2265
+ claimed_at?: string;
2266
+ }
2267
+
2268
+ interface FailedIndexJobPayload extends IndexJobPayload {
2269
+ failed_at: string;
2270
+ last_error: string;
2271
+ }
2272
+
2273
+ export interface UsageMeteringRecordInput {
2274
+ tenant_id: string;
2275
+ workspace_id?: string;
2276
+ tool_name: string;
2277
+ trace_id: string;
2278
+ status: "success" | "error";
2279
+ latency_ms: number;
2280
+ result_count: number;
2281
+ units: number;
2282
+ created_at?: string;
2283
+ }
2284
+
2285
+ export interface AuditEventInput {
2286
+ tenant_id: string;
2287
+ subject: string;
2288
+ action: string;
2289
+ resource: string;
2290
+ status: "success" | "error" | "denied";
2291
+ trace_id: string;
2292
+ details?: Record<string, unknown>;
2293
+ created_at?: string;
2294
+ }
2295
+
2296
+ export interface UsageSummary {
2297
+ tenant_id: string;
2298
+ tool_name: string;
2299
+ request_count: number;
2300
+ error_count: number;
2301
+ total_units: number;
2302
+ p95_latency_ms: number;
2303
+ last_seen_at?: string;
2304
+ }
2305
+
2306
+ export interface AuditEventRecord {
2307
+ event_id: string;
2308
+ tenant_id: string;
2309
+ subject: string;
2310
+ action: string;
2311
+ resource: string;
2312
+ status: "success" | "error" | "denied";
2313
+ trace_id: string;
2314
+ details?: Record<string, unknown>;
2315
+ created_at: string;
2316
+ }
2317
+
2318
+ export interface UsageMeterStore {
2319
+ migrate(): Promise<void>;
2320
+ recordUsage(input: UsageMeteringRecordInput): Promise<void>;
2321
+ recordAuditEvent(input: AuditEventInput): Promise<void>;
2322
+ listUsageSummary(input?: {
2323
+ tenant_id?: string;
2324
+ from?: string;
2325
+ to?: string;
2326
+ }): Promise<UsageSummary[]>;
2327
+ listAuditEvents(input?: {
2328
+ tenant_id?: string;
2329
+ limit?: number;
2330
+ }): Promise<AuditEventRecord[]>;
2331
+ }
2332
+
2333
+ export interface ClaimedIndexJob {
2334
+ raw: string;
2335
+ payload: IndexJobPayload;
2336
+ }
2337
+
2338
+ export interface IndexJobQueue {
2339
+ enqueue(job: Omit<IndexJobPayload, "job_id" | "attempts" | "enqueued_at"> & { job_id?: string }): Promise<IndexJobPayload>;
2340
+ claimNext(timeoutSeconds?: number): Promise<ClaimedIndexJob | undefined>;
2341
+ ack(claimed: ClaimedIndexJob): Promise<void>;
2342
+ retryOrDeadLetter(claimed: ClaimedIndexJob, errorMessage: string): Promise<void>;
2343
+ reclaimOrphaned(maxClaimAgeSeconds?: number): Promise<number>;
2344
+ deadLetterCount(): Promise<number>;
2345
+ pendingCount(): Promise<number>;
2346
+ processingCount(): Promise<number>;
2347
+ listDeadLetters(): Promise<FailedIndexJobPayload[]>;
2348
+ }
2349
+
2350
+ export class RedisIndexJobQueue implements IndexJobQueue {
2351
+ private readonly pendingKey: string;
2352
+ private readonly processingKey: string;
2353
+ private readonly deadLetterKey: string;
2354
+ private readonly maxAttempts: number;
2355
+ private readonly leasePrefix: string;
2356
+ private readonly claimLeaseSeconds: number;
2357
+ private readonly reconnectRetries: number;
2358
+ private readonly reconnectDelayMs: number;
2359
+
2360
+ constructor(
2361
+ private readonly redis: RedisLike,
2362
+ options?: {
2363
+ keyPrefix?: string;
2364
+ maxAttempts?: number;
2365
+ claimLeaseSeconds?: number;
2366
+ reconnectRetries?: number;
2367
+ reconnectDelayMs?: number;
2368
+ }
2369
+ ) {
2370
+ const prefix = options?.keyPrefix ?? "rce:index_jobs";
2371
+ this.pendingKey = `${prefix}:pending`;
2372
+ this.processingKey = `${prefix}:processing`;
2373
+ this.deadLetterKey = `${prefix}:dead`;
2374
+ this.leasePrefix = `${prefix}:lease`;
2375
+ this.maxAttempts = options?.maxAttempts ?? 3;
2376
+ this.claimLeaseSeconds = Math.max(5, options?.claimLeaseSeconds ?? 120);
2377
+ this.reconnectRetries = Math.max(0, options?.reconnectRetries ?? 1);
2378
+ this.reconnectDelayMs = Math.max(0, options?.reconnectDelayMs ?? 100);
2379
+ }
2380
+
2381
+ async enqueue(job: Omit<IndexJobPayload, "job_id" | "attempts" | "enqueued_at"> & { job_id?: string }): Promise<IndexJobPayload> {
2382
+ const payload: IndexJobPayload = {
2383
+ job_id: job.job_id ?? `job_${randomUUID()}`,
2384
+ tenant_id: job.tenant_id,
2385
+ workspace_id: job.workspace_id,
2386
+ index_version: job.index_version,
2387
+ manifest_key: job.manifest_key,
2388
+ attempts: 0,
2389
+ enqueued_at: new Date().toISOString()
2390
+ };
2391
+
2392
+ await this.runRedisOperation(() => this.redis.rpush(this.pendingKey, JSON.stringify(payload)));
2393
+ return payload;
2394
+ }
2395
+
2396
+ async claimNext(timeoutSeconds = 1): Promise<ClaimedIndexJob | undefined> {
2397
+ const raw = await this.runRedisOperation(() => this.redis.brpoplpush(this.pendingKey, this.processingKey, timeoutSeconds));
2398
+ if (!raw) {
2399
+ return undefined;
2400
+ }
2401
+
2402
+ const payload = JSON.parse(raw) as IndexJobPayload;
2403
+ const claimedAt = nowIso();
2404
+ payload.claimed_at = claimedAt;
2405
+ const claimedRaw = JSON.stringify(payload);
2406
+
2407
+ if (claimedRaw !== raw) {
2408
+ await this.runRedisOperation(() => this.redis.lrem(this.processingKey, 1, raw));
2409
+ await this.runRedisOperation(() => this.redis.rpush(this.processingKey, claimedRaw));
2410
+ }
2411
+
2412
+ await this.runRedisOperation(() =>
2413
+ this.redis.set(this.leaseKey(payload.job_id), claimedAt, "EX", this.claimLeaseSeconds)
2414
+ );
2415
+ return { raw: claimedRaw, payload };
2416
+ }
2417
+
2418
+ async ack(claimed: ClaimedIndexJob): Promise<void> {
2419
+ await this.runRedisOperation(() => this.redis.lrem(this.processingKey, 1, claimed.raw));
2420
+ await this.runRedisOperation(() => this.redis.del(this.leaseKey(claimed.payload.job_id)));
2421
+ }
2422
+
2423
+ async retryOrDeadLetter(claimed: ClaimedIndexJob, errorMessage: string): Promise<void> {
2424
+ await this.runRedisOperation(() => this.redis.lrem(this.processingKey, 1, claimed.raw));
2425
+ await this.runRedisOperation(() => this.redis.del(this.leaseKey(claimed.payload.job_id)));
2426
+ const attempts = claimed.payload.attempts + 1;
2427
+
2428
+ if (attempts >= this.maxAttempts) {
2429
+ const failed: FailedIndexJobPayload = {
2430
+ ...claimed.payload,
2431
+ attempts,
2432
+ failed_at: new Date().toISOString(),
2433
+ last_error: errorMessage
2434
+ };
2435
+ await this.runRedisOperation(() => this.redis.rpush(this.deadLetterKey, JSON.stringify(failed)));
2436
+ return;
2437
+ }
2438
+
2439
+ const retryPayload: IndexJobPayload = {
2440
+ ...claimed.payload,
2441
+ attempts,
2442
+ claimed_at: undefined
2443
+ };
2444
+ await this.runRedisOperation(() => this.redis.rpush(this.pendingKey, JSON.stringify(retryPayload)));
2445
+ }
2446
+
2447
+ async reclaimOrphaned(maxClaimAgeSeconds = this.claimLeaseSeconds): Promise<number> {
2448
+ const processing = await this.runRedisOperation(() => this.redis.lrange(this.processingKey, 0, -1));
2449
+ let reclaimed = 0;
2450
+ const ageThresholdMs = Math.max(1, maxClaimAgeSeconds) * 1_000;
2451
+
2452
+ for (const raw of processing) {
2453
+ const payload = JSON.parse(raw) as IndexJobPayload;
2454
+ const lease = await this.runRedisOperation(() => this.redis.get(this.leaseKey(payload.job_id)));
2455
+ const claimedAtMs = payload.claimed_at ? new Date(payload.claimed_at).getTime() : 0;
2456
+ const staleByAge = claimedAtMs > 0 && Date.now() - claimedAtMs >= ageThresholdMs;
2457
+
2458
+ if (lease && !staleByAge) {
2459
+ continue;
2460
+ }
2461
+
2462
+ if (payload.claimed_at && !staleByAge) {
2463
+ continue;
2464
+ }
2465
+
2466
+ const removed = await this.runRedisOperation(() => this.redis.lrem(this.processingKey, 1, raw));
2467
+ if (removed > 0) {
2468
+ const retryPayload: IndexJobPayload = {
2469
+ ...payload,
2470
+ claimed_at: undefined
2471
+ };
2472
+ await this.runRedisOperation(() => this.redis.rpush(this.pendingKey, JSON.stringify(retryPayload)));
2473
+ reclaimed += 1;
2474
+ }
2475
+ }
2476
+
2477
+ return reclaimed;
2478
+ }
2479
+
2480
+ async deadLetterCount(): Promise<number> {
2481
+ return this.runRedisOperation(() => this.redis.llen(this.deadLetterKey));
2482
+ }
2483
+
2484
+ async pendingCount(): Promise<number> {
2485
+ return this.runRedisOperation(() => this.redis.llen(this.pendingKey));
2486
+ }
2487
+
2488
+ async processingCount(): Promise<number> {
2489
+ return this.runRedisOperation(() => this.redis.llen(this.processingKey));
2490
+ }
2491
+
2492
+ async listDeadLetters(): Promise<FailedIndexJobPayload[]> {
2493
+ const rows = await this.runRedisOperation(() => this.redis.lrange(this.deadLetterKey, 0, -1));
2494
+ return rows.map((row) => JSON.parse(row) as FailedIndexJobPayload);
2495
+ }
2496
+
2497
+ private async runRedisOperation<T>(operation: () => Promise<T>): Promise<T> {
2498
+ for (let attempt = 0; ; attempt += 1) {
2499
+ try {
2500
+ return await operation();
2501
+ } catch (error) {
2502
+ const finalAttempt = attempt >= this.reconnectRetries;
2503
+ if (finalAttempt || !this.isRecoverableRedisError(error)) {
2504
+ throw error;
2505
+ }
2506
+
2507
+ try {
2508
+ if (typeof this.redis.connect === "function") {
2509
+ await this.redis.connect();
2510
+ }
2511
+ } catch {
2512
+ // Ignore reconnect failures here; next operation attempt will fail if redis is still unavailable.
2513
+ }
2514
+
2515
+ if (this.reconnectDelayMs > 0) {
2516
+ await waitMs(this.reconnectDelayMs);
2517
+ }
2518
+ }
2519
+ }
2520
+ }
2521
+
2522
+ private isRecoverableRedisError(error: unknown): boolean {
2523
+ if (!(error instanceof Error)) {
2524
+ return false;
2525
+ }
2526
+ const message = error.message.toLowerCase();
2527
+ return (
2528
+ message.includes("connection") ||
2529
+ message.includes("socket") ||
2530
+ message.includes("econnrefused") ||
2531
+ message.includes("econnreset") ||
2532
+ message.includes("closed")
2533
+ );
2534
+ }
2535
+
2536
+ private leaseKey(jobId: string): string {
2537
+ return `${this.leasePrefix}:${jobId}`;
2538
+ }
2539
+ }
2540
+
2541
+ export class InMemoryIndexJobQueue implements IndexJobQueue {
2542
+ private readonly pending: IndexJobPayload[] = [];
2543
+ private readonly dead: FailedIndexJobPayload[] = [];
2544
+ private readonly processing = new Map<string, IndexJobPayload>();
2545
+ private readonly maxAttempts: number;
2546
+
2547
+ constructor(options?: { maxAttempts?: number }) {
2548
+ this.maxAttempts = options?.maxAttempts ?? 3;
2549
+ }
2550
+
2551
+ async enqueue(job: Omit<IndexJobPayload, "job_id" | "attempts" | "enqueued_at"> & { job_id?: string }): Promise<IndexJobPayload> {
2552
+ const payload: IndexJobPayload = {
2553
+ job_id: job.job_id ?? `job_${randomUUID()}`,
2554
+ tenant_id: job.tenant_id,
2555
+ workspace_id: job.workspace_id,
2556
+ index_version: job.index_version,
2557
+ manifest_key: job.manifest_key,
2558
+ attempts: 0,
2559
+ enqueued_at: new Date().toISOString()
2560
+ };
2561
+ this.pending.push(payload);
2562
+ return payload;
2563
+ }
2564
+
2565
+ async claimNext(): Promise<ClaimedIndexJob | undefined> {
2566
+ const next = this.pending.shift();
2567
+ if (!next) {
2568
+ return undefined;
2569
+ }
2570
+
2571
+ const claimed: IndexJobPayload = {
2572
+ ...next,
2573
+ claimed_at: nowIso()
2574
+ };
2575
+ this.processing.set(claimed.job_id, claimed);
2576
+
2577
+ return {
2578
+ payload: claimed,
2579
+ raw: JSON.stringify(next)
2580
+ };
2581
+ }
2582
+
2583
+ async ack(claimed: ClaimedIndexJob): Promise<void> {
2584
+ this.processing.delete(claimed.payload.job_id);
2585
+ return;
2586
+ }
2587
+
2588
+ async retryOrDeadLetter(claimed: ClaimedIndexJob, errorMessage: string): Promise<void> {
2589
+ this.processing.delete(claimed.payload.job_id);
2590
+ const attempts = claimed.payload.attempts + 1;
2591
+ if (attempts >= this.maxAttempts) {
2592
+ this.dead.push({
2593
+ ...claimed.payload,
2594
+ attempts,
2595
+ failed_at: new Date().toISOString(),
2596
+ last_error: errorMessage
2597
+ });
2598
+ return;
2599
+ }
2600
+
2601
+ this.pending.push({
2602
+ ...claimed.payload,
2603
+ attempts,
2604
+ claimed_at: undefined
2605
+ });
2606
+ }
2607
+
2608
+ async reclaimOrphaned(maxClaimAgeSeconds = 120): Promise<number> {
2609
+ const cutoff = Date.now() - Math.max(1, maxClaimAgeSeconds) * 1_000;
2610
+ let reclaimed = 0;
2611
+
2612
+ for (const [jobId, payload] of this.processing.entries()) {
2613
+ const claimedAt = payload.claimed_at ? new Date(payload.claimed_at).getTime() : 0;
2614
+ if (claimedAt > cutoff) {
2615
+ continue;
2616
+ }
2617
+ this.processing.delete(jobId);
2618
+ this.pending.push({
2619
+ ...payload,
2620
+ claimed_at: undefined
2621
+ });
2622
+ reclaimed += 1;
2623
+ }
2624
+ return reclaimed;
2625
+ }
2626
+
2627
+ async deadLetterCount(): Promise<number> {
2628
+ return this.dead.length;
2629
+ }
2630
+
2631
+ async pendingCount(): Promise<number> {
2632
+ return this.pending.length;
2633
+ }
2634
+
2635
+ async processingCount(): Promise<number> {
2636
+ return this.processing.size;
2637
+ }
2638
+
2639
+ async listDeadLetters(): Promise<FailedIndexJobPayload[]> {
2640
+ return [...this.dead];
2641
+ }
2642
+ }
2643
+
2644
+ function waitMs(delay: number): Promise<void> {
2645
+ return new Promise((resolve) => setTimeout(resolve, delay));
2646
+ }
2647
+
2648
+ export class SqliteIndexJobQueue implements IndexJobQueue {
2649
+ private readonly db: SqliteDatabase;
2650
+ private readonly maxAttempts: number;
2651
+ private readonly claimTtlSeconds: number;
2652
+
2653
+ constructor(
2654
+ private readonly dbPath: string,
2655
+ options?: {
2656
+ maxAttempts?: number;
2657
+ claimTtlSeconds?: number;
2658
+ }
2659
+ ) {
2660
+ ensureSqliteParent(dbPath);
2661
+ this.db = openSqliteDatabase(dbPath);
2662
+ this.db.exec("PRAGMA journal_mode = WAL;");
2663
+ this.db.exec(`
2664
+ CREATE TABLE IF NOT EXISTS index_jobs (
2665
+ job_id TEXT PRIMARY KEY,
2666
+ tenant_id TEXT NOT NULL,
2667
+ workspace_id TEXT NOT NULL,
2668
+ index_version TEXT NOT NULL,
2669
+ manifest_key TEXT NOT NULL,
2670
+ attempts INTEGER NOT NULL,
2671
+ enqueued_at TEXT NOT NULL,
2672
+ status TEXT NOT NULL,
2673
+ claimed_at TEXT,
2674
+ failed_at TEXT,
2675
+ last_error TEXT
2676
+ );
2677
+ CREATE INDEX IF NOT EXISTS idx_sqlite_index_jobs_status_enqueued ON index_jobs(status, enqueued_at);
2678
+ CREATE INDEX IF NOT EXISTS idx_sqlite_index_jobs_status_failed ON index_jobs(status, failed_at);
2679
+ `);
2680
+ this.maxAttempts = options?.maxAttempts ?? 3;
2681
+ this.claimTtlSeconds = Math.max(5, options?.claimTtlSeconds ?? 120);
2682
+ }
2683
+
2684
+ close(): void {
2685
+ this.db.close();
2686
+ }
2687
+
2688
+ async enqueue(job: Omit<IndexJobPayload, "job_id" | "attempts" | "enqueued_at"> & { job_id?: string }): Promise<IndexJobPayload> {
2689
+ const payload: IndexJobPayload = {
2690
+ job_id: job.job_id ?? `job_${randomUUID()}`,
2691
+ tenant_id: job.tenant_id,
2692
+ workspace_id: job.workspace_id,
2693
+ index_version: job.index_version,
2694
+ manifest_key: job.manifest_key,
2695
+ attempts: 0,
2696
+ enqueued_at: nowIso()
2697
+ };
2698
+
2699
+ this.db
2700
+ .prepare(
2701
+ `
2702
+ INSERT INTO index_jobs (job_id, tenant_id, workspace_id, index_version, manifest_key, attempts, enqueued_at, status)
2703
+ VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')
2704
+ `
2705
+ )
2706
+ .run(
2707
+ payload.job_id,
2708
+ payload.tenant_id,
2709
+ payload.workspace_id,
2710
+ payload.index_version,
2711
+ payload.manifest_key,
2712
+ payload.attempts,
2713
+ payload.enqueued_at
2714
+ );
2715
+ return payload;
2716
+ }
2717
+
2718
+ async claimNext(timeoutSeconds = 1): Promise<ClaimedIndexJob | undefined> {
2719
+ const deadline = Date.now() + Math.max(0, timeoutSeconds) * 1_000;
2720
+ do {
2721
+ const claimed = this.claimNextImmediate();
2722
+ if (claimed) {
2723
+ return claimed;
2724
+ }
2725
+ if (timeoutSeconds <= 0) {
2726
+ break;
2727
+ }
2728
+ await waitMs(50);
2729
+ } while (Date.now() <= deadline);
2730
+ return undefined;
2731
+ }
2732
+
2733
+ async ack(claimed: ClaimedIndexJob): Promise<void> {
2734
+ this.db.prepare("DELETE FROM index_jobs WHERE job_id = ?").run(claimed.payload.job_id);
2735
+ }
2736
+
2737
+ async retryOrDeadLetter(claimed: ClaimedIndexJob, errorMessage: string): Promise<void> {
2738
+ const attempts = claimed.payload.attempts + 1;
2739
+ if (attempts >= this.maxAttempts) {
2740
+ this.db
2741
+ .prepare(
2742
+ `
2743
+ UPDATE index_jobs
2744
+ SET attempts = ?, status = 'dead', failed_at = ?, last_error = ?, claimed_at = NULL
2745
+ WHERE job_id = ?
2746
+ `
2747
+ )
2748
+ .run(attempts, nowIso(), errorMessage, claimed.payload.job_id);
2749
+ return;
2750
+ }
2751
+
2752
+ this.db
2753
+ .prepare(
2754
+ `
2755
+ UPDATE index_jobs
2756
+ SET attempts = ?, status = 'pending', last_error = ?, claimed_at = NULL
2757
+ WHERE job_id = ?
2758
+ `
2759
+ )
2760
+ .run(attempts, errorMessage, claimed.payload.job_id);
2761
+ }
2762
+
2763
+ async reclaimOrphaned(maxClaimAgeSeconds = this.claimTtlSeconds): Promise<number> {
2764
+ const cutoff = new Date(Date.now() - Math.max(1, maxClaimAgeSeconds) * 1_000).toISOString();
2765
+ const updated = this.db
2766
+ .prepare(
2767
+ `
2768
+ UPDATE index_jobs
2769
+ SET status = 'pending', claimed_at = NULL, last_error = COALESCE(last_error, 'reclaimed orphaned in-flight job')
2770
+ WHERE status = 'processing' AND claimed_at IS NOT NULL AND claimed_at < ?
2771
+ `
2772
+ )
2773
+ .run(cutoff);
2774
+ return Number(updated.changes ?? 0);
2775
+ }
2776
+
2777
+ async deadLetterCount(): Promise<number> {
2778
+ const row = this.db.prepare("SELECT COUNT(*) AS count FROM index_jobs WHERE status = 'dead'").get() as { count: number };
2779
+ return Number(row.count);
2780
+ }
2781
+
2782
+ async pendingCount(): Promise<number> {
2783
+ const row = this.db.prepare("SELECT COUNT(*) AS count FROM index_jobs WHERE status = 'pending'").get() as { count: number };
2784
+ return Number(row.count);
2785
+ }
2786
+
2787
+ async processingCount(): Promise<number> {
2788
+ const row = this.db.prepare("SELECT COUNT(*) AS count FROM index_jobs WHERE status = 'processing'").get() as { count: number };
2789
+ return Number(row.count);
2790
+ }
2791
+
2792
+ async listDeadLetters(): Promise<FailedIndexJobPayload[]> {
2793
+ const rows = this.db
2794
+ .prepare(
2795
+ `
2796
+ SELECT job_id, tenant_id, workspace_id, index_version, manifest_key, attempts, enqueued_at, failed_at, last_error
2797
+ FROM index_jobs
2798
+ WHERE status = 'dead'
2799
+ ORDER BY failed_at DESC
2800
+ `
2801
+ )
2802
+ .all() as Array<{
2803
+ job_id: string;
2804
+ tenant_id: string;
2805
+ workspace_id: string;
2806
+ index_version: string;
2807
+ manifest_key: string;
2808
+ attempts: number;
2809
+ enqueued_at: string;
2810
+ failed_at: string;
2811
+ last_error: string;
2812
+ }>;
2813
+ return rows.map((row) => ({
2814
+ job_id: row.job_id,
2815
+ tenant_id: row.tenant_id,
2816
+ workspace_id: row.workspace_id,
2817
+ index_version: row.index_version,
2818
+ manifest_key: row.manifest_key,
2819
+ attempts: Number(row.attempts),
2820
+ enqueued_at: row.enqueued_at,
2821
+ failed_at: row.failed_at,
2822
+ last_error: row.last_error
2823
+ }));
2824
+ }
2825
+
2826
+ private claimNextImmediate(): ClaimedIndexJob | undefined {
2827
+ const row = this.db
2828
+ .prepare(
2829
+ `
2830
+ SELECT job_id, tenant_id, workspace_id, index_version, manifest_key, attempts, enqueued_at
2831
+ FROM index_jobs
2832
+ WHERE status = 'pending'
2833
+ ORDER BY enqueued_at ASC
2834
+ LIMIT 1
2835
+ `
2836
+ )
2837
+ .get() as
2838
+ | {
2839
+ job_id: string;
2840
+ tenant_id: string;
2841
+ workspace_id: string;
2842
+ index_version: string;
2843
+ manifest_key: string;
2844
+ attempts: number;
2845
+ enqueued_at: string;
2846
+ }
2847
+ | undefined;
2848
+
2849
+ if (!row) {
2850
+ return undefined;
2851
+ }
2852
+
2853
+ const claimedAt = nowIso();
2854
+ const changed = this.db
2855
+ .prepare(
2856
+ `
2857
+ UPDATE index_jobs
2858
+ SET status = 'processing', claimed_at = ?
2859
+ WHERE job_id = ? AND status = 'pending'
2860
+ `
2861
+ )
2862
+ .run(claimedAt, row.job_id);
2863
+
2864
+ if (Number(changed.changes) !== 1) {
2865
+ return undefined;
2866
+ }
2867
+
2868
+ const payload: IndexJobPayload = {
2869
+ job_id: row.job_id,
2870
+ tenant_id: row.tenant_id,
2871
+ workspace_id: row.workspace_id,
2872
+ index_version: row.index_version,
2873
+ manifest_key: row.manifest_key,
2874
+ attempts: Number(row.attempts),
2875
+ enqueued_at: row.enqueued_at,
2876
+ claimed_at: claimedAt
2877
+ };
2878
+
2879
+ return {
2880
+ payload,
2881
+ raw: JSON.stringify(payload)
2882
+ };
2883
+ }
2884
+ }
2885
+
2886
+ export class InMemoryUsageMeterStore implements UsageMeterStore {
2887
+ private readonly usage: UsageMeteringRecordInput[] = [];
2888
+ private readonly audit: AuditEventRecord[] = [];
2889
+
2890
+ async migrate(): Promise<void> {
2891
+ return;
2892
+ }
2893
+
2894
+ async recordUsage(input: UsageMeteringRecordInput): Promise<void> {
2895
+ this.usage.push({
2896
+ ...input,
2897
+ created_at: input.created_at ?? nowIso()
2898
+ });
2899
+ }
2900
+
2901
+ async recordAuditEvent(input: AuditEventInput): Promise<void> {
2902
+ this.audit.push({
2903
+ event_id: `aud_${randomUUID()}`,
2904
+ tenant_id: input.tenant_id,
2905
+ subject: input.subject,
2906
+ action: input.action,
2907
+ resource: input.resource,
2908
+ status: input.status,
2909
+ trace_id: input.trace_id,
2910
+ ...(input.details ? { details: input.details } : {}),
2911
+ created_at: input.created_at ?? nowIso()
2912
+ });
2913
+ }
2914
+
2915
+ async listUsageSummary(input?: {
2916
+ tenant_id?: string;
2917
+ from?: string;
2918
+ to?: string;
2919
+ }): Promise<UsageSummary[]> {
2920
+ const fromMs = input?.from ? new Date(input.from).getTime() : Number.NEGATIVE_INFINITY;
2921
+ const toMs = input?.to ? new Date(input.to).getTime() : Number.POSITIVE_INFINITY;
2922
+ const rows = this.usage.filter((row) => {
2923
+ if (input?.tenant_id && row.tenant_id !== input.tenant_id) {
2924
+ return false;
2925
+ }
2926
+ const ts = new Date(row.created_at ?? nowIso()).getTime();
2927
+ return ts >= fromMs && ts <= toMs;
2928
+ });
2929
+
2930
+ const grouped = new Map<
2931
+ string,
2932
+ { tenant_id: string; tool_name: string; request_count: number; error_count: number; total_units: number; latencies: number[]; last: string }
2933
+ >();
2934
+ for (const row of rows) {
2935
+ const key = `${row.tenant_id}:${row.tool_name}`;
2936
+ const existing =
2937
+ grouped.get(key) ??
2938
+ {
2939
+ tenant_id: row.tenant_id,
2940
+ tool_name: row.tool_name,
2941
+ request_count: 0,
2942
+ error_count: 0,
2943
+ total_units: 0,
2944
+ latencies: [],
2945
+ last: row.created_at ?? nowIso()
2946
+ };
2947
+ existing.request_count += 1;
2948
+ existing.error_count += row.status === "error" ? 1 : 0;
2949
+ existing.total_units += row.units;
2950
+ existing.latencies.push(row.latency_ms);
2951
+ if ((row.created_at ?? nowIso()) > existing.last) {
2952
+ existing.last = row.created_at ?? nowIso();
2953
+ }
2954
+ grouped.set(key, existing);
2955
+ }
2956
+
2957
+ return [...grouped.values()].map((row) => ({
2958
+ tenant_id: row.tenant_id,
2959
+ tool_name: row.tool_name,
2960
+ request_count: row.request_count,
2961
+ error_count: row.error_count,
2962
+ total_units: row.total_units,
2963
+ p95_latency_ms: percentile(row.latencies, 0.95),
2964
+ last_seen_at: row.last
2965
+ }));
2966
+ }
2967
+
2968
+ async listAuditEvents(input?: {
2969
+ tenant_id?: string;
2970
+ limit?: number;
2971
+ }): Promise<AuditEventRecord[]> {
2972
+ const limit = Math.max(1, input?.limit ?? 100);
2973
+ return this.audit
2974
+ .filter((row) => (input?.tenant_id ? row.tenant_id === input.tenant_id : true))
2975
+ .sort((a, b) => b.created_at.localeCompare(a.created_at))
2976
+ .slice(0, limit);
2977
+ }
2978
+ }
2979
+
2980
+ export class SqliteUsageMeterStore implements UsageMeterStore {
2981
+ private readonly db: SqliteDatabase;
2982
+
2983
+ constructor(private readonly dbPath: string) {
2984
+ ensureSqliteParent(dbPath);
2985
+ this.db = openSqliteDatabase(dbPath);
2986
+ this.db.exec("PRAGMA journal_mode = WAL;");
2987
+ }
2988
+
2989
+ close(): void {
2990
+ this.db.close();
2991
+ }
2992
+
2993
+ async migrate(): Promise<void> {
2994
+ this.db.exec(`
2995
+ CREATE TABLE IF NOT EXISTS usage_metering (
2996
+ id TEXT PRIMARY KEY,
2997
+ tenant_id TEXT NOT NULL,
2998
+ workspace_id TEXT,
2999
+ tool_name TEXT NOT NULL,
3000
+ trace_id TEXT NOT NULL,
3001
+ status TEXT NOT NULL,
3002
+ latency_ms INTEGER NOT NULL,
3003
+ result_count INTEGER NOT NULL,
3004
+ units INTEGER NOT NULL,
3005
+ created_at TEXT NOT NULL
3006
+ );
3007
+ CREATE INDEX IF NOT EXISTS idx_usage_metering_tenant_created ON usage_metering(tenant_id, created_at DESC);
3008
+ CREATE TABLE IF NOT EXISTS audit_events (
3009
+ id TEXT PRIMARY KEY,
3010
+ tenant_id TEXT NOT NULL,
3011
+ subject TEXT NOT NULL,
3012
+ action TEXT NOT NULL,
3013
+ resource TEXT NOT NULL,
3014
+ status TEXT NOT NULL,
3015
+ trace_id TEXT NOT NULL,
3016
+ details TEXT,
3017
+ created_at TEXT NOT NULL
3018
+ );
3019
+ CREATE INDEX IF NOT EXISTS idx_audit_events_tenant_created ON audit_events(tenant_id, created_at DESC);
3020
+ `);
3021
+ }
3022
+
3023
+ async recordUsage(input: UsageMeteringRecordInput): Promise<void> {
3024
+ this.db
3025
+ .prepare(
3026
+ `
3027
+ INSERT INTO usage_metering
3028
+ (id, tenant_id, workspace_id, tool_name, trace_id, status, latency_ms, result_count, units, created_at)
3029
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
3030
+ `
3031
+ )
3032
+ .run(
3033
+ `usg_${randomUUID()}`,
3034
+ input.tenant_id,
3035
+ input.workspace_id ?? null,
3036
+ input.tool_name,
3037
+ input.trace_id,
3038
+ input.status,
3039
+ Math.max(0, Math.floor(input.latency_ms)),
3040
+ Math.max(0, Math.floor(input.result_count)),
3041
+ Math.max(0, Math.floor(input.units)),
3042
+ input.created_at ?? nowIso()
3043
+ );
3044
+ }
3045
+
3046
+ async recordAuditEvent(input: AuditEventInput): Promise<void> {
3047
+ this.db
3048
+ .prepare(
3049
+ `
3050
+ INSERT INTO audit_events
3051
+ (id, tenant_id, subject, action, resource, status, trace_id, details, created_at)
3052
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
3053
+ `
3054
+ )
3055
+ .run(
3056
+ `aud_${randomUUID()}`,
3057
+ input.tenant_id,
3058
+ input.subject,
3059
+ input.action,
3060
+ input.resource,
3061
+ input.status,
3062
+ input.trace_id,
3063
+ input.details ? JSON.stringify(input.details) : null,
3064
+ input.created_at ?? nowIso()
3065
+ );
3066
+ }
3067
+
3068
+ async listUsageSummary(input?: {
3069
+ tenant_id?: string;
3070
+ from?: string;
3071
+ to?: string;
3072
+ }): Promise<UsageSummary[]> {
3073
+ const where: string[] = [];
3074
+ const params: Array<string | number> = [];
3075
+ if (input?.tenant_id) {
3076
+ where.push("tenant_id = ?");
3077
+ params.push(input.tenant_id);
3078
+ }
3079
+ if (input?.from) {
3080
+ where.push("created_at >= ?");
3081
+ params.push(input.from);
3082
+ }
3083
+ if (input?.to) {
3084
+ where.push("created_at <= ?");
3085
+ params.push(input.to);
3086
+ }
3087
+
3088
+ const rows = this.db
3089
+ .prepare(
3090
+ `
3091
+ SELECT tenant_id, tool_name, status, latency_ms, units, created_at
3092
+ FROM usage_metering
3093
+ ${where.length > 0 ? `WHERE ${where.join(" AND ")}` : ""}
3094
+ `
3095
+ )
3096
+ .all(...params) as Array<{
3097
+ tenant_id: string;
3098
+ tool_name: string;
3099
+ status: string;
3100
+ latency_ms: number;
3101
+ units: number;
3102
+ created_at: string;
3103
+ }>;
3104
+
3105
+ const grouped = new Map<
3106
+ string,
3107
+ { tenant_id: string; tool_name: string; request_count: number; error_count: number; total_units: number; latencies: number[]; last: string }
3108
+ >();
3109
+ for (const row of rows) {
3110
+ const key = `${row.tenant_id}:${row.tool_name}`;
3111
+ const existing =
3112
+ grouped.get(key) ??
3113
+ {
3114
+ tenant_id: row.tenant_id,
3115
+ tool_name: row.tool_name,
3116
+ request_count: 0,
3117
+ error_count: 0,
3118
+ total_units: 0,
3119
+ latencies: [],
3120
+ last: row.created_at
3121
+ };
3122
+ existing.request_count += 1;
3123
+ existing.error_count += row.status === "error" ? 1 : 0;
3124
+ existing.total_units += Number(row.units);
3125
+ existing.latencies.push(Number(row.latency_ms));
3126
+ if (row.created_at > existing.last) {
3127
+ existing.last = row.created_at;
3128
+ }
3129
+ grouped.set(key, existing);
3130
+ }
3131
+
3132
+ return [...grouped.values()].map((row) => ({
3133
+ tenant_id: row.tenant_id,
3134
+ tool_name: row.tool_name,
3135
+ request_count: row.request_count,
3136
+ error_count: row.error_count,
3137
+ total_units: row.total_units,
3138
+ p95_latency_ms: percentile(row.latencies, 0.95),
3139
+ last_seen_at: row.last
3140
+ }));
3141
+ }
3142
+
3143
+ async listAuditEvents(input?: {
3144
+ tenant_id?: string;
3145
+ limit?: number;
3146
+ }): Promise<AuditEventRecord[]> {
3147
+ const where = input?.tenant_id ? "WHERE tenant_id = ?" : "";
3148
+ const limit = Math.max(1, input?.limit ?? 100);
3149
+ const rows = this.db
3150
+ .prepare(
3151
+ `
3152
+ SELECT id AS event_id, tenant_id, subject, action, resource, status, trace_id, details, created_at
3153
+ FROM audit_events
3154
+ ${where}
3155
+ ORDER BY created_at DESC
3156
+ LIMIT ?
3157
+ `
3158
+ )
3159
+ .all(...(input?.tenant_id ? [input.tenant_id, limit] : [limit])) as Array<{
3160
+ event_id: string;
3161
+ tenant_id: string;
3162
+ subject: string;
3163
+ action: string;
3164
+ resource: string;
3165
+ status: "success" | "error" | "denied";
3166
+ trace_id: string;
3167
+ details: string | null;
3168
+ created_at: string;
3169
+ }>;
3170
+
3171
+ return rows.map((row) => ({
3172
+ event_id: row.event_id,
3173
+ tenant_id: row.tenant_id,
3174
+ subject: row.subject,
3175
+ action: row.action,
3176
+ resource: row.resource,
3177
+ status: row.status,
3178
+ trace_id: row.trace_id,
3179
+ ...(row.details ? { details: JSON.parse(row.details) as Record<string, unknown> } : {}),
3180
+ created_at: row.created_at
3181
+ }));
3182
+ }
3183
+ }
3184
+
3185
+ export class PostgresUsageMeterStore implements UsageMeterStore {
3186
+ constructor(private readonly pool: Pool) {}
3187
+
3188
+ async migrate(): Promise<void> {
3189
+ await this.pool.query(`
3190
+ CREATE TABLE IF NOT EXISTS usage_metering (
3191
+ id TEXT PRIMARY KEY,
3192
+ tenant_id TEXT NOT NULL,
3193
+ workspace_id TEXT,
3194
+ tool_name TEXT NOT NULL,
3195
+ trace_id TEXT NOT NULL,
3196
+ status TEXT NOT NULL,
3197
+ latency_ms INTEGER NOT NULL,
3198
+ result_count INTEGER NOT NULL,
3199
+ units INTEGER NOT NULL,
3200
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
3201
+ );
3202
+ `);
3203
+ await this.pool.query("CREATE INDEX IF NOT EXISTS idx_usage_metering_tenant_created ON usage_metering(tenant_id, created_at DESC)");
3204
+
3205
+ await this.pool.query(`
3206
+ CREATE TABLE IF NOT EXISTS audit_events (
3207
+ id TEXT PRIMARY KEY,
3208
+ tenant_id TEXT NOT NULL,
3209
+ subject TEXT NOT NULL,
3210
+ action TEXT NOT NULL,
3211
+ resource TEXT NOT NULL,
3212
+ status TEXT NOT NULL,
3213
+ trace_id TEXT NOT NULL,
3214
+ details JSONB,
3215
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
3216
+ );
3217
+ `);
3218
+ await this.pool.query("CREATE INDEX IF NOT EXISTS idx_audit_events_tenant_created ON audit_events(tenant_id, created_at DESC)");
3219
+ }
3220
+
3221
+ async recordUsage(input: UsageMeteringRecordInput): Promise<void> {
3222
+ await this.pool.query(
3223
+ `
3224
+ INSERT INTO usage_metering
3225
+ (id, tenant_id, workspace_id, tool_name, trace_id, status, latency_ms, result_count, units, created_at)
3226
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::timestamptz)
3227
+ `,
3228
+ [
3229
+ `usg_${randomUUID()}`,
3230
+ input.tenant_id,
3231
+ input.workspace_id ?? null,
3232
+ input.tool_name,
3233
+ input.trace_id,
3234
+ input.status,
3235
+ Math.max(0, Math.floor(input.latency_ms)),
3236
+ Math.max(0, Math.floor(input.result_count)),
3237
+ Math.max(0, Math.floor(input.units)),
3238
+ input.created_at ?? nowIso()
3239
+ ]
3240
+ );
3241
+ }
3242
+
3243
+ async recordAuditEvent(input: AuditEventInput): Promise<void> {
3244
+ await this.pool.query(
3245
+ `
3246
+ INSERT INTO audit_events
3247
+ (id, tenant_id, subject, action, resource, status, trace_id, details, created_at)
3248
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9::timestamptz)
3249
+ `,
3250
+ [
3251
+ `aud_${randomUUID()}`,
3252
+ input.tenant_id,
3253
+ input.subject,
3254
+ input.action,
3255
+ input.resource,
3256
+ input.status,
3257
+ input.trace_id,
3258
+ JSON.stringify(input.details ?? null),
3259
+ input.created_at ?? nowIso()
3260
+ ]
3261
+ );
3262
+ }
3263
+
3264
+ async listUsageSummary(input?: {
3265
+ tenant_id?: string;
3266
+ from?: string;
3267
+ to?: string;
3268
+ }): Promise<UsageSummary[]> {
3269
+ const where: string[] = [];
3270
+ const params: unknown[] = [];
3271
+ if (input?.tenant_id) {
3272
+ params.push(input.tenant_id);
3273
+ where.push(`tenant_id = $${params.length}`);
3274
+ }
3275
+ if (input?.from) {
3276
+ params.push(input.from);
3277
+ where.push(`created_at >= $${params.length}::timestamptz`);
3278
+ }
3279
+ if (input?.to) {
3280
+ params.push(input.to);
3281
+ where.push(`created_at <= $${params.length}::timestamptz`);
3282
+ }
3283
+
3284
+ const result = await this.pool.query<{
3285
+ tenant_id: string;
3286
+ tool_name: string;
3287
+ request_count: number;
3288
+ error_count: number;
3289
+ total_units: number;
3290
+ p95_latency_ms: number;
3291
+ last_seen_at: unknown;
3292
+ }>(
3293
+ `
3294
+ SELECT
3295
+ tenant_id,
3296
+ tool_name,
3297
+ COUNT(*)::int AS request_count,
3298
+ COUNT(*) FILTER (WHERE status = 'error')::int AS error_count,
3299
+ COALESCE(SUM(units), 0)::int AS total_units,
3300
+ COALESCE(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY latency_ms), 0)::double precision AS p95_latency_ms,
3301
+ MAX(created_at) AS last_seen_at
3302
+ FROM usage_metering
3303
+ ${where.length > 0 ? `WHERE ${where.join(" AND ")}` : ""}
3304
+ GROUP BY tenant_id, tool_name
3305
+ ORDER BY tenant_id, tool_name
3306
+ `,
3307
+ params
3308
+ );
3309
+
3310
+ return result.rows.map((row) => ({
3311
+ tenant_id: row.tenant_id,
3312
+ tool_name: row.tool_name,
3313
+ request_count: Number(row.request_count),
3314
+ error_count: Number(row.error_count),
3315
+ total_units: Number(row.total_units),
3316
+ p95_latency_ms: Number(row.p95_latency_ms),
3317
+ ...(row.last_seen_at ? { last_seen_at: toIsoString(row.last_seen_at) } : {})
3318
+ }));
3319
+ }
3320
+
3321
+ async listAuditEvents(input?: {
3322
+ tenant_id?: string;
3323
+ limit?: number;
3324
+ }): Promise<AuditEventRecord[]> {
3325
+ const params: unknown[] = [];
3326
+ const where: string[] = [];
3327
+ if (input?.tenant_id) {
3328
+ params.push(input.tenant_id);
3329
+ where.push(`tenant_id = $${params.length}`);
3330
+ }
3331
+ params.push(Math.max(1, input?.limit ?? 100));
3332
+ const limitIndex = params.length;
3333
+
3334
+ const result = await this.pool.query<{
3335
+ event_id: string;
3336
+ tenant_id: string;
3337
+ subject: string;
3338
+ action: string;
3339
+ resource: string;
3340
+ status: "success" | "error" | "denied";
3341
+ trace_id: string;
3342
+ details: Record<string, unknown> | null;
3343
+ created_at: unknown;
3344
+ }>(
3345
+ `
3346
+ SELECT id AS event_id, tenant_id, subject, action, resource, status, trace_id, details, created_at
3347
+ FROM audit_events
3348
+ ${where.length > 0 ? `WHERE ${where.join(" AND ")}` : ""}
3349
+ ORDER BY created_at DESC
3350
+ LIMIT $${limitIndex}
3351
+ `,
3352
+ params
3353
+ );
3354
+
3355
+ return result.rows.map((row) => ({
3356
+ event_id: row.event_id,
3357
+ tenant_id: row.tenant_id,
3358
+ subject: row.subject,
3359
+ action: row.action,
3360
+ resource: row.resource,
3361
+ status: row.status,
3362
+ trace_id: row.trace_id,
3363
+ ...(row.details ? { details: row.details } : {}),
3364
+ created_at: toIsoString(row.created_at)
3365
+ }));
3366
+ }
3367
+ }
3368
+
3369
+ export function createRedisClient(url: string): RedisClient {
3370
+ return new Redis(url);
3371
+ }
3372
+
3373
+ export function buildQueryCacheKey(input: {
3374
+ workspace_id: string;
3375
+ index_version: string;
3376
+ query: string;
3377
+ top_k: number;
3378
+ filters?: {
3379
+ language?: string;
3380
+ path_prefix?: string;
3381
+ glob?: string;
3382
+ };
3383
+ }): string {
3384
+ const digest = sha256(
3385
+ JSON.stringify({
3386
+ query: input.query,
3387
+ top_k: input.top_k,
3388
+ filters: input.filters ?? null,
3389
+ index_version: input.index_version
3390
+ })
3391
+ );
3392
+ return `${input.workspace_id}:${input.index_version}:${digest}`;
3393
+ }