@denizsvnc/ai-rag-engine 0.0.0-stage → 2.2.0

package/dist/scripts/migrate-vector.js

@@ -0,0 +1,627 @@
+#!/usr/bin/env node
+
+// src/scripts/migrate-vector.ts
+import "dotenv/config";
+import { Pool as Pool2 } from "pg";
+
+// src/adapters/vector/postgres-vector.adapter.ts
+import { Pool } from "pg";
+
+// src/core/sml-engine.ts
+import { LRUCache } from "lru-cache";
+
+// src/utils/math.ts
+function dotProduct(vecA, vecB) {
+  if (vecA.length !== vecB.length) {
+    throw new Error("Vector dimensions do not match");
+  }
+  let sum = 0;
+  for (let i = 0; i < vecA.length; i++) {
+    sum += (vecA[i] ?? 0) * (vecB[i] ?? 0);
+  }
+  return sum;
+}
+
+// src/core/sml-engine.ts
+var CircuitBreakerOpenError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "CircuitBreakerOpenError";
+  }
+};
+var SMLEngine = class {
+  modelName;
+  cacheDir;
+  localFilesOnly;
+  dtype;
+  extractorPromise = null;
+  embeddingCache;
+  // Circuit Breaker State Machine
+  circuitState = "CLOSED";
+  failureCount = 0;
+  lastFailureTime = 0;
+  failureThreshold;
+  cooldownMs;
+  constructor(config) {
+    this.cacheDir = config?.cacheDir ?? process.env.RAG_MODEL_CACHE_DIR ?? "./data/models";
+    this.localFilesOnly = config?.localFilesOnly ?? false;
+    this.modelName = config?.modelName ?? "Xenova/all-MiniLM-L6-v2";
+    this.dtype = config?.dtype ?? (config?.quantized === false ? "fp32" : "q8");
+    this.failureThreshold = config?.circuitBreakerThreshold ?? 3;
+    this.cooldownMs = config?.circuitBreakerCooldownMs ?? 15e3;
+    this.embeddingCache = new LRUCache({
+      max: config?.cacheSize ?? 1e3
+    });
+  }
+  /**
+   * Resets the circuit breaker state (useful for testing and recovery).
+   */
+  resetCircuit() {
+    this.circuitState = "CLOSED";
+    this.failureCount = 0;
+    this.lastFailureTime = 0;
+    this.extractorPromise = null;
+  }
+  getCircuitState() {
+    if (this.circuitState === "OPEN") {
+      if (Date.now() - this.lastFailureTime >= this.cooldownMs) {
+        this.circuitState = "HALF_OPEN";
+      }
+    }
+    return this.circuitState;
+  }
+  getFailureCount() {
+    return this.failureCount;
+  }
+  /**
+   * Thread-safe pipeline initializer guarded by a Circuit Breaker.
+   */
+  async init() {
+    if (this.circuitState === "OPEN") {
+      const timeSinceLastFailure = Date.now() - this.lastFailureTime;
+      if (timeSinceLastFailure < this.cooldownMs) {
+        const remainingSec = Math.ceil((this.cooldownMs - timeSinceLastFailure) / 1e3);
+        throw new CircuitBreakerOpenError(
+          `[SML CircuitBreaker] Model '${this.modelName}' is temporarily tripped OPEN after ${this.failureCount} consecutive failures. Cooldown active for another ${remainingSec}s.`
+        );
+      }
+      this.circuitState = "HALF_OPEN";
+      console.log(`[SML CircuitBreaker] Cooldown expired. Transitioned to HALF_OPEN for probe.`);
+    }
+    if (!this.extractorPromise) {
+      console.log(`[SML] Loading ${this.modelName} (dtype: ${this.dtype})...`);
+      this.extractorPromise = import("@huggingface/transformers").then(async ({ pipeline, env }) => {
+        const previousCacheDir = env.cacheDir;
+        const previousAllowRemote = env.allowRemoteModels;
+        try {
+          if (this.cacheDir) {
+            env.cacheDir = this.cacheDir;
+          }
+          if (this.localFilesOnly) {
+            env.allowRemoteModels = false;
+          }
+          return await pipeline("feature-extraction", this.modelName, {
+            dtype: this.dtype
+          });
+        } finally {
+          env.cacheDir = previousCacheDir;
+          env.allowRemoteModels = previousAllowRemote;
+        }
+      }).then((pipelineInstance) => {
+        this.circuitState = "CLOSED";
+        this.failureCount = 0;
+        const extract = async (text, options) => {
+          const output = await pipelineInstance(text, options);
+          const values = output.data;
+          const data = output.data instanceof Float32Array ? output.data : Float32Array.from({ length: values.length }, (_, index) => Number(values[index]));
+          return { data, dims: output.dims };
+        };
+        return extract;
+      }).catch((err) => {
+        this.failureCount++;
+        this.lastFailureTime = Date.now();
+        this.extractorPromise = null;
+        if (this.failureCount >= this.failureThreshold) {
+          this.circuitState = "OPEN";
+          console.error(
+            `[SML CircuitBreaker] \u26A0\uFE0F Tripped to OPEN state! Model failed ${this.failureCount} times. Pausing requests for ${this.cooldownMs / 1e3}s.`
+          );
+        } else {
+          console.error(`[SML] Model loading error (attempt ${this.failureCount}/${this.failureThreshold}):`, err);
+        }
+        throw err;
+      });
+    }
+    return this.extractorPromise;
+  }
+  /**
+   * Generates a normalized embedding vector for a single text input (with LRU caching).
+   */
+  async getEmbedding(text) {
+    const cleanedText = text.trim();
+    if (!cleanedText) {
+      throw new Error("Text input cannot be empty.");
+    }
+    const cached = this.embeddingCache.get(cleanedText);
+    if (cached) return cached;
+    const extractor = await this.init();
+    const output = await extractor(cleanedText, {
+      pooling: "mean",
+      normalize: true
+    });
+    const embedding = output.data;
+    this.embeddingCache.set(cleanedText, embedding);
+    return embedding;
+  }
+  /**
+   * Generates embedding vectors for multiple texts in batch mode.
+   */
+  async getBatchEmbeddings(texts) {
+    if (texts.length === 0) return [];
+    const extractor = await this.init();
+    const output = await extractor(texts, {
+      pooling: "mean",
+      normalize: true
+    });
+    const rawData = output.data;
+    const dimensions = texts.length > 0 ? Math.floor(rawData.length / texts.length) : 384;
+    const results = [];
+    for (let i = 0; i < texts.length; i++) {
+      const start = i * dimensions;
+      results.push(rawData.subarray(start, start + dimensions));
+    }
+    return results;
+  }
+  /**
+   * Compares the query against target embeddings and returns the highest scoring match above the threshold.
+   */
+  async findBestMatch(query, targets, threshold = 0.65) {
+    if (targets.length === 0) return null;
+    const queryVec = await this.getEmbedding(query);
+    let bestTarget = null;
+    let highestScore = -1;
+    for (let i = 0; i < targets.length; i++) {
+      const target = targets[i];
+      const score = dotProduct(
+        queryVec,
+        target.embedding
+      );
+      if (score > highestScore) {
+        highestScore = score;
+        bestTarget = target;
+      }
+    }
+    if (bestTarget && highestScore >= threshold) {
+      return { target: bestTarget, score: highestScore };
+    }
+    return null;
+  }
+};
+
+// src/core/vector/hybrid-search.ts
+function reciprocalRankFusion(denseResults, sparseResults, topK = 3, rrfK = 60) {
+  const scoredDocs = /* @__PURE__ */ new Map();
+  denseResults.forEach((doc, index) => {
+    const rank = index + 1;
+    const key = JSON.stringify([doc.metadata?.tenantId ?? null, doc.id]);
+    const existing = scoredDocs.get(key) || { doc, rrfScore: 0 };
+    existing.denseRank = rank;
+    existing.rrfScore += 1 / (rrfK + rank);
+    scoredDocs.set(key, existing);
+  });
+  sparseResults.forEach((doc, index) => {
+    const rank = index + 1;
+    const key = JSON.stringify([doc.metadata?.tenantId ?? null, doc.id]);
+    const existing = scoredDocs.get(key) || { doc, rrfScore: 0 };
+    existing.sparseRank = rank;
+    existing.rrfScore += 1 / (rrfK + rank);
+    scoredDocs.set(key, existing);
+  });
+  const fusedList = Array.from(scoredDocs.values());
+  fusedList.sort((a, b) => b.rrfScore - a.rrfScore);
+  return fusedList.slice(0, topK).map(({ doc, denseRank, sparseRank, rrfScore }) => ({
+    ...doc,
+    denseRank,
+    sparseRank,
+    rrfScore,
+    score: rrfScore
+  }));
+}
+
+// src/adapters/vector/postgres-vector.adapter.ts
+var PostgresVectorAdapter = class {
+  constructor(options) {
+    this.options = options;
+    const name = options.tableName ?? "rag_documents";
+    if (!/^[a-z][a-z0-9_]{0,39}$/.test(name)) throw new Error("Invalid vector table name.");
+    this.table = `"${name}"`;
+    this.configTable = `"${name}_config"`;
+    this.manifests = `"${name}_manifests"`;
+    this.epochs = `"${name}_epochs"`;
+    this.dimensions = options.dimensions ?? 384;
+    if (!Number.isInteger(this.dimensions) || this.dimensions < 1 || this.dimensions > 2e3) {
+      throw new Error("Vector dimensions must be an integer between 1 and 2000.");
+    }
+    this.modelName = options.modelName ?? "Xenova/all-MiniLM-L6-v2";
+    if (!options.pool && !options.connectionString) throw new Error("PostgreSQL connectionString or pool is required.");
+    this.pool = options.pool ?? new Pool({
+      connectionString: options.connectionString,
+      max: 5,
+      connectionTimeoutMillis: 5e3,
+      statement_timeout: 1e4
+    });
+    if (!options.pool) {
+      this.pool.on("error", (error) => console.error("[PostgresVectorAdapter] Idle connection failed:", error.message));
+    }
+    this.embeddings = options.embeddingProvider ?? new SMLEngine({ modelName: this.modelName });
+  }
+  options;
+  pool;
+  table;
+  configTable;
+  manifests;
+  epochs;
+  modelName;
+  dimensions;
+  embeddings;
+  initPromise;
+  async init() {
+    if (!this.initPromise) {
+      this.initPromise = this.initialize().catch((error) => {
+        this.initPromise = void 0;
+        throw error;
+      });
+    }
+    return this.initPromise;
+  }
+  async initialize() {
+    await this.transaction(async (client) => {
+      if (this.options.autoMigrate !== false) {
+        await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", ["rag-vector-schema"]);
+        await client.query("CREATE EXTENSION IF NOT EXISTS vector");
+        await client.query(`CREATE TABLE IF NOT EXISTS ${this.configTable} (
+                    singleton boolean PRIMARY KEY CHECK (singleton), model_name text NOT NULL, dimensions integer NOT NULL
+                )`);
+        await client.query(`INSERT INTO ${this.configTable} VALUES (true, $1, $2) ON CONFLICT DO NOTHING`, [this.modelName, this.dimensions]);
+        await client.query(`CREATE TABLE IF NOT EXISTS ${this.table} (
+                    tenant_id text NOT NULL, id text NOT NULL, content text NOT NULL, metadata jsonb NOT NULL,
+                    embedding vector(${this.dimensions}) NOT NULL, version integer NOT NULL DEFAULT 1,
+                    updated_at timestamptz NOT NULL DEFAULT now(),
+                    search_text tsvector GENERATED ALWAYS AS (to_tsvector('simple', content)) STORED,
+                    PRIMARY KEY (tenant_id, id)
+                )`);
+        await client.query(`CREATE TABLE IF NOT EXISTS ${this.manifests} (
+                    tenant_id text NOT NULL, document_id text NOT NULL, version integer NOT NULL,
+                    PRIMARY KEY (tenant_id, document_id)
+                )`);
+        await client.query(`CREATE TABLE IF NOT EXISTS ${this.epochs} (
+                    tenant_id text PRIMARY KEY, revision bigint NOT NULL
+                )`);
+        const name = this.options.tableName ?? "rag_documents";
+        await client.query(`CREATE INDEX IF NOT EXISTS "${name}_metadata_idx" ON ${this.table} USING gin (metadata)`);
+        await client.query(`CREATE INDEX IF NOT EXISTS "${name}_text_idx" ON ${this.table} USING gin (search_text)`);
+        if (this.options.createHnswIndex) {
+          await client.query(`CREATE INDEX IF NOT EXISTS "${name}_vector_idx" ON ${this.table} USING hnsw (embedding vector_cosine_ops)`);
+        }
+      }
+      const result = await client.query(`SELECT model_name, dimensions FROM ${this.configTable} WHERE singleton = true`);
+      const config = result.rows[0];
+      if (!config || config.model_name !== this.modelName || config.dimensions !== this.dimensions) {
+        throw new Error("Embedding model/dimensions mismatch. Reindex into a new table before changing models.");
+      }
+    });
+  }
+  /** Upserts are atomic per batch and IDs are unique within a tenant. */
+  async addDocuments(documents) {
+    const rows = await this.prepare(documents);
+    if (!rows.length) return;
+    await this.init();
+    await this.transaction(async (client) => {
+      for (const tenant of await this.write(client, rows)) await this.bumpRevision(client, tenant);
+    });
+  }
+  /** Atomically replaces all chunks; retries and older document revisions cannot overwrite newer work. */
+  async replaceDocument(documentId, tenantId, version, documents) {
+    this.tenant({ tenantId });
+    if (!documentId || !Number.isSafeInteger(version) || version < 1 || !documents.length) throw new Error("Invalid document replacement.");
+    if (documents.some((doc) => doc.metadata?.tenantId !== tenantId || doc.metadata?.documentId !== documentId)) {
+      throw new Error("Replacement tenant/document mismatch.");
+    }
+    const rows = await this.prepare(documents);
+    await this.init();
+    return this.transaction(async (client) => {
+      await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [JSON.stringify([this.table, tenantId, documentId])]);
+      const previous = await client.query(`SELECT version FROM ${this.manifests} WHERE tenant_id=$1 AND document_id=$2`, [tenantId, documentId]);
+      if (previous.rows[0] && previous.rows[0].version >= version) return previous.rows[0].version === version ? "unchanged" : "superseded";
+      await client.query(`DELETE FROM ${this.table} WHERE tenant_id=$1 AND metadata @> $2::jsonb`, [tenantId, JSON.stringify({ documentId })]);
+      await this.write(client, rows);
+      await client.query(`INSERT INTO ${this.manifests} VALUES ($1,$2,$3)
+                ON CONFLICT (tenant_id,document_id) DO UPDATE SET version=EXCLUDED.version`, [tenantId, documentId, version]);
+      await this.bumpRevision(client, tenantId);
+      return "applied";
+    });
+  }
+  async getRevision(tenantId) {
+    this.tenant({ tenantId });
+    await this.init();
+    const result = await this.pool.query(`SELECT revision::text FROM ${this.epochs} WHERE tenant_id=$1`, [tenantId]);
+    return result.rows[0]?.revision ?? "0";
+  }
+  async prepare(documents) {
+    const rows = [];
+    for (const doc of documents) {
+      if (typeof doc.id !== "string" || !doc.id.trim() || typeof doc.content !== "string" || !doc.content.trim()) {
+        throw new Error("Document id and content must be non-empty strings.");
+      }
+      rows.push({ doc, tenant: this.tenant(doc.metadata), metadata: JSON.stringify(doc.metadata ?? {}), vector: this.vector(await this.embeddings.getEmbedding(doc.content)) });
+    }
+    return rows;
+  }
+  async write(client, rows) {
+    const changed = /* @__PURE__ */ new Set();
+    for (const { doc, tenant, metadata, vector } of rows) {
+      const result = await client.query(`INSERT INTO ${this.table} AS current (tenant_id,id,content,metadata,embedding)
+                VALUES ($1,$2,$3,$4::jsonb,$5::vector)
+                ON CONFLICT (tenant_id,id) DO UPDATE SET content=EXCLUDED.content,metadata=EXCLUDED.metadata,embedding=EXCLUDED.embedding,
+                    version=current.version+1,updated_at=now()
+                WHERE current.content IS DISTINCT FROM EXCLUDED.content OR current.metadata IS DISTINCT FROM EXCLUDED.metadata
+                   OR current.embedding IS DISTINCT FROM EXCLUDED.embedding`, [tenant, doc.id, doc.content, metadata, vector]);
+      if (result.rowCount) changed.add(tenant);
+    }
+    return changed;
+  }
+  async bumpRevision(client, tenant) {
+    await client.query(`INSERT INTO ${this.epochs} VALUES ($1,1) ON CONFLICT (tenant_id) DO UPDATE SET revision=${this.epochs}.revision+1`, [tenant]);
+  }
+  async search(query, topK = 3, filter) {
+    const tenant = this.tenant(filter);
+    this.validateTopK(topK);
+    const vector = this.vector(await this.embeddings.getEmbedding(query));
+    await this.init();
+    const result = await this.pool.query(
+      `SELECT id, content, metadata, version,
+                1 - (embedding <=> $3::vector) AS score FROM ${this.table}
+            WHERE tenant_id = $1 AND metadata @> $2::jsonb
+            ORDER BY embedding <=> $3::vector, id LIMIT $4`,
+      [tenant, JSON.stringify(filter), vector, topK]
+    );
+    return result.rows.map((row) => ({ ...row, score: Number(row.score) }));
+  }
+  /** PostgreSQL lexical search + dense search, fused with RRF. This is not BM25. */
+  async searchHybrid(query, topK = 3, filter) {
+    const tenant = this.tenant(filter);
+    this.validateTopK(topK);
+    const candidates = Math.min(1e3, Math.max(20, topK * 3));
+    const dense = await this.search(query, candidates, filter);
+    const sparse = await this.pool.query(`SELECT id, content, metadata, version,
+                ts_rank_cd(search_text, websearch_to_tsquery('simple', $3)) AS score FROM ${this.table}
+            WHERE tenant_id = $1 AND metadata @> $2::jsonb
+              AND search_text @@ websearch_to_tsquery('simple', $3)
+            ORDER BY score DESC, id LIMIT $4`, [tenant, JSON.stringify(filter), query, candidates]);
+    return reciprocalRankFusion(dense, sparse.rows, topK);
+  }
+  async deleteDocument(id, filter) {
+    const tenant = this.tenant(filter);
+    await this.init();
+    return this.transaction(async (client) => {
+      const result = await client.query(`DELETE FROM ${this.table} WHERE tenant_id=$1 AND id=$2 AND metadata @> $3::jsonb`, [tenant, id, JSON.stringify(filter)]);
+      if (result.rowCount) await this.bumpRevision(client, tenant);
+      return (result.rowCount ?? 0) > 0;
+    });
+  }
+  async clear(filter) {
+    const tenant = this.tenant(filter);
+    await this.init();
+    await this.transaction(async (client) => {
+      const result = await client.query(`DELETE FROM ${this.table} WHERE tenant_id=$1 AND metadata @> $2::jsonb`, [tenant, JSON.stringify(filter)]);
+      if (result.rowCount) await this.bumpRevision(client, tenant);
+    });
+  }
+  async close() {
+    if (!this.options.pool) await this.pool.end();
+  }
+  tenant(filter) {
+    const tenant = filter?.tenantId;
+    if (typeof tenant !== "string" || !tenant.trim()) throw new Error("An explicit tenantId is required for PostgreSQL vector operations.");
+    return tenant;
+  }
+  vector(embedding) {
+    const values = Array.from(embedding);
+    if (values.length !== this.dimensions || !values.every(Number.isFinite) || !values.some((value) => value !== 0)) {
+      throw new Error(`Embedding must contain ${this.dimensions} finite values and have non-zero magnitude.`);
+    }
+    return JSON.stringify(values);
+  }
+  validateTopK(topK) {
+    if (!Number.isInteger(topK) || topK < 1 || topK > 1e3) throw new Error("topK must be an integer between 1 and 1000.");
+  }
+  async transaction(work) {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      const result = await work(client);
+      await client.query("COMMIT");
+      return result;
+    } catch (error) {
+      await client.query("ROLLBACK").catch(() => {
+      });
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+};
+
+// src/core/ingestion/postgres-ingestion-store.ts
+import { createHash } from "node:crypto";
+var IngestionQuotaError = class extends Error {
+};
+var IngestionConflictError = class extends Error {
+};
+var PostgresIngestionStore = class {
+  constructor(pool, tableName = "rag_ingestion") {
+    this.pool = pool;
+    if (!/^[a-z][a-z0-9_]{0,39}$/.test(tableName)) throw new Error("Invalid ingestion table name.");
+    this.tableName = tableName;
+    this.table = `"${tableName}"`;
+    this.heads = `"${tableName}_heads"`;
+  }
+  pool;
+  table;
+  heads;
+  tableName;
+  async migrate() {
+    await this.transaction(async (client) => {
+      await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", ["rag-ingestion-schema"]);
+      await client.query(`CREATE TABLE IF NOT EXISTS ${this.heads} (
+                tenant_id text NOT NULL, document_id text NOT NULL, version integer NOT NULL,
+                PRIMARY KEY (tenant_id, document_id)
+            )`);
+      await client.query(`CREATE TABLE IF NOT EXISTS ${this.table} (
+                id text PRIMARY KEY, tenant_id text NOT NULL, user_id text NOT NULL, document_id text NOT NULL,
+                filename text NOT NULL, version integer NOT NULL, digest text NOT NULL, settings jsonb NOT NULL,
+                content bytea, idempotency_key text, status text NOT NULL CHECK (status IN ('pending','queued','processing','completed','failed')),
+                result jsonb, error text, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now()
+            )`);
+      await client.query(`CREATE UNIQUE INDEX IF NOT EXISTS "${this.tableName}_idempotency_idx" ON ${this.table} (tenant_id, idempotency_key) WHERE idempotency_key IS NOT NULL`);
+      await client.query(`CREATE INDEX IF NOT EXISTS "${this.tableName}_pending_idx" ON ${this.table} (status, updated_at)`);
+      await client.query(`CREATE INDEX IF NOT EXISTS "${this.tableName}_tenant_idx" ON ${this.table} (tenant_id)`);
+    });
+  }
+  async ready() {
+    await this.pool.query(`SELECT id FROM ${this.table} LIMIT 0`);
+  }
+  async put(input, limits) {
+    const digest = createHash("sha256").update(input.content).digest("hex");
+    return this.transaction(async (client) => {
+      await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [`ingestion:${this.tableName}:${input.tenantId}`]);
+      const previous = input.idempotencyKey ? await client.query(`SELECT * FROM ${this.table} WHERE tenant_id = $1 AND idempotency_key = $2`, [input.tenantId, input.idempotencyKey]) : await client.query(`SELECT j.* FROM ${this.table} j JOIN ${this.heads} h
+                    ON j.tenant_id = h.tenant_id AND j.document_id = h.document_id AND j.version = h.version
+                    WHERE j.tenant_id = $1 AND j.document_id = $2`, [input.tenantId, input.documentId]);
+      const old = previous.rows[0];
+      if (old && old.digest === digest && old.document_id === input.documentId && old.filename === input.filename && old.settings.chunkSize === input.settings.chunkSize && old.settings.chunkOverlap === input.settings.chunkOverlap) return this.row(old);
+      if (old && input.idempotencyKey) throw new IngestionConflictError("Idempotency key already used for different content.");
+      const quota = await client.query(`SELECT count(*) FILTER (WHERE status IN ('pending','queued','processing')) AS pending,
+                coalesce(sum(octet_length(content)), 0) AS bytes FROM ${this.table} WHERE tenant_id = $1`, [input.tenantId]);
+      if (Number(quota.rows[0].pending) >= limits.maxPending || Number(quota.rows[0].bytes) + input.content.length > limits.maxStoredBytes) {
+        throw new IngestionQuotaError("Tenant ingestion quota exceeded.");
+      }
+      const head = await client.query(`INSERT INTO ${this.heads} AS h VALUES ($1, $2, 1)
+                ON CONFLICT (tenant_id, document_id) DO UPDATE SET version = h.version + 1 RETURNING version`, [input.tenantId, input.documentId]);
+      const id = createHash("sha256").update(JSON.stringify([input.tenantId, input.documentId, head.rows[0].version, digest])).digest("hex");
+      const saved = await client.query(
+        `INSERT INTO ${this.table}
+                (id,tenant_id,user_id,document_id,filename,version,digest,settings,content,idempotency_key,status)
+                VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb,$9,$10,'pending') RETURNING *`,
+        [id, input.tenantId, input.userId, input.documentId, input.filename, head.rows[0].version, digest, JSON.stringify(input.settings), input.content, input.idempotencyKey ?? null]
+      );
+      return this.row(saved.rows[0]);
+    });
+  }
+  async get(id, tenantId) {
+    if (!tenantId) return null;
+    const result = await this.pool.query(`SELECT * FROM ${this.table} WHERE id = $1 AND tenant_id = $2`, [id, tenantId]);
+    return result.rows[0] ? this.row(result.rows[0]) : null;
+  }
+  async dispatchable(limit = 100) {
+    const result = await this.pool.query(`SELECT id, tenant_id FROM ${this.table} WHERE status IN ('pending','queued','processing')
+            ORDER BY (status = 'pending') DESC, updated_at LIMIT $1`, [limit]);
+    return result.rows.map((row) => ({ id: row.id, tenantId: row.tenant_id }));
+  }
+  async mark(id, status, error) {
+    const condition = status === "queued" ? "AND status = 'pending'" : "AND status NOT IN ('completed','failed')";
+    await this.pool.query(`UPDATE ${this.table} SET status = $2, error = $3, updated_at = now() WHERE id = $1 ${condition}`, [id, status, error ?? null]);
+  }
+  async complete(id, result) {
+    await this.pool.query(`UPDATE ${this.table} SET status = 'completed', result = $2::jsonb, error = NULL,
+            content = NULL, updated_at = now() WHERE id = $1`, [id, JSON.stringify(result)]);
+  }
+  async resetFailed(id, tenantId) {
+    const result = await this.pool.query(`UPDATE ${this.table} SET status = 'pending', error = NULL, updated_at = now()
+            WHERE id = $1 AND tenant_id = $2 AND status = 'failed' AND content IS NOT NULL`, [id, tenantId]);
+    if (!result.rowCount) throw new Error("Failed job missing or its retry payload has expired.");
+  }
+  async expireFailedBlobs(retentionDays = 7) {
+    await this.pool.query(`UPDATE ${this.table} SET content = NULL WHERE status = 'failed' AND content IS NOT NULL
+            AND updated_at < now() - ($1 * interval '1 day')`, [retentionDays]);
+  }
+  row(row) {
+    return {
+      id: row.id,
+      tenantId: row.tenant_id,
+      userId: row.user_id,
+      documentId: row.document_id,
+      filename: row.filename,
+      version: row.version,
+      digest: row.digest,
+      settings: row.settings,
+      content: row.content,
+      status: row.status,
+      result: row.result,
+      error: row.error,
+      createdAt: new Date(row.created_at).getTime(),
+      updatedAt: new Date(row.updated_at).getTime()
+    };
+  }
+  async transaction(work) {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      const result = await work(client);
+      await client.query("COMMIT");
+      return result;
+    } catch (error) {
+      await client.query("ROLLBACK").catch(() => {
+      });
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+};
+
+// src/scripts/migrate-vector.ts
+var role = process.env.RUNTIME_DB_USER;
+var password = process.env.RUNTIME_DB_PASSWORD;
+if (role && (!/^[a-z][a-z0-9_]{0,39}$/.test(role) || !password || password.length < 32)) throw new Error("Runtime role requires a safe name and a 32+ character password.");
+var table = process.env.VECTOR_TABLE_NAME || "rag_documents";
+var ingestion = process.env.INGESTION_TABLE_NAME || "rag_ingestion";
+var store = new PostgresVectorAdapter({
+  connectionString: process.env.DATABASE_URL,
+  tableName: table,
+  modelName: process.env.EMBEDDING_MODEL || "Xenova/all-MiniLM-L6-v2",
+  dimensions: Number(process.env.EMBEDDING_DIMENSIONS || 384),
+  autoMigrate: true
+});
+try {
+  await store.init();
+  const pool = new Pool2({ connectionString: process.env.DATABASE_URL });
+  try {
+    await new PostgresIngestionStore(pool, ingestion).migrate();
+    if (role) {
+      const client = await pool.connect();
+      try {
+        await client.query("BEGIN");
+        if (!(await client.query("SELECT 1 FROM pg_roles WHERE rolname=$1", [role])).rowCount) {
+          const sql = await client.query("SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', $1::text, $2::text) AS statement", [role, password]);
+          await client.query(sql.rows[0].statement);
+        }
+        await client.query(`GRANT USAGE ON SCHEMA public TO "${role}"`);
+        await client.query(`GRANT SELECT ON "${table}_config" TO "${role}"`);
+        await client.query(`GRANT SELECT,INSERT,UPDATE,DELETE ON "${table}","${table}_epochs","${table}_manifests","${ingestion}","${ingestion}_heads" TO "${role}"`);
+        await client.query("COMMIT");
+      } catch (error) {
+        await client.query("ROLLBACK").catch(() => {
+        });
+        throw error;
+      } finally {
+        client.release();
+      }
+    }
+  } finally {
+    await pool.end();
+  }
+  console.log("Vector and ingestion schemas initialized. Runtime privileges configured when requested.");
+} finally {
+  await store.close();
+}