@denizsvnc/ai-rag-engine 0.0.0-stage → 2.2.1

package/dist/worker.js

@@ -0,0 +1,1787 @@
+#!/usr/bin/env node
+
+// src/worker.ts
+import "dotenv/config";
+
+// src/config/ingestion-runtime.ts
+import { Pool as Pool2 } from "pg";
+
+// src/adapters/vector/postgres-vector.adapter.ts
+import { Pool } from "pg";
+
+// src/core/sml-engine.ts
+import { LRUCache } from "lru-cache";
+
+// src/utils/math.ts
+function dotProduct(vecA, vecB) {
+  if (vecA.length !== vecB.length) {
+    throw new Error("Vector dimensions do not match");
+  }
+  let sum = 0;
+  for (let i = 0; i < vecA.length; i++) {
+    sum += (vecA[i] ?? 0) * (vecB[i] ?? 0);
+  }
+  return sum;
+}
+
+// src/core/sml-engine.ts
+var CircuitBreakerOpenError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "CircuitBreakerOpenError";
+  }
+};
+var SMLEngine = class {
+  modelName;
+  cacheDir;
+  localFilesOnly;
+  dtype;
+  extractorPromise = null;
+  embeddingCache;
+  // Circuit Breaker State Machine
+  circuitState = "CLOSED";
+  failureCount = 0;
+  lastFailureTime = 0;
+  failureThreshold;
+  cooldownMs;
+  constructor(config) {
+    this.cacheDir = config?.cacheDir ?? process.env.RAG_MODEL_CACHE_DIR ?? "./data/models";
+    this.localFilesOnly = config?.localFilesOnly ?? false;
+    this.modelName = config?.modelName ?? "Xenova/all-MiniLM-L6-v2";
+    this.dtype = config?.dtype ?? (config?.quantized === false ? "fp32" : "q8");
+    this.failureThreshold = config?.circuitBreakerThreshold ?? 3;
+    this.cooldownMs = config?.circuitBreakerCooldownMs ?? 15e3;
+    this.embeddingCache = new LRUCache({
+      max: config?.cacheSize ?? 1e3
+    });
+  }
+  /**
+   * Resets the circuit breaker state (useful for testing and recovery).
+   */
+  resetCircuit() {
+    this.circuitState = "CLOSED";
+    this.failureCount = 0;
+    this.lastFailureTime = 0;
+    this.extractorPromise = null;
+  }
+  getCircuitState() {
+    if (this.circuitState === "OPEN") {
+      if (Date.now() - this.lastFailureTime >= this.cooldownMs) {
+        this.circuitState = "HALF_OPEN";
+      }
+    }
+    return this.circuitState;
+  }
+  getFailureCount() {
+    return this.failureCount;
+  }
+  /**
+   * Thread-safe pipeline initializer guarded by a Circuit Breaker.
+   */
+  async init() {
+    if (this.circuitState === "OPEN") {
+      const timeSinceLastFailure = Date.now() - this.lastFailureTime;
+      if (timeSinceLastFailure < this.cooldownMs) {
+        const remainingSec = Math.ceil((this.cooldownMs - timeSinceLastFailure) / 1e3);
+        throw new CircuitBreakerOpenError(
+          `[SML CircuitBreaker] Model '${this.modelName}' is temporarily tripped OPEN after ${this.failureCount} consecutive failures. Cooldown active for another ${remainingSec}s.`
+        );
+      }
+      this.circuitState = "HALF_OPEN";
+      console.log(`[SML CircuitBreaker] Cooldown expired. Transitioned to HALF_OPEN for probe.`);
+    }
+    if (!this.extractorPromise) {
+      console.log(`[SML] Loading ${this.modelName} (dtype: ${this.dtype})...`);
+      this.extractorPromise = import("@huggingface/transformers").then(async ({ pipeline: pipeline2, env }) => {
+        const previousCacheDir = env.cacheDir;
+        const previousAllowRemote = env.allowRemoteModels;
+        try {
+          if (this.cacheDir) {
+            env.cacheDir = this.cacheDir;
+          }
+          if (this.localFilesOnly) {
+            env.allowRemoteModels = false;
+          }
+          return await pipeline2("feature-extraction", this.modelName, {
+            dtype: this.dtype
+          });
+        } finally {
+          env.cacheDir = previousCacheDir;
+          env.allowRemoteModels = previousAllowRemote;
+        }
+      }).then((pipelineInstance) => {
+        this.circuitState = "CLOSED";
+        this.failureCount = 0;
+        const extract = async (text, options) => {
+          const output = await pipelineInstance(text, options);
+          const values = output.data;
+          const data = output.data instanceof Float32Array ? output.data : Float32Array.from({ length: values.length }, (_, index) => Number(values[index]));
+          return { data, dims: output.dims };
+        };
+        return extract;
+      }).catch((err) => {
+        this.failureCount++;
+        this.lastFailureTime = Date.now();
+        this.extractorPromise = null;
+        if (this.failureCount >= this.failureThreshold) {
+          this.circuitState = "OPEN";
+          console.error(
+            `[SML CircuitBreaker] \u26A0\uFE0F Tripped to OPEN state! Model failed ${this.failureCount} times. Pausing requests for ${this.cooldownMs / 1e3}s.`
+          );
+        } else {
+          console.error(`[SML] Model loading error (attempt ${this.failureCount}/${this.failureThreshold}):`, err);
+        }
+        throw err;
+      });
+    }
+    return this.extractorPromise;
+  }
+  /**
+   * Generates a normalized embedding vector for a single text input (with LRU caching).
+   */
+  async getEmbedding(text) {
+    const cleanedText = text.trim();
+    if (!cleanedText) {
+      throw new Error("Text input cannot be empty.");
+    }
+    const cached = this.embeddingCache.get(cleanedText);
+    if (cached) return cached;
+    const extractor = await this.init();
+    const output = await extractor(cleanedText, {
+      pooling: "mean",
+      normalize: true
+    });
+    const embedding = output.data;
+    this.embeddingCache.set(cleanedText, embedding);
+    return embedding;
+  }
+  /**
+   * Generates embedding vectors for multiple texts in batch mode.
+   */
+  async getBatchEmbeddings(texts) {
+    if (texts.length === 0) return [];
+    const extractor = await this.init();
+    const output = await extractor(texts, {
+      pooling: "mean",
+      normalize: true
+    });
+    const rawData = output.data;
+    const dimensions = texts.length > 0 ? Math.floor(rawData.length / texts.length) : 384;
+    const results = [];
+    for (let i = 0; i < texts.length; i++) {
+      const start = i * dimensions;
+      results.push(rawData.subarray(start, start + dimensions));
+    }
+    return results;
+  }
+  /**
+   * Compares the query against target embeddings and returns the highest scoring match above the threshold.
+   */
+  async findBestMatch(query, targets, threshold = 0.65) {
+    if (targets.length === 0) return null;
+    const queryVec = await this.getEmbedding(query);
+    let bestTarget = null;
+    let highestScore = -1;
+    for (let i = 0; i < targets.length; i++) {
+      const target = targets[i];
+      const score = dotProduct(
+        queryVec,
+        target.embedding
+      );
+      if (score > highestScore) {
+        highestScore = score;
+        bestTarget = target;
+      }
+    }
+    if (bestTarget && highestScore >= threshold) {
+      return { target: bestTarget, score: highestScore };
+    }
+    return null;
+  }
+};
+
+// src/core/vector/hybrid-search.ts
+function reciprocalRankFusion(denseResults, sparseResults, topK = 3, rrfK = 60) {
+  const scoredDocs = /* @__PURE__ */ new Map();
+  denseResults.forEach((doc, index) => {
+    const rank = index + 1;
+    const key = JSON.stringify([doc.metadata?.tenantId ?? null, doc.id]);
+    const existing = scoredDocs.get(key) || { doc, rrfScore: 0 };
+    existing.denseRank = rank;
+    existing.rrfScore += 1 / (rrfK + rank);
+    scoredDocs.set(key, existing);
+  });
+  sparseResults.forEach((doc, index) => {
+    const rank = index + 1;
+    const key = JSON.stringify([doc.metadata?.tenantId ?? null, doc.id]);
+    const existing = scoredDocs.get(key) || { doc, rrfScore: 0 };
+    existing.sparseRank = rank;
+    existing.rrfScore += 1 / (rrfK + rank);
+    scoredDocs.set(key, existing);
+  });
+  const fusedList = Array.from(scoredDocs.values());
+  fusedList.sort((a, b) => b.rrfScore - a.rrfScore);
+  return fusedList.slice(0, topK).map(({ doc, denseRank, sparseRank, rrfScore }) => ({
+    ...doc,
+    denseRank,
+    sparseRank,
+    rrfScore,
+    score: rrfScore
+  }));
+}
+
+// src/adapters/vector/postgres-vector.adapter.ts
+var PostgresVectorAdapter = class {
+  constructor(options) {
+    this.options = options;
+    const name = options.tableName ?? "rag_documents";
+    if (!/^[a-z][a-z0-9_]{0,39}$/.test(name)) throw new Error("Invalid vector table name.");
+    this.table = `"${name}"`;
+    this.configTable = `"${name}_config"`;
+    this.manifests = `"${name}_manifests"`;
+    this.epochs = `"${name}_epochs"`;
+    this.dimensions = options.dimensions ?? 384;
+    if (!Number.isInteger(this.dimensions) || this.dimensions < 1 || this.dimensions > 2e3) {
+      throw new Error("Vector dimensions must be an integer between 1 and 2000.");
+    }
+    this.modelName = options.modelName ?? "Xenova/all-MiniLM-L6-v2";
+    if (!options.pool && !options.connectionString) throw new Error("PostgreSQL connectionString or pool is required.");
+    this.pool = options.pool ?? new Pool({
+      connectionString: options.connectionString,
+      max: 5,
+      connectionTimeoutMillis: 5e3,
+      statement_timeout: 1e4
+    });
+    if (!options.pool) {
+      this.pool.on("error", (error) => console.error("[PostgresVectorAdapter] Idle connection failed:", error.message));
+    }
+    this.embeddings = options.embeddingProvider ?? new SMLEngine({ modelName: this.modelName });
+  }
+  options;
+  pool;
+  table;
+  configTable;
+  manifests;
+  epochs;
+  modelName;
+  dimensions;
+  embeddings;
+  initPromise;
+  async init() {
+    if (!this.initPromise) {
+      this.initPromise = this.initialize().catch((error) => {
+        this.initPromise = void 0;
+        throw error;
+      });
+    }
+    return this.initPromise;
+  }
+  async initialize() {
+    await this.transaction(async (client) => {
+      if (this.options.autoMigrate !== false) {
+        await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", ["rag-vector-schema"]);
+        await client.query("CREATE EXTENSION IF NOT EXISTS vector");
+        await client.query(`CREATE TABLE IF NOT EXISTS ${this.configTable} (
+                    singleton boolean PRIMARY KEY CHECK (singleton), model_name text NOT NULL, dimensions integer NOT NULL
+                )`);
+        await client.query(`INSERT INTO ${this.configTable} VALUES (true, $1, $2) ON CONFLICT DO NOTHING`, [this.modelName, this.dimensions]);
+        await client.query(`CREATE TABLE IF NOT EXISTS ${this.table} (
+                    tenant_id text NOT NULL, id text NOT NULL, content text NOT NULL, metadata jsonb NOT NULL,
+                    embedding vector(${this.dimensions}) NOT NULL, version integer NOT NULL DEFAULT 1,
+                    updated_at timestamptz NOT NULL DEFAULT now(),
+                    search_text tsvector GENERATED ALWAYS AS (to_tsvector('simple', content)) STORED,
+                    PRIMARY KEY (tenant_id, id)
+                )`);
+        await client.query(`CREATE TABLE IF NOT EXISTS ${this.manifests} (
+                    tenant_id text NOT NULL, document_id text NOT NULL, version integer NOT NULL,
+                    PRIMARY KEY (tenant_id, document_id)
+                )`);
+        await client.query(`CREATE TABLE IF NOT EXISTS ${this.epochs} (
+                    tenant_id text PRIMARY KEY, revision bigint NOT NULL
+                )`);
+        const name = this.options.tableName ?? "rag_documents";
+        await client.query(`CREATE INDEX IF NOT EXISTS "${name}_metadata_idx" ON ${this.table} USING gin (metadata)`);
+        await client.query(`CREATE INDEX IF NOT EXISTS "${name}_text_idx" ON ${this.table} USING gin (search_text)`);
+        if (this.options.createHnswIndex) {
+          await client.query(`CREATE INDEX IF NOT EXISTS "${name}_vector_idx" ON ${this.table} USING hnsw (embedding vector_cosine_ops)`);
+        }
+      }
+      const result = await client.query(`SELECT model_name, dimensions FROM ${this.configTable} WHERE singleton = true`);
+      const config = result.rows[0];
+      if (!config || config.model_name !== this.modelName || config.dimensions !== this.dimensions) {
+        throw new Error("Embedding model/dimensions mismatch. Reindex into a new table before changing models.");
+      }
+    });
+  }
+  /** Upserts are atomic per batch and IDs are unique within a tenant. */
+  async addDocuments(documents) {
+    const rows = await this.prepare(documents);
+    if (!rows.length) return;
+    await this.init();
+    await this.transaction(async (client) => {
+      for (const tenant of await this.write(client, rows)) await this.bumpRevision(client, tenant);
+    });
+  }
+  /** Atomically replaces all chunks; retries and older document revisions cannot overwrite newer work. */
+  async replaceDocument(documentId, tenantId, version, documents) {
+    this.tenant({ tenantId });
+    if (!documentId || !Number.isSafeInteger(version) || version < 1 || !documents.length) throw new Error("Invalid document replacement.");
+    if (documents.some((doc) => doc.metadata?.tenantId !== tenantId || doc.metadata?.documentId !== documentId)) {
+      throw new Error("Replacement tenant/document mismatch.");
+    }
+    const rows = await this.prepare(documents);
+    await this.init();
+    return this.transaction(async (client) => {
+      await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [JSON.stringify([this.table, tenantId, documentId])]);
+      const previous = await client.query(`SELECT version FROM ${this.manifests} WHERE tenant_id=$1 AND document_id=$2`, [tenantId, documentId]);
+      if (previous.rows[0] && previous.rows[0].version >= version) return previous.rows[0].version === version ? "unchanged" : "superseded";
+      await client.query(`DELETE FROM ${this.table} WHERE tenant_id=$1 AND metadata @> $2::jsonb`, [tenantId, JSON.stringify({ documentId })]);
+      await this.write(client, rows);
+      await client.query(`INSERT INTO ${this.manifests} VALUES ($1,$2,$3)
+                ON CONFLICT (tenant_id,document_id) DO UPDATE SET version=EXCLUDED.version`, [tenantId, documentId, version]);
+      await this.bumpRevision(client, tenantId);
+      return "applied";
+    });
+  }
+  async getRevision(tenantId) {
+    this.tenant({ tenantId });
+    await this.init();
+    const result = await this.pool.query(`SELECT revision::text FROM ${this.epochs} WHERE tenant_id=$1`, [tenantId]);
+    return result.rows[0]?.revision ?? "0";
+  }
+  async prepare(documents) {
+    const rows = [];
+    for (const doc of documents) {
+      if (typeof doc.id !== "string" || !doc.id.trim() || typeof doc.content !== "string" || !doc.content.trim()) {
+        throw new Error("Document id and content must be non-empty strings.");
+      }
+      rows.push({ doc, tenant: this.tenant(doc.metadata), metadata: JSON.stringify(doc.metadata ?? {}), vector: this.vector(await this.embeddings.getEmbedding(doc.content)) });
+    }
+    return rows;
+  }
+  async write(client, rows) {
+    const changed = /* @__PURE__ */ new Set();
+    for (const { doc, tenant, metadata, vector } of rows) {
+      const result = await client.query(`INSERT INTO ${this.table} AS current (tenant_id,id,content,metadata,embedding)
+                VALUES ($1,$2,$3,$4::jsonb,$5::vector)
+                ON CONFLICT (tenant_id,id) DO UPDATE SET content=EXCLUDED.content,metadata=EXCLUDED.metadata,embedding=EXCLUDED.embedding,
+                    version=current.version+1,updated_at=now()
+                WHERE current.content IS DISTINCT FROM EXCLUDED.content OR current.metadata IS DISTINCT FROM EXCLUDED.metadata
+                   OR current.embedding IS DISTINCT FROM EXCLUDED.embedding`, [tenant, doc.id, doc.content, metadata, vector]);
+      if (result.rowCount) changed.add(tenant);
+    }
+    return changed;
+  }
+  async bumpRevision(client, tenant) {
+    await client.query(`INSERT INTO ${this.epochs} VALUES ($1,1) ON CONFLICT (tenant_id) DO UPDATE SET revision=${this.epochs}.revision+1`, [tenant]);
+  }
+  async search(query, topK = 3, filter) {
+    const tenant = this.tenant(filter);
+    this.validateTopK(topK);
+    const vector = this.vector(await this.embeddings.getEmbedding(query));
+    await this.init();
+    const result = await this.pool.query(
+      `SELECT id, content, metadata, version,
+                1 - (embedding <=> $3::vector) AS score FROM ${this.table}
+            WHERE tenant_id = $1 AND metadata @> $2::jsonb
+            ORDER BY embedding <=> $3::vector, id LIMIT $4`,
+      [tenant, JSON.stringify(filter), vector, topK]
+    );
+    return result.rows.map((row) => ({ ...row, score: Number(row.score) }));
+  }
+  /** PostgreSQL lexical search + dense search, fused with RRF. This is not BM25. */
+  async searchHybrid(query, topK = 3, filter) {
+    const tenant = this.tenant(filter);
+    this.validateTopK(topK);
+    const candidates = Math.min(1e3, Math.max(20, topK * 3));
+    const dense = await this.search(query, candidates, filter);
+    const sparse = await this.pool.query(`SELECT id, content, metadata, version,
+                ts_rank_cd(search_text, websearch_to_tsquery('simple', $3)) AS score FROM ${this.table}
+            WHERE tenant_id = $1 AND metadata @> $2::jsonb
+              AND search_text @@ websearch_to_tsquery('simple', $3)
+            ORDER BY score DESC, id LIMIT $4`, [tenant, JSON.stringify(filter), query, candidates]);
+    return reciprocalRankFusion(dense, sparse.rows, topK);
+  }
+  async deleteDocument(id, filter) {
+    const tenant = this.tenant(filter);
+    await this.init();
+    return this.transaction(async (client) => {
+      const result = await client.query(`DELETE FROM ${this.table} WHERE tenant_id=$1 AND id=$2 AND metadata @> $3::jsonb`, [tenant, id, JSON.stringify(filter)]);
+      if (result.rowCount) await this.bumpRevision(client, tenant);
+      return (result.rowCount ?? 0) > 0;
+    });
+  }
+  async clear(filter) {
+    const tenant = this.tenant(filter);
+    await this.init();
+    await this.transaction(async (client) => {
+      const result = await client.query(`DELETE FROM ${this.table} WHERE tenant_id=$1 AND metadata @> $2::jsonb`, [tenant, JSON.stringify(filter)]);
+      if (result.rowCount) await this.bumpRevision(client, tenant);
+    });
+  }
+  async close() {
+    if (!this.options.pool) await this.pool.end();
+  }
+  tenant(filter) {
+    const tenant = filter?.tenantId;
+    if (typeof tenant !== "string" || !tenant.trim()) throw new Error("An explicit tenantId is required for PostgreSQL vector operations.");
+    return tenant;
+  }
+  vector(embedding) {
+    const values = Array.from(embedding);
+    if (values.length !== this.dimensions || !values.every(Number.isFinite) || !values.some((value) => value !== 0)) {
+      throw new Error(`Embedding must contain ${this.dimensions} finite values and have non-zero magnitude.`);
+    }
+    return JSON.stringify(values);
+  }
+  validateTopK(topK) {
+    if (!Number.isInteger(topK) || topK < 1 || topK > 1e3) throw new Error("topK must be an integer between 1 and 1000.");
+  }
+  async transaction(work) {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      const result = await work(client);
+      await client.query("COMMIT");
+      return result;
+    } catch (error) {
+      await client.query("ROLLBACK").catch(() => {
+      });
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+};
+
+// src/core/queue/bullmq-queue.ts
+import { Queue, Worker, UnrecoverableError } from "bullmq";
+import { Redis } from "ioredis";
+var BullMQJobQueue = class {
+  constructor(options = {}) {
+    this.options = options;
+    this.name = options.queueName ?? "rag-ingestion-queue";
+    if (!/^[a-zA-Z0-9_-]+$/.test(this.name)) throw new Error("Invalid queueName.");
+    this.attempts = options.attempts ?? 3;
+    for (const [name, value] of Object.entries({ attempts: this.attempts, concurrency: options.concurrency ?? 2, globalConcurrency: options.globalConcurrency ?? 4 })) {
+      if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer.`);
+    }
+    this.producer = this.connection(false);
+    this.queue = new Queue(this.name, { connection: this.producer });
+    this.queue.on("error", (error) => this.report(error));
+  }
+  options;
+  queue;
+  producer;
+  consumer;
+  worker;
+  processingType;
+  closed = false;
+  name;
+  attempts;
+  connection(worker) {
+    const settings = { maxRetriesPerRequest: worker ? null : 1, enableOfflineQueue: worker, connectTimeout: 2e3 };
+    const client = this.options.redisUrl ? new Redis(this.options.redisUrl, settings) : new Redis({ host: this.options.connection?.host ?? "127.0.0.1", port: this.options.connection?.port ?? 6379, ...this.options.connection?.password !== void 0 ? { password: this.options.connection.password } : {}, ...settings });
+    client.on("error", (error) => this.report(error));
+    return client;
+  }
+  report(error) {
+    this.options.onError?.(error);
+  }
+  async ready() {
+    let timeout;
+    try {
+      await Promise.race([this.queue.waitUntilReady(), new Promise((_, reject) => {
+        timeout = setTimeout(() => reject(new Error("Redis queue readiness timed out.")), 5e3);
+      })]);
+      await this.queue.setGlobalConcurrency(this.options.globalConcurrency ?? 4);
+    } finally {
+      if (timeout) clearTimeout(timeout);
+    }
+  }
+  async add(type, data, jobId) {
+    if (this.closed) throw new Error("Queue is closed.");
+    const job = await this.queue.add(type, data, {
+      ...jobId ? { jobId } : {},
+      attempts: this.attempts,
+      backoff: { type: "exponential", delay: this.options.backoffMs ?? 1e3 },
+      removeOnComplete: { age: 7 * 86400, count: 1e4 },
+      removeOnFail: { age: 7 * 86400, count: 1e4 }
+    });
+    return this.map(job, "queued");
+  }
+  async getJob(id) {
+    const job = await this.queue.getJob(id);
+    if (!job) return null;
+    const state = await job.getState();
+    const status = state === "active" ? "processing" : state === "completed" ? "completed" : state === "failed" ? "failed" : "queued";
+    return this.map(job, status);
+  }
+  async process(type, handler) {
+    if (this.closed) throw new Error("Queue is closed.");
+    if (this.processingType && this.processingType !== type) throw new Error("Use one job type per queue.");
+    this.processingType = type;
+    if (!this.worker) {
+      this.consumer = this.connection(true);
+      this.worker = new Worker(this.name, async (job) => {
+        if (job.name !== type) throw new UnrecoverableError(`Unsupported job type: ${job.name}`);
+        return handler(this.map(job, "processing"), (progress) => job.updateProgress(progress));
+      }, {
+        connection: this.consumer,
+        concurrency: this.options.concurrency ?? 2,
+        lockDuration: this.options.lockDurationMs ?? 3e4,
+        stalledInterval: this.options.stalledIntervalMs ?? 3e4,
+        maxStalledCount: 2
+      });
+      this.worker.on("error", (error) => this.report(error));
+    }
+    await this.worker.waitUntilReady();
+  }
+  async retry(id) {
+    const job = await this.queue.getJob(id);
+    if (!job || await job.getState() !== "failed") throw new Error("Only a failed job can be retried.");
+    await job.retry("failed", { resetAttemptsMade: true });
+  }
+  async close() {
+    if (this.closed) return;
+    this.closed = true;
+    try {
+      await this.worker?.close();
+      if (this.producer.status !== "ready") this.producer.disconnect();
+      await this.queue.close();
+    } finally {
+      this.consumer?.disconnect();
+      this.producer.disconnect();
+    }
+  }
+  map(job, status) {
+    return {
+      id: job.id,
+      type: job.name,
+      data: job.data,
+      status,
+      progress: typeof job.progress === "number" ? job.progress : 0,
+      result: job.returnvalue,
+      error: job.failedReason,
+      createdAt: job.timestamp,
+      updatedAt: job.finishedOn ?? job.processedOn ?? job.timestamp,
+      completedAt: job.finishedOn,
+      attemptsMade: job.attemptsMade,
+      maxAttempts: job.opts.attempts ?? 1
+    };
+  }
+};
+
+// src/core/ingestion/postgres-ingestion-store.ts
+import { createHash } from "node:crypto";
+var IngestionQuotaError = class extends Error {
+};
+var IngestionConflictError = class extends Error {
+};
+var PostgresIngestionStore = class {
+  constructor(pool, tableName = "rag_ingestion") {
+    this.pool = pool;
+    if (!/^[a-z][a-z0-9_]{0,39}$/.test(tableName)) throw new Error("Invalid ingestion table name.");
+    this.tableName = tableName;
+    this.table = `"${tableName}"`;
+    this.heads = `"${tableName}_heads"`;
+  }
+  pool;
+  table;
+  heads;
+  tableName;
+  async migrate() {
+    await this.transaction(async (client) => {
+      await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", ["rag-ingestion-schema"]);
+      await client.query(`CREATE TABLE IF NOT EXISTS ${this.heads} (
+                tenant_id text NOT NULL, document_id text NOT NULL, version integer NOT NULL,
+                PRIMARY KEY (tenant_id, document_id)
+            )`);
+      await client.query(`CREATE TABLE IF NOT EXISTS ${this.table} (
+                id text PRIMARY KEY, tenant_id text NOT NULL, user_id text NOT NULL, document_id text NOT NULL,
+                filename text NOT NULL, version integer NOT NULL, digest text NOT NULL, settings jsonb NOT NULL,
+                content bytea, idempotency_key text, status text NOT NULL CHECK (status IN ('pending','queued','processing','completed','failed')),
+                result jsonb, error text, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now()
+            )`);
+      await client.query(`CREATE UNIQUE INDEX IF NOT EXISTS "${this.tableName}_idempotency_idx" ON ${this.table} (tenant_id, idempotency_key) WHERE idempotency_key IS NOT NULL`);
+      await client.query(`CREATE INDEX IF NOT EXISTS "${this.tableName}_pending_idx" ON ${this.table} (status, updated_at)`);
+      await client.query(`CREATE INDEX IF NOT EXISTS "${this.tableName}_tenant_idx" ON ${this.table} (tenant_id)`);
+    });
+  }
+  async ready() {
+    await this.pool.query(`SELECT id FROM ${this.table} LIMIT 0`);
+  }
+  async put(input, limits) {
+    const digest = createHash("sha256").update(input.content).digest("hex");
+    return this.transaction(async (client) => {
+      await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [`ingestion:${this.tableName}:${input.tenantId}`]);
+      const previous = input.idempotencyKey ? await client.query(`SELECT * FROM ${this.table} WHERE tenant_id = $1 AND idempotency_key = $2`, [input.tenantId, input.idempotencyKey]) : await client.query(`SELECT j.* FROM ${this.table} j JOIN ${this.heads} h
+                    ON j.tenant_id = h.tenant_id AND j.document_id = h.document_id AND j.version = h.version
+                    WHERE j.tenant_id = $1 AND j.document_id = $2`, [input.tenantId, input.documentId]);
+      const old = previous.rows[0];
+      if (old && old.digest === digest && old.document_id === input.documentId && old.filename === input.filename && old.settings.chunkSize === input.settings.chunkSize && old.settings.chunkOverlap === input.settings.chunkOverlap) return this.row(old);
+      if (old && input.idempotencyKey) throw new IngestionConflictError("Idempotency key already used for different content.");
+      const quota = await client.query(`SELECT count(*) FILTER (WHERE status IN ('pending','queued','processing')) AS pending,
+                coalesce(sum(octet_length(content)), 0) AS bytes FROM ${this.table} WHERE tenant_id = $1`, [input.tenantId]);
+      if (Number(quota.rows[0].pending) >= limits.maxPending || Number(quota.rows[0].bytes) + input.content.length > limits.maxStoredBytes) {
+        throw new IngestionQuotaError("Tenant ingestion quota exceeded.");
+      }
+      const head = await client.query(`INSERT INTO ${this.heads} AS h VALUES ($1, $2, 1)
+                ON CONFLICT (tenant_id, document_id) DO UPDATE SET version = h.version + 1 RETURNING version`, [input.tenantId, input.documentId]);
+      const id = createHash("sha256").update(JSON.stringify([input.tenantId, input.documentId, head.rows[0].version, digest])).digest("hex");
+      const saved = await client.query(
+        `INSERT INTO ${this.table}
+                (id,tenant_id,user_id,document_id,filename,version,digest,settings,content,idempotency_key,status)
+                VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb,$9,$10,'pending') RETURNING *`,
+        [id, input.tenantId, input.userId, input.documentId, input.filename, head.rows[0].version, digest, JSON.stringify(input.settings), input.content, input.idempotencyKey ?? null]
+      );
+      return this.row(saved.rows[0]);
+    });
+  }
+  async get(id, tenantId) {
+    if (!tenantId) return null;
+    const result = await this.pool.query(`SELECT * FROM ${this.table} WHERE id = $1 AND tenant_id = $2`, [id, tenantId]);
+    return result.rows[0] ? this.row(result.rows[0]) : null;
+  }
+  async dispatchable(limit = 100) {
+    const result = await this.pool.query(`SELECT id, tenant_id FROM ${this.table} WHERE status IN ('pending','queued','processing')
+            ORDER BY (status = 'pending') DESC, updated_at LIMIT $1`, [limit]);
+    return result.rows.map((row) => ({ id: row.id, tenantId: row.tenant_id }));
+  }
+  async mark(id, status, error) {
+    const condition = status === "queued" ? "AND status = 'pending'" : "AND status NOT IN ('completed','failed')";
+    await this.pool.query(`UPDATE ${this.table} SET status = $2, error = $3, updated_at = now() WHERE id = $1 ${condition}`, [id, status, error ?? null]);
+  }
+  async complete(id, result) {
+    await this.pool.query(`UPDATE ${this.table} SET status = 'completed', result = $2::jsonb, error = NULL,
+            content = NULL, updated_at = now() WHERE id = $1`, [id, JSON.stringify(result)]);
+  }
+  async resetFailed(id, tenantId) {
+    const result = await this.pool.query(`UPDATE ${this.table} SET status = 'pending', error = NULL, updated_at = now()
+            WHERE id = $1 AND tenant_id = $2 AND status = 'failed' AND content IS NOT NULL`, [id, tenantId]);
+    if (!result.rowCount) throw new Error("Failed job missing or its retry payload has expired.");
+  }
+  async expireFailedBlobs(retentionDays = 7) {
+    await this.pool.query(`UPDATE ${this.table} SET content = NULL WHERE status = 'failed' AND content IS NOT NULL
+            AND updated_at < now() - ($1 * interval '1 day')`, [retentionDays]);
+  }
+  row(row) {
+    return {
+      id: row.id,
+      tenantId: row.tenant_id,
+      userId: row.user_id,
+      documentId: row.document_id,
+      filename: row.filename,
+      version: row.version,
+      digest: row.digest,
+      settings: row.settings,
+      content: row.content,
+      status: row.status,
+      result: row.result,
+      error: row.error,
+      createdAt: new Date(row.created_at).getTime(),
+      updatedAt: new Date(row.updated_at).getTime()
+    };
+  }
+  async transaction(work) {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      const result = await work(client);
+      await client.query("COMMIT");
+      return result;
+    } catch (error) {
+      await client.query("ROLLBACK").catch(() => {
+      });
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+};
+
+// src/services/document-ingestion.service.ts
+import { randomUUID as randomUUID2 } from "node:crypto";
+import { PassThrough, pipeline as streamPipeline } from "node:stream";
+import { promisify } from "node:util";
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+
+// src/core/security/pii-guard.ts
+var PIIGuard = class {
+  /**
+   * Validates Turkish Republic Identity Number (TCKN) using official 11-digit checksum rules:
+   * 1. 11 digits, cannot start with 0.
+   * 2. 10th digit = ((d1+d3+d5+d7+d9)*7 - (d2+d4+d6+d8)) mod 10
+   * 3. 11th digit = (d1+d2+...+d10) mod 10
+   */
+  static isValidTCKN(candidate) {
+    if (!/^[1-9]\d{10}$/.test(candidate)) {
+      return false;
+    }
+    const digits = candidate.split("").map(Number);
+    const d = digits;
+    const oddSum = d[0] + d[2] + d[4] + d[6] + d[8];
+    const evenSum = d[1] + d[3] + d[5] + d[7];
+    const tenthDigit = ((oddSum * 7 - evenSum) % 10 + 10) % 10;
+    if (d[9] !== tenthDigit) {
+      return false;
+    }
+    const sumFirstTen = digits.slice(0, 10).reduce((acc, curr) => acc + curr, 0);
+    const eleventhDigit = sumFirstTen % 10;
+    if (d[10] !== eleventhDigit) {
+      return false;
+    }
+    return true;
+  }
+  /**
+   * Validates Credit Card number using the Luhn mod-10 algorithm.
+   */
+  static isValidCreditCard(candidate) {
+    const clean = candidate.replace(/[\s-]/g, "");
+    if (!/^\d{13,19}$/.test(clean)) {
+      return false;
+    }
+    let sum = 0;
+    let shouldDouble = false;
+    for (let i = clean.length - 1; i >= 0; i--) {
+      let digit = parseInt(clean.charAt(i), 10);
+      if (shouldDouble) {
+        digit *= 2;
+        if (digit > 9) digit -= 9;
+      }
+      sum += digit;
+      shouldDouble = !shouldDouble;
+    }
+    return sum % 10 === 0;
+  }
+  /**
+   * Sanitizes input text, detecting and masking all sensitive PII entities.
+   * Respects data sovereignty policies: strictly masks for Cloud LLMs.
+   */
+  static sanitize(text, policy) {
+    if (!text || policy?.enabled === false) {
+      return {
+        sanitizedText: text || "",
+        hasPII: false,
+        detectedEntities: [],
+        bypassedForLocalLLM: false
+      };
+    }
+    const detectedEntities = [];
+    let workingText = text;
+    const maskTCKN = policy?.maskTCKN !== false;
+    const maskCards = policy?.maskCreditCards !== false;
+    const maskIBAN = policy?.maskIBAN !== false;
+    const maskEmails = policy?.maskEmails !== false;
+    const maskPhones = policy?.maskPhones !== false;
+    if (maskTCKN) {
+      const tcknRegex = /(?:\b|[^\d])([1-9]\d{10})(?:\b|[^\d])/g;
+      let match;
+      const tcknCandidates = /* @__PURE__ */ new Set();
+      while ((match = tcknRegex.exec(workingText)) !== null) {
+        const candidate = match[1];
+        if (this.isValidTCKN(candidate)) {
+          tcknCandidates.add(candidate);
+        }
+      }
+      for (const tckn of tcknCandidates) {
+        const masked = `[TCKN_${tckn.slice(0, 3)}*****${tckn.slice(9)}]`;
+        detectedEntities.push({
+          type: "TCKN",
+          originalValue: tckn,
+          maskedValue: masked
+        });
+        workingText = workingText.split(tckn).join(masked);
+      }
+    }
+    if (maskCards) {
+      const ccRegex = /\b(?:\d{4}[ -]?){3}\d{1,4}\b|\b\d{13,19}\b/g;
+      let match;
+      const cardCandidates = /* @__PURE__ */ new Set();
+      while ((match = ccRegex.exec(workingText)) !== null) {
+        const raw = match[0];
+        if (this.isValidCreditCard(raw)) {
+          cardCandidates.add(raw);
+        }
+      }
+      for (const card of cardCandidates) {
+        const clean = card.replace(/[\s-]/g, "");
+        const masked = `[CARD_****_****_****_${clean.slice(-4)}]`;
+        detectedEntities.push({
+          type: "CREDIT_CARD",
+          originalValue: card,
+          maskedValue: masked
+        });
+        workingText = workingText.split(card).join(masked);
+      }
+    }
+    if (maskIBAN) {
+      const ibanRegex = /\bTR\d{2}[\s]?\d{4}[\s]?\d{4}[\s]?\d{4}[\s]?\d{4}[\s]?\d{4}[\s]?\d{2}\b|\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b/g;
+      let match;
+      const ibans = /* @__PURE__ */ new Set();
+      while ((match = ibanRegex.exec(workingText)) !== null) {
+        ibans.add(match[0]);
+      }
+      for (const iban of ibans) {
+        const clean = iban.replace(/\s/g, "");
+        const masked = `[IBAN_${clean.slice(0, 4)}...${clean.slice(-4)}]`;
+        detectedEntities.push({
+          type: "IBAN",
+          originalValue: iban,
+          maskedValue: masked
+        });
+        workingText = workingText.split(iban).join(masked);
+      }
+    }
+    if (maskEmails) {
+      const emailRegex = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b/g;
+      let match;
+      const emails = /* @__PURE__ */ new Set();
+      while ((match = emailRegex.exec(workingText)) !== null) {
+        emails.add(match[0]);
+      }
+      for (const email of emails) {
+        const [user, domain] = email.split("@");
+        const masked = `[EMAIL_${user.slice(0, 2)}***@${domain}]`;
+        detectedEntities.push({
+          type: "EMAIL",
+          originalValue: email,
+          maskedValue: masked
+        });
+        workingText = workingText.split(email).join(masked);
+      }
+    }
+    if (maskPhones) {
+      const phoneRegex = /(?:\+?90\s?|0)?5\d{2}[\s.-]?\d{3}[\s.-]?\d{2}[\s.-]?\d{2}\b/g;
+      let match;
+      const phones = /* @__PURE__ */ new Set();
+      while ((match = phoneRegex.exec(workingText)) !== null) {
+        phones.add(match[0]);
+      }
+      for (const phone of phones) {
+        const clean = phone.replace(/[\s.-]/g, "");
+        const masked = `[PHONE_***${clean.slice(-4)}]`;
+        detectedEntities.push({
+          type: "PHONE",
+          originalValue: phone,
+          maskedValue: masked
+        });
+        workingText = workingText.split(phone).join(masked);
+      }
+    }
+    const hasPII = detectedEntities.length > 0;
+    const isLocalProvider = policy?.providerType === "local" || policy?.providerType === "custom";
+    const bypassedForLocalLLM = Boolean(hasPII && isLocalProvider && policy?.allowUnmaskedForLocalLLM);
+    return {
+      sanitizedText: bypassedForLocalLLM ? text : workingText,
+      hasPII,
+      detectedEntities,
+      bypassedForLocalLLM
+    };
+  }
+};
+
+// src/core/queue/in-memory-queue.ts
+import { randomUUID } from "node:crypto";
+var InMemoryJobQueue = class {
+  jobs = /* @__PURE__ */ new Map();
+  handlers = /* @__PURE__ */ new Map();
+  pendingQueue = [];
+  activeJobsCount = 0;
+  isClosed = false;
+  maxPendingJobs;
+  drainWaiters = [];
+  concurrency;
+  maxRetainedJobs;
+  constructor(options) {
+    this.maxPendingJobs = options?.maxPendingJobs ?? 1e3;
+    this.concurrency = options?.concurrency ?? 2;
+    this.maxRetainedJobs = options?.maxRetainedJobs ?? 500;
+    for (const value of [this.concurrency, this.maxRetainedJobs, this.maxPendingJobs]) {
+      if (!Number.isInteger(value) || value < 1) throw new Error("Queue limits must be positive integers.");
+    }
+  }
+  async add(type, data, customJobId) {
+    if (this.isClosed) {
+      throw new Error("[InMemoryJobQueue] Queue is closed. Cannot add new jobs.");
+    }
+    const id = customJobId || `job_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
+    const existing = this.jobs.get(id);
+    if (existing) return { ...existing };
+    if (this.pendingQueue.length + this.activeJobsCount >= this.maxPendingJobs) throw new Error("Pending job limit exceeded.");
+    const now = Date.now();
+    const job = {
+      id,
+      type,
+      data,
+      status: "queued",
+      progress: 0,
+      createdAt: now,
+      updatedAt: now
+    };
+    this.jobs.set(id, job);
+    this.pendingQueue.push(id);
+    this.pruneOldJobs();
+    setImmediate(() => this.dispatch());
+    return { ...job };
+  }
+  async getJob(id) {
+    const job = this.jobs.get(id);
+    if (!job) return null;
+    return { ...job };
+  }
+  process(type, handler) {
+    this.handlers.set(type, handler);
+    setImmediate(() => this.dispatch());
+  }
+  async close() {
+    this.isClosed = true;
+    this.pendingQueue = [];
+    if (this.activeJobsCount > 0) await new Promise((resolve) => this.drainWaiters.push(resolve));
+  }
+  async dispatch() {
+    if (this.isClosed || this.activeJobsCount >= this.concurrency) return;
+    if (this.pendingQueue.length === 0) return;
+    const nextJobId = this.pendingQueue.shift();
+    if (!nextJobId) return;
+    const job = this.jobs.get(nextJobId);
+    if (!job || job.status !== "queued") {
+      this.dispatch();
+      return;
+    }
+    const handler = this.handlers.get(job.type);
+    if (!handler) {
+      this.pendingQueue.unshift(nextJobId);
+      return;
+    }
+    this.activeJobsCount++;
+    job.status = "processing";
+    job.updatedAt = Date.now();
+    const updateProgress = (progress) => {
+      job.progress = Math.max(0, Math.min(100, progress));
+      job.updatedAt = Date.now();
+    };
+    try {
+      const result = await handler(job, updateProgress);
+      job.status = "completed";
+      job.progress = 100;
+      job.result = result;
+      job.completedAt = Date.now();
+      job.updatedAt = Date.now();
+    } catch (error) {
+      job.status = "failed";
+      job.error = error?.message || String(error);
+      job.completedAt = Date.now();
+      job.updatedAt = Date.now();
+    } finally {
+      this.activeJobsCount--;
+      if (this.activeJobsCount === 0) {
+        for (const resolve of this.drainWaiters) resolve();
+        this.drainWaiters = [];
+      }
+      this.pruneOldJobs();
+      setImmediate(() => this.dispatch());
+    }
+  }
+  pruneOldJobs() {
+    if (this.jobs.size <= this.maxRetainedJobs) return;
+    const terminal = [...this.jobs.values()].filter((job) => job.status === "completed" || job.status === "failed");
+    for (const job of terminal.slice(0, Math.max(0, terminal.length - this.maxRetainedJobs))) this.jobs.delete(job.id);
+  }
+};
+
+// src/services/document-ingestion.service.ts
+var pipeline = promisify(streamPipeline);
+var DANGEROUS_EXTENSIONS = /* @__PURE__ */ new Set([
+  ".exe",
+  ".dll",
+  ".so",
+  ".dylib",
+  ".bin",
+  ".sh",
+  ".bat",
+  ".cmd",
+  ".ps1",
+  ".vbs",
+  ".js",
+  ".mjs",
+  ".cjs",
+  ".py",
+  ".rb",
+  ".php",
+  ".msi",
+  ".com",
+  ".scr",
+  ".pif",
+  ".application",
+  ".gadget",
+  ".docm",
+  ".xlsm",
+  ".pptm"
+]);
+var DocumentIngestionService = class _DocumentIngestionService {
+  workerReady = Promise.resolve();
+  backend;
+  vectorStore;
+  smlEngine;
+  jobQueue;
+  chunkSize;
+  chunkOverlap;
+  piiPolicy;
+  constructor(config) {
+    this.vectorStore = config?.vectorStore;
+    this.smlEngine = config?.smlEngine;
+    this.jobQueue = config?.jobQueue || new InMemoryJobQueue();
+    this.chunkSize = config?.chunkSize ?? 500;
+    this.chunkOverlap = config?.chunkOverlap ?? 50;
+    this.piiPolicy = config?.piiPolicy;
+    this.backend = config?.backend;
+    if (!this.backend) this.initWorker();
+  }
+  /**
+   * Inspects magic bytes and file extension to block dangerous executables and scripts.
+   */
+  static validateFileSafety(filename, buffer) {
+    const extMatch = filename.toLowerCase().match(/\.[a-z0-9]+$/);
+    const ext = extMatch ? extMatch[0] : "";
+    if (DANGEROUS_EXTENSIONS.has(ext)) {
+      return {
+        safe: false,
+        reason: `Dangerous file extension detected: ${ext}. Executables, scripts, and macro files are strictly rejected.`
+      };
+    }
+    if (buffer.length >= 2) {
+      if (buffer[0] === 77 && buffer[1] === 90) {
+        return { safe: false, reason: "Malicious file detected: Windows/DOS executable header (MZ)." };
+      }
+      if (buffer[0] === 35 && buffer[1] === 33) {
+        return { safe: false, reason: "Script file detected: Unix shebang (#!) executable header." };
+      }
+    }
+    if (buffer.length >= 4) {
+      if (buffer[0] === 127 && buffer[1] === 69 && buffer[2] === 76 && buffer[3] === 70) {
+        return { safe: false, reason: "Malicious file detected: Linux ELF binary executable header." };
+      }
+      if (buffer[0] === 202 && buffer[1] === 254 && buffer[2] === 186 && buffer[3] === 190) {
+        return { safe: false, reason: "Binary file detected: Java class or Mach-O binary header." };
+      }
+      if (buffer[0] === 254 && buffer[1] === 237 && buffer[2] === 250 && (buffer[3] === 206 || buffer[3] === 207) || buffer[0] === 207 && buffer[1] === 250 && buffer[2] === 237 && buffer[3] === 254) {
+        return { safe: false, reason: "Malicious file detected: macOS Mach-O binary executable header." };
+      }
+      if (buffer[0] === 0 && buffer[1] === 97 && buffer[2] === 115 && buffer[3] === 109) {
+        return { safe: false, reason: "Binary file detected: WebAssembly binary header." };
+      }
+    }
+    return { safe: true };
+  }
+  /**
+   * Inspects magic bytes from an active stream non-destructively by peeking the initial 4KB chunk.
+   * Prevents loading multi-gigabyte files into memory just to check safety headers.
+   */
+  static async validateStreamSafety(filename, stream) {
+    return new Promise((resolve, reject) => {
+      let resolved = false;
+      const readable = stream;
+      const onData = (chunk) => {
+        if (resolved) return;
+        resolved = true;
+        const bufferChunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        const check = _DocumentIngestionService.validateFileSafety(filename, bufferChunk);
+        readable.removeListener("data", onData);
+        readable.removeListener("error", onError);
+        readable.pause();
+        if (!check.safe) {
+          if (typeof readable.destroy === "function") readable.destroy();
+          return resolve({ safe: false, reason: check.reason, safeStream: readable });
+        }
+        if (typeof readable.unshift === "function") {
+          readable.unshift(bufferChunk);
+          resolve({ safe: true, safeStream: readable });
+        } else {
+          const passThrough = new PassThrough();
+          passThrough.write(bufferChunk);
+          readable.pipe(passThrough);
+          resolve({ safe: true, safeStream: passThrough });
+        }
+      };
+      const onError = (err) => {
+        if (resolved) return;
+        resolved = true;
+        reject(err);
+      };
+      readable.once("data", onData);
+      readable.once("error", onError);
+      readable.once("end", () => {
+        if (!resolved) {
+          resolved = true;
+          resolve({ safe: true, safeStream: readable });
+        }
+      });
+    });
+  }
+  /**
+   * Enterprise Lexical Document Text Extractor.
+   * Decodes and normalizes textual content across PDF streams, Markdown, CSV, JSON, and plaintext files.
+   * 
+   * PDF Specification Compliance:
+   * - Parses text execution state matrices within BT (Begin Text) ... ET (End Text) blocks.
+   * - Evaluates 'Tj', '\'', and '"' text-showing operators with PDF octal and hexadecimal escape decoding.
+   * - Parses 'TJ' array kerning sequences, preserving natural word spacing between glyph segments.
+   * - Evaluates positional newline operators ('T*', 'Td', 'TD', 'Tm') to maintain structural paragraph integrity.
+   * - Performs binary artifact suppression and multi-lingual UTF-8 normalization (Turkish, European, and CJK diacritics).
+   */
+  static extractText(filename, buffer) {
+    const rawString = buffer.toString("utf-8");
+    if (buffer.length >= 4 && buffer.slice(0, 4).toString() === "%PDF") {
+      return _DocumentIngestionService.extractPdfText(rawString);
+    }
+    return rawString.trim();
+  }
+  /**
+   * Enterprise High-Efficiency PDF Lexical Extractor.
+   * Parses PDF content streams without heavy external C++/WASM dependencies.
+   */
+  static extractPdfText(rawContent) {
+    const extractedSegments = [];
+    const decodePdfLiteral = (encoded) => {
+      return encoded.replace(/\\([0-7]{1,3})/g, (_, oct) => String.fromCharCode(parseInt(oct, 8))).replace(/\\n/g, "\n").replace(/\\r/g, "\r").replace(/\\t/g, "	").replace(/\\b/g, "\b").replace(/\\f/g, "\f").replace(/\\\(/g, "(").replace(/\\\)/g, ")").replace(/\\\\/g, "\\");
+    };
+    const decodePdfHex = (hex) => {
+      try {
+        const cleanHex = hex.replace(/\s+/g, "");
+        return Buffer.from(cleanHex, "hex").toString("utf-8");
+      } catch {
+        return "";
+      }
+    };
+    const btEtRegex = /BT[\s\S]*?ET/g;
+    let btMatch;
+    while ((btMatch = btEtRegex.exec(rawContent)) !== null) {
+      const blockContent = btMatch[0];
+      const tjArrayRegex = /\[([\s\S]*?)\]\s*TJ/g;
+      let tjArrMatch;
+      while ((tjArrMatch = tjArrayRegex.exec(blockContent)) !== null) {
+        const arrayBody = tjArrMatch[1];
+        const elementRegex = /\((.*?)\)|<([0-9a-fA-F]+)>|(-?\d+(?:\.\d+)?)/g;
+        let elem;
+        const lineTokens = [];
+        while ((elem = elementRegex.exec(arrayBody)) !== null) {
+          if (elem[1] !== void 0) {
+            lineTokens.push(decodePdfLiteral(elem[1]));
+          } else if (elem[2] !== void 0) {
+            lineTokens.push(decodePdfHex(elem[2]));
+          } else if (elem[3] !== void 0) {
+            const kerning = parseFloat(elem[3]);
+            if (kerning < -100) {
+              lineTokens.push(" ");
+            }
+          }
+        }
+        if (lineTokens.length > 0) {
+          extractedSegments.push(lineTokens.join("").trim());
+        }
+      }
+      const tjSingleRegex = /\((.*?)\)\s*Tj/g;
+      let singleMatch;
+      while ((singleMatch = tjSingleRegex.exec(blockContent)) !== null) {
+        const text = decodePdfLiteral(singleMatch[1]).trim();
+        if (text.length > 0) {
+          extractedSegments.push(text);
+        }
+      }
+      const tjHexRegex = /<([0-9a-fA-F]+)>\s*Tj/g;
+      let hexMatch;
+      while ((hexMatch = tjHexRegex.exec(blockContent)) !== null) {
+        const text = decodePdfHex(hexMatch[1]).trim();
+        if (text.length > 0) {
+          extractedSegments.push(text);
+        }
+      }
+    }
+    if (extractedSegments.length > 0) {
+      return extractedSegments.join(" ").replace(/\s+/g, " ").trim();
+    }
+    const cleaned = rawContent.replace(/%PDF-[^\n]+/g, "").replace(/stream[\s\S]*?endstream/g, (streamObj) => {
+      const innerMatch = streamObj.match(/\((.*?)\)/g);
+      return innerMatch ? innerMatch.map((s) => decodePdfLiteral(s.slice(1, -1))).join(" ") : "";
+    }).replace(/obj[\s\S]*?endobj/g, (obj) => {
+      const innerMatch = obj.match(/\((.*?)\)/g);
+      return innerMatch ? innerMatch.map((s) => decodePdfLiteral(s.slice(1, -1))).join(" ") : "";
+    }).replace(/[^a-zA-Z0-9çÇğĞıİöÖşŞüÜäÄöÖüÜß\s.,!?:;@/\-()]/g, " ").replace(/\s+/g, " ").trim();
+    return cleaned || "PDF document with no extractable plain text stream.";
+  }
+  static chunkText(text, chunkSize = 500, overlap = 50) {
+    if (!text || text.length <= chunkSize) {
+      return text ? [text] : [];
+    }
+    const chunks = [];
+    let startIndex = 0;
+    while (startIndex < text.length) {
+      let endIndex = Math.min(startIndex + chunkSize, text.length);
+      if (endIndex < text.length) {
+        const searchWindow = text.slice(Math.max(startIndex, endIndex - 60), endIndex);
+        const lastBreak = Math.max(
+          searchWindow.lastIndexOf("\n\n"),
+          searchWindow.lastIndexOf("\n"),
+          searchWindow.lastIndexOf(". "),
+          searchWindow.lastIndexOf(" ")
+        );
+        if (lastBreak !== -1) {
+          endIndex = Math.max(startIndex, endIndex - 60) + lastBreak + 1;
+        }
+      }
+      const chunk = text.slice(startIndex, endIndex).trim();
+      if (chunk.length > 0) {
+        chunks.push(chunk);
+      }
+      startIndex = Math.max(startIndex + 1, endIndex - overlap);
+    }
+    return chunks;
+  }
+  /**
+   * Streaming chunk generator.
+   * Ingests chunks of text or buffers incrementally with natural backpressure,
+   * maintaining a constant O(1) memory footprint regardless of file size.
+   */
+  static async *chunkStream(textStream, chunkSize = 500, overlap = 50) {
+    let window = "";
+    for await (const rawChunk of textStream) {
+      const chunkStr = Buffer.isBuffer(rawChunk) ? rawChunk.toString("utf-8") : String(rawChunk);
+      window += chunkStr;
+      while (window.length >= chunkSize) {
+        let endIndex = chunkSize;
+        const searchWindow = window.slice(Math.max(0, endIndex - 60), endIndex);
+        const lastBreak = Math.max(
+          searchWindow.lastIndexOf("\n\n"),
+          searchWindow.lastIndexOf("\n"),
+          searchWindow.lastIndexOf(". "),
+          searchWindow.lastIndexOf(" ")
+        );
+        if (lastBreak !== -1) {
+          endIndex = Math.max(0, endIndex - 60) + lastBreak + 1;
+        }
+        const emitted = window.slice(0, endIndex).trim();
+        if (emitted.length > 0) {
+          yield emitted;
+        }
+        window = window.slice(Math.max(1, endIndex - overlap));
+      }
+    }
+    const remaining = window.trim();
+    if (remaining.length > 0) {
+      yield remaining;
+    }
+  }
+  /**
+   * Directly streams and chunks large documents without intermediate memory bloat.
+   * Sanitizes PII and inserts documents into the vector store with automatic backpressure.
+   */
+  async ingestStreamDirect(filename, stream, options) {
+    const safety = await _DocumentIngestionService.validateStreamSafety(filename, stream);
+    if (!safety.safe) {
+      throw new Error(`[SecurityException] Document stream rejected: ${safety.reason}`);
+    }
+    const safeStream = safety.safeStream;
+    let chunkIndex = 0;
+    let totalPIIMasked = 0;
+    const documentIds = [];
+    for await (const rawChunk of _DocumentIngestionService.chunkStream(safeStream, this.chunkSize, this.chunkOverlap)) {
+      const piiResult = PIIGuard.sanitize(rawChunk, this.piiPolicy);
+      totalPIIMasked += piiResult.detectedEntities.length;
+      const docId = `${filename}_chunk_${chunkIndex}_${randomUUID2().slice(0, 6)}`;
+      const doc = {
+        id: docId,
+        content: piiResult.sanitizedText,
+        metadata: {
+          filename,
+          chunkIndex,
+          tenantId: options?.tenantId,
+          hadPII: piiResult.hasPII,
+          maskedCount: piiResult.detectedEntities.length
+        }
+      };
+      documentIds.push(docId);
+      if (this.vectorStore && typeof this.vectorStore.addDocuments === "function") {
+        await this.vectorStore.addDocuments([doc]);
+      }
+      chunkIndex++;
+      if (options?.onProgress) {
+        options.onProgress(chunkIndex);
+      }
+    }
+    return {
+      filename,
+      chunksCount: chunkIndex,
+      piiEntitiesMasked: totalPIIMasked,
+      documentIds,
+      processedAt: (/* @__PURE__ */ new Date()).toISOString()
+    };
+  }
+  /**
+   * Submits a stream for asynchronous background queue processing.
+   * Uses backpressure-regulated disk spooling to temporary storage, preventing V8 heap exhaustion.
+   */
+  async submitStream(filename, stream, options) {
+    if (this.backend) return this.backend.submitDocument({ filename, content: stream, tenantId: options?.tenantId, userId: options?.userId });
+    await this.workerReady;
+    const safety = await _DocumentIngestionService.validateStreamSafety(filename, stream);
+    if (!safety.safe) {
+      throw new Error(`[SecurityException] Document rejected: ${safety.reason}`);
+    }
+    const safeStream = safety.safeStream;
+    const tempFilePath = path.join(os.tmpdir(), `rag_spool_${randomUUID2().replace(/-/g, "")}_${path.basename(filename)}`);
+    await pipeline(safeStream, fs.createWriteStream(tempFilePath));
+    const jobId = `stream_${randomUUID2().replace(/-/g, "").slice(0, 16)}`;
+    const job = await this.jobQueue.add("document-ingest", {
+      filename,
+      tempFilePath,
+      isStreamSpool: true,
+      mimeType: options?.mimeType,
+      tenantId: options?.tenantId,
+      userId: options?.userId
+    }, jobId);
+    return {
+      jobId: job.id,
+      status: job.status
+    };
+  }
+  /**
+   * Submits a document for background asynchronous ingestion.
+   * Supports String, Buffer, or ReadableStream payloads.
+   */
+  async submitDocument(payload) {
+    if (this.backend) return this.backend.submitDocument(payload);
+    await this.workerReady;
+    if (payload.content && typeof payload.content.pipe === "function") {
+      return this.submitStream(payload.filename, payload.content, {
+        mimeType: payload.mimeType,
+        userId: payload.userId,
+        tenantId: payload.tenantId
+      });
+    }
+    const buffer = this.toBuffer(payload.content, payload.encoding);
+    const safetyCheck = _DocumentIngestionService.validateFileSafety(payload.filename, buffer);
+    if (!safetyCheck.safe) {
+      throw new Error(`[SecurityException] Document rejected: ${safetyCheck.reason}`);
+    }
+    const jobId = `doc_${randomUUID2().replace(/-/g, "").slice(0, 16)}`;
+    const job = await this.jobQueue.add("document-ingest", {
+      filename: payload.filename,
+      contentBase64: buffer.toString("base64"),
+      mimeType: payload.mimeType,
+      userId: payload.userId,
+      tenantId: payload.tenantId
+    }, jobId);
+    return {
+      jobId: job.id,
+      status: job.status
+    };
+  }
+  /**
+   * Retrieves the status and progress of an ingestion job.
+   */
+  async getJobStatus(jobId, tenantId) {
+    if (this.backend) return this.backend.getJobStatus(jobId, tenantId);
+    const job = await this.jobQueue.getJob(jobId);
+    if (tenantId !== void 0 && job?.data?.tenantId !== tenantId) return null;
+    return job;
+  }
+  async retryJob(jobId, tenantId) {
+    if (!this.backend?.retryJob) throw new Error("Job retry requires a durable ingestion backend.");
+    await this.backend.retryJob(jobId, tenantId);
+  }
+  initWorker() {
+    this.workerReady = Promise.resolve(this.jobQueue.process("document-ingest", async (job, updateProgress) => {
+      const data = job.data;
+      if (data.isStreamSpool && data.tempFilePath) {
+        try {
+          updateProgress(20);
+          const fileReadStream = fs.createReadStream(data.tempFilePath);
+          const result2 = await this.ingestStreamDirect(data.filename, fileReadStream, {
+            tenantId: data.tenantId,
+            onProgress: (count) => {
+              updateProgress(Math.min(90, 20 + count * 2));
+            }
+          });
+          updateProgress(100);
+          return result2;
+        } finally {
+          await fsp.unlink(data.tempFilePath).catch(() => {
+          });
+        }
+      }
+      const { filename, contentBase64, tenantId } = data;
+      const buffer = Buffer.from(contentBase64, "base64");
+      updateProgress(10);
+      const safety = _DocumentIngestionService.validateFileSafety(filename, buffer);
+      if (!safety.safe) {
+        throw new Error(safety.reason);
+      }
+      updateProgress(30);
+      const rawText = _DocumentIngestionService.extractText(filename, buffer);
+      updateProgress(50);
+      const rawChunks = _DocumentIngestionService.chunkText(rawText, this.chunkSize, this.chunkOverlap);
+      updateProgress(70);
+      let totalPIIMasked = 0;
+      const sanitizedDocs = [];
+      for (let i = 0; i < rawChunks.length; i++) {
+        const chunk = rawChunks[i];
+        const piiResult = PIIGuard.sanitize(chunk, this.piiPolicy);
+        totalPIIMasked += piiResult.detectedEntities.length;
+        sanitizedDocs.push({
+          id: `${filename}_chunk_${i}_${randomUUID2().slice(0, 6)}`,
+          content: piiResult.sanitizedText,
+          metadata: {
+            filename,
+            chunkIndex: i,
+            totalChunks: rawChunks.length,
+            tenantId,
+            hadPII: piiResult.hasPII,
+            maskedCount: piiResult.detectedEntities.length
+          }
+        });
+      }
+      updateProgress(90);
+      if (this.vectorStore && typeof this.vectorStore.addDocuments === "function") {
+        await this.vectorStore.addDocuments(sanitizedDocs);
+      }
+      updateProgress(100);
+      const result = {
+        filename,
+        chunksCount: sanitizedDocs.length,
+        piiEntitiesMasked: totalPIIMasked,
+        documentIds: sanitizedDocs.map((d) => d.id),
+        processedAt: (/* @__PURE__ */ new Date()).toISOString()
+      };
+      return result;
+    }));
+    void this.workerReady.catch(() => {
+    });
+  }
+  toBuffer(content, encoding) {
+    if (encoding !== void 0 && encoding !== "utf8" && encoding !== "base64") throw new Error("Invalid encoding.");
+    if (Buffer.isBuffer(content)) return content;
+    if (encoding === "utf8") return Buffer.from(content, "utf8");
+    if (encoding === "base64") {
+      if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(content)) throw new Error("Invalid base64 content.");
+      return Buffer.from(content, "base64");
+    }
+    const base64Regex = /^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$/;
+    if (content.length > 50 && base64Regex.test(content) && !content.includes(" ")) {
+      try {
+        return Buffer.from(content, "base64");
+      } catch {
+        return Buffer.from(content, "utf-8");
+      }
+    }
+    return Buffer.from(content, "utf-8");
+  }
+};
+
+// src/core/ingestion/queued-ingestion.service.ts
+var QueuedIngestionService = class {
+  constructor(store, queue, options = {}) {
+    this.store = store;
+    this.queue = queue;
+    this.options = options;
+    this.maxBytes = options.maxUploadBytes ?? 5 * 1024 * 1024;
+    this.settings = { chunkSize: options.chunkSize ?? 500, chunkOverlap: options.chunkOverlap ?? 50 };
+    if (!Number.isInteger(this.settings.chunkSize) || this.settings.chunkSize < 32 || this.settings.chunkSize > 1e4 || !Number.isInteger(this.settings.chunkOverlap) || this.settings.chunkOverlap < 0 || this.settings.chunkOverlap >= this.settings.chunkSize) {
+      throw new Error("Invalid ingestion chunk size/overlap.");
+    }
+    for (const value of [this.maxBytes, options.maxPendingPerTenant ?? 100, options.maxStoredBytesPerTenant ?? 50 * 1024 * 1024]) {
+      if (!Number.isSafeInteger(value) || value < 1) throw new Error("Ingestion limits must be positive integers.");
+    }
+  }
+  store;
+  queue;
+  options;
+  settings;
+  maxBytes;
+  async submitDocument(payload) {
+    const { tenantId, userId, filename } = payload;
+    if (!tenantId?.trim() || !userId?.trim()) throw new Error("Trusted tenantId and userId are required.");
+    if (typeof filename !== "string" || !filename.trim() || filename.length > 255 || /[/\\\0]/.test(filename)) throw new Error("Invalid filename.");
+    const documentId = payload.documentId ?? filename;
+    if (typeof documentId !== "string" || !documentId.trim() || documentId.length > 255) throw new Error("Invalid documentId.");
+    if (payload.idempotencyKey !== void 0 && (!payload.idempotencyKey.trim() || payload.idempotencyKey.length > 200)) throw new Error("Invalid idempotency key.");
+    const content = await this.bytes(payload);
+    const safety = DocumentIngestionService.validateFileSafety(filename, content);
+    if (!safety.safe) throw new Error(`Document rejected: ${safety.reason}`);
+    const saved = await this.store.put({ tenantId, userId, documentId, filename, content, settings: this.settings, idempotencyKey: payload.idempotencyKey }, {
+      maxPending: this.options.maxPendingPerTenant ?? 100,
+      maxStoredBytes: this.options.maxStoredBytesPerTenant ?? 50 * 1024 * 1024
+    });
+    if (saved.status === "pending") {
+      try {
+        await this.dispatch({ id: saved.id, tenantId });
+      } catch {
+      }
+    }
+    return { jobId: saved.id, status: saved.status === "pending" ? (await this.store.get(saved.id, tenantId)).status : saved.status };
+  }
+  async flushOutbox() {
+    for (const reference of await this.store.dispatchable()) await this.dispatch(reference);
+  }
+  async dispatch(reference) {
+    const existing = await this.queue.getJob(reference.id);
+    if (!existing) await this.queue.add("document-ingest", reference, reference.id);
+    else if (existing.status === "failed") {
+      await this.store.mark(reference.id, "failed", existing.error);
+      return;
+    }
+    await this.store.mark(reference.id, "queued");
+  }
+  async getJobStatus(id, tenantId) {
+    const saved = await this.store.get(id, tenantId);
+    if (!saved) return null;
+    let progress = saved.status === "completed" ? 100 : 0;
+    let attemptsMade;
+    try {
+      const job = await this.queue.getJob(id);
+      progress = Math.max(progress, job?.progress ?? 0);
+      attemptsMade = job?.attemptsMade;
+    } catch {
+    }
+    return {
+      id,
+      type: "document-ingest",
+      data: { tenantId: saved.tenantId, userId: saved.userId, documentId: saved.documentId },
+      status: saved.status,
+      progress,
+      attemptsMade,
+      result: saved.result ?? void 0,
+      error: saved.error ?? void 0,
+      createdAt: saved.createdAt,
+      updatedAt: saved.updatedAt,
+      ...saved.status === "completed" || saved.status === "failed" ? { completedAt: saved.updatedAt } : {}
+    };
+  }
+  async retryJob(id, tenantId) {
+    const saved = await this.store.get(id, tenantId);
+    if (!saved || saved.status !== "failed" || !saved.content) throw new Error("Failed job missing or retry payload expired.");
+    const job = await this.queue.getJob(id);
+    if (job && job.status !== "failed") throw new Error("Job is not retryable.");
+    await this.store.resetFailed(id, tenantId);
+    if (job) await this.queue.retry(id);
+    else await this.queue.add("document-ingest", { id, tenantId }, id);
+    await this.store.mark(id, "queued");
+  }
+  async bytes(payload) {
+    if (payload.encoding !== void 0 && payload.encoding !== "base64" && payload.encoding !== "utf8") throw new Error("Invalid encoding.");
+    let data;
+    if (typeof payload.content === "string") {
+      if (Buffer.byteLength(payload.content) > (payload.encoding === "base64" ? Math.ceil(this.maxBytes / 3) * 4 : this.maxBytes)) throw new IngestionQuotaError("Upload exceeds byte limit.");
+      if (payload.encoding === "base64" && !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(payload.content)) throw new Error("Invalid base64 content.");
+      data = Buffer.from(payload.content, payload.encoding === "base64" ? "base64" : "utf8");
+    } else if (Buffer.isBuffer(payload.content)) data = payload.content;
+    else {
+      const chunks = [];
+      let length = 0;
+      for await (const chunk of payload.content) {
+        const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        length += buffer.length;
+        if (length > this.maxBytes) {
+          payload.content.destroy?.();
+          throw new IngestionQuotaError("Upload exceeds byte limit.");
+        }
+        chunks.push(buffer);
+      }
+      data = Buffer.concat(chunks);
+    }
+    if (!data.length || data.length > this.maxBytes) throw new IngestionQuotaError("Upload is empty or exceeds byte limit.");
+    return data;
+  }
+};
+
+// src/core/ingestion/ingestion-worker.ts
+import { createHash as createHash2 } from "node:crypto";
+var IngestionWorker = class {
+  constructor(service, vectors, options = {}) {
+    this.service = service;
+    this.vectors = vectors;
+    this.options = options;
+  }
+  service;
+  vectors;
+  options;
+  timer;
+  pump;
+  stopped = false;
+  async start() {
+    if (this.timer) return;
+    this.stopped = false;
+    await this.service.store.ready();
+    await this.service.queue.ready();
+    await this.service.queue.process("document-ingest", (job, progress) => this.process(job, progress));
+    await this.tick();
+    this.timer = setInterval(() => {
+      void this.tick();
+    }, this.options.pollMs ?? 1e3);
+  }
+  async tick() {
+    if (this.stopped) return;
+    if (!this.pump) {
+      this.pump = this.service.flushOutbox().then(() => this.service.store.expireFailedBlobs()).catch((error) => this.options.onError?.(error)).finally(() => {
+        this.pump = void 0;
+      });
+    }
+    await this.pump;
+  }
+  async process(job, progress) {
+    const reference = job.data;
+    const saved = await this.service.store.get(reference.id, reference.tenantId);
+    if (!saved) throw new Error("Ingestion record missing.");
+    if (saved.status === "completed") return saved.result ?? {};
+    if (!saved.content) throw new Error("Upload payload expired.");
+    try {
+      await this.service.store.mark(saved.id, "processing");
+      await progress(10);
+      if (createHash2("sha256").update(saved.content).digest("hex") !== saved.digest) throw new Error("Upload checksum mismatch.");
+      const safety = DocumentIngestionService.validateFileSafety(saved.filename, saved.content);
+      if (!safety.safe) throw new Error(`Unsafe document: ${safety.reason}`);
+      const text = DocumentIngestionService.extractText(saved.filename, saved.content);
+      if (text === "PDF document with no extractable plain text stream.") throw new Error("PDF needs OCR or a supported text parser.");
+      const sanitized = PIIGuard.sanitize(text);
+      const pieces = DocumentIngestionService.chunkText(sanitized.sanitizedText, saved.settings.chunkSize, saved.settings.chunkOverlap);
+      if (!pieces.length) throw new Error("Document contains no extractable text.");
+      const docs = pieces.map((content, index) => ({
+        id: createHash2("sha256").update(JSON.stringify([saved.documentId, index])).digest("hex"),
+        content,
+        metadata: {
+          tenantId: saved.tenantId,
+          documentId: saved.documentId,
+          documentVersion: saved.version,
+          filename: saved.filename,
+          chunkIndex: index,
+          totalChunks: pieces.length,
+          sourceDigest: saved.digest
+        }
+      }));
+      await progress(40);
+      const outcome = await this.vectors.replaceDocument(saved.documentId, saved.tenantId, saved.version, docs);
+      const result = {
+        filename: saved.filename,
+        documentId: saved.documentId,
+        version: saved.version,
+        chunksCount: docs.length,
+        piiEntitiesMasked: sanitized.detectedEntities.length,
+        documentIds: docs.map((doc) => doc.id),
+        superseded: outcome === "superseded",
+        applied: outcome === "applied",
+        processedAt: (/* @__PURE__ */ new Date()).toISOString()
+      };
+      await this.service.store.complete(saved.id, result);
+      await progress(100);
+      return result;
+    } catch (error) {
+      if ((job.attemptsMade ?? 0) + 1 >= (job.maxAttempts ?? 1)) {
+        await this.service.store.mark(saved.id, "failed", error instanceof Error ? error.message : "Processing failed.");
+      }
+      throw error;
+    }
+  }
+  async close() {
+    this.stopped = true;
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = void 0;
+    }
+    await this.pump;
+    await this.service.queue.close();
+  }
+};
+
+// src/config/ingestion-runtime.ts
+function positiveEnv(name, fallback) {
+  const value = Number(process.env[name] ?? fallback);
+  if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be a positive integer.`);
+  return value;
+}
+async function createIngestionRuntime(vectors) {
+  if (!process.env.DATABASE_URL || !process.env.REDIS_URL) throw new Error("Durable ingestion requires DATABASE_URL and REDIS_URL.");
+  const pool = new Pool2({ connectionString: process.env.DATABASE_URL, max: 5, connectionTimeoutMillis: 5e3, statement_timeout: 1e4 });
+  pool.on("error", (error) => console.error("[Ingestion DB]", error.message));
+  const store = new PostgresIngestionStore(pool, process.env.INGESTION_TABLE_NAME);
+  const queue = new BullMQJobQueue({
+    redisUrl: process.env.REDIS_URL,
+    queueName: process.env.INGESTION_QUEUE_NAME,
+    concurrency: positiveEnv("INGESTION_CONCURRENCY", 2),
+    globalConcurrency: positiveEnv("INGESTION_GLOBAL_CONCURRENCY", 4),
+    attempts: positiveEnv("INGESTION_ATTEMPTS", 3),
+    lockDurationMs: positiveEnv("INGESTION_LOCK_DURATION_MS", 3e4),
+    stalledIntervalMs: positiveEnv("INGESTION_STALLED_INTERVAL_MS", 3e4),
+    onError: (error) => console.error("[Ingestion Redis]", error.message)
+  });
+  try {
+    await store.ready();
+    await queue.ready();
+    const backend = new QueuedIngestionService(store, queue, {
+      maxUploadBytes: positiveEnv("INGESTION_MAX_UPLOAD_BYTES", 5 * 1024 * 1024),
+      maxPendingPerTenant: positiveEnv("INGESTION_MAX_PENDING_PER_TENANT", 100),
+      maxStoredBytesPerTenant: positiveEnv("INGESTION_MAX_STORED_BYTES_PER_TENANT", 50 * 1024 * 1024)
+    });
+    const worker = new IngestionWorker(backend, vectors, { onError: (error) => console.error("[Ingestion outbox]", error instanceof Error ? error.message : error) });
+    let closed = false;
+    return { backend, worker, async close() {
+      if (closed) return;
+      closed = true;
+      await worker.close();
+      await pool.end();
+    } };
+  } catch (error) {
+    await queue.close();
+    await pool.end();
+    throw error;
+  }
+}
+function persistentVectorsFromEnv() {
+  return new PostgresVectorAdapter({
+    connectionString: process.env.DATABASE_URL,
+    tableName: process.env.VECTOR_TABLE_NAME,
+    modelName: process.env.EMBEDDING_MODEL || "Xenova/all-MiniLM-L6-v2",
+    dimensions: positiveEnv("EMBEDDING_DIMENSIONS", 384),
+    autoMigrate: process.env.VECTOR_AUTO_MIGRATE === "true"
+  });
+}
+
+// src/worker.ts
+async function main() {
+  const vectors = persistentVectorsFromEnv();
+  await vectors.init();
+  const runtime = await createIngestionRuntime(vectors);
+  await runtime.worker.start();
+  console.log("[IngestionWorker] Ready");
+  let stopping = false;
+  const stop = async () => {
+    if (stopping) return;
+    stopping = true;
+    const deadline = setTimeout(() => {
+      console.error("[IngestionWorker] Shutdown timed out");
+      process.exit(1);
+    }, positiveEnv("SHUTDOWN_TIMEOUT_MS", 3e4));
+    deadline.unref();
+    try {
+      await runtime.close();
+      await vectors.close();
+    } catch (error) {
+      console.error("[IngestionWorker] Shutdown failed", error);
+      process.exitCode = 1;
+    } finally {
+      clearTimeout(deadline);
+    }
+  };
+  process.once("SIGTERM", () => {
+    void stop();
+  });
+  process.once("SIGINT", () => {
+    void stop();
+  });
+}
+main().catch((error) => {
+  console.error("[IngestionWorker]", error.message);
+  process.exit(1);
+});