@opengeni/documents 0.2.0

package/dist/index.d.ts

@@ -0,0 +1,124 @@
+import { Settings } from '@opengeni/config';
+import { FileAsset, AddDocumentRequest, Document, CreateDocumentBaseRequest, DocumentBase, DocumentSearchResult } from '@opengeni/contracts';
+import { Database } from '@opengeni/db';
+import { ObjectStorage } from '@opengeni/storage';
+
+declare const DEFAULT_DOCUMENT_PARSER = "liteparse";
+declare const DEFAULT_DOCUMENT_EMBEDDING_MODEL = "text-embedding-3-large";
+declare const DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS = 3072;
+declare const DEFAULT_DOCUMENT_CHUNK_SIZE = 1200;
+declare const DEFAULT_DOCUMENT_CHUNK_OVERLAP = 160;
+type ParsedDocument = {
+    text: string;
+    metadata?: Record<string, unknown>;
+};
+type DocumentChunk = {
+    text: string;
+    metadata: Record<string, unknown>;
+};
+type DocumentParser = {
+    name: string;
+    parse: (bytes: Uint8Array, file: FileAsset) => Promise<ParsedDocument>;
+};
+type DocumentChunker = {
+    chunk: (parsed: ParsedDocument, file: FileAsset) => DocumentChunk[];
+};
+type DocumentEmbedder = {
+    model: string;
+    dimensions: number;
+    embedMany: (texts: string[]) => Promise<number[][]>;
+    embedQuery: (text: string) => Promise<number[]>;
+};
+type DocumentServices = {
+    parser: DocumentParser;
+    chunker: DocumentChunker;
+    embedder: DocumentEmbedder;
+};
+type DocumentIndexHooks = {
+    beforeEmbed?: (input: {
+        accountId: string;
+        workspaceId: string;
+        documentId: string;
+        chunkCount: number;
+    }) => Promise<void>;
+};
+declare class LiteParseDocumentParser implements DocumentParser {
+    readonly name = "liteparse";
+    private parseQueue;
+    parse(bytes: Uint8Array, file: FileAsset): Promise<ParsedDocument>;
+    private parseWithLiteParse;
+    private enqueueParse;
+}
+declare class RecursiveTextChunker implements DocumentChunker {
+    private readonly maxChars;
+    private readonly overlapChars;
+    constructor(maxChars?: number, overlapChars?: number);
+    chunk(parsed: ParsedDocument, file: FileAsset): DocumentChunk[];
+}
+declare class OpenAIEmbeddingProvider implements DocumentEmbedder {
+    private client;
+    private readonly apiKey;
+    private readonly baseURL;
+    constructor(args: {
+        apiKey?: string | undefined;
+        baseURL?: string | undefined;
+        defaultHeaders?: Record<string, string> | undefined;
+        defaultQuery?: Record<string, string> | undefined;
+        model?: string | undefined;
+        dimensions?: number | undefined;
+    });
+    readonly model: string;
+    readonly dimensions: number;
+    private readonly defaultHeaders;
+    private readonly defaultQuery;
+    embedMany(texts: string[]): Promise<number[][]>;
+    embedQuery(text: string): Promise<number[]>;
+    private openai;
+}
+declare class DeterministicEmbeddingProvider implements DocumentEmbedder {
+    readonly dimensions: number;
+    readonly model: string;
+    constructor(dimensions?: number, model?: string);
+    embedMany(texts: string[]): Promise<number[][]>;
+    embedQuery(text: string): Promise<number[]>;
+}
+declare function createDocumentServices(settings?: Settings, overrides?: Partial<DocumentServices>): DocumentServices;
+declare function documentOpenAIEmbeddingConfig(settings?: Settings): {
+    apiKey?: string | undefined;
+    baseURL?: string | undefined;
+    defaultHeaders?: Record<string, string> | undefined;
+    defaultQuery?: Record<string, string> | undefined;
+};
+declare function createDocumentBase(db: Database, input: CreateDocumentBaseRequest & {
+    accountId: string;
+    workspaceId: string;
+}): Promise<DocumentBase>;
+declare function listDocumentBases(db: Database, workspaceId: string): Promise<DocumentBase[]>;
+declare function getDocumentBase(db: Database, workspaceId: string, baseId: string): Promise<DocumentBase | null>;
+declare function addDocumentToBase(db: Database, input: AddDocumentRequest & {
+    accountId: string;
+    workspaceId: string;
+    baseId: string;
+}): Promise<Document>;
+declare function deleteDocumentFromBase(db: Database, input: {
+    accountId: string;
+    workspaceId: string;
+    baseId: string;
+    documentId: string;
+}): Promise<void>;
+declare function listDocuments(db: Database, workspaceId: string, baseId: string): Promise<Document[]>;
+declare function getDocument(db: Database, workspaceId: string, documentId: string): Promise<Document | null>;
+declare function queueDocumentForReindex(db: Database, workspaceId: string, documentId: string): Promise<Document>;
+declare function indexDocumentNow(db: Database, objectStorage: ObjectStorage, workspaceId: string, documentId: string, services?: DocumentServices, hooks?: DocumentIndexHooks): Promise<Document>;
+declare function searchDocuments(db: Database, input: {
+    workspaceId: string;
+    query: string;
+    baseIds?: string[];
+    limit?: number;
+}, services?: Pick<DocumentServices, "embedder">): Promise<DocumentSearchResult[]>;
+declare function getDocumentChunk(db: Database, workspaceId: string, chunkId: string): Promise<DocumentSearchResult | null>;
+declare function parseDocumentBytes(bytes: Uint8Array, file: FileAsset, parser?: DocumentParser): Promise<ParsedDocument>;
+declare function chunkText(text: string, maxChars?: number, overlapChars?: number): string[];
+declare function deterministicEmbedding(text: string, dimensions?: number): number[];
+
+export { DEFAULT_DOCUMENT_CHUNK_OVERLAP, DEFAULT_DOCUMENT_CHUNK_SIZE, DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS, DEFAULT_DOCUMENT_EMBEDDING_MODEL, DEFAULT_DOCUMENT_PARSER, DeterministicEmbeddingProvider, type DocumentChunk, type DocumentChunker, type DocumentEmbedder, type DocumentIndexHooks, type DocumentParser, type DocumentServices, LiteParseDocumentParser, OpenAIEmbeddingProvider, type ParsedDocument, RecursiveTextChunker, addDocumentToBase, chunkText, createDocumentBase, createDocumentServices, deleteDocumentFromBase, deterministicEmbedding, documentOpenAIEmbeddingConfig, getDocument, getDocumentBase, getDocumentChunk, indexDocumentNow, listDocumentBases, listDocuments, parseDocumentBytes, queueDocumentForReindex, searchDocuments };

package/dist/index.js

@@ -0,0 +1,557 @@
+// src/index.ts
+import { requireFile, withRlsContext, withWorkspaceRls } from "@opengeni/db";
+import * as schema from "@opengeni/db/schema";
+import { LiteParse } from "@llamaindex/liteparse";
+import { and, asc, desc, eq, inArray, sql } from "drizzle-orm";
+import OpenAI from "openai";
+var DEFAULT_DOCUMENT_PARSER = "liteparse";
+var DEFAULT_DOCUMENT_EMBEDDING_MODEL = "text-embedding-3-large";
+var DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS = 3072;
+var DEFAULT_DOCUMENT_CHUNK_SIZE = 1200;
+var DEFAULT_DOCUMENT_CHUNK_OVERLAP = 160;
+var LiteParseDocumentParser = class {
+  name = DEFAULT_DOCUMENT_PARSER;
+  parseQueue = Promise.resolve();
+  async parse(bytes, file) {
+    const text = isTextLike(file) ? Buffer.from(bytes).toString("utf8").replace(/\0/g, " ").trim() : await this.parseWithLiteParse(bytes);
+    if (!text.trim()) {
+      throw new Error(`Parsed document is empty: ${file.filename}`);
+    }
+    return {
+      text: text.trim(),
+      metadata: {
+        parser: this.name,
+        filename: file.filename,
+        contentType: file.contentType
+      }
+    };
+  }
+  async parseWithLiteParse(bytes) {
+    return await this.enqueueParse(async () => {
+      const parser = new LiteParse({ ocrEnabled: true, numWorkers: 1 });
+      const result = await parser.parse(Buffer.from(bytes), true);
+      const text = typeof result?.text === "string" ? result.text : "";
+      return text.replace(/\0/g, " ").trim();
+    });
+  }
+  async enqueueParse(task) {
+    const previous = this.parseQueue;
+    let release = () => void 0;
+    this.parseQueue = new Promise((resolve) => {
+      release = resolve;
+    });
+    await previous.catch(() => void 0);
+    try {
+      return await task();
+    } finally {
+      release();
+    }
+  }
+};
+var RecursiveTextChunker = class {
+  constructor(maxChars = DEFAULT_DOCUMENT_CHUNK_SIZE, overlapChars = DEFAULT_DOCUMENT_CHUNK_OVERLAP) {
+    this.maxChars = maxChars;
+    this.overlapChars = overlapChars;
+    if (overlapChars >= maxChars) {
+      throw new Error("document chunk overlap must be smaller than chunk size");
+    }
+  }
+  maxChars;
+  overlapChars;
+  chunk(parsed, file) {
+    return chunkText(parsed.text, this.maxChars, this.overlapChars).map((text, index) => ({
+      text,
+      metadata: {
+        ...parsed.metadata,
+        filename: file.filename,
+        contentType: file.contentType,
+        chunkIndex: index
+      }
+    }));
+  }
+};
+var OpenAIEmbeddingProvider = class {
+  client = null;
+  apiKey;
+  baseURL;
+  constructor(args) {
+    this.apiKey = args.apiKey ?? process.env.OPENAI_API_KEY;
+    this.baseURL = args.baseURL;
+    this.defaultHeaders = args.defaultHeaders;
+    this.defaultQuery = args.defaultQuery;
+    this.model = args.model ?? DEFAULT_DOCUMENT_EMBEDDING_MODEL;
+    this.dimensions = args.dimensions ?? DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS;
+  }
+  model;
+  dimensions;
+  defaultHeaders;
+  defaultQuery;
+  async embedMany(texts) {
+    if (texts.length === 0) return [];
+    const out = [];
+    for (let start = 0; start < texts.length; start += 64) {
+      const batch = texts.slice(start, start + 64);
+      const response = await this.openai().embeddings.create({
+        model: this.model,
+        input: batch,
+        dimensions: this.dimensions
+      });
+      for (const item of response.data) {
+        out.push(validateEmbedding(item.embedding, this.dimensions, this.model));
+      }
+    }
+    return out;
+  }
+  async embedQuery(text) {
+    const [embedding] = await this.embedMany([text]);
+    if (!embedding) {
+      throw new Error("Embedding provider returned no query embedding");
+    }
+    return embedding;
+  }
+  openai() {
+    if (!this.apiKey) {
+      throw new Error("OpenAI document embeddings require an API key");
+    }
+    this.client ??= new OpenAI({
+      apiKey: this.apiKey,
+      ...this.baseURL ? { baseURL: this.baseURL } : {},
+      ...this.defaultQuery ? { defaultQuery: this.defaultQuery } : {},
+      ...this.defaultHeaders ? { defaultHeaders: this.defaultHeaders } : {}
+    });
+    return this.client;
+  }
+};
+var DeterministicEmbeddingProvider = class {
+  constructor(dimensions = DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS, model = `deterministic-local-${dimensions}`) {
+    this.dimensions = dimensions;
+    this.model = model;
+  }
+  dimensions;
+  model;
+  async embedMany(texts) {
+    return texts.map((text) => deterministicEmbedding(text, this.dimensions));
+  }
+  async embedQuery(text) {
+    return deterministicEmbedding(text, this.dimensions);
+  }
+};
+function createDocumentServices(settings, overrides = {}) {
+  const dimensions = settings?.documentEmbeddingDimensions ?? DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS;
+  const openAIEmbeddingConfig = documentOpenAIEmbeddingConfig(settings);
+  return {
+    parser: overrides.parser ?? new LiteParseDocumentParser(),
+    chunker: overrides.chunker ?? new RecursiveTextChunker(
+      settings?.documentChunkSize ?? DEFAULT_DOCUMENT_CHUNK_SIZE,
+      settings?.documentChunkOverlap ?? DEFAULT_DOCUMENT_CHUNK_OVERLAP
+    ),
+    embedder: overrides.embedder ?? (settings?.documentEmbeddingProvider === "deterministic" ? new DeterministicEmbeddingProvider(dimensions, settings.documentEmbeddingModel) : new OpenAIEmbeddingProvider({
+      ...openAIEmbeddingConfig,
+      model: settings?.documentEmbeddingModel ?? DEFAULT_DOCUMENT_EMBEDDING_MODEL,
+      dimensions
+    }))
+  };
+}
+function documentOpenAIEmbeddingConfig(settings) {
+  if (!settings) return {};
+  if (settings.documentEmbeddingApiKey || settings.documentEmbeddingBaseUrl) {
+    return {
+      apiKey: settings.documentEmbeddingApiKey ?? settings.openaiApiKey ?? settings.azureOpenaiApiKey,
+      baseURL: settings.documentEmbeddingBaseUrl ?? settings.openaiBaseUrl ?? settings.azureOpenaiBaseUrl
+    };
+  }
+  if (settings.openaiProvider === "azure") {
+    const baseURL = settings.azureOpenaiBaseUrl ?? azureDeploymentBaseUrl(settings);
+    return {
+      apiKey: settings.azureOpenaiApiKey ?? settings.azureOpenaiAdToken ?? "azure-ad-token",
+      baseURL,
+      defaultQuery: azureOpenAIDefaultQuery(settings, baseURL),
+      defaultHeaders: settings.azureOpenaiAdToken && !settings.azureOpenaiApiKey ? { Authorization: `Bearer ${settings.azureOpenaiAdToken}` } : void 0
+    };
+  }
+  return {
+    apiKey: settings.openaiApiKey,
+    baseURL: settings.openaiBaseUrl
+  };
+}
+function azureDeploymentBaseUrl(settings) {
+  const endpoint = settings.azureOpenaiEndpoint?.replace(/\/+$/, "");
+  if (!endpoint || !settings.azureOpenaiDeployment) {
+    throw new Error("Azure OpenAI endpoint/deployment settings are incomplete");
+  }
+  return `${endpoint}/openai/deployments/${settings.azureOpenaiDeployment}`;
+}
+function azureOpenAIDefaultQuery(settings, baseURL) {
+  if (!settings.azureOpenaiApiVersion) return void 0;
+  const normalized = baseURL.replace(/\/+$/, "").toLowerCase();
+  if (normalized.endsWith("/openai/v1")) {
+    return void 0;
+  }
+  return { "api-version": settings.azureOpenaiApiVersion };
+}
+async function createDocumentBase(db, input) {
+  return await withRlsContext(db, { accountId: input.accountId, workspaceId: input.workspaceId }, async (scopedDb) => {
+    const [row] = await scopedDb.insert(schema.documentBases).values({
+      accountId: input.accountId,
+      workspaceId: input.workspaceId,
+      name: input.name.trim(),
+      description: input.description?.trim() || null
+    }).returning();
+    if (!row) throw new Error("Failed to create document base");
+    return mapDocumentBase(row);
+  });
+}
+async function listDocumentBases(db, workspaceId) {
+  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
+    const rows = await scopedDb.select().from(schema.documentBases).where(eq(schema.documentBases.workspaceId, workspaceId)).orderBy(desc(schema.documentBases.createdAt));
+    return rows.map(mapDocumentBase);
+  });
+}
+async function getDocumentBase(db, workspaceId, baseId) {
+  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
+    const [row] = await scopedDb.select().from(schema.documentBases).where(and(eq(schema.documentBases.workspaceId, workspaceId), eq(schema.documentBases.id, baseId))).limit(1);
+    return row ? mapDocumentBase(row) : null;
+  });
+}
+async function addDocumentToBase(db, input) {
+  return await withRlsContext(db, { accountId: input.accountId, workspaceId: input.workspaceId }, async (scopedDb) => {
+    const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
+    if (!base) throw new Error(`Document base not found: ${input.baseId}`);
+    const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
+    const now = /* @__PURE__ */ new Date();
+    const [existing] = await scopedDb.select().from(schema.documents).where(and(eq(schema.documents.workspaceId, input.workspaceId), eq(schema.documents.baseId, input.baseId), eq(schema.documents.fileId, input.fileId))).limit(1);
+    if (existing) {
+      return mapDocument(existing);
+    }
+    const [row] = await scopedDb.insert(schema.documents).values({
+      accountId: input.accountId,
+      workspaceId: input.workspaceId,
+      baseId: input.baseId,
+      fileId: input.fileId,
+      status: "queued",
+      title: file.filename,
+      parser: DEFAULT_DOCUMENT_PARSER,
+      updatedAt: now
+    }).returning();
+    if (!row) throw new Error("Failed to create document");
+    return mapDocument(row);
+  });
+}
+async function deleteDocumentFromBase(db, input) {
+  await withRlsContext(db, { accountId: input.accountId, workspaceId: input.workspaceId }, async (scopedDb) => {
+    const [document] = await scopedDb.select().from(schema.documents).where(and(eq(schema.documents.workspaceId, input.workspaceId), eq(schema.documents.id, input.documentId))).limit(1);
+    if (!document) {
+      throw new Error(`Document not found: ${input.documentId}`);
+    }
+    if (document.baseId !== input.baseId) {
+      throw new Error(`Document not found: ${input.documentId}`);
+    }
+    await scopedDb.delete(schema.documents).where(and(eq(schema.documents.workspaceId, input.workspaceId), eq(schema.documents.id, input.documentId)));
+  });
+}
+async function listDocuments(db, workspaceId, baseId) {
+  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
+    const rows = await scopedDb.select().from(schema.documents).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.baseId, baseId))).orderBy(asc(schema.documents.createdAt));
+    return rows.map(mapDocument);
+  });
+}
+async function getDocument(db, workspaceId, documentId) {
+  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
+    const [row] = await scopedDb.select().from(schema.documents).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))).limit(1);
+    return row ? mapDocument(row) : null;
+  });
+}
+async function queueDocumentForReindex(db, workspaceId, documentId) {
+  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
+    const [row] = await scopedDb.update(schema.documents).set({
+      status: "queued",
+      error: null,
+      updatedAt: /* @__PURE__ */ new Date()
+    }).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))).returning();
+    if (!row) throw new Error(`Document not found: ${documentId}`);
+    return mapDocument(row);
+  });
+}
+async function indexDocumentNow(db, objectStorage, workspaceId, documentId, services = createDocumentServices(), hooks = {}) {
+  const [document] = await withWorkspaceRls(
+    db,
+    workspaceId,
+    async (scopedDb) => await scopedDb.select().from(schema.documents).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))).limit(1)
+  );
+  if (!document) throw new Error(`Document not found: ${documentId}`);
+  const file = await requireReadyFile(db, workspaceId, document.fileId);
+  await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
+    await scopedDb.update(schema.documents).set({
+      status: "indexing",
+      parser: services.parser.name,
+      error: null,
+      updatedAt: /* @__PURE__ */ new Date()
+    }).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)));
+  });
+  try {
+    const bytes = await objectStorage.getFileBytes(file);
+    const parsed = await services.parser.parse(bytes, file);
+    const chunks = services.chunker.chunk(parsed, file);
+    await hooks.beforeEmbed?.({
+      accountId: document.accountId,
+      workspaceId: document.workspaceId,
+      documentId,
+      chunkCount: chunks.length
+    });
+    const embeddings = await services.embedder.embedMany(chunks.map((chunk) => chunk.text));
+    if (embeddings.length !== chunks.length) {
+      throw new Error(`Embedding provider returned ${embeddings.length} embeddings for ${chunks.length} chunks`);
+    }
+    await withWorkspaceRls(db, workspaceId, async (scopedDb) => await scopedDb.transaction(async (tx) => {
+      await tx.delete(schema.documentChunks).where(and(eq(schema.documentChunks.workspaceId, workspaceId), eq(schema.documentChunks.documentId, documentId)));
+      if (chunks.length > 0) {
+        await tx.insert(schema.documentChunks).values(chunks.map((chunk, index) => ({
+          accountId: document.accountId,
+          workspaceId: document.workspaceId,
+          documentId,
+          baseId: document.baseId,
+          fileId: file.id,
+          chunkIndex: index,
+          text: chunk.text,
+          metadata: chunk.metadata,
+          embedding: validateEmbedding(embeddings[index] ?? [], services.embedder.dimensions, services.embedder.model),
+          embeddingModel: services.embedder.model
+        })));
+      }
+      await tx.update(schema.documents).set({
+        status: "ready",
+        parser: services.parser.name,
+        chunkCount: chunks.length,
+        error: null,
+        updatedAt: /* @__PURE__ */ new Date()
+      }).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)));
+    }));
+  } catch (error) {
+    const [failed] = await withWorkspaceRls(
+      db,
+      workspaceId,
+      async (scopedDb) => await scopedDb.update(schema.documents).set({
+        status: "failed",
+        error: error instanceof Error ? error.message : String(error),
+        updatedAt: /* @__PURE__ */ new Date()
+      }).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))).returning()
+    );
+    if (!failed) throw error;
+    return mapDocument(failed);
+  }
+  const updated = await getDocument(db, workspaceId, documentId);
+  if (!updated) throw new Error(`Document disappeared after indexing: ${documentId}`);
+  return updated;
+}
+async function searchDocuments(db, input, services = createDocumentServices()) {
+  const limit = Math.min(Math.max(input.limit ?? 5, 1), 20);
+  const queryEmbedding = await services.embedder.embedQuery(input.query);
+  validateEmbedding(queryEmbedding, services.embedder.dimensions, services.embedder.model);
+  const distance = sql`${schema.documentChunks.embedding} <=> ${vectorLiteral(queryEmbedding)}::vector`;
+  const rows = await withWorkspaceRls(
+    db,
+    input.workspaceId,
+    async (scopedDb) => await scopedDb.select({
+      chunkId: schema.documentChunks.id,
+      documentId: schema.documentChunks.documentId,
+      baseId: schema.documentChunks.baseId,
+      fileId: schema.documentChunks.fileId,
+      title: schema.documents.title,
+      text: schema.documentChunks.text,
+      chunkIndex: schema.documentChunks.chunkIndex,
+      metadata: schema.documentChunks.metadata,
+      distance
+    }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(and(
+      eq(schema.documents.status, "ready"),
+      eq(schema.documentChunks.workspaceId, input.workspaceId),
+      eq(schema.documentChunks.embeddingModel, services.embedder.model),
+      input.baseIds && input.baseIds.length > 0 ? inArray(schema.documentChunks.baseId, input.baseIds) : void 0
+    )).orderBy(distance).limit(limit)
+  );
+  return rows.map((row) => ({
+    chunkId: row.chunkId,
+    workspaceId: input.workspaceId,
+    documentId: row.documentId,
+    baseId: row.baseId,
+    fileId: row.fileId,
+    title: row.title,
+    text: row.text,
+    score: 1 / (1 + Number(row.distance)),
+    chunkIndex: row.chunkIndex,
+    metadata: row.metadata
+  }));
+}
+async function getDocumentChunk(db, workspaceId, chunkId) {
+  const [row] = await withWorkspaceRls(
+    db,
+    workspaceId,
+    async (scopedDb) => await scopedDb.select({
+      chunkId: schema.documentChunks.id,
+      documentId: schema.documentChunks.documentId,
+      baseId: schema.documentChunks.baseId,
+      fileId: schema.documentChunks.fileId,
+      title: schema.documents.title,
+      text: schema.documentChunks.text,
+      chunkIndex: schema.documentChunks.chunkIndex,
+      metadata: schema.documentChunks.metadata
+    }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(and(eq(schema.documentChunks.workspaceId, workspaceId), eq(schema.documentChunks.id, chunkId), eq(schema.documents.status, "ready"))).limit(1)
+  );
+  if (!row) return null;
+  return {
+    chunkId: row.chunkId,
+    workspaceId,
+    documentId: row.documentId,
+    baseId: row.baseId,
+    fileId: row.fileId,
+    title: row.title,
+    text: row.text,
+    score: 1,
+    chunkIndex: row.chunkIndex,
+    metadata: row.metadata
+  };
+}
+async function parseDocumentBytes(bytes, file, parser = new LiteParseDocumentParser()) {
+  return await parser.parse(bytes, file);
+}
+function chunkText(text, maxChars = DEFAULT_DOCUMENT_CHUNK_SIZE, overlapChars = DEFAULT_DOCUMENT_CHUNK_OVERLAP) {
+  if (overlapChars >= maxChars) {
+    throw new Error("chunk overlap must be smaller than chunk size");
+  }
+  const normalized = text.replace(/\r\n/g, "\n").replace(/[ \t]+/g, " ").trim();
+  if (!normalized) return [];
+  const paragraphs = normalized.split(/\n{2,}/).map((part) => part.replace(/\s+/g, " ").trim()).filter(Boolean);
+  const chunks = [];
+  let current = "";
+  for (const paragraph of paragraphs.length > 0 ? paragraphs : [normalized.replace(/\s+/g, " ")]) {
+    for (const part of splitOversizedText(paragraph, maxChars)) {
+      if (!current) {
+        current = part;
+      } else if (current.length + 1 + part.length <= maxChars) {
+        current = `${current} ${part}`;
+      } else {
+        chunks.push(current);
+        current = withOverlap(current, overlapChars, part, maxChars);
+      }
+    }
+  }
+  if (current) chunks.push(current);
+  return chunks.map((chunk) => chunk.trim()).filter(Boolean);
+}
+function deterministicEmbedding(text, dimensions = DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS) {
+  const values = new Array(dimensions).fill(0);
+  const tokens = text.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? [];
+  for (const token of tokens) {
+    let hash = 2166136261;
+    for (const char of token) {
+      hash ^= char.codePointAt(0) ?? 0;
+      hash = Math.imul(hash, 16777619);
+    }
+    values[Math.abs(hash) % dimensions] += 1;
+  }
+  const norm = Math.hypot(...values) || 1;
+  return values.map((value) => Number((value / norm).toFixed(6)));
+}
+async function requireReadyFile(db, workspaceId, fileId) {
+  const file = await requireFile(db, workspaceId, fileId);
+  if (file.status !== "ready") {
+    throw new Error(`File ${fileId} is ${file.status}`);
+  }
+  return file;
+}
+function validateEmbedding(values, dimensions, model) {
+  if (values.length !== dimensions) {
+    throw new Error(`Embedding model ${model} returned ${values.length} dimensions; expected ${dimensions}`);
+  }
+  if (values.some((value) => !Number.isFinite(value))) {
+    throw new Error(`Embedding model ${model} returned non-finite values`);
+  }
+  return values;
+}
+function isTextLike(file) {
+  const contentType = file.contentType.toLowerCase();
+  const filename = file.filename.toLowerCase();
+  return contentType.startsWith("text/") || contentType === "application/json" || contentType === "application/xml" || contentType === "application/x-yaml" || filename.endsWith(".md") || filename.endsWith(".markdown") || filename.endsWith(".json") || filename.endsWith(".yaml") || filename.endsWith(".yml") || filename.endsWith(".csv") || filename.endsWith(".tsv") || filename.endsWith(".xml");
+}
+function splitOversizedText(text, maxChars) {
+  if (text.length <= maxChars) return [text];
+  const out = [];
+  let remaining = text;
+  while (remaining.length > maxChars) {
+    const window = remaining.slice(0, maxChars + 1);
+    const breakAt = Math.max(
+      window.lastIndexOf(". "),
+      window.lastIndexOf("? "),
+      window.lastIndexOf("! "),
+      window.lastIndexOf("; "),
+      window.lastIndexOf(", "),
+      window.lastIndexOf(" ")
+    );
+    const end = breakAt > Math.floor(maxChars * 0.5) ? breakAt + 1 : maxChars;
+    out.push(remaining.slice(0, end).trim());
+    remaining = remaining.slice(end).trim();
+  }
+  if (remaining) out.push(remaining);
+  return out;
+}
+function withOverlap(previous, overlapChars, next, maxChars) {
+  if (overlapChars <= 0) return next;
+  const overlap = previous.slice(Math.max(0, previous.length - overlapChars)).replace(/^\S+\s+/, "").trim();
+  const candidate = overlap ? `${overlap} ${next}` : next;
+  return candidate.length <= maxChars ? candidate : next;
+}
+function vectorLiteral(values) {
+  return `[${values.join(",")}]`;
+}
+function mapDocumentBase(row) {
+  return {
+    id: row.id,
+    workspaceId: row.workspaceId,
+    name: row.name,
+    description: row.description,
+    createdAt: row.createdAt.toISOString(),
+    updatedAt: row.updatedAt.toISOString()
+  };
+}
+function mapDocument(row) {
+  return {
+    id: row.id,
+    workspaceId: row.workspaceId,
+    baseId: row.baseId,
+    fileId: row.fileId,
+    status: row.status,
+    title: row.title,
+    parser: row.parser,
+    chunkCount: row.chunkCount,
+    error: row.error,
+    createdAt: row.createdAt.toISOString(),
+    updatedAt: row.updatedAt.toISOString()
+  };
+}
+export {
+  DEFAULT_DOCUMENT_CHUNK_OVERLAP,
+  DEFAULT_DOCUMENT_CHUNK_SIZE,
+  DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS,
+  DEFAULT_DOCUMENT_EMBEDDING_MODEL,
+  DEFAULT_DOCUMENT_PARSER,
+  DeterministicEmbeddingProvider,
+  LiteParseDocumentParser,
+  OpenAIEmbeddingProvider,
+  RecursiveTextChunker,
+  addDocumentToBase,
+  chunkText,
+  createDocumentBase,
+  createDocumentServices,
+  deleteDocumentFromBase,
+  deterministicEmbedding,
+  documentOpenAIEmbeddingConfig,
+  getDocument,
+  getDocumentBase,
+  getDocumentChunk,
+  indexDocumentNow,
+  listDocumentBases,
+  listDocuments,
+  parseDocumentBytes,
+  queueDocumentForReindex,
+  searchDocuments
+};
+//# sourceMappingURL=index.js.map