@opengeni/documents 0.2.8 → 0.2.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -221,24 +221,29 @@ export class DeterministicEmbeddingProvider implements DocumentEmbedder {
221
221
  }
222
222
  }
223
223
 
224
- export function createDocumentServices(settings?: Settings, overrides: Partial<DocumentServices> = {}): DocumentServices {
224
+ export function createDocumentServices(
225
+ settings?: Settings,
226
+ overrides: Partial<DocumentServices> = {},
227
+ ): DocumentServices {
225
228
  const dimensions = settings?.documentEmbeddingDimensions ?? DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS;
226
229
  const openAIEmbeddingConfig = documentOpenAIEmbeddingConfig(settings);
227
230
  return {
228
231
  parser: overrides.parser ?? new LiteParseDocumentParser(),
229
- chunker: overrides.chunker ?? new RecursiveTextChunker(
230
- settings?.documentChunkSize ?? DEFAULT_DOCUMENT_CHUNK_SIZE,
231
- settings?.documentChunkOverlap ?? DEFAULT_DOCUMENT_CHUNK_OVERLAP,
232
- ),
233
- embedder: overrides.embedder ?? (
234
- settings?.documentEmbeddingProvider === "deterministic"
232
+ chunker:
233
+ overrides.chunker ??
234
+ new RecursiveTextChunker(
235
+ settings?.documentChunkSize ?? DEFAULT_DOCUMENT_CHUNK_SIZE,
236
+ settings?.documentChunkOverlap ?? DEFAULT_DOCUMENT_CHUNK_OVERLAP,
237
+ ),
238
+ embedder:
239
+ overrides.embedder ??
240
+ (settings?.documentEmbeddingProvider === "deterministic"
235
241
  ? new DeterministicEmbeddingProvider(dimensions, settings.documentEmbeddingModel)
236
242
  : new OpenAIEmbeddingProvider({
237
- ...openAIEmbeddingConfig,
238
- model: settings?.documentEmbeddingModel ?? DEFAULT_DOCUMENT_EMBEDDING_MODEL,
239
- dimensions,
240
- })
241
- ),
243
+ ...openAIEmbeddingConfig,
244
+ model: settings?.documentEmbeddingModel ?? DEFAULT_DOCUMENT_EMBEDDING_MODEL,
245
+ dimensions,
246
+ })),
242
247
  };
243
248
  }
244
249
 
@@ -251,8 +256,10 @@ export function documentOpenAIEmbeddingConfig(settings?: Settings): {
251
256
  if (!settings) return {};
252
257
  if (settings.documentEmbeddingApiKey || settings.documentEmbeddingBaseUrl) {
253
258
  return {
254
- apiKey: settings.documentEmbeddingApiKey ?? settings.openaiApiKey ?? settings.azureOpenaiApiKey,
255
- baseURL: settings.documentEmbeddingBaseUrl ?? settings.openaiBaseUrl ?? settings.azureOpenaiBaseUrl,
259
+ apiKey:
260
+ settings.documentEmbeddingApiKey ?? settings.openaiApiKey ?? settings.azureOpenaiApiKey,
261
+ baseURL:
262
+ settings.documentEmbeddingBaseUrl ?? settings.openaiBaseUrl ?? settings.azureOpenaiBaseUrl,
256
263
  };
257
264
  }
258
265
  if (settings.openaiProvider === "azure") {
@@ -261,9 +268,10 @@ export function documentOpenAIEmbeddingConfig(settings?: Settings): {
261
268
  apiKey: settings.azureOpenaiApiKey ?? settings.azureOpenaiAdToken ?? "azure-ad-token",
262
269
  baseURL,
263
270
  defaultQuery: azureOpenAIDefaultQuery(settings, baseURL),
264
- defaultHeaders: settings.azureOpenaiAdToken && !settings.azureOpenaiApiKey
265
- ? { Authorization: `Bearer ${settings.azureOpenaiAdToken}` }
266
- : undefined,
271
+ defaultHeaders:
272
+ settings.azureOpenaiAdToken && !settings.azureOpenaiApiKey
273
+ ? { Authorization: `Bearer ${settings.azureOpenaiAdToken}` }
274
+ : undefined,
267
275
  };
268
276
  }
269
277
  return {
@@ -292,129 +300,226 @@ function azureOpenAIDefaultQuery(
292
300
  return { "api-version": settings.azureOpenaiApiVersion };
293
301
  }
294
302
 
295
- export async function createDocumentBase(db: Database, input: CreateDocumentBaseRequest & { accountId: string; workspaceId: string }): Promise<DocumentBase> {
296
- return await withRlsContext(db, { accountId: input.accountId, workspaceId: input.workspaceId }, async (scopedDb) => {
297
- const [row] = await scopedDb.insert(schema.documentBases).values({
298
- accountId: input.accountId,
299
- workspaceId: input.workspaceId,
300
- name: input.name.trim(),
301
- description: input.description?.trim() || null,
302
- }).returning();
303
- if (!row) throw new Error("Failed to create document base");
304
- return mapDocumentBase(row);
305
- });
303
+ export async function createDocumentBase(
304
+ db: Database,
305
+ input: CreateDocumentBaseRequest & { accountId: string; workspaceId: string },
306
+ ): Promise<DocumentBase> {
307
+ return await withRlsContext(
308
+ db,
309
+ { accountId: input.accountId, workspaceId: input.workspaceId },
310
+ async (scopedDb) => {
311
+ const [row] = await scopedDb
312
+ .insert(schema.documentBases)
313
+ .values({
314
+ accountId: input.accountId,
315
+ workspaceId: input.workspaceId,
316
+ name: input.name.trim(),
317
+ description: input.description?.trim() || null,
318
+ })
319
+ .returning();
320
+ if (!row) throw new Error("Failed to create document base");
321
+ return mapDocumentBase(row);
322
+ },
323
+ );
306
324
  }
307
325
 
308
- export async function listDocumentBases(db: Database, workspaceId: string): Promise<DocumentBase[]> {
326
+ export async function listDocumentBases(
327
+ db: Database,
328
+ workspaceId: string,
329
+ ): Promise<DocumentBase[]> {
309
330
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
310
- const rows = await scopedDb.select().from(schema.documentBases)
331
+ const rows = await scopedDb
332
+ .select()
333
+ .from(schema.documentBases)
311
334
  .where(eq(schema.documentBases.workspaceId, workspaceId))
312
335
  .orderBy(desc(schema.documentBases.createdAt));
313
336
  return rows.map(mapDocumentBase);
314
337
  });
315
338
  }
316
339
 
317
- export async function getDocumentBase(db: Database, workspaceId: string, baseId: string): Promise<DocumentBase | null> {
340
+ export async function getDocumentBase(
341
+ db: Database,
342
+ workspaceId: string,
343
+ baseId: string,
344
+ ): Promise<DocumentBase | null> {
318
345
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
319
- const [row] = await scopedDb.select().from(schema.documentBases).where(and(eq(schema.documentBases.workspaceId, workspaceId), eq(schema.documentBases.id, baseId))).limit(1);
346
+ const [row] = await scopedDb
347
+ .select()
348
+ .from(schema.documentBases)
349
+ .where(
350
+ and(eq(schema.documentBases.workspaceId, workspaceId), eq(schema.documentBases.id, baseId)),
351
+ )
352
+ .limit(1);
320
353
  return row ? mapDocumentBase(row) : null;
321
354
  });
322
355
  }
323
356
 
324
- export async function addDocumentToBase(db: Database, input: AddDocumentRequest & { accountId: string; workspaceId: string; baseId: string }): Promise<Document> {
325
- return await withRlsContext(db, { accountId: input.accountId, workspaceId: input.workspaceId }, async (scopedDb) => {
326
- const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
327
- if (!base) throw new Error(`Document base not found: ${input.baseId}`);
328
- const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
329
- const now = new Date();
330
- const [existing] = await scopedDb.select().from(schema.documents)
331
- .where(and(eq(schema.documents.workspaceId, input.workspaceId), eq(schema.documents.baseId, input.baseId), eq(schema.documents.fileId, input.fileId)))
332
- .limit(1);
333
- if (existing) {
334
- // Idempotent re-add: refresh caller-supplied source metadata on the
335
- // existing row instead of silently discarding it (aclTags especially —
336
- // a re-add that tightens tags must not be a no-op).
337
- const [updated] = await scopedDb.update(schema.documents).set({
338
- title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? existing.title,
339
- ...(input.sourceKind !== undefined ? { sourceKind: input.sourceKind } : {}),
340
- sourceUri: cleanString(input.sourceUri) ?? existing.sourceUri,
341
- sourceExternalId: cleanString(input.sourceExternalId) ?? existing.sourceExternalId,
342
- sourceTitle: cleanString(input.sourceTitle) ?? existing.sourceTitle,
343
- sourceAuthor: cleanString(input.sourceAuthor) ?? existing.sourceAuthor,
344
- sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt) ?? existing.sourceCreatedAt,
345
- sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt) ?? existing.sourceUpdatedAt,
346
- sourceVersion: cleanString(input.sourceVersion) ?? existing.sourceVersion,
347
- ...(input.aclTags !== undefined ? { aclTags: cleanStringArray(input.aclTags) } : {}),
348
- updatedAt: now,
349
- }).where(and(eq(schema.documents.workspaceId, input.workspaceId), eq(schema.documents.id, existing.id))).returning();
350
- return mapDocument(updated ?? existing);
351
- }
352
- const [row] = await scopedDb.insert(schema.documents).values({
353
- accountId: input.accountId,
354
- workspaceId: input.workspaceId,
355
- baseId: input.baseId,
356
- fileId: input.fileId,
357
- status: "queued",
358
- title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? file.filename,
359
- parser: DEFAULT_DOCUMENT_PARSER,
360
- sourceKind: input.sourceKind ?? "manual_upload",
361
- sourceUri: cleanString(input.sourceUri) ?? null,
362
- sourceExternalId: cleanString(input.sourceExternalId) ?? null,
363
- sourceTitle: cleanString(input.sourceTitle) ?? null,
364
- sourceAuthor: cleanString(input.sourceAuthor) ?? null,
365
- sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
366
- sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
367
- sourceVersion: cleanString(input.sourceVersion) ?? null,
368
- aclTags: cleanStringArray(input.aclTags),
369
- updatedAt: now,
370
- }).returning();
371
- if (!row) throw new Error("Failed to create document");
372
- return mapDocument(row);
373
- });
357
+ export async function addDocumentToBase(
358
+ db: Database,
359
+ input: AddDocumentRequest & { accountId: string; workspaceId: string; baseId: string },
360
+ ): Promise<Document> {
361
+ return await withRlsContext(
362
+ db,
363
+ { accountId: input.accountId, workspaceId: input.workspaceId },
364
+ async (scopedDb) => {
365
+ const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
366
+ if (!base) throw new Error(`Document base not found: ${input.baseId}`);
367
+ const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
368
+ const now = new Date();
369
+ const [existing] = await scopedDb
370
+ .select()
371
+ .from(schema.documents)
372
+ .where(
373
+ and(
374
+ eq(schema.documents.workspaceId, input.workspaceId),
375
+ eq(schema.documents.baseId, input.baseId),
376
+ eq(schema.documents.fileId, input.fileId),
377
+ ),
378
+ )
379
+ .limit(1);
380
+ if (existing) {
381
+ // Idempotent re-add: refresh caller-supplied source metadata on the
382
+ // existing row instead of silently discarding it (aclTags especially —
383
+ // a re-add that tightens tags must not be a no-op).
384
+ const [updated] = await scopedDb
385
+ .update(schema.documents)
386
+ .set({
387
+ title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? existing.title,
388
+ ...(input.sourceKind !== undefined ? { sourceKind: input.sourceKind } : {}),
389
+ sourceUri: cleanString(input.sourceUri) ?? existing.sourceUri,
390
+ sourceExternalId: cleanString(input.sourceExternalId) ?? existing.sourceExternalId,
391
+ sourceTitle: cleanString(input.sourceTitle) ?? existing.sourceTitle,
392
+ sourceAuthor: cleanString(input.sourceAuthor) ?? existing.sourceAuthor,
393
+ sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt) ?? existing.sourceCreatedAt,
394
+ sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt) ?? existing.sourceUpdatedAt,
395
+ sourceVersion: cleanString(input.sourceVersion) ?? existing.sourceVersion,
396
+ ...(input.aclTags !== undefined ? { aclTags: cleanStringArray(input.aclTags) } : {}),
397
+ updatedAt: now,
398
+ })
399
+ .where(
400
+ and(
401
+ eq(schema.documents.workspaceId, input.workspaceId),
402
+ eq(schema.documents.id, existing.id),
403
+ ),
404
+ )
405
+ .returning();
406
+ return mapDocument(updated ?? existing);
407
+ }
408
+ const [row] = await scopedDb
409
+ .insert(schema.documents)
410
+ .values({
411
+ accountId: input.accountId,
412
+ workspaceId: input.workspaceId,
413
+ baseId: input.baseId,
414
+ fileId: input.fileId,
415
+ status: "queued",
416
+ title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? file.filename,
417
+ parser: DEFAULT_DOCUMENT_PARSER,
418
+ sourceKind: input.sourceKind ?? "manual_upload",
419
+ sourceUri: cleanString(input.sourceUri) ?? null,
420
+ sourceExternalId: cleanString(input.sourceExternalId) ?? null,
421
+ sourceTitle: cleanString(input.sourceTitle) ?? null,
422
+ sourceAuthor: cleanString(input.sourceAuthor) ?? null,
423
+ sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
424
+ sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
425
+ sourceVersion: cleanString(input.sourceVersion) ?? null,
426
+ aclTags: cleanStringArray(input.aclTags),
427
+ updatedAt: now,
428
+ })
429
+ .returning();
430
+ if (!row) throw new Error("Failed to create document");
431
+ return mapDocument(row);
432
+ },
433
+ );
374
434
  }
375
435
 
376
436
  export async function deleteDocumentFromBase(
377
437
  db: Database,
378
438
  input: { accountId: string; workspaceId: string; baseId: string; documentId: string },
379
439
  ): Promise<void> {
380
- await withRlsContext(db, { accountId: input.accountId, workspaceId: input.workspaceId }, async (scopedDb) => {
381
- const [document] = await scopedDb.select().from(schema.documents)
382
- .where(and(eq(schema.documents.workspaceId, input.workspaceId), eq(schema.documents.id, input.documentId)))
383
- .limit(1);
384
- if (!document) {
385
- throw new Error(`Document not found: ${input.documentId}`);
386
- }
387
- if (document.baseId !== input.baseId) {
388
- throw new Error(`Document not found: ${input.documentId}`);
389
- }
390
- await scopedDb.delete(schema.documents)
391
- .where(and(eq(schema.documents.workspaceId, input.workspaceId), eq(schema.documents.id, input.documentId)));
392
- });
440
+ await withRlsContext(
441
+ db,
442
+ { accountId: input.accountId, workspaceId: input.workspaceId },
443
+ async (scopedDb) => {
444
+ const [document] = await scopedDb
445
+ .select()
446
+ .from(schema.documents)
447
+ .where(
448
+ and(
449
+ eq(schema.documents.workspaceId, input.workspaceId),
450
+ eq(schema.documents.id, input.documentId),
451
+ ),
452
+ )
453
+ .limit(1);
454
+ if (!document) {
455
+ throw new Error(`Document not found: ${input.documentId}`);
456
+ }
457
+ if (document.baseId !== input.baseId) {
458
+ throw new Error(`Document not found: ${input.documentId}`);
459
+ }
460
+ await scopedDb
461
+ .delete(schema.documents)
462
+ .where(
463
+ and(
464
+ eq(schema.documents.workspaceId, input.workspaceId),
465
+ eq(schema.documents.id, input.documentId),
466
+ ),
467
+ );
468
+ },
469
+ );
393
470
  }
394
471
 
395
- export async function listDocuments(db: Database, workspaceId: string, baseId: string): Promise<Document[]> {
472
+ export async function listDocuments(
473
+ db: Database,
474
+ workspaceId: string,
475
+ baseId: string,
476
+ ): Promise<Document[]> {
396
477
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
397
- const rows = await scopedDb.select().from(schema.documents)
398
- .where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.baseId, baseId)))
478
+ const rows = await scopedDb
479
+ .select()
480
+ .from(schema.documents)
481
+ .where(
482
+ and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.baseId, baseId)),
483
+ )
399
484
  .orderBy(asc(schema.documents.createdAt));
400
485
  return rows.map(mapDocument);
401
486
  });
402
487
  }
403
488
 
404
- export async function getDocument(db: Database, workspaceId: string, documentId: string): Promise<Document | null> {
489
+ export async function getDocument(
490
+ db: Database,
491
+ workspaceId: string,
492
+ documentId: string,
493
+ ): Promise<Document | null> {
405
494
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
406
- const [row] = await scopedDb.select().from(schema.documents).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))).limit(1);
495
+ const [row] = await scopedDb
496
+ .select()
497
+ .from(schema.documents)
498
+ .where(
499
+ and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
500
+ )
501
+ .limit(1);
407
502
  return row ? mapDocument(row) : null;
408
503
  });
409
504
  }
410
505
 
411
- export async function queueDocumentForReindex(db: Database, workspaceId: string, documentId: string): Promise<Document> {
506
+ export async function queueDocumentForReindex(
507
+ db: Database,
508
+ workspaceId: string,
509
+ documentId: string,
510
+ ): Promise<Document> {
412
511
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
413
- const [row] = await scopedDb.update(schema.documents).set({
414
- status: "queued",
415
- error: null,
416
- updatedAt: new Date(),
417
- }).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))).returning();
512
+ const [row] = await scopedDb
513
+ .update(schema.documents)
514
+ .set({
515
+ status: "queued",
516
+ error: null,
517
+ updatedAt: new Date(),
518
+ })
519
+ .where(
520
+ and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
521
+ )
522
+ .returning();
418
523
  if (!row) throw new Error(`Document not found: ${documentId}`);
419
524
  return mapDocument(row);
420
525
  });
@@ -428,18 +533,32 @@ export async function indexDocumentNow(
428
533
  services: DocumentServices = createDocumentServices(),
429
534
  hooks: DocumentIndexHooks = {},
430
535
  ): Promise<Document> {
431
- const [document] = await withWorkspaceRls(db, workspaceId, async (scopedDb) =>
432
- await scopedDb.select().from(schema.documents).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))).limit(1)
536
+ const [document] = await withWorkspaceRls(
537
+ db,
538
+ workspaceId,
539
+ async (scopedDb) =>
540
+ await scopedDb
541
+ .select()
542
+ .from(schema.documents)
543
+ .where(
544
+ and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
545
+ )
546
+ .limit(1),
433
547
  );
434
548
  if (!document) throw new Error(`Document not found: ${documentId}`);
435
549
  const file = await requireReadyFile(db, workspaceId, document.fileId);
436
550
  await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
437
- await scopedDb.update(schema.documents).set({
438
- status: "indexing",
439
- parser: services.parser.name,
440
- error: null,
441
- updatedAt: new Date(),
442
- }).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)));
551
+ await scopedDb
552
+ .update(schema.documents)
553
+ .set({
554
+ status: "indexing",
555
+ parser: services.parser.name,
556
+ error: null,
557
+ updatedAt: new Date(),
558
+ })
559
+ .where(
560
+ and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
561
+ );
443
562
  });
444
563
  try {
445
564
  const bytes = await objectStorage.getFileBytes(file);
@@ -453,51 +572,88 @@ export async function indexDocumentNow(
453
572
  });
454
573
  const embeddings = await services.embedder.embedMany(chunks.map((chunk) => chunk.text));
455
574
  if (embeddings.length !== chunks.length) {
456
- throw new Error(`Embedding provider returned ${embeddings.length} embeddings for ${chunks.length} chunks`);
575
+ throw new Error(
576
+ `Embedding provider returned ${embeddings.length} embeddings for ${chunks.length} chunks`,
577
+ );
457
578
  }
458
- await withWorkspaceRls(db, workspaceId, async (scopedDb) => await scopedDb.transaction(async (tx) => {
459
- await tx.delete(schema.documentChunks).where(and(eq(schema.documentChunks.workspaceId, workspaceId), eq(schema.documentChunks.documentId, documentId)));
460
- if (chunks.length > 0) {
461
- await tx.insert(schema.documentChunks).values(chunks.map((chunk, index) => ({
462
- accountId: document.accountId,
463
- workspaceId: document.workspaceId,
464
- documentId,
465
- baseId: document.baseId,
466
- fileId: file.id,
467
- chunkIndex: index,
468
- text: chunk.text,
469
- metadata: {
470
- ...chunk.metadata,
471
- documentTitle: document.title,
472
- sourceKind: document.sourceKind,
473
- sourceUri: document.sourceUri,
474
- sourceExternalId: document.sourceExternalId,
475
- sourceTitle: document.sourceTitle,
476
- sourceAuthor: document.sourceAuthor,
477
- sourceCreatedAt: document.sourceCreatedAt?.toISOString() ?? null,
478
- sourceUpdatedAt: document.sourceUpdatedAt?.toISOString() ?? null,
479
- sourceVersion: document.sourceVersion,
480
- aclTags: document.aclTags,
481
- },
482
- embedding: validateEmbedding(embeddings[index] ?? [], services.embedder.dimensions, services.embedder.model),
483
- embeddingModel: services.embedder.model,
484
- })));
485
- }
486
- await tx.update(schema.documents).set({
487
- status: "ready",
488
- parser: services.parser.name,
489
- chunkCount: chunks.length,
490
- error: null,
491
- updatedAt: new Date(),
492
- }).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)));
493
- }));
579
+ await withWorkspaceRls(
580
+ db,
581
+ workspaceId,
582
+ async (scopedDb) =>
583
+ await scopedDb.transaction(async (tx) => {
584
+ await tx
585
+ .delete(schema.documentChunks)
586
+ .where(
587
+ and(
588
+ eq(schema.documentChunks.workspaceId, workspaceId),
589
+ eq(schema.documentChunks.documentId, documentId),
590
+ ),
591
+ );
592
+ if (chunks.length > 0) {
593
+ await tx.insert(schema.documentChunks).values(
594
+ chunks.map((chunk, index) => ({
595
+ accountId: document.accountId,
596
+ workspaceId: document.workspaceId,
597
+ documentId,
598
+ baseId: document.baseId,
599
+ fileId: file.id,
600
+ chunkIndex: index,
601
+ text: chunk.text,
602
+ metadata: {
603
+ ...chunk.metadata,
604
+ documentTitle: document.title,
605
+ sourceKind: document.sourceKind,
606
+ sourceUri: document.sourceUri,
607
+ sourceExternalId: document.sourceExternalId,
608
+ sourceTitle: document.sourceTitle,
609
+ sourceAuthor: document.sourceAuthor,
610
+ sourceCreatedAt: document.sourceCreatedAt?.toISOString() ?? null,
611
+ sourceUpdatedAt: document.sourceUpdatedAt?.toISOString() ?? null,
612
+ sourceVersion: document.sourceVersion,
613
+ aclTags: document.aclTags,
614
+ },
615
+ embedding: validateEmbedding(
616
+ embeddings[index] ?? [],
617
+ services.embedder.dimensions,
618
+ services.embedder.model,
619
+ ),
620
+ embeddingModel: services.embedder.model,
621
+ })),
622
+ );
623
+ }
624
+ await tx
625
+ .update(schema.documents)
626
+ .set({
627
+ status: "ready",
628
+ parser: services.parser.name,
629
+ chunkCount: chunks.length,
630
+ error: null,
631
+ updatedAt: new Date(),
632
+ })
633
+ .where(
634
+ and(
635
+ eq(schema.documents.workspaceId, workspaceId),
636
+ eq(schema.documents.id, documentId),
637
+ ),
638
+ );
639
+ }),
640
+ );
494
641
  } catch (error) {
495
- const [failed] = await withWorkspaceRls(db, workspaceId, async (scopedDb) =>
496
- await scopedDb.update(schema.documents).set({
497
- status: "failed",
498
- error: error instanceof Error ? error.message : String(error),
499
- updatedAt: new Date(),
500
- }).where(and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))).returning()
642
+ const [failed] = await withWorkspaceRls(
643
+ db,
644
+ workspaceId,
645
+ async (scopedDb) =>
646
+ await scopedDb
647
+ .update(schema.documents)
648
+ .set({
649
+ status: "failed",
650
+ error: error instanceof Error ? error.message : String(error),
651
+ updatedAt: new Date(),
652
+ })
653
+ .where(
654
+ and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
655
+ )
656
+ .returning(),
501
657
  );
502
658
  if (!failed) throw error;
503
659
  return mapDocument(failed);
@@ -518,19 +674,22 @@ export async function searchDocuments(
518
674
  const rows: CombinedSearchRow[] = [];
519
675
  if (mode === "vector" || mode === "hybrid") {
520
676
  try {
521
- rows.push(...await vectorSearchDocuments(db, input, candidateLimit, services));
677
+ rows.push(...(await vectorSearchDocuments(db, input, candidateLimit, services)));
522
678
  } catch (error) {
523
679
  if (mode === "vector") {
524
680
  throw error;
525
681
  }
526
- console.warn("document hybrid search vector component failed; falling back to keyword search", {
527
- workspaceId: input.workspaceId,
528
- error: error instanceof Error ? error.message : String(error),
529
- });
682
+ console.warn(
683
+ "document hybrid search vector component failed; falling back to keyword search",
684
+ {
685
+ workspaceId: input.workspaceId,
686
+ error: error instanceof Error ? error.message : String(error),
687
+ },
688
+ );
530
689
  }
531
690
  }
532
691
  if (mode === "keyword" || mode === "hybrid") {
533
- rows.push(...await keywordSearchDocuments(db, input, candidateLimit));
692
+ rows.push(...(await keywordSearchDocuments(db, input, candidateLimit)));
534
693
  }
535
694
  return mergeDocumentSearchRows(rows, mode).slice(0, limit);
536
695
  }
@@ -544,31 +703,36 @@ async function vectorSearchDocuments(
544
703
  const queryEmbedding = await services.embedder.embedQuery(input.query);
545
704
  validateEmbedding(queryEmbedding, services.embedder.dimensions, services.embedder.model);
546
705
  const distance = sql<number>`${schema.documentChunks.embedding} <=> ${vectorLiteral(queryEmbedding)}::vector`;
547
- const rows = await withWorkspaceRls(db, input.workspaceId, async (scopedDb) =>
548
- await scopedDb.select({
549
- chunkId: schema.documentChunks.id,
550
- documentId: schema.documentChunks.documentId,
551
- baseId: schema.documentChunks.baseId,
552
- fileId: schema.documentChunks.fileId,
553
- title: schema.documents.title,
554
- text: schema.documentChunks.text,
555
- chunkIndex: schema.documentChunks.chunkIndex,
556
- metadata: schema.documentChunks.metadata,
557
- sourceKind: schema.documents.sourceKind,
558
- sourceUri: schema.documents.sourceUri,
559
- sourceExternalId: schema.documents.sourceExternalId,
560
- sourceTitle: schema.documents.sourceTitle,
561
- sourceAuthor: schema.documents.sourceAuthor,
562
- sourceCreatedAt: schema.documents.sourceCreatedAt,
563
- sourceUpdatedAt: schema.documents.sourceUpdatedAt,
564
- sourceVersion: schema.documents.sourceVersion,
565
- aclTags: schema.documents.aclTags,
566
- distance,
567
- }).from(schema.documentChunks)
568
- .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
569
- .where(and(...documentSearchConditions(input, services.embedder.model)))
570
- .orderBy(distance)
571
- .limit(limit)
706
+ const rows = await withWorkspaceRls(
707
+ db,
708
+ input.workspaceId,
709
+ async (scopedDb) =>
710
+ await scopedDb
711
+ .select({
712
+ chunkId: schema.documentChunks.id,
713
+ documentId: schema.documentChunks.documentId,
714
+ baseId: schema.documentChunks.baseId,
715
+ fileId: schema.documentChunks.fileId,
716
+ title: schema.documents.title,
717
+ text: schema.documentChunks.text,
718
+ chunkIndex: schema.documentChunks.chunkIndex,
719
+ metadata: schema.documentChunks.metadata,
720
+ sourceKind: schema.documents.sourceKind,
721
+ sourceUri: schema.documents.sourceUri,
722
+ sourceExternalId: schema.documents.sourceExternalId,
723
+ sourceTitle: schema.documents.sourceTitle,
724
+ sourceAuthor: schema.documents.sourceAuthor,
725
+ sourceCreatedAt: schema.documents.sourceCreatedAt,
726
+ sourceUpdatedAt: schema.documents.sourceUpdatedAt,
727
+ sourceVersion: schema.documents.sourceVersion,
728
+ aclTags: schema.documents.aclTags,
729
+ distance,
730
+ })
731
+ .from(schema.documentChunks)
732
+ .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
733
+ .where(and(...documentSearchConditions(input, services.embedder.model)))
734
+ .orderBy(distance)
735
+ .limit(limit),
572
736
  );
573
737
  return rows.map((row) => ({
574
738
  ...mapSearchRowBase(row, input.workspaceId),
@@ -577,36 +741,47 @@ async function vectorSearchDocuments(
577
741
  }));
578
742
  }
579
743
 
580
- async function keywordSearchDocuments(db: Database, input: DocumentSearchInput, limit: number): Promise<CombinedSearchRow[]> {
744
+ async function keywordSearchDocuments(
745
+ db: Database,
746
+ input: DocumentSearchInput,
747
+ limit: number,
748
+ ): Promise<CombinedSearchRow[]> {
581
749
  const rank = sql<number>`ts_rank_cd(to_tsvector('simple', ${schema.documentChunks.text}), plainto_tsquery('simple', ${input.query}))`;
582
- const rows = await withWorkspaceRls(db, input.workspaceId, async (scopedDb) =>
583
- await scopedDb.select({
584
- chunkId: schema.documentChunks.id,
585
- documentId: schema.documentChunks.documentId,
586
- baseId: schema.documentChunks.baseId,
587
- fileId: schema.documentChunks.fileId,
588
- title: schema.documents.title,
589
- text: schema.documentChunks.text,
590
- chunkIndex: schema.documentChunks.chunkIndex,
591
- metadata: schema.documentChunks.metadata,
592
- sourceKind: schema.documents.sourceKind,
593
- sourceUri: schema.documents.sourceUri,
594
- sourceExternalId: schema.documents.sourceExternalId,
595
- sourceTitle: schema.documents.sourceTitle,
596
- sourceAuthor: schema.documents.sourceAuthor,
597
- sourceCreatedAt: schema.documents.sourceCreatedAt,
598
- sourceUpdatedAt: schema.documents.sourceUpdatedAt,
599
- sourceVersion: schema.documents.sourceVersion,
600
- aclTags: schema.documents.aclTags,
601
- rank,
602
- }).from(schema.documentChunks)
603
- .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
604
- .where(and(
605
- ...documentSearchConditions(input),
606
- sql`to_tsvector('simple', ${schema.documentChunks.text}) @@ plainto_tsquery('simple', ${input.query})`,
607
- ))
608
- .orderBy(desc(rank))
609
- .limit(limit)
750
+ const rows = await withWorkspaceRls(
751
+ db,
752
+ input.workspaceId,
753
+ async (scopedDb) =>
754
+ await scopedDb
755
+ .select({
756
+ chunkId: schema.documentChunks.id,
757
+ documentId: schema.documentChunks.documentId,
758
+ baseId: schema.documentChunks.baseId,
759
+ fileId: schema.documentChunks.fileId,
760
+ title: schema.documents.title,
761
+ text: schema.documentChunks.text,
762
+ chunkIndex: schema.documentChunks.chunkIndex,
763
+ metadata: schema.documentChunks.metadata,
764
+ sourceKind: schema.documents.sourceKind,
765
+ sourceUri: schema.documents.sourceUri,
766
+ sourceExternalId: schema.documents.sourceExternalId,
767
+ sourceTitle: schema.documents.sourceTitle,
768
+ sourceAuthor: schema.documents.sourceAuthor,
769
+ sourceCreatedAt: schema.documents.sourceCreatedAt,
770
+ sourceUpdatedAt: schema.documents.sourceUpdatedAt,
771
+ sourceVersion: schema.documents.sourceVersion,
772
+ aclTags: schema.documents.aclTags,
773
+ rank,
774
+ })
775
+ .from(schema.documentChunks)
776
+ .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
777
+ .where(
778
+ and(
779
+ ...documentSearchConditions(input),
780
+ sql`to_tsvector('simple', ${schema.documentChunks.text}) @@ plainto_tsquery('simple', ${input.query})`,
781
+ ),
782
+ )
783
+ .orderBy(desc(rank))
784
+ .limit(limit),
610
785
  );
611
786
  return rows.map((row) => ({
612
787
  ...mapSearchRowBase(row, input.workspaceId),
@@ -615,30 +790,45 @@ async function keywordSearchDocuments(db: Database, input: DocumentSearchInput,
615
790
  }));
616
791
  }
617
792
 
618
- export async function getDocumentChunk(db: Database, workspaceId: string, chunkId: string): Promise<DocumentSearchResult | null> {
619
- const [row] = await withWorkspaceRls(db, workspaceId, async (scopedDb) =>
620
- await scopedDb.select({
621
- chunkId: schema.documentChunks.id,
622
- documentId: schema.documentChunks.documentId,
623
- baseId: schema.documentChunks.baseId,
624
- fileId: schema.documentChunks.fileId,
625
- title: schema.documents.title,
626
- text: schema.documentChunks.text,
627
- chunkIndex: schema.documentChunks.chunkIndex,
628
- metadata: schema.documentChunks.metadata,
629
- sourceKind: schema.documents.sourceKind,
630
- sourceUri: schema.documents.sourceUri,
631
- sourceExternalId: schema.documents.sourceExternalId,
632
- sourceTitle: schema.documents.sourceTitle,
633
- sourceAuthor: schema.documents.sourceAuthor,
634
- sourceCreatedAt: schema.documents.sourceCreatedAt,
635
- sourceUpdatedAt: schema.documents.sourceUpdatedAt,
636
- sourceVersion: schema.documents.sourceVersion,
637
- aclTags: schema.documents.aclTags,
638
- }).from(schema.documentChunks)
639
- .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
640
- .where(and(eq(schema.documentChunks.workspaceId, workspaceId), eq(schema.documentChunks.id, chunkId), eq(schema.documents.status, "ready")))
641
- .limit(1)
793
+ export async function getDocumentChunk(
794
+ db: Database,
795
+ workspaceId: string,
796
+ chunkId: string,
797
+ ): Promise<DocumentSearchResult | null> {
798
+ const [row] = await withWorkspaceRls(
799
+ db,
800
+ workspaceId,
801
+ async (scopedDb) =>
802
+ await scopedDb
803
+ .select({
804
+ chunkId: schema.documentChunks.id,
805
+ documentId: schema.documentChunks.documentId,
806
+ baseId: schema.documentChunks.baseId,
807
+ fileId: schema.documentChunks.fileId,
808
+ title: schema.documents.title,
809
+ text: schema.documentChunks.text,
810
+ chunkIndex: schema.documentChunks.chunkIndex,
811
+ metadata: schema.documentChunks.metadata,
812
+ sourceKind: schema.documents.sourceKind,
813
+ sourceUri: schema.documents.sourceUri,
814
+ sourceExternalId: schema.documents.sourceExternalId,
815
+ sourceTitle: schema.documents.sourceTitle,
816
+ sourceAuthor: schema.documents.sourceAuthor,
817
+ sourceCreatedAt: schema.documents.sourceCreatedAt,
818
+ sourceUpdatedAt: schema.documents.sourceUpdatedAt,
819
+ sourceVersion: schema.documents.sourceVersion,
820
+ aclTags: schema.documents.aclTags,
821
+ })
822
+ .from(schema.documentChunks)
823
+ .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
824
+ .where(
825
+ and(
826
+ eq(schema.documentChunks.workspaceId, workspaceId),
827
+ eq(schema.documentChunks.id, chunkId),
828
+ eq(schema.documents.status, "ready"),
829
+ ),
830
+ )
831
+ .limit(1),
642
832
  );
643
833
  if (!row) return null;
644
834
  return {
@@ -650,7 +840,10 @@ export async function getDocumentChunk(db: Database, workspaceId: string, chunkI
650
840
  };
651
841
  }
652
842
 
653
- type SearchRowBase = Omit<DocumentSearchResult, "score" | "matchType" | "vectorScore" | "keywordScore">;
843
+ type SearchRowBase = Omit<
844
+ DocumentSearchResult,
845
+ "score" | "matchType" | "vectorScore" | "keywordScore"
846
+ >;
654
847
  type CombinedSearchRow = SearchRowBase & {
655
848
  vectorScore: number | null;
656
849
  keywordScore: number | null;
@@ -677,25 +870,28 @@ function documentSearchConditions(input: DocumentSearchInput, embeddingModel?: s
677
870
  return conditions;
678
871
  }
679
872
 
680
- function mapSearchRowBase(row: {
681
- chunkId: string;
682
- documentId: string;
683
- baseId: string;
684
- fileId: string;
685
- title: string;
686
- text: string;
687
- chunkIndex: number;
688
- metadata: Record<string, unknown>;
689
- sourceKind: string;
690
- sourceUri: string | null;
691
- sourceExternalId: string | null;
692
- sourceTitle: string | null;
693
- sourceAuthor: string | null;
694
- sourceCreatedAt: Date | null;
695
- sourceUpdatedAt: Date | null;
696
- sourceVersion: string | null;
697
- aclTags: string[];
698
- }, workspaceId: string): SearchRowBase {
873
+ function mapSearchRowBase(
874
+ row: {
875
+ chunkId: string;
876
+ documentId: string;
877
+ baseId: string;
878
+ fileId: string;
879
+ title: string;
880
+ text: string;
881
+ chunkIndex: number;
882
+ metadata: Record<string, unknown>;
883
+ sourceKind: string;
884
+ sourceUri: string | null;
885
+ sourceExternalId: string | null;
886
+ sourceTitle: string | null;
887
+ sourceAuthor: string | null;
888
+ sourceCreatedAt: Date | null;
889
+ sourceUpdatedAt: Date | null;
890
+ sourceVersion: string | null;
891
+ aclTags: string[];
892
+ },
893
+ workspaceId: string,
894
+ ): SearchRowBase {
699
895
  return {
700
896
  chunkId: row.chunkId,
701
897
  workspaceId,
@@ -718,7 +914,10 @@ function mapSearchRowBase(row: {
718
914
  };
719
915
  }
720
916
 
721
- function mergeDocumentSearchRows(rows: CombinedSearchRow[], mode: DocumentSearchMode): DocumentSearchResult[] {
917
+ function mergeDocumentSearchRows(
918
+ rows: CombinedSearchRow[],
919
+ mode: DocumentSearchMode,
920
+ ): DocumentSearchResult[] {
722
921
  const byChunk = new Map<string, CombinedSearchRow>();
723
922
  for (const row of rows) {
724
923
  const existing = byChunk.get(row.chunkId);
@@ -732,25 +931,29 @@ function mergeDocumentSearchRows(rows: CombinedSearchRow[], mode: DocumentSearch
732
931
  keywordScore: Math.max(existing.keywordScore ?? 0, row.keywordScore ?? 0) || null,
733
932
  });
734
933
  }
735
- return [...byChunk.values()].map((row) => {
736
- const vectorScore = row.vectorScore;
737
- const keywordScore = row.keywordScore;
738
- const matchType: DocumentSearchMode = vectorScore !== null && keywordScore !== null
739
- ? "hybrid"
740
- : vectorScore !== null
741
- ? "vector"
742
- : "keyword";
743
- return {
744
- ...row,
745
- score: combinedSearchScore(mode, vectorScore, keywordScore, matchType),
746
- matchType,
747
- };
748
- }).sort((left, right) =>
749
- right.score - left.score
750
- || (right.vectorScore ?? 0) - (left.vectorScore ?? 0)
751
- || (right.keywordScore ?? 0) - (left.keywordScore ?? 0)
752
- || left.chunkIndex - right.chunkIndex
753
- );
934
+ return [...byChunk.values()]
935
+ .map((row) => {
936
+ const vectorScore = row.vectorScore;
937
+ const keywordScore = row.keywordScore;
938
+ const matchType: DocumentSearchMode =
939
+ vectorScore !== null && keywordScore !== null
940
+ ? "hybrid"
941
+ : vectorScore !== null
942
+ ? "vector"
943
+ : "keyword";
944
+ return {
945
+ ...row,
946
+ score: combinedSearchScore(mode, vectorScore, keywordScore, matchType),
947
+ matchType,
948
+ };
949
+ })
950
+ .sort(
951
+ (left, right) =>
952
+ right.score - left.score ||
953
+ (right.vectorScore ?? 0) - (left.vectorScore ?? 0) ||
954
+ (right.keywordScore ?? 0) - (left.keywordScore ?? 0) ||
955
+ left.chunkIndex - right.chunkIndex,
956
+ );
754
957
  }
755
958
 
756
959
  function combinedSearchScore(
@@ -763,7 +966,9 @@ function combinedSearchScore(
763
966
  const keyword = keywordScore ?? 0;
764
967
  if (mode === "vector") return roundScore(vector);
765
968
  if (mode === "keyword") return roundScore(keyword);
766
- return roundScore(Math.min(1, (0.65 * vector) + (0.35 * keyword) + (matchType === "hybrid" ? 0.1 : 0)));
969
+ return roundScore(
970
+ Math.min(1, 0.65 * vector + 0.35 * keyword + (matchType === "hybrid" ? 0.1 : 0)),
971
+ );
767
972
  }
768
973
 
769
974
  function normalizeKeywordScore(rank: number): number {
@@ -777,17 +982,31 @@ function roundScore(value: number): number {
777
982
  return Number(value.toFixed(6));
778
983
  }
779
984
 
780
- export async function parseDocumentBytes(bytes: Uint8Array, file: FileAsset, parser: DocumentParser = new LiteParseDocumentParser()): Promise<ParsedDocument> {
985
+ export async function parseDocumentBytes(
986
+ bytes: Uint8Array,
987
+ file: FileAsset,
988
+ parser: DocumentParser = new LiteParseDocumentParser(),
989
+ ): Promise<ParsedDocument> {
781
990
  return await parser.parse(bytes, file);
782
991
  }
783
992
 
784
- export function chunkText(text: string, maxChars = DEFAULT_DOCUMENT_CHUNK_SIZE, overlapChars = DEFAULT_DOCUMENT_CHUNK_OVERLAP): string[] {
993
+ export function chunkText(
994
+ text: string,
995
+ maxChars = DEFAULT_DOCUMENT_CHUNK_SIZE,
996
+ overlapChars = DEFAULT_DOCUMENT_CHUNK_OVERLAP,
997
+ ): string[] {
785
998
  if (overlapChars >= maxChars) {
786
999
  throw new Error("chunk overlap must be smaller than chunk size");
787
1000
  }
788
- const normalized = text.replace(/\r\n/g, "\n").replace(/[ \t]+/g, " ").trim();
1001
+ const normalized = text
1002
+ .replace(/\r\n/g, "\n")
1003
+ .replace(/[ \t]+/g, " ")
1004
+ .trim();
789
1005
  if (!normalized) return [];
790
- const paragraphs = normalized.split(/\n{2,}/).map((part) => part.replace(/\s+/g, " ").trim()).filter(Boolean);
1006
+ const paragraphs = normalized
1007
+ .split(/\n{2,}/)
1008
+ .map((part) => part.replace(/\s+/g, " ").trim())
1009
+ .filter(Boolean);
791
1010
  const chunks: string[] = [];
792
1011
  let current = "";
793
1012
  for (const paragraph of paragraphs.length > 0 ? paragraphs : [normalized.replace(/\s+/g, " ")]) {
@@ -806,7 +1025,10 @@ export function chunkText(text: string, maxChars = DEFAULT_DOCUMENT_CHUNK_SIZE,
806
1025
  return chunks.map((chunk) => chunk.trim()).filter(Boolean);
807
1026
  }
808
1027
 
809
- export function deterministicEmbedding(text: string, dimensions = DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS): number[] {
1028
+ export function deterministicEmbedding(
1029
+ text: string,
1030
+ dimensions = DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS,
1031
+ ): number[] {
810
1032
  const values = new Array(dimensions).fill(0);
811
1033
  const tokens = text.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? [];
812
1034
  for (const token of tokens) {
@@ -821,7 +1043,11 @@ export function deterministicEmbedding(text: string, dimensions = DEFAULT_DOCUME
821
1043
  return values.map((value) => Number((value / norm).toFixed(6)));
822
1044
  }
823
1045
 
824
- async function requireReadyFile(db: Database, workspaceId: string, fileId: string): Promise<FileAsset> {
1046
+ async function requireReadyFile(
1047
+ db: Database,
1048
+ workspaceId: string,
1049
+ fileId: string,
1050
+ ): Promise<FileAsset> {
825
1051
  const file = await requireFile(db, workspaceId, fileId);
826
1052
  if (file.status !== "ready") {
827
1053
  throw new Error(`File ${fileId} is ${file.status}`);
@@ -831,7 +1057,9 @@ async function requireReadyFile(db: Database, workspaceId: string, fileId: strin
831
1057
 
832
1058
  function validateEmbedding(values: number[], dimensions: number, model: string): number[] {
833
1059
  if (values.length !== dimensions) {
834
- throw new Error(`Embedding model ${model} returned ${values.length} dimensions; expected ${dimensions}`);
1060
+ throw new Error(
1061
+ `Embedding model ${model} returned ${values.length} dimensions; expected ${dimensions}`,
1062
+ );
835
1063
  }
836
1064
  if (values.some((value) => !Number.isFinite(value))) {
837
1065
  throw new Error(`Embedding model ${model} returned non-finite values`);
@@ -842,18 +1070,20 @@ function validateEmbedding(values: number[], dimensions: number, model: string):
842
1070
  function isTextLike(file: FileAsset): boolean {
843
1071
  const contentType = file.contentType.toLowerCase();
844
1072
  const filename = file.filename.toLowerCase();
845
- return contentType.startsWith("text/")
846
- || contentType === "application/json"
847
- || contentType === "application/xml"
848
- || contentType === "application/x-yaml"
849
- || filename.endsWith(".md")
850
- || filename.endsWith(".markdown")
851
- || filename.endsWith(".json")
852
- || filename.endsWith(".yaml")
853
- || filename.endsWith(".yml")
854
- || filename.endsWith(".csv")
855
- || filename.endsWith(".tsv")
856
- || filename.endsWith(".xml");
1073
+ return (
1074
+ contentType.startsWith("text/") ||
1075
+ contentType === "application/json" ||
1076
+ contentType === "application/xml" ||
1077
+ contentType === "application/x-yaml" ||
1078
+ filename.endsWith(".md") ||
1079
+ filename.endsWith(".markdown") ||
1080
+ filename.endsWith(".json") ||
1081
+ filename.endsWith(".yaml") ||
1082
+ filename.endsWith(".yml") ||
1083
+ filename.endsWith(".csv") ||
1084
+ filename.endsWith(".tsv") ||
1085
+ filename.endsWith(".xml")
1086
+ );
857
1087
  }
858
1088
 
859
1089
  function splitOversizedText(text: string, maxChars: number): string[] {
@@ -878,9 +1108,17 @@ function splitOversizedText(text: string, maxChars: number): string[] {
878
1108
  return out;
879
1109
  }
880
1110
 
881
- function withOverlap(previous: string, overlapChars: number, next: string, maxChars: number): string {
1111
+ function withOverlap(
1112
+ previous: string,
1113
+ overlapChars: number,
1114
+ next: string,
1115
+ maxChars: number,
1116
+ ): string {
882
1117
  if (overlapChars <= 0) return next;
883
- const overlap = previous.slice(Math.max(0, previous.length - overlapChars)).replace(/^\S+\s+/, "").trim();
1118
+ const overlap = previous
1119
+ .slice(Math.max(0, previous.length - overlapChars))
1120
+ .replace(/^\S+\s+/, "")
1121
+ .trim();
884
1122
  const candidate = overlap ? `${overlap} ${next}` : next;
885
1123
  return candidate.length <= maxChars ? candidate : next;
886
1124
  }