@opengeni/documents 0.2.8 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +200 -120
- package/dist/index.js.map +1 -1
- package/package.json +16 -16
- package/src/index.ts +541 -303
package/src/index.ts
CHANGED
|
@@ -221,24 +221,29 @@ export class DeterministicEmbeddingProvider implements DocumentEmbedder {
|
|
|
221
221
|
}
|
|
222
222
|
}
|
|
223
223
|
|
|
224
|
-
export function createDocumentServices(
|
|
224
|
+
export function createDocumentServices(
|
|
225
|
+
settings?: Settings,
|
|
226
|
+
overrides: Partial<DocumentServices> = {},
|
|
227
|
+
): DocumentServices {
|
|
225
228
|
const dimensions = settings?.documentEmbeddingDimensions ?? DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS;
|
|
226
229
|
const openAIEmbeddingConfig = documentOpenAIEmbeddingConfig(settings);
|
|
227
230
|
return {
|
|
228
231
|
parser: overrides.parser ?? new LiteParseDocumentParser(),
|
|
229
|
-
chunker:
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
232
|
+
chunker:
|
|
233
|
+
overrides.chunker ??
|
|
234
|
+
new RecursiveTextChunker(
|
|
235
|
+
settings?.documentChunkSize ?? DEFAULT_DOCUMENT_CHUNK_SIZE,
|
|
236
|
+
settings?.documentChunkOverlap ?? DEFAULT_DOCUMENT_CHUNK_OVERLAP,
|
|
237
|
+
),
|
|
238
|
+
embedder:
|
|
239
|
+
overrides.embedder ??
|
|
240
|
+
(settings?.documentEmbeddingProvider === "deterministic"
|
|
235
241
|
? new DeterministicEmbeddingProvider(dimensions, settings.documentEmbeddingModel)
|
|
236
242
|
: new OpenAIEmbeddingProvider({
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
),
|
|
243
|
+
...openAIEmbeddingConfig,
|
|
244
|
+
model: settings?.documentEmbeddingModel ?? DEFAULT_DOCUMENT_EMBEDDING_MODEL,
|
|
245
|
+
dimensions,
|
|
246
|
+
})),
|
|
242
247
|
};
|
|
243
248
|
}
|
|
244
249
|
|
|
@@ -251,8 +256,10 @@ export function documentOpenAIEmbeddingConfig(settings?: Settings): {
|
|
|
251
256
|
if (!settings) return {};
|
|
252
257
|
if (settings.documentEmbeddingApiKey || settings.documentEmbeddingBaseUrl) {
|
|
253
258
|
return {
|
|
254
|
-
apiKey:
|
|
255
|
-
|
|
259
|
+
apiKey:
|
|
260
|
+
settings.documentEmbeddingApiKey ?? settings.openaiApiKey ?? settings.azureOpenaiApiKey,
|
|
261
|
+
baseURL:
|
|
262
|
+
settings.documentEmbeddingBaseUrl ?? settings.openaiBaseUrl ?? settings.azureOpenaiBaseUrl,
|
|
256
263
|
};
|
|
257
264
|
}
|
|
258
265
|
if (settings.openaiProvider === "azure") {
|
|
@@ -261,9 +268,10 @@ export function documentOpenAIEmbeddingConfig(settings?: Settings): {
|
|
|
261
268
|
apiKey: settings.azureOpenaiApiKey ?? settings.azureOpenaiAdToken ?? "azure-ad-token",
|
|
262
269
|
baseURL,
|
|
263
270
|
defaultQuery: azureOpenAIDefaultQuery(settings, baseURL),
|
|
264
|
-
defaultHeaders:
|
|
265
|
-
|
|
266
|
-
|
|
271
|
+
defaultHeaders:
|
|
272
|
+
settings.azureOpenaiAdToken && !settings.azureOpenaiApiKey
|
|
273
|
+
? { Authorization: `Bearer ${settings.azureOpenaiAdToken}` }
|
|
274
|
+
: undefined,
|
|
267
275
|
};
|
|
268
276
|
}
|
|
269
277
|
return {
|
|
@@ -292,129 +300,226 @@ function azureOpenAIDefaultQuery(
|
|
|
292
300
|
return { "api-version": settings.azureOpenaiApiVersion };
|
|
293
301
|
}
|
|
294
302
|
|
|
295
|
-
export async function createDocumentBase(
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
303
|
+
export async function createDocumentBase(
|
|
304
|
+
db: Database,
|
|
305
|
+
input: CreateDocumentBaseRequest & { accountId: string; workspaceId: string },
|
|
306
|
+
): Promise<DocumentBase> {
|
|
307
|
+
return await withRlsContext(
|
|
308
|
+
db,
|
|
309
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
310
|
+
async (scopedDb) => {
|
|
311
|
+
const [row] = await scopedDb
|
|
312
|
+
.insert(schema.documentBases)
|
|
313
|
+
.values({
|
|
314
|
+
accountId: input.accountId,
|
|
315
|
+
workspaceId: input.workspaceId,
|
|
316
|
+
name: input.name.trim(),
|
|
317
|
+
description: input.description?.trim() || null,
|
|
318
|
+
})
|
|
319
|
+
.returning();
|
|
320
|
+
if (!row) throw new Error("Failed to create document base");
|
|
321
|
+
return mapDocumentBase(row);
|
|
322
|
+
},
|
|
323
|
+
);
|
|
306
324
|
}
|
|
307
325
|
|
|
308
|
-
export async function listDocumentBases(
|
|
326
|
+
export async function listDocumentBases(
|
|
327
|
+
db: Database,
|
|
328
|
+
workspaceId: string,
|
|
329
|
+
): Promise<DocumentBase[]> {
|
|
309
330
|
return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
|
|
310
|
-
const rows = await scopedDb
|
|
331
|
+
const rows = await scopedDb
|
|
332
|
+
.select()
|
|
333
|
+
.from(schema.documentBases)
|
|
311
334
|
.where(eq(schema.documentBases.workspaceId, workspaceId))
|
|
312
335
|
.orderBy(desc(schema.documentBases.createdAt));
|
|
313
336
|
return rows.map(mapDocumentBase);
|
|
314
337
|
});
|
|
315
338
|
}
|
|
316
339
|
|
|
317
|
-
export async function getDocumentBase(
|
|
340
|
+
export async function getDocumentBase(
|
|
341
|
+
db: Database,
|
|
342
|
+
workspaceId: string,
|
|
343
|
+
baseId: string,
|
|
344
|
+
): Promise<DocumentBase | null> {
|
|
318
345
|
return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
|
|
319
|
-
const [row] = await scopedDb
|
|
346
|
+
const [row] = await scopedDb
|
|
347
|
+
.select()
|
|
348
|
+
.from(schema.documentBases)
|
|
349
|
+
.where(
|
|
350
|
+
and(eq(schema.documentBases.workspaceId, workspaceId), eq(schema.documentBases.id, baseId)),
|
|
351
|
+
)
|
|
352
|
+
.limit(1);
|
|
320
353
|
return row ? mapDocumentBase(row) : null;
|
|
321
354
|
});
|
|
322
355
|
}
|
|
323
356
|
|
|
324
|
-
export async function addDocumentToBase(
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
.
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
357
|
+
export async function addDocumentToBase(
|
|
358
|
+
db: Database,
|
|
359
|
+
input: AddDocumentRequest & { accountId: string; workspaceId: string; baseId: string },
|
|
360
|
+
): Promise<Document> {
|
|
361
|
+
return await withRlsContext(
|
|
362
|
+
db,
|
|
363
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
364
|
+
async (scopedDb) => {
|
|
365
|
+
const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
|
|
366
|
+
if (!base) throw new Error(`Document base not found: ${input.baseId}`);
|
|
367
|
+
const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
|
|
368
|
+
const now = new Date();
|
|
369
|
+
const [existing] = await scopedDb
|
|
370
|
+
.select()
|
|
371
|
+
.from(schema.documents)
|
|
372
|
+
.where(
|
|
373
|
+
and(
|
|
374
|
+
eq(schema.documents.workspaceId, input.workspaceId),
|
|
375
|
+
eq(schema.documents.baseId, input.baseId),
|
|
376
|
+
eq(schema.documents.fileId, input.fileId),
|
|
377
|
+
),
|
|
378
|
+
)
|
|
379
|
+
.limit(1);
|
|
380
|
+
if (existing) {
|
|
381
|
+
// Idempotent re-add: refresh caller-supplied source metadata on the
|
|
382
|
+
// existing row instead of silently discarding it (aclTags especially —
|
|
383
|
+
// a re-add that tightens tags must not be a no-op).
|
|
384
|
+
const [updated] = await scopedDb
|
|
385
|
+
.update(schema.documents)
|
|
386
|
+
.set({
|
|
387
|
+
title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? existing.title,
|
|
388
|
+
...(input.sourceKind !== undefined ? { sourceKind: input.sourceKind } : {}),
|
|
389
|
+
sourceUri: cleanString(input.sourceUri) ?? existing.sourceUri,
|
|
390
|
+
sourceExternalId: cleanString(input.sourceExternalId) ?? existing.sourceExternalId,
|
|
391
|
+
sourceTitle: cleanString(input.sourceTitle) ?? existing.sourceTitle,
|
|
392
|
+
sourceAuthor: cleanString(input.sourceAuthor) ?? existing.sourceAuthor,
|
|
393
|
+
sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt) ?? existing.sourceCreatedAt,
|
|
394
|
+
sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt) ?? existing.sourceUpdatedAt,
|
|
395
|
+
sourceVersion: cleanString(input.sourceVersion) ?? existing.sourceVersion,
|
|
396
|
+
...(input.aclTags !== undefined ? { aclTags: cleanStringArray(input.aclTags) } : {}),
|
|
397
|
+
updatedAt: now,
|
|
398
|
+
})
|
|
399
|
+
.where(
|
|
400
|
+
and(
|
|
401
|
+
eq(schema.documents.workspaceId, input.workspaceId),
|
|
402
|
+
eq(schema.documents.id, existing.id),
|
|
403
|
+
),
|
|
404
|
+
)
|
|
405
|
+
.returning();
|
|
406
|
+
return mapDocument(updated ?? existing);
|
|
407
|
+
}
|
|
408
|
+
const [row] = await scopedDb
|
|
409
|
+
.insert(schema.documents)
|
|
410
|
+
.values({
|
|
411
|
+
accountId: input.accountId,
|
|
412
|
+
workspaceId: input.workspaceId,
|
|
413
|
+
baseId: input.baseId,
|
|
414
|
+
fileId: input.fileId,
|
|
415
|
+
status: "queued",
|
|
416
|
+
title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? file.filename,
|
|
417
|
+
parser: DEFAULT_DOCUMENT_PARSER,
|
|
418
|
+
sourceKind: input.sourceKind ?? "manual_upload",
|
|
419
|
+
sourceUri: cleanString(input.sourceUri) ?? null,
|
|
420
|
+
sourceExternalId: cleanString(input.sourceExternalId) ?? null,
|
|
421
|
+
sourceTitle: cleanString(input.sourceTitle) ?? null,
|
|
422
|
+
sourceAuthor: cleanString(input.sourceAuthor) ?? null,
|
|
423
|
+
sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
|
|
424
|
+
sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
|
|
425
|
+
sourceVersion: cleanString(input.sourceVersion) ?? null,
|
|
426
|
+
aclTags: cleanStringArray(input.aclTags),
|
|
427
|
+
updatedAt: now,
|
|
428
|
+
})
|
|
429
|
+
.returning();
|
|
430
|
+
if (!row) throw new Error("Failed to create document");
|
|
431
|
+
return mapDocument(row);
|
|
432
|
+
},
|
|
433
|
+
);
|
|
374
434
|
}
|
|
375
435
|
|
|
376
436
|
export async function deleteDocumentFromBase(
|
|
377
437
|
db: Database,
|
|
378
438
|
input: { accountId: string; workspaceId: string; baseId: string; documentId: string },
|
|
379
439
|
): Promise<void> {
|
|
380
|
-
await withRlsContext(
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
440
|
+
await withRlsContext(
|
|
441
|
+
db,
|
|
442
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
443
|
+
async (scopedDb) => {
|
|
444
|
+
const [document] = await scopedDb
|
|
445
|
+
.select()
|
|
446
|
+
.from(schema.documents)
|
|
447
|
+
.where(
|
|
448
|
+
and(
|
|
449
|
+
eq(schema.documents.workspaceId, input.workspaceId),
|
|
450
|
+
eq(schema.documents.id, input.documentId),
|
|
451
|
+
),
|
|
452
|
+
)
|
|
453
|
+
.limit(1);
|
|
454
|
+
if (!document) {
|
|
455
|
+
throw new Error(`Document not found: ${input.documentId}`);
|
|
456
|
+
}
|
|
457
|
+
if (document.baseId !== input.baseId) {
|
|
458
|
+
throw new Error(`Document not found: ${input.documentId}`);
|
|
459
|
+
}
|
|
460
|
+
await scopedDb
|
|
461
|
+
.delete(schema.documents)
|
|
462
|
+
.where(
|
|
463
|
+
and(
|
|
464
|
+
eq(schema.documents.workspaceId, input.workspaceId),
|
|
465
|
+
eq(schema.documents.id, input.documentId),
|
|
466
|
+
),
|
|
467
|
+
);
|
|
468
|
+
},
|
|
469
|
+
);
|
|
393
470
|
}
|
|
394
471
|
|
|
395
|
-
export async function listDocuments(
|
|
472
|
+
export async function listDocuments(
|
|
473
|
+
db: Database,
|
|
474
|
+
workspaceId: string,
|
|
475
|
+
baseId: string,
|
|
476
|
+
): Promise<Document[]> {
|
|
396
477
|
return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
|
|
397
|
-
const rows = await scopedDb
|
|
398
|
-
.
|
|
478
|
+
const rows = await scopedDb
|
|
479
|
+
.select()
|
|
480
|
+
.from(schema.documents)
|
|
481
|
+
.where(
|
|
482
|
+
and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.baseId, baseId)),
|
|
483
|
+
)
|
|
399
484
|
.orderBy(asc(schema.documents.createdAt));
|
|
400
485
|
return rows.map(mapDocument);
|
|
401
486
|
});
|
|
402
487
|
}
|
|
403
488
|
|
|
404
|
-
export async function getDocument(
|
|
489
|
+
export async function getDocument(
|
|
490
|
+
db: Database,
|
|
491
|
+
workspaceId: string,
|
|
492
|
+
documentId: string,
|
|
493
|
+
): Promise<Document | null> {
|
|
405
494
|
return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
|
|
406
|
-
const [row] = await scopedDb
|
|
495
|
+
const [row] = await scopedDb
|
|
496
|
+
.select()
|
|
497
|
+
.from(schema.documents)
|
|
498
|
+
.where(
|
|
499
|
+
and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
|
|
500
|
+
)
|
|
501
|
+
.limit(1);
|
|
407
502
|
return row ? mapDocument(row) : null;
|
|
408
503
|
});
|
|
409
504
|
}
|
|
410
505
|
|
|
411
|
-
export async function queueDocumentForReindex(
|
|
506
|
+
export async function queueDocumentForReindex(
|
|
507
|
+
db: Database,
|
|
508
|
+
workspaceId: string,
|
|
509
|
+
documentId: string,
|
|
510
|
+
): Promise<Document> {
|
|
412
511
|
return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
|
|
413
|
-
const [row] = await scopedDb
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
512
|
+
const [row] = await scopedDb
|
|
513
|
+
.update(schema.documents)
|
|
514
|
+
.set({
|
|
515
|
+
status: "queued",
|
|
516
|
+
error: null,
|
|
517
|
+
updatedAt: new Date(),
|
|
518
|
+
})
|
|
519
|
+
.where(
|
|
520
|
+
and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
|
|
521
|
+
)
|
|
522
|
+
.returning();
|
|
418
523
|
if (!row) throw new Error(`Document not found: ${documentId}`);
|
|
419
524
|
return mapDocument(row);
|
|
420
525
|
});
|
|
@@ -428,18 +533,32 @@ export async function indexDocumentNow(
|
|
|
428
533
|
services: DocumentServices = createDocumentServices(),
|
|
429
534
|
hooks: DocumentIndexHooks = {},
|
|
430
535
|
): Promise<Document> {
|
|
431
|
-
const [document] = await withWorkspaceRls(
|
|
432
|
-
|
|
536
|
+
const [document] = await withWorkspaceRls(
|
|
537
|
+
db,
|
|
538
|
+
workspaceId,
|
|
539
|
+
async (scopedDb) =>
|
|
540
|
+
await scopedDb
|
|
541
|
+
.select()
|
|
542
|
+
.from(schema.documents)
|
|
543
|
+
.where(
|
|
544
|
+
and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
|
|
545
|
+
)
|
|
546
|
+
.limit(1),
|
|
433
547
|
);
|
|
434
548
|
if (!document) throw new Error(`Document not found: ${documentId}`);
|
|
435
549
|
const file = await requireReadyFile(db, workspaceId, document.fileId);
|
|
436
550
|
await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
|
|
437
|
-
await scopedDb
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
551
|
+
await scopedDb
|
|
552
|
+
.update(schema.documents)
|
|
553
|
+
.set({
|
|
554
|
+
status: "indexing",
|
|
555
|
+
parser: services.parser.name,
|
|
556
|
+
error: null,
|
|
557
|
+
updatedAt: new Date(),
|
|
558
|
+
})
|
|
559
|
+
.where(
|
|
560
|
+
and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
|
|
561
|
+
);
|
|
443
562
|
});
|
|
444
563
|
try {
|
|
445
564
|
const bytes = await objectStorage.getFileBytes(file);
|
|
@@ -453,51 +572,88 @@ export async function indexDocumentNow(
|
|
|
453
572
|
});
|
|
454
573
|
const embeddings = await services.embedder.embedMany(chunks.map((chunk) => chunk.text));
|
|
455
574
|
if (embeddings.length !== chunks.length) {
|
|
456
|
-
throw new Error(
|
|
575
|
+
throw new Error(
|
|
576
|
+
`Embedding provider returned ${embeddings.length} embeddings for ${chunks.length} chunks`,
|
|
577
|
+
);
|
|
457
578
|
}
|
|
458
|
-
await withWorkspaceRls(
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
579
|
+
await withWorkspaceRls(
|
|
580
|
+
db,
|
|
581
|
+
workspaceId,
|
|
582
|
+
async (scopedDb) =>
|
|
583
|
+
await scopedDb.transaction(async (tx) => {
|
|
584
|
+
await tx
|
|
585
|
+
.delete(schema.documentChunks)
|
|
586
|
+
.where(
|
|
587
|
+
and(
|
|
588
|
+
eq(schema.documentChunks.workspaceId, workspaceId),
|
|
589
|
+
eq(schema.documentChunks.documentId, documentId),
|
|
590
|
+
),
|
|
591
|
+
);
|
|
592
|
+
if (chunks.length > 0) {
|
|
593
|
+
await tx.insert(schema.documentChunks).values(
|
|
594
|
+
chunks.map((chunk, index) => ({
|
|
595
|
+
accountId: document.accountId,
|
|
596
|
+
workspaceId: document.workspaceId,
|
|
597
|
+
documentId,
|
|
598
|
+
baseId: document.baseId,
|
|
599
|
+
fileId: file.id,
|
|
600
|
+
chunkIndex: index,
|
|
601
|
+
text: chunk.text,
|
|
602
|
+
metadata: {
|
|
603
|
+
...chunk.metadata,
|
|
604
|
+
documentTitle: document.title,
|
|
605
|
+
sourceKind: document.sourceKind,
|
|
606
|
+
sourceUri: document.sourceUri,
|
|
607
|
+
sourceExternalId: document.sourceExternalId,
|
|
608
|
+
sourceTitle: document.sourceTitle,
|
|
609
|
+
sourceAuthor: document.sourceAuthor,
|
|
610
|
+
sourceCreatedAt: document.sourceCreatedAt?.toISOString() ?? null,
|
|
611
|
+
sourceUpdatedAt: document.sourceUpdatedAt?.toISOString() ?? null,
|
|
612
|
+
sourceVersion: document.sourceVersion,
|
|
613
|
+
aclTags: document.aclTags,
|
|
614
|
+
},
|
|
615
|
+
embedding: validateEmbedding(
|
|
616
|
+
embeddings[index] ?? [],
|
|
617
|
+
services.embedder.dimensions,
|
|
618
|
+
services.embedder.model,
|
|
619
|
+
),
|
|
620
|
+
embeddingModel: services.embedder.model,
|
|
621
|
+
})),
|
|
622
|
+
);
|
|
623
|
+
}
|
|
624
|
+
await tx
|
|
625
|
+
.update(schema.documents)
|
|
626
|
+
.set({
|
|
627
|
+
status: "ready",
|
|
628
|
+
parser: services.parser.name,
|
|
629
|
+
chunkCount: chunks.length,
|
|
630
|
+
error: null,
|
|
631
|
+
updatedAt: new Date(),
|
|
632
|
+
})
|
|
633
|
+
.where(
|
|
634
|
+
and(
|
|
635
|
+
eq(schema.documents.workspaceId, workspaceId),
|
|
636
|
+
eq(schema.documents.id, documentId),
|
|
637
|
+
),
|
|
638
|
+
);
|
|
639
|
+
}),
|
|
640
|
+
);
|
|
494
641
|
} catch (error) {
|
|
495
|
-
const [failed] = await withWorkspaceRls(
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
642
|
+
const [failed] = await withWorkspaceRls(
|
|
643
|
+
db,
|
|
644
|
+
workspaceId,
|
|
645
|
+
async (scopedDb) =>
|
|
646
|
+
await scopedDb
|
|
647
|
+
.update(schema.documents)
|
|
648
|
+
.set({
|
|
649
|
+
status: "failed",
|
|
650
|
+
error: error instanceof Error ? error.message : String(error),
|
|
651
|
+
updatedAt: new Date(),
|
|
652
|
+
})
|
|
653
|
+
.where(
|
|
654
|
+
and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
|
|
655
|
+
)
|
|
656
|
+
.returning(),
|
|
501
657
|
);
|
|
502
658
|
if (!failed) throw error;
|
|
503
659
|
return mapDocument(failed);
|
|
@@ -518,19 +674,22 @@ export async function searchDocuments(
|
|
|
518
674
|
const rows: CombinedSearchRow[] = [];
|
|
519
675
|
if (mode === "vector" || mode === "hybrid") {
|
|
520
676
|
try {
|
|
521
|
-
rows.push(...await vectorSearchDocuments(db, input, candidateLimit, services));
|
|
677
|
+
rows.push(...(await vectorSearchDocuments(db, input, candidateLimit, services)));
|
|
522
678
|
} catch (error) {
|
|
523
679
|
if (mode === "vector") {
|
|
524
680
|
throw error;
|
|
525
681
|
}
|
|
526
|
-
console.warn(
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
682
|
+
console.warn(
|
|
683
|
+
"document hybrid search vector component failed; falling back to keyword search",
|
|
684
|
+
{
|
|
685
|
+
workspaceId: input.workspaceId,
|
|
686
|
+
error: error instanceof Error ? error.message : String(error),
|
|
687
|
+
},
|
|
688
|
+
);
|
|
530
689
|
}
|
|
531
690
|
}
|
|
532
691
|
if (mode === "keyword" || mode === "hybrid") {
|
|
533
|
-
rows.push(...await keywordSearchDocuments(db, input, candidateLimit));
|
|
692
|
+
rows.push(...(await keywordSearchDocuments(db, input, candidateLimit)));
|
|
534
693
|
}
|
|
535
694
|
return mergeDocumentSearchRows(rows, mode).slice(0, limit);
|
|
536
695
|
}
|
|
@@ -544,31 +703,36 @@ async function vectorSearchDocuments(
|
|
|
544
703
|
const queryEmbedding = await services.embedder.embedQuery(input.query);
|
|
545
704
|
validateEmbedding(queryEmbedding, services.embedder.dimensions, services.embedder.model);
|
|
546
705
|
const distance = sql<number>`${schema.documentChunks.embedding} <=> ${vectorLiteral(queryEmbedding)}::vector`;
|
|
547
|
-
const rows = await withWorkspaceRls(
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
706
|
+
const rows = await withWorkspaceRls(
|
|
707
|
+
db,
|
|
708
|
+
input.workspaceId,
|
|
709
|
+
async (scopedDb) =>
|
|
710
|
+
await scopedDb
|
|
711
|
+
.select({
|
|
712
|
+
chunkId: schema.documentChunks.id,
|
|
713
|
+
documentId: schema.documentChunks.documentId,
|
|
714
|
+
baseId: schema.documentChunks.baseId,
|
|
715
|
+
fileId: schema.documentChunks.fileId,
|
|
716
|
+
title: schema.documents.title,
|
|
717
|
+
text: schema.documentChunks.text,
|
|
718
|
+
chunkIndex: schema.documentChunks.chunkIndex,
|
|
719
|
+
metadata: schema.documentChunks.metadata,
|
|
720
|
+
sourceKind: schema.documents.sourceKind,
|
|
721
|
+
sourceUri: schema.documents.sourceUri,
|
|
722
|
+
sourceExternalId: schema.documents.sourceExternalId,
|
|
723
|
+
sourceTitle: schema.documents.sourceTitle,
|
|
724
|
+
sourceAuthor: schema.documents.sourceAuthor,
|
|
725
|
+
sourceCreatedAt: schema.documents.sourceCreatedAt,
|
|
726
|
+
sourceUpdatedAt: schema.documents.sourceUpdatedAt,
|
|
727
|
+
sourceVersion: schema.documents.sourceVersion,
|
|
728
|
+
aclTags: schema.documents.aclTags,
|
|
729
|
+
distance,
|
|
730
|
+
})
|
|
731
|
+
.from(schema.documentChunks)
|
|
732
|
+
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
733
|
+
.where(and(...documentSearchConditions(input, services.embedder.model)))
|
|
734
|
+
.orderBy(distance)
|
|
735
|
+
.limit(limit),
|
|
572
736
|
);
|
|
573
737
|
return rows.map((row) => ({
|
|
574
738
|
...mapSearchRowBase(row, input.workspaceId),
|
|
@@ -577,36 +741,47 @@ async function vectorSearchDocuments(
|
|
|
577
741
|
}));
|
|
578
742
|
}
|
|
579
743
|
|
|
580
|
-
async function keywordSearchDocuments(
|
|
744
|
+
async function keywordSearchDocuments(
|
|
745
|
+
db: Database,
|
|
746
|
+
input: DocumentSearchInput,
|
|
747
|
+
limit: number,
|
|
748
|
+
): Promise<CombinedSearchRow[]> {
|
|
581
749
|
const rank = sql<number>`ts_rank_cd(to_tsvector('simple', ${schema.documentChunks.text}), plainto_tsquery('simple', ${input.query}))`;
|
|
582
|
-
const rows = await withWorkspaceRls(
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
750
|
+
const rows = await withWorkspaceRls(
|
|
751
|
+
db,
|
|
752
|
+
input.workspaceId,
|
|
753
|
+
async (scopedDb) =>
|
|
754
|
+
await scopedDb
|
|
755
|
+
.select({
|
|
756
|
+
chunkId: schema.documentChunks.id,
|
|
757
|
+
documentId: schema.documentChunks.documentId,
|
|
758
|
+
baseId: schema.documentChunks.baseId,
|
|
759
|
+
fileId: schema.documentChunks.fileId,
|
|
760
|
+
title: schema.documents.title,
|
|
761
|
+
text: schema.documentChunks.text,
|
|
762
|
+
chunkIndex: schema.documentChunks.chunkIndex,
|
|
763
|
+
metadata: schema.documentChunks.metadata,
|
|
764
|
+
sourceKind: schema.documents.sourceKind,
|
|
765
|
+
sourceUri: schema.documents.sourceUri,
|
|
766
|
+
sourceExternalId: schema.documents.sourceExternalId,
|
|
767
|
+
sourceTitle: schema.documents.sourceTitle,
|
|
768
|
+
sourceAuthor: schema.documents.sourceAuthor,
|
|
769
|
+
sourceCreatedAt: schema.documents.sourceCreatedAt,
|
|
770
|
+
sourceUpdatedAt: schema.documents.sourceUpdatedAt,
|
|
771
|
+
sourceVersion: schema.documents.sourceVersion,
|
|
772
|
+
aclTags: schema.documents.aclTags,
|
|
773
|
+
rank,
|
|
774
|
+
})
|
|
775
|
+
.from(schema.documentChunks)
|
|
776
|
+
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
777
|
+
.where(
|
|
778
|
+
and(
|
|
779
|
+
...documentSearchConditions(input),
|
|
780
|
+
sql`to_tsvector('simple', ${schema.documentChunks.text}) @@ plainto_tsquery('simple', ${input.query})`,
|
|
781
|
+
),
|
|
782
|
+
)
|
|
783
|
+
.orderBy(desc(rank))
|
|
784
|
+
.limit(limit),
|
|
610
785
|
);
|
|
611
786
|
return rows.map((row) => ({
|
|
612
787
|
...mapSearchRowBase(row, input.workspaceId),
|
|
@@ -615,30 +790,45 @@ async function keywordSearchDocuments(db: Database, input: DocumentSearchInput,
|
|
|
615
790
|
}));
|
|
616
791
|
}
|
|
617
792
|
|
|
618
|
-
export async function getDocumentChunk(
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
793
|
+
export async function getDocumentChunk(
|
|
794
|
+
db: Database,
|
|
795
|
+
workspaceId: string,
|
|
796
|
+
chunkId: string,
|
|
797
|
+
): Promise<DocumentSearchResult | null> {
|
|
798
|
+
const [row] = await withWorkspaceRls(
|
|
799
|
+
db,
|
|
800
|
+
workspaceId,
|
|
801
|
+
async (scopedDb) =>
|
|
802
|
+
await scopedDb
|
|
803
|
+
.select({
|
|
804
|
+
chunkId: schema.documentChunks.id,
|
|
805
|
+
documentId: schema.documentChunks.documentId,
|
|
806
|
+
baseId: schema.documentChunks.baseId,
|
|
807
|
+
fileId: schema.documentChunks.fileId,
|
|
808
|
+
title: schema.documents.title,
|
|
809
|
+
text: schema.documentChunks.text,
|
|
810
|
+
chunkIndex: schema.documentChunks.chunkIndex,
|
|
811
|
+
metadata: schema.documentChunks.metadata,
|
|
812
|
+
sourceKind: schema.documents.sourceKind,
|
|
813
|
+
sourceUri: schema.documents.sourceUri,
|
|
814
|
+
sourceExternalId: schema.documents.sourceExternalId,
|
|
815
|
+
sourceTitle: schema.documents.sourceTitle,
|
|
816
|
+
sourceAuthor: schema.documents.sourceAuthor,
|
|
817
|
+
sourceCreatedAt: schema.documents.sourceCreatedAt,
|
|
818
|
+
sourceUpdatedAt: schema.documents.sourceUpdatedAt,
|
|
819
|
+
sourceVersion: schema.documents.sourceVersion,
|
|
820
|
+
aclTags: schema.documents.aclTags,
|
|
821
|
+
})
|
|
822
|
+
.from(schema.documentChunks)
|
|
823
|
+
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
824
|
+
.where(
|
|
825
|
+
and(
|
|
826
|
+
eq(schema.documentChunks.workspaceId, workspaceId),
|
|
827
|
+
eq(schema.documentChunks.id, chunkId),
|
|
828
|
+
eq(schema.documents.status, "ready"),
|
|
829
|
+
),
|
|
830
|
+
)
|
|
831
|
+
.limit(1),
|
|
642
832
|
);
|
|
643
833
|
if (!row) return null;
|
|
644
834
|
return {
|
|
@@ -650,7 +840,10 @@ export async function getDocumentChunk(db: Database, workspaceId: string, chunkI
|
|
|
650
840
|
};
|
|
651
841
|
}
|
|
652
842
|
|
|
653
|
-
type SearchRowBase = Omit<
|
|
843
|
+
type SearchRowBase = Omit<
|
|
844
|
+
DocumentSearchResult,
|
|
845
|
+
"score" | "matchType" | "vectorScore" | "keywordScore"
|
|
846
|
+
>;
|
|
654
847
|
type CombinedSearchRow = SearchRowBase & {
|
|
655
848
|
vectorScore: number | null;
|
|
656
849
|
keywordScore: number | null;
|
|
@@ -677,25 +870,28 @@ function documentSearchConditions(input: DocumentSearchInput, embeddingModel?: s
|
|
|
677
870
|
return conditions;
|
|
678
871
|
}
|
|
679
872
|
|
|
680
|
-
function mapSearchRowBase(
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
873
|
+
function mapSearchRowBase(
|
|
874
|
+
row: {
|
|
875
|
+
chunkId: string;
|
|
876
|
+
documentId: string;
|
|
877
|
+
baseId: string;
|
|
878
|
+
fileId: string;
|
|
879
|
+
title: string;
|
|
880
|
+
text: string;
|
|
881
|
+
chunkIndex: number;
|
|
882
|
+
metadata: Record<string, unknown>;
|
|
883
|
+
sourceKind: string;
|
|
884
|
+
sourceUri: string | null;
|
|
885
|
+
sourceExternalId: string | null;
|
|
886
|
+
sourceTitle: string | null;
|
|
887
|
+
sourceAuthor: string | null;
|
|
888
|
+
sourceCreatedAt: Date | null;
|
|
889
|
+
sourceUpdatedAt: Date | null;
|
|
890
|
+
sourceVersion: string | null;
|
|
891
|
+
aclTags: string[];
|
|
892
|
+
},
|
|
893
|
+
workspaceId: string,
|
|
894
|
+
): SearchRowBase {
|
|
699
895
|
return {
|
|
700
896
|
chunkId: row.chunkId,
|
|
701
897
|
workspaceId,
|
|
@@ -718,7 +914,10 @@ function mapSearchRowBase(row: {
|
|
|
718
914
|
};
|
|
719
915
|
}
|
|
720
916
|
|
|
721
|
-
function mergeDocumentSearchRows(
|
|
917
|
+
function mergeDocumentSearchRows(
|
|
918
|
+
rows: CombinedSearchRow[],
|
|
919
|
+
mode: DocumentSearchMode,
|
|
920
|
+
): DocumentSearchResult[] {
|
|
722
921
|
const byChunk = new Map<string, CombinedSearchRow>();
|
|
723
922
|
for (const row of rows) {
|
|
724
923
|
const existing = byChunk.get(row.chunkId);
|
|
@@ -732,25 +931,29 @@ function mergeDocumentSearchRows(rows: CombinedSearchRow[], mode: DocumentSearch
|
|
|
732
931
|
keywordScore: Math.max(existing.keywordScore ?? 0, row.keywordScore ?? 0) || null,
|
|
733
932
|
});
|
|
734
933
|
}
|
|
735
|
-
return [...byChunk.values()]
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
934
|
+
return [...byChunk.values()]
|
|
935
|
+
.map((row) => {
|
|
936
|
+
const vectorScore = row.vectorScore;
|
|
937
|
+
const keywordScore = row.keywordScore;
|
|
938
|
+
const matchType: DocumentSearchMode =
|
|
939
|
+
vectorScore !== null && keywordScore !== null
|
|
940
|
+
? "hybrid"
|
|
941
|
+
: vectorScore !== null
|
|
942
|
+
? "vector"
|
|
943
|
+
: "keyword";
|
|
944
|
+
return {
|
|
945
|
+
...row,
|
|
946
|
+
score: combinedSearchScore(mode, vectorScore, keywordScore, matchType),
|
|
947
|
+
matchType,
|
|
948
|
+
};
|
|
949
|
+
})
|
|
950
|
+
.sort(
|
|
951
|
+
(left, right) =>
|
|
952
|
+
right.score - left.score ||
|
|
953
|
+
(right.vectorScore ?? 0) - (left.vectorScore ?? 0) ||
|
|
954
|
+
(right.keywordScore ?? 0) - (left.keywordScore ?? 0) ||
|
|
955
|
+
left.chunkIndex - right.chunkIndex,
|
|
956
|
+
);
|
|
754
957
|
}
|
|
755
958
|
|
|
756
959
|
function combinedSearchScore(
|
|
@@ -763,7 +966,9 @@ function combinedSearchScore(
|
|
|
763
966
|
const keyword = keywordScore ?? 0;
|
|
764
967
|
if (mode === "vector") return roundScore(vector);
|
|
765
968
|
if (mode === "keyword") return roundScore(keyword);
|
|
766
|
-
return roundScore(
|
|
969
|
+
return roundScore(
|
|
970
|
+
Math.min(1, 0.65 * vector + 0.35 * keyword + (matchType === "hybrid" ? 0.1 : 0)),
|
|
971
|
+
);
|
|
767
972
|
}
|
|
768
973
|
|
|
769
974
|
function normalizeKeywordScore(rank: number): number {
|
|
@@ -777,17 +982,31 @@ function roundScore(value: number): number {
|
|
|
777
982
|
return Number(value.toFixed(6));
|
|
778
983
|
}
|
|
779
984
|
|
|
780
|
-
export async function parseDocumentBytes(
|
|
985
|
+
export async function parseDocumentBytes(
|
|
986
|
+
bytes: Uint8Array,
|
|
987
|
+
file: FileAsset,
|
|
988
|
+
parser: DocumentParser = new LiteParseDocumentParser(),
|
|
989
|
+
): Promise<ParsedDocument> {
|
|
781
990
|
return await parser.parse(bytes, file);
|
|
782
991
|
}
|
|
783
992
|
|
|
784
|
-
export function chunkText(
|
|
993
|
+
export function chunkText(
|
|
994
|
+
text: string,
|
|
995
|
+
maxChars = DEFAULT_DOCUMENT_CHUNK_SIZE,
|
|
996
|
+
overlapChars = DEFAULT_DOCUMENT_CHUNK_OVERLAP,
|
|
997
|
+
): string[] {
|
|
785
998
|
if (overlapChars >= maxChars) {
|
|
786
999
|
throw new Error("chunk overlap must be smaller than chunk size");
|
|
787
1000
|
}
|
|
788
|
-
const normalized = text
|
|
1001
|
+
const normalized = text
|
|
1002
|
+
.replace(/\r\n/g, "\n")
|
|
1003
|
+
.replace(/[ \t]+/g, " ")
|
|
1004
|
+
.trim();
|
|
789
1005
|
if (!normalized) return [];
|
|
790
|
-
const paragraphs = normalized
|
|
1006
|
+
const paragraphs = normalized
|
|
1007
|
+
.split(/\n{2,}/)
|
|
1008
|
+
.map((part) => part.replace(/\s+/g, " ").trim())
|
|
1009
|
+
.filter(Boolean);
|
|
791
1010
|
const chunks: string[] = [];
|
|
792
1011
|
let current = "";
|
|
793
1012
|
for (const paragraph of paragraphs.length > 0 ? paragraphs : [normalized.replace(/\s+/g, " ")]) {
|
|
@@ -806,7 +1025,10 @@ export function chunkText(text: string, maxChars = DEFAULT_DOCUMENT_CHUNK_SIZE,
|
|
|
806
1025
|
return chunks.map((chunk) => chunk.trim()).filter(Boolean);
|
|
807
1026
|
}
|
|
808
1027
|
|
|
809
|
-
export function deterministicEmbedding(
|
|
1028
|
+
export function deterministicEmbedding(
|
|
1029
|
+
text: string,
|
|
1030
|
+
dimensions = DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS,
|
|
1031
|
+
): number[] {
|
|
810
1032
|
const values = new Array(dimensions).fill(0);
|
|
811
1033
|
const tokens = text.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? [];
|
|
812
1034
|
for (const token of tokens) {
|
|
@@ -821,7 +1043,11 @@ export function deterministicEmbedding(text: string, dimensions = DEFAULT_DOCUME
|
|
|
821
1043
|
return values.map((value) => Number((value / norm).toFixed(6)));
|
|
822
1044
|
}
|
|
823
1045
|
|
|
824
|
-
async function requireReadyFile(
|
|
1046
|
+
async function requireReadyFile(
|
|
1047
|
+
db: Database,
|
|
1048
|
+
workspaceId: string,
|
|
1049
|
+
fileId: string,
|
|
1050
|
+
): Promise<FileAsset> {
|
|
825
1051
|
const file = await requireFile(db, workspaceId, fileId);
|
|
826
1052
|
if (file.status !== "ready") {
|
|
827
1053
|
throw new Error(`File ${fileId} is ${file.status}`);
|
|
@@ -831,7 +1057,9 @@ async function requireReadyFile(db: Database, workspaceId: string, fileId: strin
|
|
|
831
1057
|
|
|
832
1058
|
function validateEmbedding(values: number[], dimensions: number, model: string): number[] {
|
|
833
1059
|
if (values.length !== dimensions) {
|
|
834
|
-
throw new Error(
|
|
1060
|
+
throw new Error(
|
|
1061
|
+
`Embedding model ${model} returned ${values.length} dimensions; expected ${dimensions}`,
|
|
1062
|
+
);
|
|
835
1063
|
}
|
|
836
1064
|
if (values.some((value) => !Number.isFinite(value))) {
|
|
837
1065
|
throw new Error(`Embedding model ${model} returned non-finite values`);
|
|
@@ -842,18 +1070,20 @@ function validateEmbedding(values: number[], dimensions: number, model: string):
|
|
|
842
1070
|
function isTextLike(file: FileAsset): boolean {
|
|
843
1071
|
const contentType = file.contentType.toLowerCase();
|
|
844
1072
|
const filename = file.filename.toLowerCase();
|
|
845
|
-
return
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
1073
|
+
return (
|
|
1074
|
+
contentType.startsWith("text/") ||
|
|
1075
|
+
contentType === "application/json" ||
|
|
1076
|
+
contentType === "application/xml" ||
|
|
1077
|
+
contentType === "application/x-yaml" ||
|
|
1078
|
+
filename.endsWith(".md") ||
|
|
1079
|
+
filename.endsWith(".markdown") ||
|
|
1080
|
+
filename.endsWith(".json") ||
|
|
1081
|
+
filename.endsWith(".yaml") ||
|
|
1082
|
+
filename.endsWith(".yml") ||
|
|
1083
|
+
filename.endsWith(".csv") ||
|
|
1084
|
+
filename.endsWith(".tsv") ||
|
|
1085
|
+
filename.endsWith(".xml")
|
|
1086
|
+
);
|
|
857
1087
|
}
|
|
858
1088
|
|
|
859
1089
|
function splitOversizedText(text: string, maxChars: number): string[] {
|
|
@@ -878,9 +1108,17 @@ function splitOversizedText(text: string, maxChars: number): string[] {
|
|
|
878
1108
|
return out;
|
|
879
1109
|
}
|
|
880
1110
|
|
|
881
|
-
function withOverlap(
|
|
1111
|
+
function withOverlap(
|
|
1112
|
+
previous: string,
|
|
1113
|
+
overlapChars: number,
|
|
1114
|
+
next: string,
|
|
1115
|
+
maxChars: number,
|
|
1116
|
+
): string {
|
|
882
1117
|
if (overlapChars <= 0) return next;
|
|
883
|
-
const overlap = previous
|
|
1118
|
+
const overlap = previous
|
|
1119
|
+
.slice(Math.max(0, previous.length - overlapChars))
|
|
1120
|
+
.replace(/^\S+\s+/, "")
|
|
1121
|
+
.trim();
|
|
884
1122
|
const candidate = overlap ? `${overlap} ${next}` : next;
|
|
885
1123
|
return candidate.length <= maxChars ? candidate : next;
|
|
886
1124
|
}
|