@opengeni/documents 0.2.72 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +36 -7
- package/dist/index.js +224 -59
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
- package/src/index.ts +326 -86
package/dist/index.js
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
// src/index.ts
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
requireFile,
|
|
4
|
+
rlsContextForWorkspace,
|
|
5
|
+
setSubjectRlsContext,
|
|
6
|
+
withRlsContext,
|
|
7
|
+
withWorkspaceRls,
|
|
8
|
+
withWorkspaceSubjectRls
|
|
9
|
+
} from "@opengeni/db";
|
|
3
10
|
import * as schema from "@opengeni/db/schema";
|
|
4
11
|
import { LiteParse } from "@llamaindex/liteparse";
|
|
5
12
|
import { and, asc, desc, eq, inArray, or, sql } from "drizzle-orm";
|
|
@@ -11,6 +18,7 @@ var DEFAULT_DOCUMENT_CHUNK_SIZE = 1200;
|
|
|
11
18
|
var DEFAULT_DOCUMENT_CHUNK_OVERLAP = 160;
|
|
12
19
|
var DEFAULT_DOCUMENT_CURATION_MODEL = "gpt-4o-mini";
|
|
13
20
|
var DOCUMENT_CURATION_MAX_INPUT_CHARS = 24e3;
|
|
21
|
+
var DOCUMENT_AUTHORITY_SUBJECT_MAX_BYTES = 1024;
|
|
14
22
|
var DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE = 0.75;
|
|
15
23
|
var DEFAULT_BASE_NAME = "Default";
|
|
16
24
|
var DEFAULT_BASE_DESCRIPTION = "Default base for dropped files and notes.";
|
|
@@ -362,12 +370,12 @@ async function getDocumentInventory(db, workspaceId, input) {
|
|
|
362
370
|
DOCUMENT_INVENTORY_MAX_TOPIC_CHARS,
|
|
363
371
|
"topicMaxChars"
|
|
364
372
|
);
|
|
365
|
-
return await
|
|
373
|
+
return await withDocumentRls(db, workspaceId, input.access, async (scopedDb) => {
|
|
366
374
|
const [baseTotal] = await scopedDb.select({ count: sql`count(*)::int` }).from(schema.documentBases).where(eq(schema.documentBases.workspaceId, workspaceId));
|
|
367
375
|
const baseJoin = and(
|
|
368
376
|
eq(schema.documents.workspaceId, workspaceId),
|
|
369
377
|
eq(schema.documents.baseId, schema.documentBases.id),
|
|
370
|
-
...documentAccessConditions(input.access)
|
|
378
|
+
...documentAccessConditions(workspaceId, input.access)
|
|
371
379
|
);
|
|
372
380
|
const baseRows = await scopedDb.select({
|
|
373
381
|
id: schema.documentBases.id,
|
|
@@ -381,7 +389,7 @@ async function getDocumentInventory(db, workspaceId, input) {
|
|
|
381
389
|
}).from(schema.documentBases).leftJoin(schema.documents, baseJoin).where(eq(schema.documentBases.workspaceId, workspaceId)).groupBy(schema.documentBases.id, schema.documentBases.name, schema.documentBases.createdAt).orderBy(desc(schema.documentBases.createdAt), asc(schema.documentBases.id)).limit(baseLimit);
|
|
382
390
|
const documentWhere = and(
|
|
383
391
|
eq(schema.documents.workspaceId, workspaceId),
|
|
384
|
-
...documentAccessConditions(input.access)
|
|
392
|
+
...documentAccessConditions(workspaceId, input.access)
|
|
385
393
|
);
|
|
386
394
|
const [summary] = await scopedDb.select({
|
|
387
395
|
visibleDocumentCount: sql`count(${schema.documents.id})::int`,
|
|
@@ -501,12 +509,23 @@ async function addDocumentToBase(db, input) {
|
|
|
501
509
|
db,
|
|
502
510
|
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
503
511
|
async (scopedDb) => {
|
|
512
|
+
const viewerSubjectId = cleanString(input.access?.viewerSubjectId ?? null);
|
|
513
|
+
const authority = resolveDocumentAuthority({
|
|
514
|
+
kind: input.authorityKind,
|
|
515
|
+
legacyVisibility: input.visibility,
|
|
516
|
+
workspaceId: input.workspaceId,
|
|
517
|
+
initiatingSubjectId: input.initiatingSubjectId
|
|
518
|
+
});
|
|
519
|
+
if (authority.kind === "organization" && input.organizationAuthorityGranted !== true) {
|
|
520
|
+
throw new Error("organization document writes require exact account authority");
|
|
521
|
+
}
|
|
522
|
+
if (authority.kind === "personal" && (viewerSubjectId !== authority.subjectId || cleanString(input.createdBy ?? null) !== authority.subjectId)) {
|
|
523
|
+
throw new Error("personal document writes require the exact initiating subject");
|
|
524
|
+
}
|
|
525
|
+
if (viewerSubjectId) await setSubjectRlsContext(scopedDb, viewerSubjectId);
|
|
504
526
|
const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
|
|
505
527
|
if (!base) throw new Error(`Document base not found: ${input.baseId}`);
|
|
506
528
|
const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
|
|
507
|
-
if (input.visibility === "private" && !cleanString(input.createdBy ?? null)) {
|
|
508
|
-
throw new Error("private documents require a creating subject");
|
|
509
|
-
}
|
|
510
529
|
const now = /* @__PURE__ */ new Date();
|
|
511
530
|
const [existing] = await scopedDb.select().from(schema.documents).where(
|
|
512
531
|
and(
|
|
@@ -519,6 +538,10 @@ async function addDocumentToBase(db, input) {
|
|
|
519
538
|
if (!documentMatchesAccess(existing, input.access)) {
|
|
520
539
|
throw new Error(`Document not found: ${existing.id}`);
|
|
521
540
|
}
|
|
541
|
+
assertOrganizationDocumentAuthority(
|
|
542
|
+
existing.authorityKind,
|
|
543
|
+
input.organizationAuthorityGranted
|
|
544
|
+
);
|
|
522
545
|
const [updated] = await scopedDb.update(schema.documents).set({
|
|
523
546
|
title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? existing.title,
|
|
524
547
|
...input.sourceKind !== void 0 ? { sourceKind: input.sourceKind } : {},
|
|
@@ -535,7 +558,7 @@ async function addDocumentToBase(db, input) {
|
|
|
535
558
|
and(
|
|
536
559
|
eq(schema.documents.workspaceId, input.workspaceId),
|
|
537
560
|
eq(schema.documents.id, existing.id),
|
|
538
|
-
...documentAccessConditions(input.access)
|
|
561
|
+
...documentAccessConditions(input.workspaceId, input.access)
|
|
539
562
|
)
|
|
540
563
|
).returning();
|
|
541
564
|
return mapDocument(updated ?? existing);
|
|
@@ -557,7 +580,10 @@ async function addDocumentToBase(db, input) {
|
|
|
557
580
|
sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
|
|
558
581
|
sourceVersion: cleanString(input.sourceVersion) ?? null,
|
|
559
582
|
aclTags: cleanStringArray(input.aclTags),
|
|
560
|
-
|
|
583
|
+
authorityKind: authority.kind,
|
|
584
|
+
authorityWorkspaceId: authority.workspaceId,
|
|
585
|
+
authoritySubjectId: authority.subjectId,
|
|
586
|
+
visibility: authority.kind === "personal" ? "private" : "workspace",
|
|
561
587
|
agentAccess: input.agentAccess ?? true,
|
|
562
588
|
createdBy: input.createdBy ?? null,
|
|
563
589
|
curationStatus: input.curationStatus ?? "none",
|
|
@@ -573,14 +599,17 @@ async function moveDocumentToBase(db, input) {
|
|
|
573
599
|
db,
|
|
574
600
|
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
575
601
|
async (scopedDb) => {
|
|
602
|
+
const viewerSubjectId = cleanString(input.access?.viewerSubjectId ?? null);
|
|
603
|
+
if (viewerSubjectId) await setSubjectRlsContext(scopedDb, viewerSubjectId);
|
|
576
604
|
const [row] = await scopedDb.select().from(schema.documents).where(
|
|
577
605
|
and(
|
|
578
606
|
eq(schema.documents.workspaceId, input.workspaceId),
|
|
579
607
|
eq(schema.documents.id, input.documentId),
|
|
580
|
-
...documentAccessConditions(input.access)
|
|
608
|
+
...documentAccessConditions(input.workspaceId, input.access)
|
|
581
609
|
)
|
|
582
610
|
).limit(1);
|
|
583
611
|
if (!row) throw new Error(`Document not found: ${input.documentId}`);
|
|
612
|
+
assertOrganizationDocumentAuthority(row.authorityKind, input.organizationAuthorityGranted);
|
|
584
613
|
const suggestion = row.curation?.suggestedBaseId;
|
|
585
614
|
const targetBaseId = input.targetBaseId ?? suggestion;
|
|
586
615
|
if (!targetBaseId) {
|
|
@@ -609,7 +638,7 @@ async function moveDocumentToBase(db, input) {
|
|
|
609
638
|
and(
|
|
610
639
|
eq(schema.documents.workspaceId, input.workspaceId),
|
|
611
640
|
eq(schema.documents.id, input.documentId),
|
|
612
|
-
...documentAccessConditions(input.access)
|
|
641
|
+
...documentAccessConditions(input.workspaceId, input.access)
|
|
613
642
|
)
|
|
614
643
|
).returning();
|
|
615
644
|
if (updated) {
|
|
@@ -632,16 +661,22 @@ async function deleteDocumentFromBase(db, input) {
|
|
|
632
661
|
db,
|
|
633
662
|
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
634
663
|
async (scopedDb) => {
|
|
664
|
+
const viewerSubjectId = cleanString(input.access?.viewerSubjectId ?? null);
|
|
665
|
+
if (viewerSubjectId) await setSubjectRlsContext(scopedDb, viewerSubjectId);
|
|
635
666
|
const [document] = await scopedDb.select().from(schema.documents).where(
|
|
636
667
|
and(
|
|
637
668
|
eq(schema.documents.workspaceId, input.workspaceId),
|
|
638
669
|
eq(schema.documents.id, input.documentId),
|
|
639
|
-
...documentAccessConditions(input.access)
|
|
670
|
+
...documentAccessConditions(input.workspaceId, input.access)
|
|
640
671
|
)
|
|
641
672
|
).limit(1);
|
|
642
673
|
if (!document) {
|
|
643
674
|
throw new Error(`Document not found: ${input.documentId}`);
|
|
644
675
|
}
|
|
676
|
+
assertOrganizationDocumentAuthority(
|
|
677
|
+
document.authorityKind,
|
|
678
|
+
input.organizationAuthorityGranted
|
|
679
|
+
);
|
|
645
680
|
if (document.baseId !== input.baseId) {
|
|
646
681
|
throw new Error(`Document not found: ${input.documentId}`);
|
|
647
682
|
}
|
|
@@ -649,38 +684,47 @@ async function deleteDocumentFromBase(db, input) {
|
|
|
649
684
|
and(
|
|
650
685
|
eq(schema.documents.workspaceId, input.workspaceId),
|
|
651
686
|
eq(schema.documents.id, input.documentId),
|
|
652
|
-
...documentAccessConditions(input.access)
|
|
687
|
+
...documentAccessConditions(input.workspaceId, input.access)
|
|
653
688
|
)
|
|
654
689
|
);
|
|
655
690
|
}
|
|
656
691
|
);
|
|
657
692
|
}
|
|
658
693
|
async function listDocuments(db, workspaceId, baseId, access) {
|
|
659
|
-
return await
|
|
694
|
+
return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
|
|
660
695
|
const rows = await scopedDb.select().from(schema.documents).where(
|
|
661
696
|
and(
|
|
662
697
|
eq(schema.documents.workspaceId, workspaceId),
|
|
663
698
|
eq(schema.documents.baseId, baseId),
|
|
664
|
-
...documentAccessConditions(access)
|
|
699
|
+
...documentAccessConditions(workspaceId, access)
|
|
665
700
|
)
|
|
666
701
|
).orderBy(asc(schema.documents.createdAt));
|
|
667
702
|
return rows.map(mapDocument);
|
|
668
703
|
});
|
|
669
704
|
}
|
|
670
705
|
async function getDocument(db, workspaceId, documentId, access) {
|
|
671
|
-
return await
|
|
706
|
+
return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
|
|
672
707
|
const [row] = await scopedDb.select().from(schema.documents).where(
|
|
673
708
|
and(
|
|
674
709
|
eq(schema.documents.workspaceId, workspaceId),
|
|
675
710
|
eq(schema.documents.id, documentId),
|
|
676
|
-
...documentAccessConditions(access)
|
|
711
|
+
...documentAccessConditions(workspaceId, access)
|
|
677
712
|
)
|
|
678
713
|
).limit(1);
|
|
679
714
|
return row ? mapDocument(row) : null;
|
|
680
715
|
});
|
|
681
716
|
}
|
|
682
|
-
async function queueDocumentForReindex(db, workspaceId, documentId, access) {
|
|
683
|
-
return await
|
|
717
|
+
async function queueDocumentForReindex(db, workspaceId, documentId, access, organizationAuthorityGranted) {
|
|
718
|
+
return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
|
|
719
|
+
const [document] = await scopedDb.select({ authorityKind: schema.documents.authorityKind }).from(schema.documents).where(
|
|
720
|
+
and(
|
|
721
|
+
eq(schema.documents.workspaceId, workspaceId),
|
|
722
|
+
eq(schema.documents.id, documentId),
|
|
723
|
+
...documentAccessConditions(workspaceId, access)
|
|
724
|
+
)
|
|
725
|
+
).limit(1);
|
|
726
|
+
if (!document) throw new Error(`Document not found: ${documentId}`);
|
|
727
|
+
assertOrganizationDocumentAuthority(document.authorityKind, organizationAuthorityGranted);
|
|
684
728
|
const [row] = await scopedDb.update(schema.documents).set({
|
|
685
729
|
status: "queued",
|
|
686
730
|
error: null,
|
|
@@ -689,17 +733,23 @@ async function queueDocumentForReindex(db, workspaceId, documentId, access) {
|
|
|
689
733
|
and(
|
|
690
734
|
eq(schema.documents.workspaceId, workspaceId),
|
|
691
735
|
eq(schema.documents.id, documentId),
|
|
692
|
-
...documentAccessConditions(access)
|
|
736
|
+
...documentAccessConditions(workspaceId, access)
|
|
693
737
|
)
|
|
694
738
|
).returning();
|
|
695
739
|
if (!row) throw new Error(`Document not found: ${documentId}`);
|
|
696
740
|
return mapDocument(row);
|
|
697
741
|
});
|
|
698
742
|
}
|
|
699
|
-
|
|
700
|
-
|
|
743
|
+
function assertOrganizationDocumentAuthority(authorityKind, organizationAuthorityGranted) {
|
|
744
|
+
if (authorityKind === "organization" && organizationAuthorityGranted !== true) {
|
|
745
|
+
throw new Error("organization document mutations require exact account authority");
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
async function indexDocumentNow(db, objectStorage, workspaceId, documentId, services = createDocumentServices(), hooks = {}, access) {
|
|
749
|
+
const [loadedDocument] = await withDocumentRls(
|
|
701
750
|
db,
|
|
702
751
|
workspaceId,
|
|
752
|
+
access,
|
|
703
753
|
async (scopedDb) => await scopedDb.select().from(schema.documents).where(
|
|
704
754
|
and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))
|
|
705
755
|
).limit(1)
|
|
@@ -707,7 +757,7 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
|
|
|
707
757
|
if (!loadedDocument) throw new Error(`Document not found: ${documentId}`);
|
|
708
758
|
let document = loadedDocument;
|
|
709
759
|
const file = await requireReadyFile(db, workspaceId, document.fileId);
|
|
710
|
-
await
|
|
760
|
+
await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
|
|
711
761
|
await scopedDb.update(schema.documents).set({
|
|
712
762
|
status: "indexing",
|
|
713
763
|
parser: services.parser.name,
|
|
@@ -736,9 +786,10 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
|
|
|
736
786
|
`Embedding provider returned ${embeddings.length} embeddings for ${chunks.length} chunks`
|
|
737
787
|
);
|
|
738
788
|
}
|
|
739
|
-
await
|
|
789
|
+
await withDocumentRls(
|
|
740
790
|
db,
|
|
741
791
|
workspaceId,
|
|
792
|
+
access,
|
|
742
793
|
async (scopedDb) => await scopedDb.transaction(async (tx) => {
|
|
743
794
|
await tx.delete(schema.documentChunks).where(
|
|
744
795
|
and(
|
|
@@ -754,6 +805,9 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
|
|
|
754
805
|
documentId,
|
|
755
806
|
baseId: document.baseId,
|
|
756
807
|
fileId: file.id,
|
|
808
|
+
authorityKind: document.authorityKind,
|
|
809
|
+
authorityWorkspaceId: document.authorityWorkspaceId,
|
|
810
|
+
authoritySubjectId: document.authoritySubjectId,
|
|
757
811
|
chunkIndex: index,
|
|
758
812
|
text: chunk.text,
|
|
759
813
|
metadata: {
|
|
@@ -793,9 +847,10 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
|
|
|
793
847
|
})
|
|
794
848
|
);
|
|
795
849
|
} catch (error) {
|
|
796
|
-
const [failed] = await
|
|
850
|
+
const [failed] = await withDocumentRls(
|
|
797
851
|
db,
|
|
798
852
|
workspaceId,
|
|
853
|
+
access,
|
|
799
854
|
async (scopedDb) => await scopedDb.update(schema.documents).set({
|
|
800
855
|
status: "failed",
|
|
801
856
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -808,16 +863,17 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
|
|
|
808
863
|
return mapDocument(failed);
|
|
809
864
|
}
|
|
810
865
|
const updated = await getDocument(db, workspaceId, documentId, {
|
|
811
|
-
viewerSubjectId: document.
|
|
866
|
+
viewerSubjectId: document.authoritySubjectId
|
|
812
867
|
});
|
|
813
868
|
if (!updated) throw new Error(`Document disappeared after indexing: ${documentId}`);
|
|
814
869
|
return updated;
|
|
815
870
|
}
|
|
816
871
|
async function curateDroppedDocument(db, services, document, parsed, file) {
|
|
817
872
|
if (!services.curator) {
|
|
818
|
-
const [updated2] = await
|
|
873
|
+
const [updated2] = await withDocumentRls(
|
|
819
874
|
db,
|
|
820
875
|
document.workspaceId,
|
|
876
|
+
{ viewerSubjectId: document.authoritySubjectId },
|
|
821
877
|
async (scopedDb) => await scopedDb.update(schema.documents).set({
|
|
822
878
|
curationStatus: "none",
|
|
823
879
|
summary: null,
|
|
@@ -860,9 +916,10 @@ async function curateDroppedDocument(db, services, document, parsed, file) {
|
|
|
860
916
|
const suggestedBase = candidates.find((base) => base.id === outcome.targetBaseId) ?? null;
|
|
861
917
|
let moveToBaseId = null;
|
|
862
918
|
if (suggestedBase && failure === null && outcome.confidence >= DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE) {
|
|
863
|
-
const conflict = await
|
|
919
|
+
const conflict = await withDocumentRls(
|
|
864
920
|
db,
|
|
865
921
|
document.workspaceId,
|
|
922
|
+
{ viewerSubjectId: document.authoritySubjectId },
|
|
866
923
|
async (scopedDb) => await scopedDb.select({ id: schema.documents.id }).from(schema.documents).where(
|
|
867
924
|
and(
|
|
868
925
|
eq(schema.documents.workspaceId, document.workspaceId),
|
|
@@ -884,9 +941,10 @@ async function curateDroppedDocument(db, services, document, parsed, file) {
|
|
|
884
941
|
model
|
|
885
942
|
};
|
|
886
943
|
const curationStatus = failure ? "failed" : moveToBaseId ? "auto_filed" : "suggested";
|
|
887
|
-
const [updated] = await
|
|
944
|
+
const [updated] = await withDocumentRls(
|
|
888
945
|
db,
|
|
889
946
|
document.workspaceId,
|
|
947
|
+
{ viewerSubjectId: document.authoritySubjectId },
|
|
890
948
|
async (scopedDb) => await scopedDb.update(schema.documents).set({
|
|
891
949
|
title: outcome.title ?? document.title,
|
|
892
950
|
summary: outcome.summary,
|
|
@@ -906,6 +964,7 @@ async function curateDroppedDocument(db, services, document, parsed, file) {
|
|
|
906
964
|
return updated ?? document;
|
|
907
965
|
}
|
|
908
966
|
async function searchDocuments(db, input, services = createDocumentServices()) {
|
|
967
|
+
await assertDocumentAccountWorkspace(db, input.accountId, input.workspaceId);
|
|
909
968
|
const mode = input.mode ?? "hybrid";
|
|
910
969
|
const limit = Math.min(Math.max(input.limit ?? 5, 1), 50);
|
|
911
970
|
const candidateLimit = mode === "hybrid" ? Math.min(limit * 4, 100) : limit;
|
|
@@ -931,15 +990,44 @@ async function searchDocuments(db, input, services = createDocumentServices()) {
|
|
|
931
990
|
}
|
|
932
991
|
return mergeDocumentSearchRows(rows, mode).slice(0, limit);
|
|
933
992
|
}
|
|
993
|
+
async function searchEffectiveDocuments(db, input, services = createDocumentServices()) {
|
|
994
|
+
const initiatingSubjectId = cleanString(input.initiatingSubjectId);
|
|
995
|
+
if (!initiatingSubjectId) {
|
|
996
|
+
throw new Error("effective document retrieval requires an initiating subject");
|
|
997
|
+
}
|
|
998
|
+
return await searchDocuments(
|
|
999
|
+
db,
|
|
1000
|
+
{
|
|
1001
|
+
accountId: input.accountId,
|
|
1002
|
+
workspaceId: input.workspaceId,
|
|
1003
|
+
query: input.query,
|
|
1004
|
+
baseIds: input.baseIds,
|
|
1005
|
+
limit: input.limit,
|
|
1006
|
+
mode: input.mode,
|
|
1007
|
+
sourceKinds: input.sourceKinds,
|
|
1008
|
+
aclTags: input.aclTags,
|
|
1009
|
+
// Construct the lower-level access filter here instead of spreading the
|
|
1010
|
+
// caller input, so an untyped/legacy access override is always ignored.
|
|
1011
|
+
access: {
|
|
1012
|
+
viewerSubjectId: initiatingSubjectId,
|
|
1013
|
+
...input.surface === "agent" ? { agentOnly: true } : {}
|
|
1014
|
+
}
|
|
1015
|
+
},
|
|
1016
|
+
services
|
|
1017
|
+
);
|
|
1018
|
+
}
|
|
934
1019
|
async function vectorSearchDocuments(db, input, limit, services) {
|
|
935
1020
|
const queryEmbedding = await services.embedder.embedQuery(input.query);
|
|
936
1021
|
validateEmbedding(queryEmbedding, services.embedder.dimensions, services.embedder.model);
|
|
937
1022
|
const distance = sql`${schema.documentChunks.embedding} <=> ${vectorLiteral(queryEmbedding)}::vector`;
|
|
938
|
-
const rows = await
|
|
1023
|
+
const rows = await withDocumentAccountRls(
|
|
939
1024
|
db,
|
|
1025
|
+
input.accountId,
|
|
940
1026
|
input.workspaceId,
|
|
1027
|
+
input.access,
|
|
941
1028
|
async (scopedDb) => await scopedDb.select({
|
|
942
1029
|
chunkId: schema.documentChunks.id,
|
|
1030
|
+
workspaceId: schema.documentChunks.workspaceId,
|
|
943
1031
|
documentId: schema.documentChunks.documentId,
|
|
944
1032
|
baseId: schema.documentChunks.baseId,
|
|
945
1033
|
fileId: schema.documentChunks.fileId,
|
|
@@ -956,22 +1044,28 @@ async function vectorSearchDocuments(db, input, limit, services) {
|
|
|
956
1044
|
sourceUpdatedAt: schema.documents.sourceUpdatedAt,
|
|
957
1045
|
sourceVersion: schema.documents.sourceVersion,
|
|
958
1046
|
aclTags: schema.documents.aclTags,
|
|
1047
|
+
authorityKind: schema.documents.authorityKind,
|
|
1048
|
+
authorityWorkspaceId: schema.documents.authorityWorkspaceId,
|
|
1049
|
+
authoritySubjectId: schema.documents.authoritySubjectId,
|
|
959
1050
|
distance
|
|
960
1051
|
}).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(and(...documentSearchConditions(input, services.embedder.model))).orderBy(distance).limit(limit)
|
|
961
1052
|
);
|
|
962
1053
|
return rows.map((row) => ({
|
|
963
|
-
...mapSearchRowBase(row
|
|
1054
|
+
...mapSearchRowBase(row),
|
|
964
1055
|
vectorScore: 1 / (1 + Number(row.distance)),
|
|
965
1056
|
keywordScore: null
|
|
966
1057
|
}));
|
|
967
1058
|
}
|
|
968
1059
|
async function keywordSearchDocuments(db, input, limit) {
|
|
969
1060
|
const rank = sql`ts_rank_cd(to_tsvector('simple', ${schema.documentChunks.text}), plainto_tsquery('simple', ${input.query}))`;
|
|
970
|
-
const rows = await
|
|
1061
|
+
const rows = await withDocumentAccountRls(
|
|
971
1062
|
db,
|
|
1063
|
+
input.accountId,
|
|
972
1064
|
input.workspaceId,
|
|
1065
|
+
input.access,
|
|
973
1066
|
async (scopedDb) => await scopedDb.select({
|
|
974
1067
|
chunkId: schema.documentChunks.id,
|
|
1068
|
+
workspaceId: schema.documentChunks.workspaceId,
|
|
975
1069
|
documentId: schema.documentChunks.documentId,
|
|
976
1070
|
baseId: schema.documentChunks.baseId,
|
|
977
1071
|
fileId: schema.documentChunks.fileId,
|
|
@@ -988,6 +1082,9 @@ async function keywordSearchDocuments(db, input, limit) {
|
|
|
988
1082
|
sourceUpdatedAt: schema.documents.sourceUpdatedAt,
|
|
989
1083
|
sourceVersion: schema.documents.sourceVersion,
|
|
990
1084
|
aclTags: schema.documents.aclTags,
|
|
1085
|
+
authorityKind: schema.documents.authorityKind,
|
|
1086
|
+
authorityWorkspaceId: schema.documents.authorityWorkspaceId,
|
|
1087
|
+
authoritySubjectId: schema.documents.authoritySubjectId,
|
|
991
1088
|
rank
|
|
992
1089
|
}).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
|
|
993
1090
|
and(
|
|
@@ -997,17 +1094,20 @@ async function keywordSearchDocuments(db, input, limit) {
|
|
|
997
1094
|
).orderBy(desc(rank)).limit(limit)
|
|
998
1095
|
);
|
|
999
1096
|
return rows.map((row) => ({
|
|
1000
|
-
...mapSearchRowBase(row
|
|
1097
|
+
...mapSearchRowBase(row),
|
|
1001
1098
|
vectorScore: null,
|
|
1002
1099
|
keywordScore: normalizeKeywordScore(Number(row.rank))
|
|
1003
1100
|
}));
|
|
1004
1101
|
}
|
|
1005
|
-
async function getDocumentChunk(db, workspaceId, chunkId, access) {
|
|
1006
|
-
const [row] = await
|
|
1102
|
+
async function getDocumentChunk(db, accountId, workspaceId, chunkId, access) {
|
|
1103
|
+
const [row] = await withDocumentAccountRls(
|
|
1007
1104
|
db,
|
|
1105
|
+
accountId,
|
|
1008
1106
|
workspaceId,
|
|
1107
|
+
access,
|
|
1009
1108
|
async (scopedDb) => await scopedDb.select({
|
|
1010
1109
|
chunkId: schema.documentChunks.id,
|
|
1110
|
+
workspaceId: schema.documentChunks.workspaceId,
|
|
1011
1111
|
documentId: schema.documentChunks.documentId,
|
|
1012
1112
|
baseId: schema.documentChunks.baseId,
|
|
1013
1113
|
fileId: schema.documentChunks.fileId,
|
|
@@ -1023,55 +1123,102 @@ async function getDocumentChunk(db, workspaceId, chunkId, access) {
|
|
|
1023
1123
|
sourceCreatedAt: schema.documents.sourceCreatedAt,
|
|
1024
1124
|
sourceUpdatedAt: schema.documents.sourceUpdatedAt,
|
|
1025
1125
|
sourceVersion: schema.documents.sourceVersion,
|
|
1026
|
-
aclTags: schema.documents.aclTags
|
|
1126
|
+
aclTags: schema.documents.aclTags,
|
|
1127
|
+
authorityKind: schema.documents.authorityKind,
|
|
1128
|
+
authorityWorkspaceId: schema.documents.authorityWorkspaceId,
|
|
1129
|
+
authoritySubjectId: schema.documents.authoritySubjectId
|
|
1027
1130
|
}).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
|
|
1028
1131
|
and(
|
|
1029
|
-
eq(schema.
|
|
1132
|
+
eq(schema.documents.accountId, accountId),
|
|
1133
|
+
eq(schema.documentChunks.accountId, accountId),
|
|
1030
1134
|
eq(schema.documentChunks.id, chunkId),
|
|
1031
1135
|
eq(schema.documents.status, "ready"),
|
|
1032
|
-
...documentAccessConditions(access)
|
|
1136
|
+
...documentAccessConditions(workspaceId, access)
|
|
1033
1137
|
)
|
|
1034
1138
|
).limit(1)
|
|
1035
1139
|
);
|
|
1036
1140
|
if (!row) return null;
|
|
1037
1141
|
return {
|
|
1038
|
-
...mapSearchRowBase(row
|
|
1142
|
+
...mapSearchRowBase(row),
|
|
1039
1143
|
score: 1,
|
|
1040
1144
|
matchType: "hybrid",
|
|
1041
1145
|
vectorScore: null,
|
|
1042
1146
|
keywordScore: null
|
|
1043
1147
|
};
|
|
1044
1148
|
}
|
|
1045
|
-
function
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1149
|
+
function resolveDocumentAuthority(input) {
|
|
1150
|
+
const legacyKind = input.legacyVisibility === "private" ? "personal" : "workspace";
|
|
1151
|
+
const kind = input.kind ?? legacyKind;
|
|
1152
|
+
if (input.kind && input.legacyVisibility && input.legacyVisibility === "private" !== (input.kind === "personal")) {
|
|
1153
|
+
throw new Error("document authorityKind conflicts with legacy visibility");
|
|
1154
|
+
}
|
|
1155
|
+
if (kind === "organization") {
|
|
1156
|
+
return { kind, workspaceId: null, subjectId: null };
|
|
1050
1157
|
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1158
|
+
if (kind === "workspace") {
|
|
1159
|
+
return { kind, workspaceId: input.workspaceId, subjectId: null };
|
|
1160
|
+
}
|
|
1161
|
+
const subjectId = cleanString(input.initiatingSubjectId ?? null);
|
|
1162
|
+
if (!subjectId) throw new Error("personal documents require an initiating subject");
|
|
1163
|
+
if (new TextEncoder().encode(subjectId).byteLength > DOCUMENT_AUTHORITY_SUBJECT_MAX_BYTES) {
|
|
1164
|
+
throw new Error(
|
|
1165
|
+
`personal document initiating subject exceeds ${DOCUMENT_AUTHORITY_SUBJECT_MAX_BYTES} UTF-8 bytes`
|
|
1056
1166
|
);
|
|
1057
|
-
return condition ? [condition] : [];
|
|
1058
1167
|
}
|
|
1059
|
-
return
|
|
1168
|
+
return { kind, workspaceId: input.workspaceId, subjectId };
|
|
1169
|
+
}
|
|
1170
|
+
async function withDocumentRls(db, workspaceId, access, fn) {
|
|
1171
|
+
const subjectId = cleanString(access?.viewerSubjectId ?? null);
|
|
1172
|
+
return subjectId ? await withWorkspaceSubjectRls(db, workspaceId, subjectId, fn) : await withWorkspaceRls(db, workspaceId, fn);
|
|
1173
|
+
}
|
|
1174
|
+
async function withDocumentAccountRls(db, accountId, workspaceId, access, fn) {
|
|
1175
|
+
const context = await assertDocumentAccountWorkspace(db, accountId, workspaceId);
|
|
1176
|
+
const subjectId = cleanString(access?.viewerSubjectId ?? null);
|
|
1177
|
+
return await withRlsContext(db, context, async (scopedDb) => {
|
|
1178
|
+
if (subjectId) await setSubjectRlsContext(scopedDb, subjectId);
|
|
1179
|
+
return await fn(scopedDb);
|
|
1180
|
+
});
|
|
1181
|
+
}
|
|
1182
|
+
async function assertDocumentAccountWorkspace(db, accountId, workspaceId) {
|
|
1183
|
+
const context = await rlsContextForWorkspace(db, workspaceId);
|
|
1184
|
+
if (context.accountId !== accountId) {
|
|
1185
|
+
throw new Error("document account/workspace authority mismatch");
|
|
1186
|
+
}
|
|
1187
|
+
return context;
|
|
1188
|
+
}
|
|
1189
|
+
function documentAccessConditions(workspaceId, access) {
|
|
1190
|
+
const organization = eq(schema.documents.authorityKind, "organization");
|
|
1191
|
+
const workspace = and(
|
|
1192
|
+
eq(schema.documents.authorityKind, "workspace"),
|
|
1193
|
+
eq(schema.documents.authorityWorkspaceId, workspaceId)
|
|
1194
|
+
);
|
|
1195
|
+
const viewer = cleanString(access?.viewerSubjectId ?? null);
|
|
1196
|
+
const personal = viewer ? and(
|
|
1197
|
+
eq(schema.documents.authorityKind, "personal"),
|
|
1198
|
+
eq(schema.documents.authorityWorkspaceId, workspaceId),
|
|
1199
|
+
eq(schema.documents.authoritySubjectId, viewer)
|
|
1200
|
+
) : void 0;
|
|
1201
|
+
const authority = viewer ? or(organization, workspace, personal) ?? organization : or(organization, workspace) ?? organization;
|
|
1202
|
+
if (access?.agentOnly) {
|
|
1203
|
+
return [eq(schema.documents.agentAccess, true), authority];
|
|
1204
|
+
}
|
|
1205
|
+
return [authority];
|
|
1060
1206
|
}
|
|
1061
1207
|
function documentMatchesAccess(document, access) {
|
|
1062
1208
|
if (access?.agentOnly) {
|
|
1063
|
-
return document.agentAccess && (document
|
|
1209
|
+
return document.agentAccess && canViewDocument(document, access.viewerSubjectId);
|
|
1064
1210
|
}
|
|
1065
1211
|
return canViewDocument(document, access?.viewerSubjectId);
|
|
1066
1212
|
}
|
|
1067
1213
|
function canViewDocument(document, viewerSubjectId) {
|
|
1068
|
-
return document.
|
|
1214
|
+
return document.authorityKind !== "personal" || !!viewerSubjectId && document.authoritySubjectId === viewerSubjectId;
|
|
1069
1215
|
}
|
|
1070
1216
|
function documentSearchConditions(input, embeddingModel) {
|
|
1071
1217
|
const conditions = [
|
|
1072
1218
|
eq(schema.documents.status, "ready"),
|
|
1073
|
-
eq(schema.
|
|
1074
|
-
|
|
1219
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1220
|
+
eq(schema.documentChunks.accountId, input.accountId),
|
|
1221
|
+
...documentAccessConditions(input.workspaceId, input.access)
|
|
1075
1222
|
];
|
|
1076
1223
|
if (embeddingModel) {
|
|
1077
1224
|
conditions.push(eq(schema.documentChunks.embeddingModel, embeddingModel));
|
|
@@ -1088,10 +1235,10 @@ function documentSearchConditions(input, embeddingModel) {
|
|
|
1088
1235
|
}
|
|
1089
1236
|
return conditions;
|
|
1090
1237
|
}
|
|
1091
|
-
function mapSearchRowBase(row
|
|
1238
|
+
function mapSearchRowBase(row) {
|
|
1092
1239
|
return {
|
|
1093
1240
|
chunkId: row.chunkId,
|
|
1094
|
-
workspaceId,
|
|
1241
|
+
workspaceId: row.workspaceId,
|
|
1095
1242
|
documentId: row.documentId,
|
|
1096
1243
|
baseId: row.baseId,
|
|
1097
1244
|
fileId: row.fileId,
|
|
@@ -1107,7 +1254,10 @@ function mapSearchRowBase(row, workspaceId) {
|
|
|
1107
1254
|
sourceCreatedAt: row.sourceCreatedAt?.toISOString() ?? null,
|
|
1108
1255
|
sourceUpdatedAt: row.sourceUpdatedAt?.toISOString() ?? null,
|
|
1109
1256
|
sourceVersion: row.sourceVersion,
|
|
1110
|
-
aclTags: cleanStringArray(row.aclTags)
|
|
1257
|
+
aclTags: cleanStringArray(row.aclTags),
|
|
1258
|
+
authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
|
|
1259
|
+
authorityWorkspaceId: row.authorityWorkspaceId,
|
|
1260
|
+
authoritySubjectId: row.authoritySubjectId
|
|
1111
1261
|
};
|
|
1112
1262
|
}
|
|
1113
1263
|
function mergeDocumentSearchRows(rows, mode) {
|
|
@@ -1312,6 +1462,9 @@ function mapDocument(row) {
|
|
|
1312
1462
|
sourceUpdatedAt: row.sourceUpdatedAt?.toISOString() ?? null,
|
|
1313
1463
|
sourceVersion: row.sourceVersion,
|
|
1314
1464
|
aclTags: cleanStringArray(row.aclTags),
|
|
1465
|
+
authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
|
|
1466
|
+
authorityWorkspaceId: row.authorityWorkspaceId,
|
|
1467
|
+
authoritySubjectId: row.authoritySubjectId,
|
|
1315
1468
|
visibility: normalizeDocumentVisibility(row.visibility),
|
|
1316
1469
|
createdBy: row.createdBy,
|
|
1317
1470
|
agentAccess: row.agentAccess,
|
|
@@ -1323,6 +1476,15 @@ function mapDocument(row) {
|
|
|
1323
1476
|
updatedAt: row.updatedAt.toISOString()
|
|
1324
1477
|
};
|
|
1325
1478
|
}
|
|
1479
|
+
function normalizeDocumentAuthorityKind(value) {
|
|
1480
|
+
switch (value) {
|
|
1481
|
+
case "organization":
|
|
1482
|
+
case "personal":
|
|
1483
|
+
return value;
|
|
1484
|
+
default:
|
|
1485
|
+
return "workspace";
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1326
1488
|
function normalizeDocumentVisibility(value) {
|
|
1327
1489
|
return value === "private" ? "private" : "workspace";
|
|
1328
1490
|
}
|
|
@@ -1346,6 +1508,7 @@ export {
|
|
|
1346
1508
|
DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS,
|
|
1347
1509
|
DEFAULT_DOCUMENT_EMBEDDING_MODEL,
|
|
1348
1510
|
DEFAULT_DOCUMENT_PARSER,
|
|
1511
|
+
DOCUMENT_AUTHORITY_SUBJECT_MAX_BYTES,
|
|
1349
1512
|
DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE,
|
|
1350
1513
|
DOCUMENT_CURATION_MAX_INPUT_CHARS,
|
|
1351
1514
|
DeterministicEmbeddingProvider,
|
|
@@ -1376,6 +1539,8 @@ export {
|
|
|
1376
1539
|
parseCurationOutcome,
|
|
1377
1540
|
parseDocumentBytes,
|
|
1378
1541
|
queueDocumentForReindex,
|
|
1379
|
-
|
|
1542
|
+
resolveDocumentAuthority,
|
|
1543
|
+
searchDocuments,
|
|
1544
|
+
searchEffectiveDocuments
|
|
1380
1545
|
};
|
|
1381
1546
|
//# sourceMappingURL=index.js.map
|