@absolutejs/rag 0.0.19 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-kit/index.js +31179 -0
- package/dist/adapter-kit/index.js.map +27 -0
- package/dist/index.js +139 -2170
- package/dist/index.js.map +5 -8
- package/dist/src/adapter-kit/index.d.ts +8 -0
- package/dist/src/adapters/sync.d.ts +21 -0
- package/dist/src/index.d.ts +2 -4
- package/dist/src/sync/sync.d.ts +1 -1
- package/dist/src/vue/useRAG.d.ts +58 -18
- package/dist/src/vue/useRAGIndexAdmin.d.ts +56 -16
- package/dist/src/vue/useRAGOps.d.ts +2 -2
- package/dist/types/adapters.d.ts +13 -30
- package/dist/types/index.d.ts +0 -1
- package/package.json +13 -3
- package/dist/src/adapters/postgres.d.ts +0 -3
- package/dist/src/adapters/sqlite.d.ts +0 -3
- package/dist/src/internal/resolveAbsoluteSQLiteVec.d.ts +0 -3
- package/dist/types/sync.d.ts +0 -23
package/dist/index.js
CHANGED
|
@@ -35670,2204 +35670,176 @@ var createInMemoryRAGStore = (options = {}) => {
|
|
|
35670
35670
|
getStatus: () => createInMemoryStatus(dimensions)
|
|
35671
35671
|
};
|
|
35672
35672
|
};
|
|
35673
|
-
// src/adapters/
|
|
35674
|
-
|
|
35675
|
-
|
|
35676
|
-
|
|
35677
|
-
|
|
35678
|
-
|
|
35679
|
-
|
|
35680
|
-
|
|
35681
|
-
|
|
35682
|
-
|
|
35683
|
-
|
|
35684
|
-
|
|
35685
|
-
|
|
35686
|
-
const
|
|
35687
|
-
|
|
35688
|
-
|
|
35689
|
-
|
|
35690
|
-
|
|
35691
|
-
const baseFloor = Math.max(input.topK, input.topK * Math.max(1, Math.floor(input.queryMultiplier)));
|
|
35692
|
-
let tuned = input.plannerProfile === "latency" ? clamp(Math.max(input.topK * 2, baseFloor)) : input.plannerProfile === "recall" ? clamp(Math.max(input.topK * 12, baseFloor * 4)) : clamp(Math.max(input.topK * 6, baseFloor * 2));
|
|
35693
|
-
if (filteredCap !== undefined) {
|
|
35694
|
-
if (filteredCap === 0) {
|
|
35695
|
-
return 0;
|
|
35696
|
-
}
|
|
35697
|
-
tuned = Math.min(tuned, filteredCap);
|
|
35698
|
-
}
|
|
35699
|
-
return tuned;
|
|
35700
|
-
};
|
|
35701
|
-
var planNativeCandidateSearchBackfillK = (input) => {
|
|
35702
|
-
if (typeof input.maxBackfills === "number" && Number.isFinite(input.maxBackfills) && (input.backfillCount ?? 0) >= Math.max(0, Math.floor(input.maxBackfills))) {
|
|
35703
|
-
return input.currentSearchK;
|
|
35704
|
-
}
|
|
35705
|
-
const cappedLimit = input.filteredCandidateCount === undefined || !Number.isFinite(input.filteredCandidateCount) ? input.candidateLimit : Math.min(input.candidateLimit, Math.max(0, Math.floor(input.filteredCandidateCount)));
|
|
35706
|
-
if (input.currentSearchK >= cappedLimit) {
|
|
35707
|
-
return input.currentSearchK;
|
|
35708
|
-
}
|
|
35709
|
-
return Math.min(cappedLimit, Math.max(input.currentSearchK + 1, input.currentSearchK * 2));
|
|
35710
|
-
};
|
|
35711
|
-
var summarizeSQLiteCandidateCoverage = (input) => {
|
|
35712
|
-
const basis = typeof input.filteredCandidateCount === "number" && Number.isFinite(input.filteredCandidateCount) ? Math.max(0, Math.floor(input.filteredCandidateCount)) : typeof input.returnedCount === "number" && Number.isFinite(input.returnedCount) ? Math.max(0, Math.floor(input.returnedCount)) : 0;
|
|
35713
|
-
if (basis === 0) {
|
|
35714
|
-
return "empty";
|
|
35715
|
-
}
|
|
35716
|
-
if (basis < input.topK) {
|
|
35717
|
-
return "under_target";
|
|
35718
|
-
}
|
|
35719
|
-
if (basis >= input.topK * 3) {
|
|
35720
|
-
return "broad";
|
|
35721
|
-
}
|
|
35722
|
-
return "target_sized";
|
|
35723
|
-
};
|
|
35724
|
-
|
|
35725
|
-
// src/adapters/postgres.ts
|
|
35726
|
-
var DEFAULT_DIMENSIONS = RAG_VECTOR_DIMENSIONS_DEFAULT;
|
|
35727
|
-
var DEFAULT_TABLE_NAME = "rag_chunks";
|
|
35728
|
-
var DEFAULT_SCHEMA_NAME = "public";
|
|
35729
|
-
var DEFAULT_QUERY_MULTIPLIER = 4;
|
|
35730
|
-
var MAX_QUERY_MULTIPLIER = 16;
|
|
35731
|
-
var DEFAULT_POSTGRES_INDEX_TYPE = "hnsw";
|
|
35732
|
-
var DEFAULT_POSTGRES_IVFFLAT_LISTS = 100;
|
|
35733
|
-
var DEFAULT_POSTGRES_HNSW_M = 16;
|
|
35734
|
-
var DEFAULT_POSTGRES_HNSW_EF_CONSTRUCTION = 64;
|
|
35735
|
-
var IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
35736
|
-
var FILTER_PATH_SEGMENT_RE = /^[a-zA-Z0-9_]+$/;
|
|
35737
|
-
var isObjectFilterRecord = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
35738
|
-
var isNestedFilterArray2 = (value) => Array.isArray(value) && value.every((entry) => isObjectFilterRecord(entry));
|
|
35739
|
-
var isOperatorFilterRecord = (value) => isObjectFilterRecord(value) && Object.keys(value).some((key) => key.startsWith("$"));
|
|
35740
|
-
var countFilterClauses = (filter) => {
|
|
35741
|
-
if (!filter) {
|
|
35742
|
-
return 0;
|
|
35743
|
-
}
|
|
35744
|
-
let count = 0;
|
|
35745
|
-
for (const [key, value] of Object.entries(filter)) {
|
|
35746
|
-
if (key === "$and" || key === "$or") {
|
|
35747
|
-
if (isNestedFilterArray2(value)) {
|
|
35748
|
-
count += value.reduce((total, entry) => total + countFilterClauses(entry), 0);
|
|
35749
|
-
}
|
|
35750
|
-
continue;
|
|
35751
|
-
}
|
|
35752
|
-
if (key === "$not") {
|
|
35753
|
-
if (isObjectFilterRecord(value)) {
|
|
35754
|
-
count += countFilterClauses(value);
|
|
35755
|
-
}
|
|
35756
|
-
continue;
|
|
35757
|
-
}
|
|
35758
|
-
count += 1;
|
|
35759
|
-
}
|
|
35760
|
-
return count;
|
|
35761
|
-
};
|
|
35762
|
-
var toPostgresJsonPath = (key) => {
|
|
35763
|
-
const segments = key.split(".").filter(Boolean);
|
|
35764
|
-
if (segments.length === 0 || !segments.every((segment) => FILTER_PATH_SEGMENT_RE.test(segment))) {
|
|
35765
|
-
return null;
|
|
35766
|
-
}
|
|
35767
|
-
return segments;
|
|
35768
|
-
};
|
|
35769
|
-
var toPostgresFilterBinding = (value) => {
|
|
35770
|
-
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean" || value === null) {
|
|
35771
|
-
return value;
|
|
35772
|
-
}
|
|
35773
|
-
return;
|
|
35774
|
-
};
|
|
35775
|
-
var buildPostgresJsonbScalarEquality = (input) => {
|
|
35776
|
-
const comparison = input.comparison ?? "=";
|
|
35777
|
-
return comparison === "=" ? `jsonb_typeof(${input.valueSql}) = 'null'` : `coalesce(jsonb_typeof(${input.valueSql}), 'missing') <> 'null'`;
|
|
35778
|
-
};
|
|
35779
|
-
var buildPostgresMetadataScalarEquality = (input) => input.value === null ? buildPostgresJsonbScalarEquality({
|
|
35780
|
-
comparison: input.comparison,
|
|
35781
|
-
valueSql: input.valueSql
|
|
35782
|
-
}) : `${input.actualSql} ${input.comparison ?? "="} ${input.bind(String(input.value))}`;
|
|
35783
|
-
var buildPostgresFilterPlan = (filter, startIndex = 0) => {
|
|
35784
|
-
if (!filter) {
|
|
35785
|
-
return { clause: "", params: [] };
|
|
35786
|
-
}
|
|
35787
|
-
const params = [];
|
|
35788
|
-
const bind = (value) => {
|
|
35789
|
-
params.push(value);
|
|
35790
|
-
return `$${params.length + startIndex}`;
|
|
35791
|
-
};
|
|
35792
|
-
const build = (entry) => {
|
|
35793
|
-
const clauses = [];
|
|
35794
|
-
for (const [key, value] of Object.entries(entry)) {
|
|
35795
|
-
if (key === "$and" || key === "$or") {
|
|
35796
|
-
if (!isNestedFilterArray2(value) || value.length === 0) {
|
|
35797
|
-
return null;
|
|
35798
|
-
}
|
|
35799
|
-
const nested = value.map((item) => build(item));
|
|
35800
|
-
if (nested.some((item) => item === null)) {
|
|
35801
|
-
return null;
|
|
35802
|
-
}
|
|
35803
|
-
clauses.push(`(${nested.filter((item) => Boolean(item)).join(key === "$and" ? " AND " : " OR ")})`);
|
|
35804
|
-
continue;
|
|
35805
|
-
}
|
|
35806
|
-
if (key === "$not") {
|
|
35807
|
-
if (!isObjectFilterRecord(value)) {
|
|
35808
|
-
return null;
|
|
35809
|
-
}
|
|
35810
|
-
const nested = build(value);
|
|
35811
|
-
if (!nested) {
|
|
35812
|
-
return null;
|
|
35813
|
-
}
|
|
35814
|
-
clauses.push(`NOT (${nested})`);
|
|
35815
|
-
continue;
|
|
35816
|
-
}
|
|
35817
|
-
const isScalarField = key === "chunkId" || key === "source" || key === "title";
|
|
35818
|
-
const jsonPath = isScalarField ? null : toPostgresJsonPath(key);
|
|
35819
|
-
if (!isScalarField && !jsonPath) {
|
|
35820
|
-
return null;
|
|
35821
|
-
}
|
|
35822
|
-
let actualSql;
|
|
35823
|
-
let metadataPathSegments = [];
|
|
35824
|
-
let metadataValueSql;
|
|
35825
|
-
if (isScalarField) {
|
|
35826
|
-
actualSql = key === "chunkId" ? "chunk_id" : key;
|
|
35827
|
-
} else {
|
|
35828
|
-
metadataPathSegments = jsonPath ?? [];
|
|
35829
|
-
actualSql = `jsonb_extract_path_text(metadata, ${metadataPathSegments.map((segment) => `'${segment}'`).join(", ")})`;
|
|
35830
|
-
metadataValueSql = `metadata #> '{${metadataPathSegments.join(",")}}'`;
|
|
35831
|
-
}
|
|
35832
|
-
if (!isOperatorFilterRecord(value)) {
|
|
35833
|
-
const binding = toPostgresFilterBinding(value);
|
|
35834
|
-
if (binding === undefined) {
|
|
35835
|
-
return null;
|
|
35836
|
-
}
|
|
35837
|
-
clauses.push(isScalarField ? `${actualSql} = ${bind(String(binding))}` : buildPostgresMetadataScalarEquality({
|
|
35838
|
-
actualSql,
|
|
35839
|
-
bind,
|
|
35840
|
-
value: binding,
|
|
35841
|
-
valueSql: metadataValueSql
|
|
35842
|
-
}));
|
|
35843
|
-
continue;
|
|
35844
|
-
}
|
|
35845
|
-
const operatorClauses = Object.entries(value).map(([operator, expected]) => {
|
|
35846
|
-
switch (operator) {
|
|
35847
|
-
case "$exists":
|
|
35848
|
-
return isScalarField ? expected ? `${actualSql} IS NOT NULL` : `${actualSql} IS NULL` : expected ? `${metadataValueSql} IS NOT NULL` : `${metadataValueSql} IS NULL`;
|
|
35849
|
-
case "$in": {
|
|
35850
|
-
if (!Array.isArray(expected) || expected.length === 0) {
|
|
35851
|
-
return null;
|
|
35852
|
-
}
|
|
35853
|
-
const bindings = expected.map((entry2) => toPostgresFilterBinding(entry2)).filter((entry2) => entry2 !== undefined);
|
|
35854
|
-
if (bindings.length !== expected.length) {
|
|
35855
|
-
return null;
|
|
35856
|
-
}
|
|
35857
|
-
return isScalarField ? `${actualSql} IN (${bindings.map((entry2) => bind(String(entry2))).join(", ")})` : `(${bindings.map((entry2) => buildPostgresMetadataScalarEquality({
|
|
35858
|
-
actualSql,
|
|
35859
|
-
bind,
|
|
35860
|
-
value: entry2,
|
|
35861
|
-
valueSql: metadataValueSql
|
|
35862
|
-
})).join(" OR ")})`;
|
|
35863
|
-
}
|
|
35864
|
-
case "$ne": {
|
|
35865
|
-
const binding = toPostgresFilterBinding(expected);
|
|
35866
|
-
return binding === undefined ? null : isScalarField ? `${actualSql} <> ${bind(String(binding))}` : buildPostgresMetadataScalarEquality({
|
|
35867
|
-
actualSql,
|
|
35868
|
-
bind,
|
|
35869
|
-
comparison: "<>",
|
|
35870
|
-
value: binding,
|
|
35871
|
-
valueSql: metadataValueSql
|
|
35872
|
-
});
|
|
35873
|
-
}
|
|
35874
|
-
case "$gt":
|
|
35875
|
-
case "$gte":
|
|
35876
|
-
case "$lt":
|
|
35877
|
-
case "$lte": {
|
|
35878
|
-
if (typeof expected !== "number" || !Number.isFinite(expected)) {
|
|
35879
|
-
return null;
|
|
35880
|
-
}
|
|
35881
|
-
const comparison = operator === "$gt" ? ">" : operator === "$gte" ? ">=" : operator === "$lt" ? "<" : "<=";
|
|
35882
|
-
return `((${actualSql}) ~ '^-?[0-9]+(\\.[0-9]+)?$' AND (${actualSql})::double precision ${comparison} ${bind(expected)})`;
|
|
35883
|
-
}
|
|
35884
|
-
case "$contains":
|
|
35885
|
-
if (isScalarField) {
|
|
35886
|
-
return null;
|
|
35887
|
-
}
|
|
35888
|
-
if (toPostgresFilterBinding(expected) === undefined) {
|
|
35889
|
-
return null;
|
|
35890
|
-
}
|
|
35891
|
-
return `(${metadataValueSql} IS NOT NULL AND ${metadataValueSql} ? ${bind(String(expected))})`;
|
|
35892
|
-
case "$containsAny":
|
|
35893
|
-
case "$containsAll": {
|
|
35894
|
-
if (isScalarField || !Array.isArray(expected)) {
|
|
35895
|
-
return null;
|
|
35896
|
-
}
|
|
35897
|
-
const values = expected.map((entry2) => toPostgresFilterBinding(entry2)).filter((entry2) => entry2 !== undefined);
|
|
35898
|
-
if (values.length === 0 || values.length !== expected.length) {
|
|
35899
|
-
return null;
|
|
35900
|
-
}
|
|
35901
|
-
const sqlArray = `ARRAY[${values.map((value2) => bind(String(value2))).join(", ")}]::text[]`;
|
|
35902
|
-
return `(${metadataValueSql} IS NOT NULL AND ${metadataValueSql} ${operator === "$containsAny" ? "?|" : "?&"} ${sqlArray})`;
|
|
35903
|
-
}
|
|
35904
|
-
default:
|
|
35905
|
-
return null;
|
|
35906
|
-
}
|
|
35907
|
-
});
|
|
35908
|
-
if (operatorClauses.some((clause2) => clause2 === null)) {
|
|
35909
|
-
return null;
|
|
35910
|
-
}
|
|
35911
|
-
clauses.push(operatorClauses.filter((clause2) => Boolean(clause2)).map((clause2) => `(${clause2})`).join(" AND "));
|
|
35912
|
-
}
|
|
35913
|
-
return clauses.length > 0 ? clauses.map((clause2) => `(${clause2})`).join(" AND ") : "";
|
|
35914
|
-
};
|
|
35915
|
-
const clause = build(filter);
|
|
35916
|
-
return clause === null || clause.trim().length === 0 ? null : { clause, params };
|
|
35917
|
-
};
|
|
35918
|
-
var buildPostgresPushdownFilter = (filter) => {
|
|
35919
|
-
if (!filter) {
|
|
35920
|
-
return;
|
|
35921
|
-
}
|
|
35922
|
-
const hasPushdownFilterPlan = (entry) => {
|
|
35923
|
-
const plan = buildPostgresFilterPlan(entry);
|
|
35924
|
-
return plan !== null && Boolean(plan.clause) && plan.clause.trim().length > 0;
|
|
35925
|
-
};
|
|
35926
|
-
const hasPushdownFilterPlanEntry = (entry) => {
|
|
35927
|
-
if (!isObjectFilterRecord(entry)) {
|
|
35928
|
-
return false;
|
|
35929
|
-
}
|
|
35930
|
-
return hasPushdownFilterPlan(entry);
|
|
35931
|
-
};
|
|
35932
|
-
const nextEntries = [];
|
|
35933
|
-
for (const [key, value] of Object.entries(filter)) {
|
|
35934
|
-
if (key === "$and" || key === "$or") {
|
|
35935
|
-
if (!isNestedFilterArray2(value)) {
|
|
35936
|
-
continue;
|
|
35937
|
-
}
|
|
35938
|
-
const nested = value.map((entry) => buildPostgresPushdownFilter(entry)).filter((entry) => hasPushdownFilterPlanEntry(entry));
|
|
35939
|
-
if (nested.length > 0) {
|
|
35940
|
-
nextEntries.push([key, nested]);
|
|
35941
|
-
}
|
|
35942
|
-
continue;
|
|
35943
|
-
}
|
|
35944
|
-
if (key === "$not") {
|
|
35945
|
-
if (!isObjectFilterRecord(value)) {
|
|
35946
|
-
continue;
|
|
35947
|
-
}
|
|
35948
|
-
const nested = buildPostgresPushdownFilter(value);
|
|
35949
|
-
if (hasPushdownFilterPlanEntry(nested)) {
|
|
35950
|
-
nextEntries.push([key, nested]);
|
|
35951
|
-
}
|
|
35952
|
-
continue;
|
|
35953
|
-
}
|
|
35954
|
-
if (Array.isArray(value) || isOperatorFilterRecord(value) && Object.keys(value).some((operator) => !(operator === "$exists" || operator === "$in" || operator === "$contains" || operator === "$containsAny" || operator === "$containsAll" || operator === "$ne" || operator === "$gt" || operator === "$gte" || operator === "$lt" || operator === "$lte"))) {
|
|
35955
|
-
continue;
|
|
35956
|
-
}
|
|
35957
|
-
const isScalarColumnKey = ["chunkId", "source", "title"].includes(key);
|
|
35958
|
-
const jsonPath = isScalarColumnKey ? null : toPostgresJsonPath(key);
|
|
35959
|
-
if (!isScalarColumnKey && !jsonPath) {
|
|
35960
|
-
continue;
|
|
35961
|
-
}
|
|
35962
|
-
if (!hasPushdownFilterPlan({ [key]: value })) {
|
|
35963
|
-
continue;
|
|
35964
|
-
}
|
|
35965
|
-
nextEntries.push([key, value]);
|
|
35966
|
-
}
|
|
35967
|
-
return nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined;
|
|
35968
|
-
};
|
|
35969
|
-
var resolvePostgresPushdownMode = (input) => {
|
|
35970
|
-
const totalFilterClauseCount = countFilterClauses(input.filter);
|
|
35971
|
-
const pushdownClauseCount = countFilterClauses(input.pushdownFilter);
|
|
35972
|
-
const jsRemainderClauseCount = Math.max(0, totalFilterClauseCount - pushdownClauseCount);
|
|
35973
|
-
const pushdownMode = pushdownClauseCount === 0 ? "none" : pushdownClauseCount >= totalFilterClauseCount ? "full" : "partial";
|
|
35974
|
-
return {
|
|
35975
|
-
jsRemainderClauseCount,
|
|
35976
|
-
jsRemainderRatio: totalFilterClauseCount > 0 ? jsRemainderClauseCount / totalFilterClauseCount : undefined,
|
|
35977
|
-
pushdownClauseCount,
|
|
35978
|
-
pushdownCoverageRatio: totalFilterClauseCount > 0 ? pushdownClauseCount / totalFilterClauseCount : undefined,
|
|
35979
|
-
pushdownMode,
|
|
35980
|
-
totalFilterClauseCount
|
|
35981
|
-
};
|
|
35982
|
-
};
|
|
35983
|
-
var assertSupportedIdentifier2 = (name) => {
|
|
35984
|
-
if (!IDENTIFIER_RE.test(name)) {
|
|
35985
|
-
throw new Error(`Invalid identifier "${name}". Only alphanumeric and underscore names are allowed.`);
|
|
35986
|
-
}
|
|
35987
|
-
};
|
|
35988
|
-
var normalizePostgresIndexType = (value) => {
|
|
35989
|
-
if (value === undefined) {
|
|
35990
|
-
return DEFAULT_POSTGRES_INDEX_TYPE;
|
|
35991
|
-
}
|
|
35992
|
-
if (value === "none" || value === "hnsw" || value === "ivfflat") {
|
|
35993
|
-
return value;
|
|
35994
|
-
}
|
|
35995
|
-
throw new Error(`Invalid postgres index type "${String(value)}". Expected "none", "hnsw", or "ivfflat".`);
|
|
35996
|
-
};
|
|
35997
|
-
var normalizePositiveInteger = (value, fallback) => {
|
|
35998
|
-
if (value === undefined || !Number.isFinite(value)) {
|
|
35999
|
-
return fallback;
|
|
36000
|
-
}
|
|
36001
|
-
return Math.max(1, Math.floor(value));
|
|
36002
|
-
};
|
|
36003
|
-
var getPostgresIndexOperatorClass = (metric) => metric === "cosine" ? "vector_cosine_ops" : metric === "inner_product" ? "vector_ip_ops" : "vector_l2_ops";
|
|
36004
|
-
var getPostgresIndexName = (qualifiedTableName, indexType) => indexType === "none" ? undefined : `${qualifiedTableName.replace(".", "_")}_embedding_${indexType}_idx`;
|
|
36005
|
-
var buildPostgresIndexSql = (input) => {
|
|
36006
|
-
if (input.indexType === "none") {
|
|
36007
|
-
return;
|
|
36008
|
-
}
|
|
36009
|
-
const opclass = getPostgresIndexOperatorClass(input.distanceMetric);
|
|
36010
|
-
const indexName = getPostgresIndexName(input.qualifiedTableName, input.indexType);
|
|
36011
|
-
const optionsSql = input.indexType === "hnsw" ? ` with (m = ${input.hnswM}, ef_construction = ${input.hnswEfConstruction})` : ` with (lists = ${input.indexLists})`;
|
|
36012
|
-
const createPrefix = input.ifNotExists === false ? "create index" : "create index if not exists";
|
|
36013
|
-
return `${createPrefix} ${indexName} on ${input.qualifiedTableName} using ${input.indexType} (embedding ${opclass})${optionsSql}`;
|
|
36014
|
-
};
|
|
36015
|
-
var normalizeQueryMultiplier = (value) => {
|
|
36016
|
-
if (value === undefined || !Number.isFinite(value)) {
|
|
36017
|
-
return DEFAULT_QUERY_MULTIPLIER;
|
|
36018
|
-
}
|
|
36019
|
-
return Math.min(MAX_QUERY_MULTIPLIER, Math.max(1, Math.floor(value)));
|
|
36020
|
-
};
|
|
36021
|
-
var normalizeMaxBackfills = (value) => {
|
|
36022
|
-
if (value === undefined || !Number.isFinite(value)) {
|
|
36023
|
-
return;
|
|
36024
|
-
}
|
|
36025
|
-
return Math.max(0, Math.floor(value));
|
|
36026
|
-
};
|
|
36027
|
-
var normalizeMinResults = (value, topK) => {
|
|
36028
|
-
if (value === undefined || !Number.isFinite(value)) {
|
|
36029
|
-
return topK;
|
|
36030
|
-
}
|
|
36031
|
-
return Math.min(topK, Math.max(1, Math.floor(value)));
|
|
36032
|
-
};
|
|
36033
|
-
var resolveFillTarget = (input) => {
|
|
36034
|
-
const fillPolicy = input.fillPolicy ?? "satisfy_min_results";
|
|
36035
|
-
return {
|
|
36036
|
-
fillPolicy,
|
|
36037
|
-
targetResults: fillPolicy === "strict_topk" ? input.topK : input.minResults
|
|
36038
|
-
};
|
|
36039
|
-
};
|
|
36040
|
-
var toQualifiedTableName = (schemaName, tableName) => `${schemaName}.${tableName}`;
|
|
36041
|
-
var toVectorLiteral = (vector) => `[${vector.join(",")}]`;
|
|
36042
|
-
var parseMetadata = (value) => {
|
|
36043
|
-
if (typeof value === "string") {
|
|
36044
|
-
try {
|
|
36045
|
-
const parsed = JSON.parse(value);
|
|
36046
|
-
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
36047
|
-
return parsed;
|
|
36048
|
-
}
|
|
36049
|
-
} catch {
|
|
36050
|
-
return;
|
|
36051
|
-
}
|
|
36052
|
-
}
|
|
36053
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
36054
|
-
return;
|
|
36055
|
-
}
|
|
36056
|
-
return value;
|
|
36057
|
-
};
|
|
36058
|
-
var parseVectorText = (value) => {
|
|
36059
|
-
if (!value) {
|
|
36060
|
-
return [];
|
|
36061
|
-
}
|
|
36062
|
-
const normalized = value.trim();
|
|
36063
|
-
const wrapped = normalized.startsWith("[") ? normalized : `[${normalized.replace(/[()]/g, "")}]`;
|
|
36064
|
-
try {
|
|
36065
|
-
const parsed = JSON.parse(wrapped);
|
|
36066
|
-
return Array.isArray(parsed) ? parsed.filter((entry) => typeof entry === "number" && Number.isFinite(entry)) : [];
|
|
36067
|
-
} catch {
|
|
36068
|
-
return [];
|
|
36069
|
-
}
|
|
36070
|
-
};
|
|
36071
|
-
var parseCountValue = (value) => {
|
|
36072
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
36073
|
-
return value;
|
|
36074
|
-
}
|
|
36075
|
-
if (typeof value === "bigint") {
|
|
36076
|
-
return Number(value);
|
|
36077
|
-
}
|
|
36078
|
-
if (typeof value === "string") {
|
|
36079
|
-
const parsed = Number(value);
|
|
36080
|
-
return Number.isFinite(parsed) ? parsed : 0;
|
|
36081
|
-
}
|
|
36082
|
-
return 0;
|
|
36083
|
-
};
|
|
36084
|
-
var parseBooleanValue = (value) => {
|
|
36085
|
-
if (typeof value === "boolean") {
|
|
36086
|
-
return value;
|
|
36087
|
-
}
|
|
36088
|
-
if (typeof value === "number") {
|
|
36089
|
-
return value !== 0;
|
|
36090
|
-
}
|
|
36091
|
-
if (typeof value === "bigint") {
|
|
36092
|
-
return value !== 0n;
|
|
36093
|
-
}
|
|
36094
|
-
if (typeof value === "string") {
|
|
36095
|
-
const normalized = value.trim().toLowerCase();
|
|
36096
|
-
return normalized === "true" || normalized === "t" || normalized === "1";
|
|
36097
|
-
}
|
|
36098
|
-
return false;
|
|
36099
|
-
};
|
|
36100
|
-
var refreshPostgresRuntimeDiagnostics = async (db, nativeDiagnostics, input) => {
|
|
36101
|
-
try {
|
|
36102
|
-
const rows = await db.unsafe(`select
|
|
36103
|
-
c.reltuples::bigint as estimated_row_count,
|
|
36104
|
-
pg_relation_size($1::regclass) as table_bytes,
|
|
36105
|
-
pg_indexes_size($1::regclass) as index_bytes,
|
|
36106
|
-
pg_total_relation_size($1::regclass) as total_bytes,
|
|
36107
|
-
exists(
|
|
36108
|
-
select 1
|
|
36109
|
-
from pg_indexes
|
|
36110
|
-
where schemaname = $2
|
|
36111
|
-
and tablename = $3
|
|
36112
|
-
and indexname = $4
|
|
36113
|
-
) as index_present
|
|
36114
|
-
from pg_class c
|
|
36115
|
-
join pg_namespace n on n.oid = c.relnamespace
|
|
36116
|
-
where n.nspname = $2
|
|
36117
|
-
and c.relname = $3
|
|
36118
|
-
limit 1`, [
|
|
36119
|
-
input.qualifiedTableName,
|
|
36120
|
-
input.schemaName,
|
|
36121
|
-
input.tableName,
|
|
36122
|
-
input.indexName ?? ""
|
|
36123
|
-
]);
|
|
36124
|
-
const row = rows[0];
|
|
36125
|
-
nativeDiagnostics.indexName = input.indexName;
|
|
36126
|
-
nativeDiagnostics.indexPresent = input.indexName ? parseBooleanValue(row?.index_present) : undefined;
|
|
36127
|
-
nativeDiagnostics.estimatedRowCount = parseCountValue(row?.estimated_row_count);
|
|
36128
|
-
nativeDiagnostics.tableBytes = parseCountValue(row?.table_bytes);
|
|
36129
|
-
nativeDiagnostics.indexBytes = parseCountValue(row?.index_bytes);
|
|
36130
|
-
nativeDiagnostics.totalBytes = parseCountValue(row?.total_bytes);
|
|
36131
|
-
nativeDiagnostics.lastHealthCheckAt = Date.now();
|
|
36132
|
-
nativeDiagnostics.lastHealthError = undefined;
|
|
36133
|
-
} catch (error) {
|
|
36134
|
-
nativeDiagnostics.lastHealthCheckAt = Date.now();
|
|
36135
|
-
nativeDiagnostics.lastHealthError = error instanceof Error ? error.message : String(error);
|
|
36136
|
-
}
|
|
36137
|
-
};
|
|
36138
|
-
var analyzePostgresTable = async (db, nativeDiagnostics, input) => {
|
|
36139
|
-
try {
|
|
36140
|
-
await db.unsafe(`analyze ${input.qualifiedTableName}`);
|
|
36141
|
-
nativeDiagnostics.lastAnalyzeAt = Date.now();
|
|
36142
|
-
nativeDiagnostics.lastAnalyzeError = undefined;
|
|
36143
|
-
await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, input);
|
|
36144
|
-
} catch (error) {
|
|
36145
|
-
nativeDiagnostics.lastAnalyzeAt = Date.now();
|
|
36146
|
-
nativeDiagnostics.lastAnalyzeError = error instanceof Error ? error.message : String(error);
|
|
36147
|
-
throw error;
|
|
36148
|
-
}
|
|
36149
|
-
};
|
|
36150
|
-
var rebuildPostgresNativeIndex = async (db, nativeDiagnostics, input) => {
|
|
36151
|
-
if (!input.indexName || input.indexType === "none") {
|
|
36152
|
-
throw new Error("Postgres native index rebuild is not configured");
|
|
36153
|
-
}
|
|
36154
|
-
try {
|
|
36155
|
-
await db.unsafe(`drop index if exists ${input.indexName}`);
|
|
36156
|
-
await db.unsafe(buildPostgresIndexSql({
|
|
36157
|
-
distanceMetric: input.distanceMetric,
|
|
36158
|
-
hnswEfConstruction: input.hnswEfConstruction,
|
|
36159
|
-
hnswM: input.hnswM,
|
|
36160
|
-
ifNotExists: false,
|
|
36161
|
-
indexLists: input.indexLists,
|
|
36162
|
-
indexType: input.indexType,
|
|
36163
|
-
qualifiedTableName: input.qualifiedTableName
|
|
36164
|
-
}));
|
|
36165
|
-
nativeDiagnostics.lastReindexAt = Date.now();
|
|
36166
|
-
nativeDiagnostics.lastReindexError = undefined;
|
|
36167
|
-
await analyzePostgresTable(db, nativeDiagnostics, input);
|
|
36168
|
-
} catch (error) {
|
|
36169
|
-
nativeDiagnostics.lastReindexAt = Date.now();
|
|
36170
|
-
nativeDiagnostics.lastReindexError = error instanceof Error ? error.message : String(error);
|
|
36171
|
-
throw error;
|
|
36172
|
-
}
|
|
36173
|
-
};
|
|
36174
|
-
var getPostgresChunkIdsByChunkIds = async (db, qualifiedTableName, chunkIds) => {
|
|
36175
|
-
const normalized = [...new Set(chunkIds)].filter((chunkId) => chunkId.length > 0);
|
|
36176
|
-
if (normalized.length === 0) {
|
|
36177
|
-
return [];
|
|
36178
|
-
}
|
|
36179
|
-
const placeholders = normalized.map((_, index) => `$${index + 1}`).join(", ");
|
|
36180
|
-
const rows = await db.unsafe(`select chunk_id from ${qualifiedTableName} where chunk_id in (${placeholders})`, normalized);
|
|
36181
|
-
return rows.map((row) => row.chunk_id).filter((chunkId) => typeof chunkId === "string");
|
|
36182
|
-
};
|
|
36183
|
-
var getPostgresCandidateChunkIdsByFilter = async (db, qualifiedTableName, filter) => {
|
|
36184
|
-
if (!filter || Object.keys(filter).length === 0) {
|
|
36185
|
-
return [];
|
|
36186
|
-
}
|
|
36187
|
-
const pushdownFilter = buildPostgresPushdownFilter(filter);
|
|
36188
|
-
const filterPlan = buildPostgresFilterPlan(pushdownFilter);
|
|
36189
|
-
const rowsSql = filterPlan?.clause ? `select chunk_id, text, title, source, metadata from ${qualifiedTableName} where ${filterPlan.clause}` : `select chunk_id, text, title, source, metadata from ${qualifiedTableName}`;
|
|
36190
|
-
const rows = await db.unsafe(rowsSql, filterPlan?.clause ? filterPlan.params ?? [] : []);
|
|
36191
|
-
const chunks = rows.map((row) => mapRowToChunk(row)).filter((chunk) => matchesFilter(chunk, filter));
|
|
36192
|
-
return chunks.map((chunk) => chunk.chunkId);
|
|
36193
|
-
};
|
|
36194
|
-
var getPostgresCandidateChunkIds = async (db, qualifiedTableName, input) => {
|
|
36195
|
-
const chunkIdSet = new Set;
|
|
36196
|
-
if (input.filter && Object.keys(input.filter).length > 0) {
|
|
36197
|
-
for (const chunkId of await getPostgresCandidateChunkIdsByFilter(db, qualifiedTableName, input.filter)) {
|
|
36198
|
-
chunkIdSet.add(chunkId);
|
|
36199
|
-
}
|
|
36200
|
-
}
|
|
36201
|
-
if (input.chunkIds && input.chunkIds.length > 0) {
|
|
36202
|
-
for (const chunkId of await getPostgresChunkIdsByChunkIds(db, qualifiedTableName, input.chunkIds)) {
|
|
36203
|
-
chunkIdSet.add(chunkId);
|
|
36204
|
-
}
|
|
36205
|
-
}
|
|
36206
|
-
return [...chunkIdSet];
|
|
36207
|
-
};
|
|
36208
|
-
var normalizeDistance = (distance, metric) => {
|
|
36209
|
-
if (!Number.isFinite(distance)) {
|
|
36210
|
-
return 0;
|
|
36211
|
-
}
|
|
36212
|
-
if (metric === "cosine") {
|
|
36213
|
-
return Math.min(1, Math.max(0, 1 - distance));
|
|
36214
|
-
}
|
|
36215
|
-
if (metric === "inner_product") {
|
|
36216
|
-
return Math.max(0, -distance);
|
|
36217
|
-
}
|
|
36218
|
-
return Math.max(0, 1 / (1 + Math.abs(distance)));
|
|
36219
|
-
};
|
|
36220
|
-
var getDistanceOperator = (metric) => metric === "cosine" ? "<=>" : metric === "inner_product" ? "<#>" : "<->";
|
|
36221
|
-
var createPostgresStatus = (dimensions, nativeDiagnostics) => ({
|
|
36222
|
-
backend: "postgres",
|
|
36223
|
-
dimensions,
|
|
36224
|
-
native: nativeDiagnostics,
|
|
36225
|
-
vectorMode: "native_pgvector"
|
|
36226
|
-
});
|
|
36227
|
-
var createPostgresCapabilities = () => ({
|
|
36228
|
-
backend: "postgres",
|
|
36229
|
-
nativeVectorSearch: true,
|
|
36230
|
-
persistence: "external",
|
|
36231
|
-
serverSideFiltering: true,
|
|
36232
|
-
streamingIngestStatus: false
|
|
36233
|
-
});
|
|
36234
|
-
var updatePostgresLastQueryPlan = (input) => {
|
|
36235
|
-
const pushdown = resolvePostgresPushdownMode({
|
|
36236
|
-
filter: input.filter,
|
|
36237
|
-
pushdownFilter: input.pushdownFilter
|
|
36238
|
-
});
|
|
36239
|
-
input.nativeDiagnostics.lastQueryPlan = {
|
|
36240
|
-
backfillCount: input.backfillCount,
|
|
36241
|
-
candidateBudgetExhausted: input.candidateBudgetExhausted,
|
|
36242
|
-
candidateCoverage: summarizeSQLiteCandidateCoverage({
|
|
36243
|
-
filteredCandidateCount: input.filteredCandidateCount,
|
|
36244
|
-
returnedCount: input.returnedCount,
|
|
36245
|
-
topK: input.topK
|
|
36246
|
-
}),
|
|
36247
|
-
filteredCandidateCount: input.filteredCandidateCount,
|
|
36248
|
-
finalSearchK: input.finalSearchK,
|
|
36249
|
-
initialSearchK: input.initialSearchK,
|
|
36250
|
-
searchExpansionRatio: typeof input.initialSearchK === "number" && typeof input.finalSearchK === "number" && input.initialSearchK > 0 ? input.finalSearchK / input.initialSearchK : undefined,
|
|
36251
|
-
candidateLimitUsed: input.candidateLimitUsed,
|
|
36252
|
-
maxBackfillsUsed: input.maxBackfillsUsed,
|
|
36253
|
-
minResultsUsed: input.minResultsUsed,
|
|
36254
|
-
fillPolicyUsed: input.fillPolicyUsed,
|
|
36255
|
-
plannerProfileUsed: input.plannerProfileUsed,
|
|
36256
|
-
jsRemainderClauseCount: pushdown.jsRemainderClauseCount,
|
|
36257
|
-
queryMultiplierUsed: input.queryMultiplierUsed,
|
|
36258
|
-
jsRemainderRatio: pushdown.jsRemainderRatio,
|
|
36259
|
-
pushdownApplied: pushdown.pushdownClauseCount > 0,
|
|
36260
|
-
pushdownClauseCount: pushdown.pushdownClauseCount,
|
|
36261
|
-
pushdownCoverageRatio: pushdown.pushdownCoverageRatio,
|
|
36262
|
-
pushdownMode: pushdown.pushdownMode,
|
|
36263
|
-
queryMode: "native_pgvector",
|
|
36264
|
-
candidateYieldRatio: typeof input.returnedCount === "number" && typeof input.finalSearchK === "number" && input.finalSearchK > 0 ? input.returnedCount / input.finalSearchK : undefined,
|
|
36265
|
-
returnedCount: input.returnedCount,
|
|
36266
|
-
backfillLimitReached: input.backfillLimitReached,
|
|
36267
|
-
minResultsSatisfied: input.minResultsSatisfied,
|
|
36268
|
-
topKFillRatio: typeof input.returnedCount === "number" && input.topK > 0 ? input.returnedCount / input.topK : undefined,
|
|
36269
|
-
totalFilterClauseCount: pushdown.totalFilterClauseCount,
|
|
36270
|
-
underfilledTopK: input.underfilledTopK
|
|
36271
|
-
};
|
|
36272
|
-
};
|
|
36273
|
-
var matchesFilter = (record, filter) => matchesMetadataFilterRecord({
|
|
36274
|
-
chunkId: record.chunkId,
|
|
36275
|
-
metadata: record.metadata,
|
|
36276
|
-
source: record.source,
|
|
36277
|
-
title: record.title,
|
|
36278
|
-
...record.metadata ?? {}
|
|
36279
|
-
}, filter);
|
|
36280
|
-
var mapRowToChunk = (row) => ({
|
|
36281
|
-
chunkId: row.chunk_id,
|
|
36282
|
-
metadata: parseMetadata(row.metadata),
|
|
36283
|
-
source: row.source ?? undefined,
|
|
36284
|
-
text: row.text,
|
|
36285
|
-
title: row.title ?? undefined,
|
|
36286
|
-
vector: parseVectorText(row.embedding)
|
|
36287
|
-
});
|
|
36288
|
-
var ensurePostgresSchema = async (db, input) => {
|
|
36289
|
-
await db.unsafe("create extension if not exists vector");
|
|
36290
|
-
const [schemaName] = input.qualifiedTableName.split(".");
|
|
36291
|
-
if (schemaName) {
|
|
36292
|
-
await db.unsafe(`create schema if not exists ${schemaName}`);
|
|
36293
|
-
}
|
|
36294
|
-
await db.unsafe(`
|
|
36295
|
-
create table if not exists ${input.qualifiedTableName} (
|
|
36296
|
-
chunk_id text primary key,
|
|
36297
|
-
text text not null,
|
|
36298
|
-
title text,
|
|
36299
|
-
source text,
|
|
36300
|
-
metadata jsonb,
|
|
36301
|
-
embedding vector(${input.dimensions}) not null
|
|
36302
|
-
)
|
|
36303
|
-
`);
|
|
36304
|
-
const indexSql = buildPostgresIndexSql(input);
|
|
36305
|
-
if (indexSql) {
|
|
36306
|
-
await db.unsafe(indexSql);
|
|
36307
|
-
}
|
|
36308
|
-
};
|
|
36309
|
-
var createPostgresRAGStore = (options = {}) => {
|
|
36310
|
-
const dimensions = options.dimensions ?? DEFAULT_DIMENSIONS;
|
|
36311
|
-
const distanceMetric = options.distanceMetric ?? "cosine";
|
|
36312
|
-
const queryMultiplier = normalizeQueryMultiplier(options.queryMultiplier);
|
|
36313
|
-
const indexType = normalizePostgresIndexType(options.indexType);
|
|
36314
|
-
const indexLists = normalizePositiveInteger(options.indexLists, DEFAULT_POSTGRES_IVFFLAT_LISTS);
|
|
36315
|
-
const hnswM = normalizePositiveInteger(options.hnswM, DEFAULT_POSTGRES_HNSW_M);
|
|
36316
|
-
const hnswEfConstruction = normalizePositiveInteger(options.hnswEfConstruction, DEFAULT_POSTGRES_HNSW_EF_CONSTRUCTION);
|
|
36317
|
-
const tableName = options.tableName ?? DEFAULT_TABLE_NAME;
|
|
36318
|
-
const schemaName = options.schemaName ?? DEFAULT_SCHEMA_NAME;
|
|
36319
|
-
assertSupportedIdentifier2(tableName);
|
|
36320
|
-
assertSupportedIdentifier2(schemaName);
|
|
36321
|
-
const qualifiedTableName = toQualifiedTableName(schemaName, tableName);
|
|
36322
|
-
const indexName = getPostgresIndexName(qualifiedTableName, indexType);
|
|
36323
|
-
const db = options.sql ?? new Bun.SQL(options.connectionString ?? process.env.RAG_POSTGRES_URL ?? process.env.DATABASE_URL ?? "postgres://postgres:postgres@localhost:55433/absolute_rag_demo");
|
|
36324
|
-
const nativeDiagnostics = {
|
|
36325
|
-
active: true,
|
|
36326
|
-
available: true,
|
|
36327
|
-
distanceMetric,
|
|
36328
|
-
extensionName: "vector",
|
|
36329
|
-
indexName,
|
|
36330
|
-
indexType,
|
|
36331
|
-
mode: "pgvector",
|
|
36332
|
-
requested: true,
|
|
36333
|
-
schemaName,
|
|
36334
|
-
tableName
|
|
36335
|
-
};
|
|
36336
|
-
const capabilities = createPostgresCapabilities();
|
|
36337
|
-
const distanceOperator = getDistanceOperator(distanceMetric);
|
|
36338
|
-
let initialized;
|
|
36339
|
-
const init = () => {
|
|
36340
|
-
initialized ??= ensurePostgresSchema(db, {
|
|
36341
|
-
dimensions,
|
|
36342
|
-
distanceMetric,
|
|
36343
|
-
hnswEfConstruction,
|
|
36344
|
-
hnswM,
|
|
36345
|
-
indexLists,
|
|
36346
|
-
indexType,
|
|
36347
|
-
qualifiedTableName
|
|
36348
|
-
}).then(() => refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
|
|
36349
|
-
indexName,
|
|
36350
|
-
qualifiedTableName,
|
|
36351
|
-
schemaName,
|
|
36352
|
-
tableName
|
|
36353
|
-
})).catch((error) => {
|
|
36354
|
-
nativeDiagnostics.active = false;
|
|
36355
|
-
nativeDiagnostics.available = false;
|
|
36356
|
-
nativeDiagnostics.lastInitError = error instanceof Error ? error.message : String(error);
|
|
36357
|
-
nativeDiagnostics.lastMigrationError = error instanceof Error ? error.message : String(error);
|
|
36358
|
-
nativeDiagnostics.fallbackReason = nativeDiagnostics.lastInitError;
|
|
36359
|
-
throw error;
|
|
36360
|
-
});
|
|
36361
|
-
return initialized;
|
|
36362
|
-
};
|
|
36363
|
-
const embed = async (input) => {
|
|
36364
|
-
input.model;
|
|
36365
|
-
input.signal;
|
|
36366
|
-
if (options.mockEmbedding) {
|
|
36367
|
-
return options.mockEmbedding(input.text);
|
|
36368
|
-
}
|
|
36369
|
-
return normalizeVector(createRAGVector(input.text, dimensions));
|
|
36370
|
-
};
|
|
36371
|
-
const query = async (input) => {
|
|
36372
|
-
await init();
|
|
36373
|
-
const queryVector = normalizeVector(input.queryVector);
|
|
36374
|
-
const queryMultiplier2 = normalizeQueryMultiplier(input.queryMultiplier ?? options.queryMultiplier);
|
|
36375
|
-
const maxBackfills = normalizeMaxBackfills(input.maxBackfills);
|
|
36376
|
-
const minResults = normalizeMinResults(input.minResults, input.topK);
|
|
36377
|
-
const fillTarget = resolveFillTarget({
|
|
36378
|
-
fillPolicy: input.fillPolicy,
|
|
36379
|
-
minResults,
|
|
36380
|
-
topK: input.topK
|
|
36381
|
-
});
|
|
36382
|
-
const queryVectorLiteral = toVectorLiteral(queryVector);
|
|
36383
|
-
const pushdownFilter = buildPostgresPushdownFilter(input.filter);
|
|
36384
|
-
const queryFilterPlan = buildPostgresFilterPlan(pushdownFilter);
|
|
36385
|
-
const effectivePushdownFilter = queryFilterPlan ? pushdownFilter : undefined;
|
|
36386
|
-
const countFilterPlan = queryFilterPlan;
|
|
36387
|
-
const countSql = countFilterPlan?.clause ? `select count(*)::int as count from ${qualifiedTableName} where ${countFilterPlan.clause}` : `select count(*)::int as count from ${qualifiedTableName}`;
|
|
36388
|
-
const totalRowsResult = await db.unsafe(countSql, countFilterPlan?.params ?? []);
|
|
36389
|
-
nativeDiagnostics.lastFilterDebug = {
|
|
36390
|
-
countParams: countFilterPlan?.params ?? [],
|
|
36391
|
-
countResultRaw: totalRowsResult?.[0],
|
|
36392
|
-
countSql,
|
|
36393
|
-
filter: input.filter,
|
|
36394
|
-
pushdownFilter: effectivePushdownFilter
|
|
36395
|
-
};
|
|
36396
|
-
const totalRows = parseCountValue(totalRowsResult?.[0]?.count);
|
|
36397
|
-
const candidateLimit = resolveAdaptiveNativeCandidateLimit({
|
|
36398
|
-
defaultCandidateLimit: RAG_NATIVE_QUERY_CANDIDATE_LIMIT,
|
|
36399
|
-
explicitCandidateLimit: input.candidateLimit,
|
|
36400
|
-
filteredCandidateCount: totalRows,
|
|
36401
|
-
plannerProfile: input.plannerProfile,
|
|
36402
|
-
queryMultiplier: queryMultiplier2,
|
|
36403
|
-
topK: input.topK
|
|
36404
|
-
});
|
|
36405
|
-
const hasPushdownFilter = Boolean(effectivePushdownFilter);
|
|
36406
|
-
const plannedFilteredCandidateCount = hasPushdownFilter && totalRows === 0 ? undefined : totalRows;
|
|
36407
|
-
const initialSearchK = planNativeCandidateSearchK({
|
|
36408
|
-
candidateLimit,
|
|
36409
|
-
filteredCandidateCount: plannedFilteredCandidateCount,
|
|
36410
|
-
queryMultiplier: queryMultiplier2,
|
|
36411
|
-
topK: input.topK
|
|
36412
|
-
});
|
|
36413
|
-
if (initialSearchK === 0) {
|
|
36414
|
-
return [];
|
|
36415
|
-
}
|
|
36416
|
-
let currentSearchK = initialSearchK;
|
|
36417
|
-
let backfillCount = 0;
|
|
36418
|
-
let candidateBudgetExhausted = false;
|
|
36419
|
-
let backfillLimitReached = false;
|
|
36420
|
-
let effectiveFilteredCandidateCount = plannedFilteredCandidateCount;
|
|
36421
|
-
let mapped = [];
|
|
36422
|
-
for (;; ) {
|
|
36423
|
-
const rowsSql = queryFilterPlan?.clause ? `select chunk_id, text, title, source, metadata, embedding::text as embedding, embedding ${distanceOperator} '${queryVectorLiteral}'::vector as distance from ${qualifiedTableName} where ${queryFilterPlan.clause} order by embedding ${distanceOperator} '${queryVectorLiteral}'::vector limit $${queryFilterPlan.params.length + 1}` : `select chunk_id, text, title, source, metadata, embedding::text as embedding, embedding ${distanceOperator} '${queryVectorLiteral}'::vector as distance from ${qualifiedTableName} order by embedding ${distanceOperator} '${queryVectorLiteral}'::vector limit $1`;
|
|
36424
|
-
const rows = await db.unsafe(rowsSql, queryFilterPlan?.clause ? [...queryFilterPlan.params ?? [], currentSearchK] : [currentSearchK]);
|
|
36425
|
-
nativeDiagnostics.lastFilterDebug = {
|
|
36426
|
-
...nativeDiagnostics.lastFilterDebug,
|
|
36427
|
-
queryParams: queryFilterPlan?.clause ? [...queryFilterPlan.params ?? [], currentSearchK] : [currentSearchK],
|
|
36428
|
-
queryRowCount: rows.length,
|
|
36429
|
-
querySql: rowsSql
|
|
36430
|
-
};
|
|
36431
|
-
if (hasPushdownFilter && effectiveFilteredCandidateCount === undefined && rows.length <= currentSearchK) {
|
|
36432
|
-
effectiveFilteredCandidateCount = rows.length;
|
|
36433
|
-
}
|
|
36434
|
-
mapped = rows.map((row) => {
|
|
36435
|
-
const chunk = mapRowToChunk(row);
|
|
36436
|
-
return {
|
|
36437
|
-
chunk,
|
|
36438
|
-
score: normalizeDistance(Number(row.distance ?? 0), distanceMetric)
|
|
36439
|
-
};
|
|
36440
|
-
}).filter(({ chunk }) => matchesFilter(chunk, input.filter)).map((entry) => ({
|
|
36441
|
-
chunkId: entry.chunk.chunkId,
|
|
36442
|
-
chunkText: entry.chunk.text,
|
|
36443
|
-
embedding: entry.chunk.vector,
|
|
36444
|
-
metadata: entry.chunk.metadata,
|
|
36445
|
-
score: entry.score,
|
|
36446
|
-
source: entry.chunk.source,
|
|
36447
|
-
title: entry.chunk.title
|
|
36448
|
-
})).sort((left, right) => right.score - left.score);
|
|
36449
|
-
if (mapped.length >= fillTarget.targetResults) {
|
|
36450
|
-
break;
|
|
36451
|
-
}
|
|
36452
|
-
const nextSearchK = planNativeCandidateSearchBackfillK({
|
|
36453
|
-
backfillCount,
|
|
36454
|
-
candidateLimit,
|
|
36455
|
-
currentSearchK,
|
|
36456
|
-
filteredCandidateCount: effectiveFilteredCandidateCount,
|
|
36457
|
-
maxBackfills
|
|
36458
|
-
});
|
|
36459
|
-
if (nextSearchK <= currentSearchK) {
|
|
36460
|
-
backfillLimitReached = typeof maxBackfills === "number" && backfillCount >= maxBackfills && mapped.length < fillTarget.targetResults;
|
|
36461
|
-
candidateBudgetExhausted = mapped.length < fillTarget.targetResults;
|
|
36462
|
-
break;
|
|
36463
|
-
}
|
|
36464
|
-
currentSearchK = nextSearchK;
|
|
36465
|
-
backfillCount += 1;
|
|
36466
|
-
}
|
|
36467
|
-
nativeDiagnostics.lastQueryError = undefined;
|
|
36468
|
-
const returned = mapped.slice(0, input.topK);
|
|
36469
|
-
updatePostgresLastQueryPlan({
|
|
36470
|
-
backfillCount,
|
|
36471
|
-
backfillLimitReached,
|
|
36472
|
-
candidateBudgetExhausted,
|
|
36473
|
-
candidateLimitUsed: candidateLimit,
|
|
36474
|
-
maxBackfillsUsed: maxBackfills,
|
|
36475
|
-
minResultsUsed: minResults,
|
|
36476
|
-
fillPolicyUsed: fillTarget.fillPolicy,
|
|
36477
|
-
plannerProfileUsed: input.plannerProfile,
|
|
36478
|
-
filter: input.filter,
|
|
36479
|
-
pushdownFilter: effectivePushdownFilter,
|
|
36480
|
-
queryMultiplierUsed: queryMultiplier2,
|
|
36481
|
-
filteredCandidateCount: effectiveFilteredCandidateCount,
|
|
36482
|
-
finalSearchK: currentSearchK,
|
|
36483
|
-
initialSearchK,
|
|
36484
|
-
nativeDiagnostics,
|
|
36485
|
-
minResultsSatisfied: returned.length >= minResults,
|
|
36486
|
-
returnedCount: returned.length,
|
|
36487
|
-
topK: input.topK,
|
|
36488
|
-
underfilledTopK: returned.length < input.topK
|
|
36489
|
-
});
|
|
36490
|
-
return returned;
|
|
36491
|
-
};
|
|
36492
|
-
const queryLexical = async (input) => {
|
|
36493
|
-
await init();
|
|
36494
|
-
const pushdownFilter = buildPostgresPushdownFilter(input.filter);
|
|
36495
|
-
const lexicalFilterPlan = buildPostgresFilterPlan(pushdownFilter);
|
|
36496
|
-
const rowsSql = lexicalFilterPlan?.clause ? `select chunk_id, text, title, source, metadata from ${qualifiedTableName} where ${lexicalFilterPlan.clause}` : `select chunk_id, text, title, source, metadata from ${qualifiedTableName}`;
|
|
36497
|
-
const rows = await db.unsafe(rowsSql, lexicalFilterPlan?.params ?? []);
|
|
36498
|
-
const chunks = rows.map((row) => mapRowToChunk(row)).filter((chunk) => matchesFilter(chunk, input.filter));
|
|
36499
|
-
const ranked = rankRAGLexicalMatches(input.query, chunks);
|
|
36500
|
-
return ranked.slice(0, input.topK).map(({ result, score }) => ({
|
|
36501
|
-
chunkId: result.chunkId,
|
|
36502
|
-
chunkText: result.text,
|
|
36503
|
-
metadata: result.metadata,
|
|
36504
|
-
score,
|
|
36505
|
-
source: result.source,
|
|
36506
|
-
title: result.title
|
|
36507
|
-
}));
|
|
36508
|
-
};
|
|
36509
|
-
const upsert = async (input) => {
|
|
36510
|
-
await init();
|
|
36511
|
-
const chunks = input.chunks.length > 0 ? await Promise.all(input.chunks.map(async (chunk) => ({
|
|
36512
|
-
chunkId: chunk.chunkId,
|
|
36513
|
-
metadata: chunk.metadata,
|
|
36514
|
-
source: chunk.source,
|
|
36515
|
-
text: chunk.text,
|
|
36516
|
-
title: chunk.title,
|
|
36517
|
-
vector: chunk.embedding ? normalizeVector(chunk.embedding) : normalizeVector(await embed({ text: chunk.text }))
|
|
36518
|
-
}))) : [];
|
|
36519
|
-
for (const chunk of chunks) {
|
|
36520
|
-
await db.unsafe(`insert into ${qualifiedTableName} (chunk_id, text, title, source, metadata, embedding)
|
|
36521
|
-
values ($1, $2, $3, $4, $5::jsonb, $6::vector)
|
|
36522
|
-
on conflict (chunk_id) do update set
|
|
36523
|
-
text = excluded.text,
|
|
36524
|
-
title = excluded.title,
|
|
36525
|
-
source = excluded.source,
|
|
36526
|
-
metadata = excluded.metadata,
|
|
36527
|
-
embedding = excluded.embedding`, [
|
|
36528
|
-
chunk.chunkId,
|
|
36529
|
-
chunk.text,
|
|
36530
|
-
chunk.title ?? null,
|
|
36531
|
-
chunk.source ?? null,
|
|
36532
|
-
chunk.metadata ?? null,
|
|
36533
|
-
toVectorLiteral(chunk.vector)
|
|
36534
|
-
]);
|
|
36535
|
-
}
|
|
36536
|
-
await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
|
|
36537
|
-
indexName,
|
|
36538
|
-
qualifiedTableName,
|
|
36539
|
-
schemaName,
|
|
36540
|
-
tableName
|
|
36541
|
-
});
|
|
36542
|
-
};
|
|
36543
|
-
const count = async (input = {}) => {
|
|
36544
|
-
await init();
|
|
36545
|
-
const filter = input.filter;
|
|
36546
|
-
const chunkIds = input.chunkIds;
|
|
36547
|
-
const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
|
|
36548
|
-
const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
|
|
36549
|
-
if (!hasFilter && !hasChunkIds) {
|
|
36550
|
-
const countResult = await db.unsafe(`select count(*)::int as count from ${qualifiedTableName}`);
|
|
36551
|
-
return parseCountValue(countResult[0]?.count);
|
|
36552
|
-
}
|
|
36553
|
-
return (await getPostgresCandidateChunkIds(db, qualifiedTableName, {
|
|
36554
|
-
filter,
|
|
36555
|
-
chunkIds
|
|
36556
|
-
})).length;
|
|
36557
|
-
};
|
|
36558
|
-
const remove = async (input = {}) => {
|
|
36559
|
-
await init();
|
|
36560
|
-
const filter = input.filter;
|
|
36561
|
-
const chunkIds = input.chunkIds;
|
|
36562
|
-
const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
|
|
36563
|
-
const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
|
|
36564
|
-
if (!hasFilter && !hasChunkIds) {
|
|
36565
|
-
return 0;
|
|
36566
|
-
}
|
|
36567
|
-
const ids = await getPostgresCandidateChunkIds(db, qualifiedTableName, {
|
|
36568
|
-
filter,
|
|
36569
|
-
chunkIds
|
|
36570
|
-
});
|
|
36571
|
-
if (ids.length === 0) {
|
|
36572
|
-
return 0;
|
|
36573
|
-
}
|
|
36574
|
-
const placeholders = ids.map((_, index) => `$${index + 1}`).join(", ");
|
|
36575
|
-
await db.unsafe(`delete from ${qualifiedTableName} where chunk_id in (${placeholders})`, ids);
|
|
36576
|
-
await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
|
|
36577
|
-
indexName,
|
|
36578
|
-
qualifiedTableName,
|
|
36579
|
-
schemaName,
|
|
36580
|
-
tableName
|
|
36581
|
-
});
|
|
36582
|
-
return ids.length;
|
|
36583
|
-
};
|
|
36584
|
-
const clear = async () => {
|
|
36585
|
-
await init();
|
|
36586
|
-
await db.unsafe(`truncate table ${qualifiedTableName}`);
|
|
36587
|
-
await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
|
|
36588
|
-
indexName,
|
|
36589
|
-
qualifiedTableName,
|
|
36590
|
-
schemaName,
|
|
36591
|
-
tableName
|
|
36592
|
-
});
|
|
36593
|
-
};
|
|
36594
|
-
const analyze = async () => {
|
|
36595
|
-
await init();
|
|
36596
|
-
await analyzePostgresTable(db, nativeDiagnostics, {
|
|
36597
|
-
indexName,
|
|
36598
|
-
qualifiedTableName,
|
|
36599
|
-
schemaName,
|
|
36600
|
-
tableName
|
|
36601
|
-
});
|
|
36602
|
-
};
|
|
36603
|
-
const rebuildNativeIndex = async () => {
|
|
36604
|
-
await init();
|
|
36605
|
-
await rebuildPostgresNativeIndex(db, nativeDiagnostics, {
|
|
36606
|
-
distanceMetric,
|
|
36607
|
-
hnswEfConstruction,
|
|
36608
|
-
hnswM,
|
|
36609
|
-
indexLists,
|
|
36610
|
-
indexName,
|
|
36611
|
-
indexType,
|
|
36612
|
-
qualifiedTableName,
|
|
36613
|
-
schemaName,
|
|
36614
|
-
tableName
|
|
36615
|
-
});
|
|
36616
|
-
};
|
|
36617
|
-
const close = async () => {
|
|
36618
|
-
await db.close?.();
|
|
36619
|
-
};
|
|
36620
|
-
return {
|
|
36621
|
-
analyze,
|
|
36622
|
-
clear,
|
|
36623
|
-
close,
|
|
36624
|
-
embed,
|
|
36625
|
-
getCapabilities: () => capabilities,
|
|
36626
|
-
getStatus: () => createPostgresStatus(dimensions, nativeDiagnostics),
|
|
36627
|
-
query,
|
|
36628
|
-
queryLexical,
|
|
36629
|
-
rebuildNativeIndex: indexName ? rebuildNativeIndex : undefined,
|
|
36630
|
-
count,
|
|
36631
|
-
delete: remove,
|
|
36632
|
-
upsert
|
|
36633
|
-
};
|
|
36634
|
-
};
|
|
36635
|
-
// src/adapters/sqlite.ts
|
|
36636
|
-
import { Database } from "bun:sqlite";
|
|
36637
|
-
import { existsSync as existsSync2 } from "fs";
|
|
36638
|
-
|
|
36639
|
-
// src/internal/resolveAbsoluteSQLiteVec.ts
|
|
36640
|
-
import { existsSync, readFileSync } from "fs";
|
|
36641
|
-
import { arch, platform } from "os";
|
|
36642
|
-
import { dirname as dirname5, join as join3 } from "path";
|
|
36643
|
-
var PLATFORM_PACKAGE_MAP = {
|
|
36644
|
-
"darwin-arm64": {
|
|
36645
|
-
libraryFile: "vec0.dylib",
|
|
36646
|
-
packageName: "@absolutejs/absolute-rag-sqlite-darwin-arm64"
|
|
36647
|
-
},
|
|
36648
|
-
"darwin-x64": {
|
|
36649
|
-
libraryFile: "vec0.dylib",
|
|
36650
|
-
packageName: "@absolutejs/absolute-rag-sqlite-darwin-x64"
|
|
36651
|
-
},
|
|
36652
|
-
"linux-arm64": {
|
|
36653
|
-
libraryFile: "vec0.so",
|
|
36654
|
-
packageName: "@absolutejs/absolute-rag-sqlite-linux-arm64"
|
|
36655
|
-
},
|
|
36656
|
-
"linux-x64": {
|
|
36657
|
-
libraryFile: "vec0.so",
|
|
36658
|
-
packageName: "@absolutejs/absolute-rag-sqlite-linux-x64"
|
|
36659
|
-
},
|
|
36660
|
-
"win32-x64": {
|
|
36661
|
-
libraryFile: "vec0.dll",
|
|
36662
|
-
packageName: "@absolutejs/absolute-rag-sqlite-windows-x64"
|
|
36663
|
-
}
|
|
36664
|
-
};
|
|
36665
|
-
var currentPlatformKey = () => `${platform()}-${arch()}`;
|
|
36666
|
-
var getErrorMessage = (error) => error instanceof Error ? error.message : String(error);
|
|
36667
|
-
var isPackageJsonShape = (value) => Boolean(value) && typeof value === "object";
|
|
36668
|
-
var readPackageVersion = (packageJsonPath) => {
|
|
36669
|
-
try {
|
|
36670
|
-
const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8"));
|
|
36671
|
-
if (!isPackageJsonShape(packageJson)) {
|
|
36672
|
-
return;
|
|
36673
|
-
}
|
|
36674
|
-
return typeof packageJson.version === "string" ? packageJson.version : undefined;
|
|
36675
|
-
} catch {
|
|
36676
|
-
return;
|
|
36677
|
-
}
|
|
36678
|
-
};
|
|
36679
|
-
var resolveAbsoluteSQLiteVec = () => {
|
|
36680
|
-
const platformKey = currentPlatformKey();
|
|
36681
|
-
const packageInfo = PLATFORM_PACKAGE_MAP[platformKey];
|
|
36682
|
-
if (!packageInfo) {
|
|
36683
|
-
return {
|
|
36684
|
-
platformKey,
|
|
36685
|
-
reason: `No AbsoluteJS sqlite-vec package is defined for ${platformKey}.`,
|
|
36686
|
-
source: "absolute-package",
|
|
36687
|
-
status: "unsupported_platform"
|
|
36688
|
-
};
|
|
36689
|
-
}
|
|
36690
|
-
try {
|
|
36691
|
-
const resolve4 = import.meta.resolve;
|
|
36692
|
-
if (typeof resolve4 !== "function") {
|
|
36693
|
-
throw new Error("AbsoluteJS sqlite-vec package resolution requires import.meta.resolve support.");
|
|
36694
|
-
}
|
|
36695
|
-
const packageJsonPath = new URL(resolve4(`${packageInfo.packageName}/package.json`)).pathname;
|
|
36696
|
-
const packageRoot = dirname5(packageJsonPath);
|
|
36697
|
-
const libraryPath = join3(packageRoot, packageInfo.libraryFile);
|
|
36698
|
-
const packageVersion = readPackageVersion(packageJsonPath);
|
|
36699
|
-
if (!existsSync(libraryPath)) {
|
|
36700
|
-
return {
|
|
36701
|
-
libraryFile: packageInfo.libraryFile,
|
|
36702
|
-
libraryPath,
|
|
36703
|
-
packageName: packageInfo.packageName,
|
|
36704
|
-
packageRoot,
|
|
36705
|
-
packageVersion,
|
|
36706
|
-
platformKey,
|
|
36707
|
-
reason: `Resolved ${packageInfo.packageName} but ${packageInfo.libraryFile} was not found.`,
|
|
36708
|
-
source: "absolute-package",
|
|
36709
|
-
status: "binary_missing"
|
|
36710
|
-
};
|
|
36711
|
-
}
|
|
36712
|
-
return {
|
|
36713
|
-
libraryFile: packageInfo.libraryFile,
|
|
36714
|
-
libraryPath,
|
|
36715
|
-
packageName: packageInfo.packageName,
|
|
36716
|
-
packageRoot,
|
|
36717
|
-
packageVersion,
|
|
36718
|
-
platformKey,
|
|
36719
|
-
source: "absolute-package",
|
|
36720
|
-
status: "resolved"
|
|
36721
|
-
};
|
|
36722
|
-
} catch (error) {
|
|
36723
|
-
return {
|
|
36724
|
-
libraryFile: packageInfo.libraryFile,
|
|
36725
|
-
packageName: packageInfo.packageName,
|
|
36726
|
-
platformKey,
|
|
36727
|
-
reason: getErrorMessage(error),
|
|
36728
|
-
source: "absolute-package",
|
|
36729
|
-
status: "package_not_installed"
|
|
36730
|
-
};
|
|
36731
|
-
}
|
|
36732
|
-
};
|
|
36733
|
-
var resolveAbsoluteSQLiteVecExtensionPath = () => {
|
|
36734
|
-
const resolution = resolveAbsoluteSQLiteVec();
|
|
36735
|
-
return resolution.status === "resolved" ? resolution.libraryPath ?? null : null;
|
|
36736
|
-
};
|
|
36737
|
-
|
|
36738
|
-
// src/adapters/sqlite.ts
|
|
36739
|
-
var DEFAULT_DIMENSIONS2 = RAG_VECTOR_DIMENSIONS_DEFAULT;
|
|
36740
|
-
var DEFAULT_TABLE_NAME2 = "rag_chunks";
|
|
36741
|
-
var DEFAULT_NATIVE_TABLE_SUFFIX = "_vec0";
|
|
36742
|
-
var DEFAULT_QUERY_MULTIPLIER2 = 4;
|
|
36743
|
-
var MAX_QUERY_MULTIPLIER2 = 16;
|
|
36744
|
-
var IDENTIFIER_RE2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
36745
|
-
var isParsedMetadata = (value) => Boolean(value) && typeof value === "object";
|
|
36746
|
-
var isObjectRecord5 = (value) => Boolean(value) && typeof value === "object";
|
|
36747
|
-
var isStoredRow = (value) => isObjectRecord5(value) && typeof value.chunk_id === "string" && typeof value.text === "string" && (typeof value.title === "string" || value.title === null) && (typeof value.source === "string" || value.source === null) && (typeof value.metadata === "string" || value.metadata === null) && typeof value.embedding === "string";
|
|
36748
|
-
var isNativeStoredRow = (value) => isObjectRecord5(value) && typeof value.chunk_id === "string" && typeof value.chunk_text === "string" && (typeof value.title === "string" || value.title === null) && (typeof value.source === "string" || value.source === null) && (typeof value.metadata === "string" || value.metadata === null) && typeof value.embedding === "string" && typeof value.distance === "number";
|
|
36749
|
-
var toStoredRows = (value) => Array.isArray(value) ? value.filter((row) => isStoredRow(row)) : [];
|
|
36750
|
-
var toNativeStoredRows = (value) => Array.isArray(value) ? value.filter((row) => isNativeStoredRow(row)) : [];
|
|
36751
|
-
var createSQLiteStatus = (dimensions, nativeDiagnostics, useNative) => ({
|
|
36752
|
-
backend: "sqlite",
|
|
36753
|
-
dimensions,
|
|
36754
|
-
native: nativeDiagnostics,
|
|
36755
|
-
vectorMode: useNative ? "native_vec0" : "json_fallback"
|
|
36756
|
-
});
|
|
36757
|
-
var createSQLiteCapabilities = (useNative) => ({
|
|
36758
|
-
backend: "sqlite",
|
|
36759
|
-
nativeVectorSearch: useNative,
|
|
36760
|
-
persistence: "embedded",
|
|
36761
|
-
serverSideFiltering: useNative,
|
|
36762
|
-
streamingIngestStatus: false
|
|
36763
|
-
});
|
|
36764
|
-
var assertSupportedIdentifier3 = (name) => {
|
|
36765
|
-
if (!IDENTIFIER_RE2.test(name)) {
|
|
36766
|
-
throw new Error(`Invalid table name "${name}". Only alphanumeric and underscore names are allowed.`);
|
|
36767
|
-
}
|
|
36768
|
-
};
|
|
36769
|
-
var normalizeQueryMultiplier2 = (value) => {
|
|
36770
|
-
if (value === undefined || !Number.isFinite(value)) {
|
|
36771
|
-
return DEFAULT_QUERY_MULTIPLIER2;
|
|
36772
|
-
}
|
|
36773
|
-
const minMultiplier = Math.max(1, Math.floor(value));
|
|
36774
|
-
return Math.min(minMultiplier, MAX_QUERY_MULTIPLIER2);
|
|
36775
|
-
};
|
|
36776
|
-
var normalizeMaxBackfills2 = (value) => {
|
|
36777
|
-
if (value === undefined || !Number.isFinite(value)) {
|
|
36778
|
-
return;
|
|
36779
|
-
}
|
|
36780
|
-
return Math.max(0, Math.floor(value));
|
|
36781
|
-
};
|
|
36782
|
-
var normalizeMinResults2 = (value, topK) => {
|
|
36783
|
-
if (value === undefined || !Number.isFinite(value)) {
|
|
36784
|
-
return topK;
|
|
36785
|
-
}
|
|
36786
|
-
return Math.min(topK, Math.max(1, Math.floor(value)));
|
|
36787
|
-
};
|
|
36788
|
-
var resolveFillTarget2 = (input) => {
|
|
36789
|
-
const fillPolicy = input.fillPolicy ?? "satisfy_min_results";
|
|
36790
|
-
return {
|
|
36791
|
-
fillPolicy,
|
|
36792
|
-
targetResults: fillPolicy === "strict_topk" ? input.topK : input.minResults
|
|
36793
|
-
};
|
|
36794
|
-
};
|
|
36795
|
-
var toJSONString = (metadata) => metadata === undefined ? null : JSON.stringify(metadata);
|
|
36796
|
-
var parseMetadata2 = (value) => {
|
|
36797
|
-
if (value === null) {
|
|
36798
|
-
return;
|
|
36799
|
-
}
|
|
36800
|
-
try {
|
|
36801
|
-
const parsed = JSON.parse(value);
|
|
36802
|
-
if (isParsedMetadata(parsed)) {
|
|
36803
|
-
return parsed;
|
|
36804
|
-
}
|
|
36805
|
-
} catch {}
|
|
36806
|
-
return;
|
|
36807
|
-
};
|
|
36808
|
-
var parseVector = (value) => {
|
|
36809
|
-
try {
|
|
36810
|
-
const parsed = JSON.parse(value);
|
|
36811
|
-
if (Array.isArray(parsed)) {
|
|
36812
|
-
return parsed.filter((element) => typeof element === "number" && Number.isFinite(element));
|
|
36813
|
-
}
|
|
36814
|
-
} catch {}
|
|
36815
|
-
return [];
|
|
36816
|
-
};
|
|
36817
|
-
var toSQLiteFilterBinding = (value) => value === null || typeof value === "string" || typeof value === "number" || typeof value === "bigint" || typeof value === "boolean" ? value : undefined;
|
|
36818
|
-
var FILTER_PATH_SEGMENT_RE2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
36819
|
-
var isObjectFilterRecord2 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
36820
|
-
var isOperatorFilterRecord2 = (value) => isObjectFilterRecord2(value) && Object.keys(value).some((key) => key.startsWith("$"));
|
|
36821
|
-
var isNestedFilterArray3 = (value) => Array.isArray(value) && value.every((entry) => isObjectFilterRecord2(entry));
|
|
36822
|
-
var toSQLiteJsonPath = (key) => {
|
|
36823
|
-
const segments = key.split(".").filter(Boolean);
|
|
36824
|
-
if (segments.length === 0 || !segments.every((segment) => FILTER_PATH_SEGMENT_RE2.test(segment))) {
|
|
36825
|
-
return null;
|
|
36826
|
-
}
|
|
36827
|
-
return `$.${segments.join(".")}`;
|
|
36828
|
-
};
|
|
36829
|
-
var resolveSQLiteFilterColumn = (key, aliases) => {
|
|
36830
|
-
if (key === "chunkId")
|
|
36831
|
-
return { actualSql: aliases.chunkId, kind: "scalar" };
|
|
36832
|
-
if (key === "source")
|
|
36833
|
-
return { actualSql: aliases.source, kind: "scalar" };
|
|
36834
|
-
if (key === "title")
|
|
36835
|
-
return { actualSql: aliases.title, kind: "scalar" };
|
|
36836
|
-
const path = toSQLiteJsonPath(key);
|
|
36837
|
-
if (!path) {
|
|
36838
|
-
return null;
|
|
36839
|
-
}
|
|
36840
|
-
return {
|
|
36841
|
-
actualSql: `json_extract(${aliases.metadata}, ?)`,
|
|
36842
|
-
kind: "metadata",
|
|
36843
|
-
path
|
|
36844
|
-
};
|
|
36845
|
-
};
|
|
36846
|
-
var buildSQLiteScalarClause = (actualSql, expected, params) => {
|
|
36847
|
-
if (!isOperatorFilterRecord2(expected)) {
|
|
36848
|
-
const binding = toSQLiteFilterBinding(expected);
|
|
36849
|
-
if (binding === undefined) {
|
|
36850
|
-
return null;
|
|
36851
|
-
}
|
|
36852
|
-
params.push(binding);
|
|
36853
|
-
return `${actualSql} = ?`;
|
|
36854
|
-
}
|
|
36855
|
-
const clauses = [];
|
|
36856
|
-
for (const [operator, value] of Object.entries(expected)) {
|
|
36857
|
-
switch (operator) {
|
|
36858
|
-
case "$exists":
|
|
36859
|
-
clauses.push(Boolean(value) ? `${actualSql} IS NOT NULL` : `${actualSql} IS NULL`);
|
|
36860
|
-
break;
|
|
36861
|
-
case "$ne":
|
|
36862
|
-
const neBinding = toSQLiteFilterBinding(value);
|
|
36863
|
-
if (neBinding === undefined) {
|
|
36864
|
-
return null;
|
|
36865
|
-
}
|
|
36866
|
-
params.push(neBinding);
|
|
36867
|
-
clauses.push(`(${actualSql} IS NULL OR ${actualSql} != ?)`);
|
|
36868
|
-
break;
|
|
36869
|
-
case "$in":
|
|
36870
|
-
if (!Array.isArray(value) || value.length === 0 || value.some((entry) => toSQLiteFilterBinding(entry) === undefined)) {
|
|
36871
|
-
return null;
|
|
36872
|
-
}
|
|
36873
|
-
params.push(...value);
|
|
36874
|
-
clauses.push(`${actualSql} IN (${value.map(() => "?").join(", ")})`);
|
|
36875
|
-
break;
|
|
36876
|
-
case "$gt":
|
|
36877
|
-
case "$gte":
|
|
36878
|
-
case "$lt":
|
|
36879
|
-
case "$lte": {
|
|
36880
|
-
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
36881
|
-
return null;
|
|
36882
|
-
}
|
|
36883
|
-
params.push(value);
|
|
36884
|
-
const comparison = operator === "$gt" ? ">" : operator === "$gte" ? ">=" : operator === "$lt" ? "<" : "<=";
|
|
36885
|
-
clauses.push(`${actualSql} ${comparison} ?`);
|
|
36886
|
-
break;
|
|
36887
|
-
}
|
|
36888
|
-
default:
|
|
36889
|
-
return null;
|
|
36890
|
-
}
|
|
36891
|
-
}
|
|
36892
|
-
return clauses.length > 0 ? clauses.map((clause) => `(${clause})`).join(" AND ") : null;
|
|
36893
|
-
};
|
|
36894
|
-
var buildSQLiteArrayClause = (metadataSql, path, expected, params) => {
|
|
36895
|
-
if (!isOperatorFilterRecord2(expected)) {
|
|
36896
|
-
return null;
|
|
36897
|
-
}
|
|
36898
|
-
const arrayTypeClause = `json_type(${metadataSql}, ?) = 'array'`;
|
|
36899
|
-
const clauses = [];
|
|
36900
|
-
for (const [operator, value] of Object.entries(expected)) {
|
|
36901
|
-
switch (operator) {
|
|
36902
|
-
case "$contains":
|
|
36903
|
-
const containsBinding = toSQLiteFilterBinding(value);
|
|
36904
|
-
if (containsBinding === undefined) {
|
|
36905
|
-
return null;
|
|
36906
|
-
}
|
|
36907
|
-
params.push(path, path, containsBinding);
|
|
36908
|
-
clauses.push(`(${arrayTypeClause} AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value = ?))`);
|
|
36909
|
-
break;
|
|
36910
|
-
case "$containsAny":
|
|
36911
|
-
if (!Array.isArray(value) || value.length === 0 || value.some((entry) => toSQLiteFilterBinding(entry) === undefined)) {
|
|
36912
|
-
return null;
|
|
36913
|
-
}
|
|
36914
|
-
params.push(path, path, ...value);
|
|
36915
|
-
clauses.push(`(${arrayTypeClause} AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value IN (${value.map(() => "?").join(", ")})))`);
|
|
36916
|
-
break;
|
|
36917
|
-
case "$containsAll":
|
|
36918
|
-
if (!Array.isArray(value) || value.length === 0 || value.some((entry) => toSQLiteFilterBinding(entry) === undefined)) {
|
|
36919
|
-
return null;
|
|
36920
|
-
}
|
|
36921
|
-
clauses.push(...value.map((entry) => {
|
|
36922
|
-
params.push(path, path, entry);
|
|
36923
|
-
return `(${arrayTypeClause} AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value = ?))`;
|
|
36924
|
-
}));
|
|
36925
|
-
break;
|
|
36926
|
-
default:
|
|
36927
|
-
return null;
|
|
36928
|
-
}
|
|
36929
|
-
}
|
|
36930
|
-
return clauses.length > 0 ? clauses.map((clause) => `(${clause})`).join(" AND ") : null;
|
|
36931
|
-
};
|
|
36932
|
-
var buildSQLiteMetadataScalarMatchClause = (metadataSql, actualSql, path, expected, params) => {
|
|
36933
|
-
const binding = toSQLiteFilterBinding(expected);
|
|
36934
|
-
if (binding === undefined) {
|
|
36935
|
-
return null;
|
|
36936
|
-
}
|
|
36937
|
-
params.push(path, binding, path, path, binding);
|
|
36938
|
-
return `(${actualSql} = ? OR (json_type(${metadataSql}, ?) = 'array' AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value = ?)))`;
|
|
36939
|
-
};
|
|
36940
|
-
var buildSQLiteFilterPlan = (filter, aliases) => {
|
|
36941
|
-
if (!filter) {
|
|
36942
|
-
return { clause: "", params: [] };
|
|
36943
|
-
}
|
|
36944
|
-
const params = [];
|
|
36945
|
-
const build = (entry) => {
|
|
36946
|
-
const clauses = [];
|
|
36947
|
-
for (const [key, value] of Object.entries(entry)) {
|
|
36948
|
-
if (key === "$and" || key === "$or") {
|
|
36949
|
-
if (!isNestedFilterArray3(value) || value.length === 0) {
|
|
36950
|
-
return null;
|
|
36951
|
-
}
|
|
36952
|
-
const nested = value.map((item) => build(item));
|
|
36953
|
-
if (nested.some((item) => item === null)) {
|
|
36954
|
-
return null;
|
|
36955
|
-
}
|
|
36956
|
-
clauses.push(`(${nested.filter((item) => Boolean(item)).join(key === "$and" ? " AND " : " OR ")})`);
|
|
36957
|
-
continue;
|
|
36958
|
-
}
|
|
36959
|
-
if (key === "$not") {
|
|
36960
|
-
if (!isObjectFilterRecord2(value)) {
|
|
36961
|
-
return null;
|
|
36962
|
-
}
|
|
36963
|
-
const nested = build(value);
|
|
36964
|
-
if (!nested) {
|
|
36965
|
-
return null;
|
|
36966
|
-
}
|
|
36967
|
-
clauses.push(`NOT (${nested})`);
|
|
36968
|
-
continue;
|
|
36969
|
-
}
|
|
36970
|
-
const resolved = resolveSQLiteFilterColumn(key, aliases);
|
|
36971
|
-
if (!resolved) {
|
|
36972
|
-
return null;
|
|
36973
|
-
}
|
|
36974
|
-
if (resolved.kind === "metadata") {
|
|
36975
|
-
const metadataScalarClause = buildSQLiteMetadataScalarMatchClause(aliases.metadata, resolved.actualSql, resolved.path, value, params);
|
|
36976
|
-
if (metadataScalarClause) {
|
|
36977
|
-
clauses.push(metadataScalarClause);
|
|
36978
|
-
continue;
|
|
36979
|
-
}
|
|
36980
|
-
const arrayClause = buildSQLiteArrayClause(aliases.metadata, resolved.path, value, params);
|
|
36981
|
-
if (arrayClause) {
|
|
36982
|
-
clauses.push(arrayClause);
|
|
36983
|
-
continue;
|
|
36984
|
-
}
|
|
36985
|
-
params.push(resolved.path);
|
|
36986
|
-
}
|
|
36987
|
-
const scalarClause = buildSQLiteScalarClause(resolved.actualSql, value, params);
|
|
36988
|
-
if (!scalarClause) {
|
|
36989
|
-
return null;
|
|
36990
|
-
}
|
|
36991
|
-
clauses.push(scalarClause);
|
|
36992
|
-
}
|
|
36993
|
-
return clauses.length > 0 ? clauses.map((clause2) => `(${clause2})`).join(" AND ") : "";
|
|
36994
|
-
};
|
|
36995
|
-
const clause = build(filter);
|
|
36996
|
-
return clause === null ? null : { clause, params };
|
|
36997
|
-
};
|
|
36998
|
-
var buildSQLitePushdownFilter = (filter) => {
|
|
36999
|
-
if (!filter) {
|
|
37000
|
-
return;
|
|
37001
|
-
}
|
|
37002
|
-
const nextEntries = [];
|
|
37003
|
-
for (const [key, value] of Object.entries(filter)) {
|
|
37004
|
-
if (key === "$and" || key === "$or") {
|
|
37005
|
-
if (!isNestedFilterArray3(value)) {
|
|
37006
|
-
continue;
|
|
37007
|
-
}
|
|
37008
|
-
const nested = value.map((entry) => buildSQLitePushdownFilter(entry)).filter((entry) => Boolean(entry));
|
|
37009
|
-
if (nested.length > 0) {
|
|
37010
|
-
nextEntries.push([key, nested]);
|
|
37011
|
-
}
|
|
37012
|
-
continue;
|
|
37013
|
-
}
|
|
37014
|
-
if (key === "$not") {
|
|
37015
|
-
if (!isObjectFilterRecord2(value)) {
|
|
37016
|
-
continue;
|
|
37017
|
-
}
|
|
37018
|
-
const nested = buildSQLitePushdownFilter(value);
|
|
37019
|
-
if (nested) {
|
|
37020
|
-
nextEntries.push([key, nested]);
|
|
37021
|
-
}
|
|
37022
|
-
continue;
|
|
37023
|
-
}
|
|
37024
|
-
if (Array.isArray(value) || isOperatorFilterRecord2(value) && Object.keys(value).some((operator) => operator === "$contains" || operator === "$containsAny" || operator === "$containsAll")) {
|
|
37025
|
-
continue;
|
|
37026
|
-
}
|
|
37027
|
-
nextEntries.push([key, value]);
|
|
37028
|
-
}
|
|
37029
|
-
return nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined;
|
|
37030
|
-
};
|
|
37031
|
-
var normalizeDistance2 = (distance, metric) => {
|
|
37032
|
-
if (!Number.isFinite(distance)) {
|
|
37033
|
-
return 0;
|
|
37034
|
-
}
|
|
37035
|
-
if (metric === "cosine") {
|
|
37036
|
-
return Math.min(1, Math.max(0, 1 - distance));
|
|
37037
|
-
}
|
|
37038
|
-
return Math.max(0, 1 / (1 + Math.abs(distance)));
|
|
37039
|
-
};
|
|
37040
|
-
var matchesFilter2 = (record, filter) => matchesMetadataFilterRecord({
|
|
37041
|
-
chunkId: record.chunkId,
|
|
37042
|
-
metadata: record.metadata,
|
|
37043
|
-
source: record.source,
|
|
37044
|
-
title: record.title,
|
|
37045
|
-
...record.metadata ?? {}
|
|
37046
|
-
}, filter);
|
|
37047
|
-
var mapFilterToRows = (rows) => rows.map((row) => ({
|
|
37048
|
-
chunkId: row.chunk_id,
|
|
37049
|
-
metadata: parseMetadata2(row.metadata),
|
|
37050
|
-
source: row.source ?? undefined,
|
|
37051
|
-
text: row.text,
|
|
37052
|
-
title: row.title ?? undefined,
|
|
37053
|
-
vector: parseVector(row.embedding)
|
|
37054
|
-
}));
|
|
37055
|
-
var buildJsonQuerySql = (tableName, whereClause) => `
|
|
37056
|
-
SELECT chunk_id, text, title, source, metadata, embedding FROM ${tableName}
|
|
37057
|
-
${whereClause ? `WHERE ${whereClause}` : ""}
|
|
37058
|
-
`;
|
|
37059
|
-
var buildJsonCountSql = (tableName, whereClause) => `
|
|
37060
|
-
SELECT COUNT(*) AS count FROM ${tableName}
|
|
37061
|
-
${whereClause ? `WHERE ${whereClause}` : ""}
|
|
37062
|
-
`;
|
|
37063
|
-
var getChunkCountFromSql = (db, sql, params = []) => {
|
|
37064
|
-
const result = db.prepare(sql).get(...params);
|
|
37065
|
-
const count = result?.count;
|
|
37066
|
-
return typeof count === "number" && Number.isFinite(count) ? count : 0;
|
|
37067
|
-
};
|
|
37068
|
-
var getPragmaNumericValue = (db, pragma) => {
|
|
37069
|
-
const row = db.prepare(`PRAGMA ${pragma}`).get();
|
|
37070
|
-
const value = row ? row[Object.keys(row)[0] ?? ""] : undefined;
|
|
37071
|
-
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
37072
|
-
};
|
|
37073
|
-
var refreshSQLiteRuntimeDiagnostics = (db, nativeDiagnostics, diagnosticTableName) => {
|
|
37074
|
-
if (!nativeDiagnostics) {
|
|
37075
|
-
return;
|
|
37076
|
-
}
|
|
37077
|
-
try {
|
|
37078
|
-
const rowCount = getChunkCountFromSql(db, buildJsonCountSql(diagnosticTableName));
|
|
37079
|
-
const pageCount = getPragmaNumericValue(db, "page_count");
|
|
37080
|
-
const pageSize = getPragmaNumericValue(db, "page_size");
|
|
37081
|
-
const freelistCount = getPragmaNumericValue(db, "freelist_count");
|
|
37082
|
-
nativeDiagnostics.rowCount = rowCount;
|
|
37083
|
-
nativeDiagnostics.pageCount = pageCount;
|
|
37084
|
-
nativeDiagnostics.freelistCount = freelistCount;
|
|
37085
|
-
nativeDiagnostics.databaseBytes = pageCount > 0 && pageSize > 0 ? pageCount * pageSize : 0;
|
|
37086
|
-
nativeDiagnostics.lastHealthCheckAt = Date.now();
|
|
37087
|
-
nativeDiagnostics.lastHealthError = undefined;
|
|
37088
|
-
} catch (error) {
|
|
37089
|
-
nativeDiagnostics.lastHealthCheckAt = Date.now();
|
|
37090
|
-
nativeDiagnostics.lastHealthError = getErrorMessage2(error);
|
|
37091
|
-
}
|
|
37092
|
-
};
|
|
37093
|
-
var buildNativeQuerySql = (tableName, whereClause) => `
|
|
37094
|
-
SELECT
|
|
37095
|
-
chunk_id,
|
|
37096
|
-
embedding,
|
|
37097
|
-
chunk_text,
|
|
37098
|
-
title,
|
|
37099
|
-
source,
|
|
37100
|
-
metadata,
|
|
37101
|
-
distance
|
|
37102
|
-
FROM ${tableName}
|
|
37103
|
-
WHERE embedding MATCH vec_f32(?)
|
|
37104
|
-
AND k = ?
|
|
37105
|
-
${whereClause ? `AND (${whereClause})` : ""}
|
|
37106
|
-
ORDER BY distance
|
|
37107
|
-
`;
|
|
37108
|
-
var getFilteredSQLiteCandidateCount = (db, tableName, filterPlan) => {
|
|
37109
|
-
if (!filterPlan) {
|
|
37110
|
-
return;
|
|
37111
|
-
}
|
|
37112
|
-
const result = db.prepare(buildJsonCountSql(tableName, filterPlan.clause)).get(...filterPlan.params);
|
|
37113
|
-
const count = result?.count;
|
|
37114
|
-
return typeof count === "number" && Number.isFinite(count) ? count : undefined;
|
|
37115
|
-
};
|
|
37116
|
-
var createJsonStatements = (db, tableName) => {
|
|
37117
|
-
const insertSql = `
|
|
37118
|
-
INSERT INTO ${tableName} (
|
|
37119
|
-
chunk_id,
|
|
37120
|
-
text,
|
|
37121
|
-
title,
|
|
37122
|
-
source,
|
|
37123
|
-
metadata,
|
|
37124
|
-
embedding
|
|
37125
|
-
) VALUES (?, ?, ?, ?, ?, ?)
|
|
37126
|
-
ON CONFLICT(chunk_id) DO UPDATE SET
|
|
37127
|
-
text = excluded.text,
|
|
37128
|
-
title = excluded.title,
|
|
37129
|
-
source = excluded.source,
|
|
37130
|
-
metadata = excluded.metadata,
|
|
37131
|
-
embedding = excluded.embedding
|
|
37132
|
-
`;
|
|
37133
|
-
const querySql = `
|
|
37134
|
-
SELECT chunk_id, text, title, source, metadata, embedding FROM ${tableName}
|
|
37135
|
-
`;
|
|
37136
|
-
const clearSql = `DELETE FROM ${tableName}`;
|
|
37137
|
-
const deleteSql = `DELETE FROM ${tableName} WHERE chunk_id = ?`;
|
|
37138
|
-
const init = () => db.exec(`
|
|
37139
|
-
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
37140
|
-
chunk_id TEXT PRIMARY KEY,
|
|
37141
|
-
text TEXT NOT NULL,
|
|
37142
|
-
title TEXT,
|
|
37143
|
-
source TEXT,
|
|
37144
|
-
metadata TEXT,
|
|
37145
|
-
embedding TEXT NOT NULL
|
|
37146
|
-
)
|
|
37147
|
-
`);
|
|
37148
|
-
init();
|
|
37149
|
-
return {
|
|
37150
|
-
clear: db.prepare(clearSql),
|
|
37151
|
-
delete: db.prepare(deleteSql),
|
|
37152
|
-
init,
|
|
37153
|
-
insert: db.prepare(insertSql),
|
|
37154
|
-
query: db.prepare(querySql)
|
|
37155
|
-
};
|
|
37156
|
-
};
|
|
37157
|
-
var getSQLiteChunkIdsByChunkIds = (db, tableName, chunkIds) => {
|
|
37158
|
-
const uniqueChunkIds = [...new Set(chunkIds)];
|
|
37159
|
-
if (uniqueChunkIds.length === 0) {
|
|
37160
|
-
return [];
|
|
37161
|
-
}
|
|
37162
|
-
const whereClause = uniqueChunkIds.map(() => "?").join(", ");
|
|
37163
|
-
const rows = db.prepare(`SELECT chunk_id FROM ${tableName} WHERE chunk_id IN (${whereClause})`).all(...uniqueChunkIds);
|
|
37164
|
-
return rows.map((row) => row.chunk_id).filter((chunkId) => typeof chunkId === "string");
|
|
37165
|
-
};
|
|
37166
|
-
var getSQLiteCandidateChunkIdsByFilter = (db, tableName, filter, jsonStatements) => {
|
|
37167
|
-
if (!filter || Object.keys(filter).length === 0) {
|
|
37168
|
-
return [];
|
|
37169
|
-
}
|
|
37170
|
-
const pushdownFilter = buildSQLitePushdownFilter(filter);
|
|
37171
|
-
const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
|
|
37172
|
-
chunkId: "chunk_id",
|
|
37173
|
-
metadata: "metadata",
|
|
37174
|
-
source: "source",
|
|
37175
|
-
title: "title"
|
|
35673
|
+
// src/adapters/sync.ts
|
|
35674
|
+
import {
|
|
35675
|
+
createSyncEngine,
|
|
35676
|
+
createTextIndex,
|
|
35677
|
+
createVectorIndex,
|
|
35678
|
+
defineSearchCollection
|
|
35679
|
+
} from "@absolutejs/sync/engine";
|
|
35680
|
+
var createSyncRAGStore = (options = {}) => {
|
|
35681
|
+
const dimensions = options.dimensions ?? RAG_VECTOR_DIMENSIONS_DEFAULT;
|
|
35682
|
+
const engine = options.engine ?? createSyncEngine();
|
|
35683
|
+
const retrievalCollection = options.collection ?? "ragRetrieval";
|
|
35684
|
+
const table = options.table ?? "ragChunks";
|
|
35685
|
+
const records = new Map;
|
|
35686
|
+
const toLight = (record) => ({
|
|
35687
|
+
chunkId: record.chunkId,
|
|
35688
|
+
chunkText: record.text,
|
|
35689
|
+
source: record.source,
|
|
35690
|
+
title: record.title
|
|
37176
35691
|
});
|
|
37177
|
-
const
|
|
37178
|
-
|
|
37179
|
-
|
|
37180
|
-
};
|
|
37181
|
-
var getSQLiteCandidateChunkIds = (db, tableName, jsonStatements, input) => {
|
|
37182
|
-
const chunkIdSet = new Set;
|
|
37183
|
-
if (input.filter && Object.keys(input.filter).length > 0) {
|
|
37184
|
-
for (const chunkId of getSQLiteCandidateChunkIdsByFilter(db, tableName, input.filter, jsonStatements)) {
|
|
37185
|
-
chunkIdSet.add(chunkId);
|
|
37186
|
-
}
|
|
37187
|
-
}
|
|
37188
|
-
if (input.chunkIds && input.chunkIds.length > 0) {
|
|
37189
|
-
for (const chunkId of getSQLiteChunkIdsByChunkIds(db, tableName, input.chunkIds)) {
|
|
37190
|
-
chunkIdSet.add(chunkId);
|
|
37191
|
-
}
|
|
37192
|
-
}
|
|
37193
|
-
return [...chunkIdSet];
|
|
37194
|
-
};
|
|
37195
|
-
var toVectorText = (vector) => JSON.stringify(vector);
|
|
37196
|
-
var createNativeVec0Table = (db, tableName, dimensions, metric) => {
|
|
37197
|
-
const metricSuffix = metric === "cosine" ? " distance_metric=cosine" : "";
|
|
37198
|
-
db.exec(`
|
|
37199
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName} USING vec0(
|
|
37200
|
-
chunk_id TEXT,
|
|
37201
|
-
embedding float[${dimensions}]${metricSuffix},
|
|
37202
|
-
+chunk_text TEXT,
|
|
37203
|
-
title TEXT,
|
|
37204
|
-
source TEXT,
|
|
37205
|
-
metadata TEXT
|
|
37206
|
-
)
|
|
37207
|
-
`);
|
|
37208
|
-
};
|
|
37209
|
-
var createNativeVec0Statements = (db, tableName) => {
|
|
37210
|
-
const upsertSql = `
|
|
37211
|
-
INSERT INTO ${tableName} (
|
|
37212
|
-
chunk_id,
|
|
37213
|
-
embedding,
|
|
37214
|
-
chunk_text,
|
|
37215
|
-
title,
|
|
37216
|
-
source,
|
|
37217
|
-
metadata
|
|
37218
|
-
) VALUES (?, vec_f32(?), ?, ?, ?, ?)
|
|
37219
|
-
`;
|
|
37220
|
-
const deleteSql = `DELETE FROM ${tableName} WHERE chunk_id = ?`;
|
|
37221
|
-
const querySql = `
|
|
37222
|
-
SELECT
|
|
37223
|
-
chunk_id,
|
|
37224
|
-
embedding,
|
|
37225
|
-
chunk_text,
|
|
37226
|
-
title,
|
|
37227
|
-
source,
|
|
37228
|
-
metadata,
|
|
37229
|
-
distance
|
|
37230
|
-
FROM ${tableName}
|
|
37231
|
-
WHERE embedding MATCH vec_f32(?)
|
|
37232
|
-
AND k = ?
|
|
37233
|
-
ORDER BY distance
|
|
37234
|
-
`;
|
|
37235
|
-
return {
|
|
37236
|
-
clear: db.prepare(`DELETE FROM ${tableName}`),
|
|
37237
|
-
delete: db.prepare(deleteSql),
|
|
37238
|
-
insert: db.prepare(upsertSql),
|
|
37239
|
-
query: db.prepare(querySql)
|
|
37240
|
-
};
|
|
37241
|
-
};
|
|
37242
|
-
var mapToRows = (vector, chunks, filter) => chunks.map((chunk) => ({
|
|
37243
|
-
chunk,
|
|
37244
|
-
score: querySimilarity(vector, normalizeVector(chunk.vector))
|
|
37245
|
-
})).filter(({ chunk }) => matchesFilter2(chunk, filter)).sort((left, right) => right.score - left.score);
|
|
37246
|
-
var executeNativeInitSql = (db, initSql) => {
|
|
37247
|
-
if (!initSql) {
|
|
37248
|
-
return;
|
|
37249
|
-
}
|
|
37250
|
-
if (typeof initSql === "string") {
|
|
37251
|
-
db.exec(initSql);
|
|
37252
|
-
return;
|
|
37253
|
-
}
|
|
37254
|
-
for (const command of initSql) {
|
|
37255
|
-
db.exec(command);
|
|
37256
|
-
}
|
|
37257
|
-
};
|
|
37258
|
-
var getErrorMessage2 = (error) => error instanceof Error ? error.message : String(error);
|
|
37259
|
-
var resolveConfiguredNativeExtension = (nativeConfig) => {
|
|
37260
|
-
const platformKey = `${process.platform}-${process.arch}`;
|
|
37261
|
-
if (nativeConfig?.extensionPath) {
|
|
37262
|
-
return existsSync2(nativeConfig.extensionPath) ? {
|
|
37263
|
-
libraryPath: nativeConfig.extensionPath,
|
|
37264
|
-
platformKey,
|
|
37265
|
-
source: "explicit",
|
|
37266
|
-
status: "resolved"
|
|
37267
|
-
} : {
|
|
37268
|
-
libraryPath: nativeConfig.extensionPath,
|
|
37269
|
-
platformKey,
|
|
37270
|
-
reason: `Configured native.extensionPath was not found: ${nativeConfig.extensionPath}`,
|
|
37271
|
-
source: "explicit",
|
|
37272
|
-
status: "binary_missing"
|
|
37273
|
-
};
|
|
37274
|
-
}
|
|
37275
|
-
if (nativeConfig?.resolveFromAbsolutePackages !== false) {
|
|
37276
|
-
return resolveAbsolutePackageNativeExtension(platformKey);
|
|
37277
|
-
}
|
|
37278
|
-
const envResolution = resolveNativeExtensionFromEnv(platformKey);
|
|
37279
|
-
if (envResolution)
|
|
37280
|
-
return envResolution;
|
|
37281
|
-
return {
|
|
37282
|
-
platformKey,
|
|
37283
|
-
reason: "No native sqlite-vec path was configured. AbsoluteJS will still attempt vec0 initialization in case the extension is already registered on the Database connection.",
|
|
37284
|
-
source: "database",
|
|
37285
|
-
status: "not_configured"
|
|
37286
|
-
};
|
|
37287
|
-
};
|
|
37288
|
-
var describeNativeFallbackReason = (resolution) => {
|
|
37289
|
-
if (!resolution) {
|
|
37290
|
-
return "Native sqlite vec0 was not configured.";
|
|
37291
|
-
}
|
|
37292
|
-
switch (resolution.status) {
|
|
37293
|
-
case "resolved":
|
|
37294
|
-
return;
|
|
37295
|
-
case "package_not_installed":
|
|
37296
|
-
return `Install ${resolution.packageName ?? "@absolutejs/absolute-rag-sqlite"} for ${resolution.platformKey}, or provide native.extensionPath.`;
|
|
37297
|
-
case "binary_missing":
|
|
37298
|
-
return resolution.reason ?? "Resolved sqlite-vec binary was missing.";
|
|
37299
|
-
case "unsupported_platform":
|
|
37300
|
-
return resolution.reason ?? "This platform is not yet supported by AbsoluteJS sqlite-vec packages.";
|
|
37301
|
-
case "not_configured":
|
|
37302
|
-
return resolution.reason ?? "No sqlite-vec binary path was configured.";
|
|
37303
|
-
case "package_invalid":
|
|
37304
|
-
return resolution.reason ?? "The sqlite-vec package manifest was invalid.";
|
|
37305
|
-
default:
|
|
37306
|
-
return "Native sqlite vec0 could not be initialized.";
|
|
37307
|
-
}
|
|
37308
|
-
};
|
|
37309
|
-
var resolveNativeExtensionFromEnv = (platformKey) => {
|
|
37310
|
-
const envPath = process.env.SQLITE_VEC_EXTENSION_PATH;
|
|
37311
|
-
if (!envPath) {
|
|
37312
|
-
return null;
|
|
37313
|
-
}
|
|
37314
|
-
if (existsSync2(envPath)) {
|
|
37315
|
-
return {
|
|
37316
|
-
libraryPath: envPath,
|
|
37317
|
-
platformKey,
|
|
37318
|
-
source: "env",
|
|
37319
|
-
status: "resolved"
|
|
37320
|
-
};
|
|
37321
|
-
}
|
|
37322
|
-
return {
|
|
37323
|
-
libraryPath: envPath,
|
|
37324
|
-
platformKey,
|
|
37325
|
-
reason: `SQLITE_VEC_EXTENSION_PATH was set but not found: ${envPath}`,
|
|
37326
|
-
source: "env",
|
|
37327
|
-
status: "binary_missing"
|
|
37328
|
-
};
|
|
37329
|
-
};
|
|
37330
|
-
var shouldResolveNativeFromEnv = (resolution) => resolution.status === "binary_missing" || resolution.status === "package_not_installed" || resolution.status === "unsupported_platform";
|
|
37331
|
-
var resolveAbsolutePackageNativeExtension = (platformKey) => {
|
|
37332
|
-
const packageResolution = resolveAbsoluteSQLiteVec();
|
|
37333
|
-
if (!shouldResolveNativeFromEnv(packageResolution)) {
|
|
37334
|
-
return packageResolution;
|
|
37335
|
-
}
|
|
37336
|
-
const envResolution = resolveNativeExtensionFromEnv(platformKey);
|
|
37337
|
-
if (envResolution) {
|
|
37338
|
-
return envResolution;
|
|
37339
|
-
}
|
|
37340
|
-
return packageResolution;
|
|
37341
|
-
};
|
|
37342
|
-
var activateNativeDiagnostics = (nativeDiagnostics) => {
|
|
37343
|
-
if (!nativeDiagnostics) {
|
|
37344
|
-
return;
|
|
37345
|
-
}
|
|
37346
|
-
nativeDiagnostics.available = true;
|
|
37347
|
-
nativeDiagnostics.active = true;
|
|
37348
|
-
nativeDiagnostics.fallbackReason = undefined;
|
|
37349
|
-
if (nativeDiagnostics.resolution && nativeDiagnostics.resolution.status === "resolved") {
|
|
37350
|
-
return;
|
|
37351
|
-
}
|
|
37352
|
-
nativeDiagnostics.resolution = {
|
|
37353
|
-
platformKey: `${process.platform}-${process.arch}`,
|
|
37354
|
-
reason: "sqlite-vec was already available on the Database connection or loaded by native.extensionInitSql.",
|
|
37355
|
-
source: "database",
|
|
37356
|
-
status: "resolved"
|
|
37357
|
-
};
|
|
37358
|
-
};
|
|
37359
|
-
var loadNativeExtension = (db, nativeResolution) => {
|
|
37360
|
-
if (nativeResolution?.status !== "resolved") {
|
|
37361
|
-
return;
|
|
37362
|
-
}
|
|
37363
|
-
if (!nativeResolution.libraryPath) {
|
|
37364
|
-
return;
|
|
37365
|
-
}
|
|
37366
|
-
db.loadExtension(nativeResolution.libraryPath);
|
|
37367
|
-
};
|
|
37368
|
-
var markNativeLoadFailure = (nativeDiagnostics, error, nativeResolution) => {
|
|
37369
|
-
if (!nativeDiagnostics) {
|
|
37370
|
-
return;
|
|
37371
|
-
}
|
|
37372
|
-
nativeDiagnostics.available = false;
|
|
37373
|
-
nativeDiagnostics.active = false;
|
|
37374
|
-
nativeDiagnostics.lastLoadError = getErrorMessage2(error);
|
|
37375
|
-
nativeDiagnostics.fallbackReason = describeNativeFallbackReason(nativeResolution) ?? nativeDiagnostics.lastLoadError;
|
|
37376
|
-
};
|
|
37377
|
-
var markNativeQueryFailure = (nativeDiagnostics, error) => {
|
|
37378
|
-
if (!nativeDiagnostics) {
|
|
37379
|
-
return;
|
|
37380
|
-
}
|
|
37381
|
-
nativeDiagnostics.lastQueryError = getErrorMessage2(error);
|
|
37382
|
-
nativeDiagnostics.active = false;
|
|
37383
|
-
nativeDiagnostics.fallbackReason = nativeDiagnostics.lastQueryError;
|
|
37384
|
-
};
|
|
37385
|
-
var countFilterClauses2 = (filter) => {
|
|
37386
|
-
if (!filter) {
|
|
37387
|
-
return 0;
|
|
37388
|
-
}
|
|
37389
|
-
let count = 0;
|
|
37390
|
-
for (const [key, value] of Object.entries(filter)) {
|
|
37391
|
-
if (key === "$and" || key === "$or") {
|
|
37392
|
-
if (isNestedFilterArray3(value)) {
|
|
37393
|
-
count += value.reduce((total, entry) => total + countFilterClauses2(entry), 0);
|
|
37394
|
-
}
|
|
37395
|
-
continue;
|
|
37396
|
-
}
|
|
37397
|
-
if (key === "$not") {
|
|
37398
|
-
if (isObjectFilterRecord2(value)) {
|
|
37399
|
-
count += countFilterClauses2(value);
|
|
37400
|
-
}
|
|
37401
|
-
continue;
|
|
37402
|
-
}
|
|
37403
|
-
count += 1;
|
|
37404
|
-
}
|
|
37405
|
-
return count;
|
|
37406
|
-
};
|
|
37407
|
-
var resolveSQLitePushdownMode = (input) => {
|
|
37408
|
-
const totalFilterClauseCount = countFilterClauses2(input.filter);
|
|
37409
|
-
const pushdownClauseCount = countFilterClauses2(input.pushdownFilter);
|
|
37410
|
-
const jsRemainderClauseCount = Math.max(0, totalFilterClauseCount - pushdownClauseCount);
|
|
37411
|
-
const pushdownMode = pushdownClauseCount === 0 ? "none" : pushdownClauseCount >= totalFilterClauseCount ? "full" : "partial";
|
|
37412
|
-
return {
|
|
37413
|
-
jsRemainderClauseCount,
|
|
37414
|
-
jsRemainderRatio: totalFilterClauseCount > 0 ? jsRemainderClauseCount / totalFilterClauseCount : undefined,
|
|
37415
|
-
pushdownClauseCount,
|
|
37416
|
-
pushdownCoverageRatio: totalFilterClauseCount > 0 ? pushdownClauseCount / totalFilterClauseCount : undefined,
|
|
37417
|
-
pushdownMode,
|
|
37418
|
-
totalFilterClauseCount
|
|
37419
|
-
};
|
|
37420
|
-
};
|
|
37421
|
-
var updateSQLiteLastQueryPlan = (input) => {
|
|
37422
|
-
if (!input.nativeDiagnostics) {
|
|
37423
|
-
return;
|
|
37424
|
-
}
|
|
37425
|
-
const pushdown = resolveSQLitePushdownMode({
|
|
37426
|
-
filter: input.filter,
|
|
37427
|
-
pushdownFilter: input.pushdownFilter
|
|
35692
|
+
const textIndex = createTextIndex({
|
|
35693
|
+
fields: ["chunkText"],
|
|
35694
|
+
key: (chunk) => chunk.chunkId
|
|
37428
35695
|
});
|
|
37429
|
-
|
|
37430
|
-
|
|
37431
|
-
|
|
37432
|
-
|
|
37433
|
-
|
|
37434
|
-
|
|
37435
|
-
|
|
37436
|
-
|
|
37437
|
-
|
|
37438
|
-
|
|
37439
|
-
|
|
37440
|
-
searchExpansionRatio: typeof input.initialSearchK === "number" && typeof input.finalSearchK === "number" && input.initialSearchK > 0 ? input.finalSearchK / input.initialSearchK : undefined,
|
|
37441
|
-
jsRemainderClauseCount: pushdown.jsRemainderClauseCount,
|
|
37442
|
-
plannerProfileUsed: input.plannerProfileUsed,
|
|
37443
|
-
candidateLimitUsed: input.candidateLimitUsed,
|
|
37444
|
-
maxBackfillsUsed: input.maxBackfillsUsed,
|
|
37445
|
-
minResultsUsed: input.minResultsUsed,
|
|
37446
|
-
fillPolicyUsed: input.fillPolicyUsed,
|
|
37447
|
-
queryMultiplierUsed: input.queryMultiplierUsed,
|
|
37448
|
-
jsRemainderRatio: pushdown.jsRemainderRatio,
|
|
37449
|
-
pushdownApplied: pushdown.pushdownClauseCount > 0,
|
|
37450
|
-
pushdownClauseCount: pushdown.pushdownClauseCount,
|
|
37451
|
-
pushdownCoverageRatio: pushdown.pushdownCoverageRatio,
|
|
37452
|
-
pushdownMode: pushdown.pushdownMode,
|
|
37453
|
-
queryMode: input.queryMode,
|
|
37454
|
-
candidateYieldRatio: typeof input.returnedCount === "number" && typeof input.finalSearchK === "number" && input.finalSearchK > 0 ? input.returnedCount / input.finalSearchK : undefined,
|
|
37455
|
-
returnedCount: input.returnedCount,
|
|
37456
|
-
backfillLimitReached: input.backfillLimitReached,
|
|
37457
|
-
minResultsSatisfied: input.minResultsSatisfied,
|
|
37458
|
-
topKFillRatio: typeof input.returnedCount === "number" && input.topK > 0 ? input.returnedCount / input.topK : undefined,
|
|
37459
|
-
totalFilterClauseCount: pushdown.totalFilterClauseCount,
|
|
37460
|
-
underfilledTopK: input.underfilledTopK
|
|
37461
|
-
};
|
|
37462
|
-
};
|
|
37463
|
-
var markNativeUpsertFailure = (nativeDiagnostics, error) => {
|
|
37464
|
-
if (!nativeDiagnostics) {
|
|
37465
|
-
return;
|
|
37466
|
-
}
|
|
37467
|
-
nativeDiagnostics.lastUpsertError = getErrorMessage2(error);
|
|
37468
|
-
nativeDiagnostics.active = false;
|
|
37469
|
-
nativeDiagnostics.fallbackReason = nativeDiagnostics.lastUpsertError;
|
|
37470
|
-
};
|
|
37471
|
-
var analyzeSQLiteBackend = (input) => {
|
|
37472
|
-
try {
|
|
37473
|
-
input.db.exec("PRAGMA optimize");
|
|
37474
|
-
input.db.exec(`ANALYZE ${input.tableName}`);
|
|
37475
|
-
input.nativeDiagnostics && (input.nativeDiagnostics.lastAnalyzeAt = Date.now());
|
|
37476
|
-
if (input.nativeDiagnostics) {
|
|
37477
|
-
input.nativeDiagnostics.lastAnalyzeError = undefined;
|
|
37478
|
-
}
|
|
37479
|
-
refreshSQLiteRuntimeDiagnostics(input.db, input.nativeDiagnostics, input.diagnosticTableName);
|
|
37480
|
-
} catch (error) {
|
|
37481
|
-
if (input.nativeDiagnostics) {
|
|
37482
|
-
input.nativeDiagnostics.lastAnalyzeAt = Date.now();
|
|
37483
|
-
input.nativeDiagnostics.lastAnalyzeError = getErrorMessage2(error);
|
|
37484
|
-
}
|
|
37485
|
-
throw error;
|
|
37486
|
-
}
|
|
37487
|
-
};
|
|
37488
|
-
var initializeNativeBackend = (input) => {
|
|
37489
|
-
const {
|
|
37490
|
-
db,
|
|
37491
|
-
dimensions,
|
|
37492
|
-
nativeConfig,
|
|
37493
|
-
nativeDiagnostics,
|
|
37494
|
-
nativeDistanceMetric,
|
|
37495
|
-
nativeResolution,
|
|
37496
|
-
nativeTableName
|
|
37497
|
-
} = input;
|
|
37498
|
-
loadNativeExtension(db, nativeResolution);
|
|
37499
|
-
executeNativeInitSql(db, nativeConfig.extensionInitSql);
|
|
37500
|
-
createNativeVec0Table(db, nativeTableName, dimensions, nativeDistanceMetric);
|
|
37501
|
-
const nativeStatements = createNativeVec0Statements(db, nativeTableName);
|
|
37502
|
-
activateNativeDiagnostics(nativeDiagnostics);
|
|
37503
|
-
return nativeStatements;
|
|
37504
|
-
};
|
|
37505
|
-
var createNativeInitializationError = (error, nativeTableName) => new Error(`Failed to initialize sqlite vec0 backend for table "${nativeTableName}". ` + `Install @absolutejs/absolute-rag-sqlite for your platform, set native.extensionPath, or pre-register the sqlite-vec extension in the Database connection. ` + `Details: ${getErrorMessage2(error)}`);
|
|
37506
|
-
var initializeNativeBackendSafely = (input) => {
|
|
37507
|
-
const { nativeConfig, nativeDiagnostics, nativeResolution, nativeTableName } = input;
|
|
37508
|
-
try {
|
|
37509
|
-
return initializeNativeBackend(input);
|
|
37510
|
-
} catch (error) {
|
|
37511
|
-
markNativeLoadFailure(nativeDiagnostics, error, nativeResolution);
|
|
37512
|
-
if (nativeConfig.requireAvailable) {
|
|
37513
|
-
throw createNativeInitializationError(error, nativeTableName);
|
|
37514
|
-
}
|
|
37515
|
-
return;
|
|
37516
|
-
}
|
|
37517
|
-
};
|
|
37518
|
-
var fallbackToJsonUpsert = (chunks, jsonStatements) => {
|
|
37519
|
-
for (const chunk of chunks) {
|
|
37520
|
-
jsonStatements.insert.run(chunk.chunkId, chunk.text, chunk.title ?? null, chunk.source ?? null, toJSONString(chunk.metadata), toVectorText(chunk.vector));
|
|
37521
|
-
}
|
|
37522
|
-
};
|
|
37523
|
-
var upsertNativeChunks = (chunks, nativeStatements) => {
|
|
37524
|
-
if (!nativeStatements) {
|
|
37525
|
-
throw new Error("Native vector statements unavailable");
|
|
37526
|
-
}
|
|
37527
|
-
for (const chunk of chunks) {
|
|
37528
|
-
nativeStatements.delete.run(chunk.chunkId);
|
|
37529
|
-
nativeStatements.insert.run(chunk.chunkId, toVectorText(chunk.vector), chunk.text, chunk.title ?? null, chunk.source ?? null, toJSONString(chunk.metadata));
|
|
37530
|
-
}
|
|
37531
|
-
};
|
|
37532
|
-
var createSQLiteRAGStore = (options = {}) => {
|
|
37533
|
-
const dimensions = options.dimensions ?? DEFAULT_DIMENSIONS2;
|
|
37534
|
-
const tableName = options.tableName ?? DEFAULT_TABLE_NAME2;
|
|
37535
|
-
assertSupportedIdentifier3(tableName);
|
|
37536
|
-
const nativeConfig = options.native;
|
|
37537
|
-
const nativeTableName = nativeConfig?.tableName ?? `${tableName}${DEFAULT_NATIVE_TABLE_SUFFIX}`;
|
|
37538
|
-
if (nativeConfig?.mode === "vec0" && nativeConfig.tableName) {
|
|
37539
|
-
assertSupportedIdentifier3(nativeConfig.tableName);
|
|
37540
|
-
}
|
|
37541
|
-
if (!Number.isInteger(dimensions) || dimensions <= 0) {
|
|
37542
|
-
throw new Error(`Invalid dimension "${dimensions}". dimensions must be a positive integer.`);
|
|
37543
|
-
}
|
|
37544
|
-
const db = options.db ?? new Database(options.path ?? ":memory:");
|
|
37545
|
-
const nativeDistanceMetric = nativeConfig?.distanceMetric === "l2" ? "l2" : "cosine";
|
|
37546
|
-
const nativeQueryMultiplier = normalizeQueryMultiplier2(nativeConfig?.queryMultiplier);
|
|
37547
|
-
const nativeResolution = nativeConfig?.mode === "vec0" ? resolveConfiguredNativeExtension(nativeConfig) : undefined;
|
|
37548
|
-
const nativeDiagnostics = nativeConfig?.mode === "vec0" ? {
|
|
37549
|
-
active: false,
|
|
37550
|
-
available: false,
|
|
37551
|
-
distanceMetric: nativeDistanceMetric,
|
|
37552
|
-
fallbackReason: describeNativeFallbackReason(nativeResolution),
|
|
37553
|
-
mode: nativeConfig.mode,
|
|
37554
|
-
requested: true,
|
|
37555
|
-
resolution: nativeResolution,
|
|
37556
|
-
tableName: nativeTableName
|
|
37557
|
-
} : undefined;
|
|
37558
|
-
const jsonStatements = createJsonStatements(db, tableName);
|
|
37559
|
-
jsonStatements.init();
|
|
37560
|
-
let useNative = false;
|
|
37561
|
-
let nativeStatements;
|
|
37562
|
-
if (nativeConfig?.mode === "vec0") {
|
|
37563
|
-
nativeStatements = initializeNativeBackendSafely({
|
|
37564
|
-
db,
|
|
37565
|
-
dimensions,
|
|
37566
|
-
nativeConfig,
|
|
37567
|
-
nativeDiagnostics,
|
|
37568
|
-
nativeDistanceMetric,
|
|
37569
|
-
nativeResolution,
|
|
37570
|
-
nativeTableName
|
|
37571
|
-
});
|
|
37572
|
-
useNative = nativeStatements !== undefined;
|
|
37573
|
-
}
|
|
37574
|
-
refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
|
|
35696
|
+
const vectorIndex = createVectorIndex({
|
|
35697
|
+
embedding: (chunk) => records.get(chunk.chunkId)?.vector ?? [],
|
|
35698
|
+
key: (chunk) => chunk.chunkId
|
|
35699
|
+
});
|
|
35700
|
+
engine.registerSearch(defineSearchCollection({
|
|
35701
|
+
index: () => textIndex,
|
|
35702
|
+
key: (chunk) => chunk.chunkId,
|
|
35703
|
+
name: retrievalCollection,
|
|
35704
|
+
source: () => [...records.values()].map(toLight),
|
|
35705
|
+
table
|
|
35706
|
+
}));
|
|
37575
35707
|
const embed = async (input) => {
|
|
37576
|
-
|
|
37577
|
-
|
|
37578
|
-
throw new DOMException("Aborted", "AbortError");
|
|
35708
|
+
if (options.embedding) {
|
|
35709
|
+
return options.embedding(input);
|
|
37579
35710
|
}
|
|
37580
35711
|
if (options.mockEmbedding) {
|
|
37581
|
-
return options.mockEmbedding(input.text)
|
|
35712
|
+
return options.mockEmbedding(input.text);
|
|
37582
35713
|
}
|
|
37583
|
-
return normalizeVector(
|
|
37584
|
-
};
|
|
37585
|
-
const queryFallback = async (input) => {
|
|
37586
|
-
const queryVector = normalizeVector(input.queryVector);
|
|
37587
|
-
const pushdownFilter = buildSQLitePushdownFilter(input.filter);
|
|
37588
|
-
const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
|
|
37589
|
-
chunkId: "chunk_id",
|
|
37590
|
-
metadata: "metadata",
|
|
37591
|
-
source: "source",
|
|
37592
|
-
title: "title"
|
|
37593
|
-
});
|
|
37594
|
-
const rawRows = toStoredRows(filterPlan ? db.prepare(buildJsonQuerySql(tableName, filterPlan.clause)).all(...filterPlan.params) : jsonStatements.query.all());
|
|
37595
|
-
const chunks = mapFilterToRows(rawRows);
|
|
37596
|
-
const filtered = mapToRows(queryVector, chunks, input.filter);
|
|
37597
|
-
const limited = filtered.slice(0, input.topK);
|
|
37598
|
-
updateSQLiteLastQueryPlan({
|
|
37599
|
-
backfillCount: 0,
|
|
37600
|
-
candidateBudgetExhausted: false,
|
|
37601
|
-
filter: input.filter,
|
|
37602
|
-
filteredCandidateCount: rawRows.length,
|
|
37603
|
-
finalSearchK: rawRows.length,
|
|
37604
|
-
initialSearchK: rawRows.length,
|
|
37605
|
-
nativeDiagnostics,
|
|
37606
|
-
pushdownFilter,
|
|
37607
|
-
plannerProfileUsed: input.plannerProfile,
|
|
37608
|
-
queryMultiplierUsed: input.queryMultiplier,
|
|
37609
|
-
queryMode: "json_fallback",
|
|
37610
|
-
returnedCount: limited.length,
|
|
37611
|
-
topK: input.topK,
|
|
37612
|
-
underfilledTopK: limited.length < input.topK
|
|
37613
|
-
});
|
|
37614
|
-
return limited.map(({ chunk, score }) => ({
|
|
37615
|
-
chunkId: chunk.chunkId,
|
|
37616
|
-
chunkText: chunk.text,
|
|
37617
|
-
embedding: chunk.vector,
|
|
37618
|
-
metadata: chunk.metadata,
|
|
37619
|
-
score,
|
|
37620
|
-
source: chunk.source,
|
|
37621
|
-
title: chunk.title
|
|
37622
|
-
}));
|
|
35714
|
+
return normalizeVector(createRAGVector(input.text, dimensions));
|
|
37623
35715
|
};
|
|
37624
|
-
const
|
|
37625
|
-
|
|
37626
|
-
|
|
37627
|
-
|
|
37628
|
-
|
|
37629
|
-
|
|
37630
|
-
|
|
37631
|
-
|
|
37632
|
-
|
|
37633
|
-
|
|
37634
|
-
|
|
37635
|
-
|
|
37636
|
-
|
|
37637
|
-
|
|
37638
|
-
|
|
37639
|
-
|
|
37640
|
-
|
|
37641
|
-
|
|
37642
|
-
|
|
37643
|
-
|
|
37644
|
-
|
|
37645
|
-
|
|
37646
|
-
|
|
37647
|
-
|
|
37648
|
-
|
|
37649
|
-
|
|
37650
|
-
|
|
37651
|
-
|
|
37652
|
-
|
|
37653
|
-
|
|
37654
|
-
|
|
37655
|
-
|
|
37656
|
-
filteredCandidateCount,
|
|
37657
|
-
queryMultiplier,
|
|
37658
|
-
topK: input.topK
|
|
37659
|
-
});
|
|
37660
|
-
if (searchK === 0) {
|
|
37661
|
-
return [];
|
|
37662
|
-
}
|
|
37663
|
-
const runNativeQuery = (candidateK) => toNativeStoredRows(filterPlan ? db.prepare(buildNativeQuerySql(nativeTableName, filterPlan.clause)).all(queryVectorText, candidateK, ...filterPlan.params) : nativeStatements.query.all(queryVectorText, candidateK));
|
|
37664
|
-
let currentSearchK = searchK;
|
|
37665
|
-
let backfillCount = 0;
|
|
37666
|
-
let candidateBudgetExhausted = false;
|
|
37667
|
-
let backfillLimitReached = false;
|
|
37668
|
-
let mapped = [];
|
|
37669
|
-
for (;; ) {
|
|
37670
|
-
const rawRows = runNativeQuery(currentSearchK);
|
|
37671
|
-
mapped = rawRows.map((row) => {
|
|
37672
|
-
const chunk = {
|
|
37673
|
-
chunkId: row.chunk_id,
|
|
37674
|
-
metadata: parseMetadata2(row.metadata),
|
|
37675
|
-
source: row.source ?? undefined,
|
|
37676
|
-
text: row.chunk_text,
|
|
37677
|
-
title: row.title ?? undefined,
|
|
37678
|
-
vector: parseVector(row.embedding)
|
|
37679
|
-
};
|
|
37680
|
-
return {
|
|
37681
|
-
chunk,
|
|
37682
|
-
score: normalizeDistance2(row.distance, nativeDistanceMetric)
|
|
37683
|
-
};
|
|
37684
|
-
}).filter(({ chunk }) => matchesFilter2(chunk, input.filter)).map((entry) => ({
|
|
37685
|
-
chunkId: entry.chunk.chunkId,
|
|
37686
|
-
chunkText: entry.chunk.text,
|
|
37687
|
-
embedding: entry.chunk.vector,
|
|
37688
|
-
metadata: entry.chunk.metadata,
|
|
37689
|
-
score: entry.score,
|
|
37690
|
-
source: entry.chunk.source,
|
|
37691
|
-
title: entry.chunk.title
|
|
37692
|
-
})).sort((left, right) => right.score - left.score);
|
|
37693
|
-
if (mapped.length >= fillTarget.targetResults) {
|
|
37694
|
-
break;
|
|
37695
|
-
}
|
|
37696
|
-
const nextSearchK = planNativeCandidateSearchBackfillK({
|
|
37697
|
-
backfillCount,
|
|
37698
|
-
candidateLimit,
|
|
37699
|
-
currentSearchK,
|
|
37700
|
-
filteredCandidateCount,
|
|
37701
|
-
maxBackfills
|
|
37702
|
-
});
|
|
37703
|
-
if (nextSearchK <= currentSearchK) {
|
|
37704
|
-
backfillLimitReached = typeof maxBackfills === "number" && backfillCount >= maxBackfills && mapped.length < fillTarget.targetResults;
|
|
37705
|
-
candidateBudgetExhausted = mapped.length < fillTarget.targetResults;
|
|
37706
|
-
break;
|
|
37707
|
-
}
|
|
37708
|
-
currentSearchK = nextSearchK;
|
|
37709
|
-
backfillCount += 1;
|
|
35716
|
+
const matchesFilter = (record, filter) => matchesMetadataFilterRecord({
|
|
35717
|
+
chunkId: record.chunkId,
|
|
35718
|
+
metadata: record.metadata,
|
|
35719
|
+
source: record.source,
|
|
35720
|
+
title: record.title,
|
|
35721
|
+
...record.metadata ?? {}
|
|
35722
|
+
}, filter);
|
|
35723
|
+
const toResult = (record, score) => ({
|
|
35724
|
+
chunkId: record.chunkId,
|
|
35725
|
+
chunkText: record.text,
|
|
35726
|
+
embedding: record.vector,
|
|
35727
|
+
metadata: record.metadata,
|
|
35728
|
+
score,
|
|
35729
|
+
source: record.source,
|
|
35730
|
+
title: record.title
|
|
35731
|
+
});
|
|
35732
|
+
const upsert = async (input) => {
|
|
35733
|
+
for (const chunk of input.chunks) {
|
|
35734
|
+
const vector = chunk.embedding ? normalizeVector(chunk.embedding) : normalizeVector(await embed({ text: chunk.text }));
|
|
35735
|
+
const record = {
|
|
35736
|
+
chunkId: chunk.chunkId,
|
|
35737
|
+
metadata: chunk.metadata,
|
|
35738
|
+
source: chunk.source,
|
|
35739
|
+
text: chunk.text,
|
|
35740
|
+
title: chunk.title,
|
|
35741
|
+
vector
|
|
35742
|
+
};
|
|
35743
|
+
records.set(record.chunkId, record);
|
|
35744
|
+
const light = toLight(record);
|
|
35745
|
+
vectorIndex.add(light);
|
|
35746
|
+
textIndex.add(light);
|
|
35747
|
+
await engine.applyChange(table, { op: "insert", row: light });
|
|
37710
35748
|
}
|
|
37711
|
-
updateSQLiteLastQueryPlan({
|
|
37712
|
-
backfillCount,
|
|
37713
|
-
backfillLimitReached,
|
|
37714
|
-
candidateBudgetExhausted,
|
|
37715
|
-
filter: input.filter,
|
|
37716
|
-
filteredCandidateCount,
|
|
37717
|
-
finalSearchK: currentSearchK,
|
|
37718
|
-
initialSearchK: searchK,
|
|
37719
|
-
nativeDiagnostics,
|
|
37720
|
-
pushdownFilter,
|
|
37721
|
-
plannerProfileUsed: input.plannerProfile,
|
|
37722
|
-
candidateLimitUsed: candidateLimit,
|
|
37723
|
-
maxBackfillsUsed: maxBackfills,
|
|
37724
|
-
minResultsUsed: minResults,
|
|
37725
|
-
fillPolicyUsed: fillTarget.fillPolicy,
|
|
37726
|
-
queryMultiplierUsed: queryMultiplier,
|
|
37727
|
-
queryMode: "native_vec0",
|
|
37728
|
-
returnedCount: Math.min(mapped.length, input.topK),
|
|
37729
|
-
minResultsSatisfied: mapped.length >= minResults,
|
|
37730
|
-
topK: input.topK,
|
|
37731
|
-
underfilledTopK: mapped.length < input.topK
|
|
37732
|
-
});
|
|
37733
|
-
return mapped.slice(0, input.topK);
|
|
37734
35749
|
};
|
|
37735
35750
|
const query = async (input) => {
|
|
37736
|
-
|
|
37737
|
-
|
|
37738
|
-
|
|
37739
|
-
|
|
37740
|
-
|
|
37741
|
-
|
|
37742
|
-
|
|
37743
|
-
if (
|
|
37744
|
-
|
|
35751
|
+
const queryVector = normalizeVector(input.queryVector);
|
|
35752
|
+
const results = [];
|
|
35753
|
+
for (const record of records.values()) {
|
|
35754
|
+
if (!matchesFilter(record, input.filter)) {
|
|
35755
|
+
continue;
|
|
35756
|
+
}
|
|
35757
|
+
const score = querySimilarity(queryVector, normalizeVector(record.vector));
|
|
35758
|
+
if (Number.isFinite(score)) {
|
|
35759
|
+
results.push(toResult(record, score));
|
|
37745
35760
|
}
|
|
37746
|
-
return queryFallback(input);
|
|
37747
35761
|
}
|
|
35762
|
+
results.sort((first, second) => second.score - first.score);
|
|
35763
|
+
return results.slice(0, input.topK);
|
|
37748
35764
|
};
|
|
37749
35765
|
const queryLexical = async (input) => {
|
|
37750
|
-
const
|
|
37751
|
-
const
|
|
37752
|
-
|
|
37753
|
-
|
|
37754
|
-
|
|
37755
|
-
|
|
37756
|
-
});
|
|
37757
|
-
const rawRows = toStoredRows(filterPlan ? db.prepare(buildJsonQuerySql(tableName, filterPlan.clause)).all(...filterPlan.params) : jsonStatements.query.all());
|
|
37758
|
-
const chunks = mapFilterToRows(rawRows).filter((chunk) => matchesFilter2(chunk, input.filter));
|
|
37759
|
-
const ranked = rankRAGLexicalMatches(input.query, chunks);
|
|
37760
|
-
return ranked.slice(0, input.topK).map(({ result, score }) => ({
|
|
37761
|
-
chunkId: result.chunkId,
|
|
37762
|
-
chunkText: result.text,
|
|
37763
|
-
metadata: result.metadata,
|
|
37764
|
-
score,
|
|
37765
|
-
source: result.source,
|
|
37766
|
-
title: result.title
|
|
37767
|
-
}));
|
|
37768
|
-
};
|
|
37769
|
-
const upsert = async (input) => {
|
|
37770
|
-
const chunks = input.chunks.length > 0 ? await Promise.all(input.chunks.map(async (chunk) => ({
|
|
37771
|
-
chunkId: chunk.chunkId,
|
|
37772
|
-
metadata: chunk.metadata,
|
|
37773
|
-
source: chunk.source,
|
|
37774
|
-
text: chunk.text,
|
|
37775
|
-
title: chunk.title,
|
|
37776
|
-
vector: chunk.embedding ? normalizeVector(chunk.embedding) : normalizeVector(await embed({ text: chunk.text }))
|
|
37777
|
-
}))) : [];
|
|
37778
|
-
if (!useNative) {
|
|
37779
|
-
fallbackToJsonUpsert(chunks, jsonStatements);
|
|
37780
|
-
refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, tableName);
|
|
37781
|
-
return;
|
|
37782
|
-
}
|
|
37783
|
-
try {
|
|
37784
|
-
upsertNativeChunks(chunks, nativeStatements);
|
|
37785
|
-
} catch (error) {
|
|
37786
|
-
markNativeUpsertFailure(nativeDiagnostics, error);
|
|
37787
|
-
if (nativeConfig?.requireAvailable) {
|
|
37788
|
-
throw new Error(`Native vector upsert failed for table "${nativeTableName}". ${getErrorMessage2(error)}`, { cause: error });
|
|
35766
|
+
const hits = textIndex.search(input.query, input.topK * 5);
|
|
35767
|
+
const results = [];
|
|
35768
|
+
for (const hit of hits) {
|
|
35769
|
+
const record = records.get(hit.row.chunkId);
|
|
35770
|
+
if (record && matchesFilter(record, input.filter)) {
|
|
35771
|
+
results.push(toResult(record, hit.score));
|
|
37789
35772
|
}
|
|
37790
|
-
useNative = false;
|
|
37791
|
-
fallbackToJsonUpsert(chunks, jsonStatements);
|
|
37792
35773
|
}
|
|
37793
|
-
|
|
35774
|
+
return results.slice(0, input.topK);
|
|
37794
35775
|
};
|
|
35776
|
+
const hasFilters = (chunkIds, filter) => ({
|
|
35777
|
+
filtered: Boolean(filter && Object.keys(filter).length > 0),
|
|
35778
|
+
ided: chunkIds.size > 0
|
|
35779
|
+
});
|
|
37795
35780
|
const count = async (input = {}) => {
|
|
37796
|
-
const
|
|
37797
|
-
const
|
|
37798
|
-
|
|
37799
|
-
|
|
37800
|
-
if (!hasFilter && !hasChunkIds) {
|
|
37801
|
-
return getChunkCountFromSql(db, buildJsonCountSql(tableName), []);
|
|
35781
|
+
const chunkIds = new Set(input.chunkIds ?? []);
|
|
35782
|
+
const { filtered, ided } = hasFilters(chunkIds, input.filter);
|
|
35783
|
+
if (!filtered && !ided) {
|
|
35784
|
+
return records.size;
|
|
37802
35785
|
}
|
|
37803
|
-
|
|
37804
|
-
return getSQLiteCandidateChunkIdsByFilter(db, tableName, filter, jsonStatements).length;
|
|
37805
|
-
}
|
|
37806
|
-
return getSQLiteCandidateChunkIds(db, tableName, jsonStatements, {
|
|
37807
|
-
chunkIds,
|
|
37808
|
-
filter
|
|
37809
|
-
}).length;
|
|
35786
|
+
return [...records.values()].filter((record) => ided && chunkIds.has(record.chunkId) || filtered && matchesFilter(record, input.filter)).length;
|
|
37810
35787
|
};
|
|
37811
35788
|
const remove = async (input = {}) => {
|
|
37812
|
-
const
|
|
37813
|
-
const
|
|
37814
|
-
|
|
37815
|
-
const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
|
|
37816
|
-
if (!hasFilter && !hasChunkIds) {
|
|
37817
|
-
return 0;
|
|
37818
|
-
}
|
|
37819
|
-
const toDelete = getSQLiteCandidateChunkIds(db, tableName, jsonStatements, {
|
|
37820
|
-
chunkIds,
|
|
37821
|
-
filter
|
|
37822
|
-
});
|
|
37823
|
-
if (toDelete.length === 0) {
|
|
35789
|
+
const chunkIds = new Set(input.chunkIds ?? []);
|
|
35790
|
+
const { filtered, ided } = hasFilters(chunkIds, input.filter);
|
|
35791
|
+
if (!filtered && !ided) {
|
|
37824
35792
|
return 0;
|
|
37825
35793
|
}
|
|
37826
|
-
|
|
37827
|
-
|
|
37828
|
-
|
|
37829
|
-
|
|
37830
|
-
|
|
37831
|
-
|
|
37832
|
-
|
|
37833
|
-
|
|
37834
|
-
|
|
35794
|
+
let removed = 0;
|
|
35795
|
+
for (const record of [...records.values()]) {
|
|
35796
|
+
const matches = ided && chunkIds.has(record.chunkId) || filtered && matchesFilter(record, input.filter);
|
|
35797
|
+
if (!matches) {
|
|
35798
|
+
continue;
|
|
35799
|
+
}
|
|
35800
|
+
records.delete(record.chunkId);
|
|
35801
|
+
vectorIndex.remove(record.chunkId);
|
|
35802
|
+
textIndex.remove(record.chunkId);
|
|
35803
|
+
await engine.applyChange(table, {
|
|
35804
|
+
op: "delete",
|
|
35805
|
+
row: toLight(record)
|
|
35806
|
+
});
|
|
35807
|
+
removed += 1;
|
|
37835
35808
|
}
|
|
37836
|
-
|
|
37837
|
-
return toDelete.length;
|
|
35809
|
+
return removed;
|
|
37838
35810
|
};
|
|
37839
|
-
const clear = () => {
|
|
37840
|
-
|
|
37841
|
-
|
|
37842
|
-
|
|
37843
|
-
|
|
37844
|
-
|
|
37845
|
-
try {
|
|
37846
|
-
nativeStatements.clear.run();
|
|
37847
|
-
} catch {
|
|
37848
|
-
jsonStatements.clear.run();
|
|
35811
|
+
const clear = async () => {
|
|
35812
|
+
for (const record of [...records.values()]) {
|
|
35813
|
+
records.delete(record.chunkId);
|
|
35814
|
+
vectorIndex.remove(record.chunkId);
|
|
35815
|
+
textIndex.remove(record.chunkId);
|
|
35816
|
+
await engine.applyChange(table, { op: "delete", row: toLight(record) });
|
|
37849
35817
|
}
|
|
37850
|
-
refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
|
|
37851
35818
|
};
|
|
37852
|
-
const
|
|
37853
|
-
|
|
37854
|
-
|
|
37855
|
-
|
|
37856
|
-
|
|
37857
|
-
|
|
37858
|
-
|
|
35819
|
+
const status = {
|
|
35820
|
+
backend: "in_memory",
|
|
35821
|
+
dimensions,
|
|
35822
|
+
vectorMode: "in_memory"
|
|
35823
|
+
};
|
|
35824
|
+
const capabilities = {
|
|
35825
|
+
backend: "in_memory",
|
|
35826
|
+
nativeVectorSearch: false,
|
|
35827
|
+
persistence: "memory_only",
|
|
35828
|
+
serverSideFiltering: false,
|
|
35829
|
+
streamingIngestStatus: false
|
|
37859
35830
|
};
|
|
37860
35831
|
return {
|
|
37861
|
-
analyze,
|
|
37862
35832
|
clear,
|
|
35833
|
+
count,
|
|
35834
|
+
delete: remove,
|
|
37863
35835
|
embed,
|
|
35836
|
+
engine,
|
|
35837
|
+
getCapabilities: () => capabilities,
|
|
35838
|
+
getStatus: () => status,
|
|
37864
35839
|
query,
|
|
37865
35840
|
queryLexical,
|
|
37866
|
-
|
|
37867
|
-
|
|
37868
|
-
upsert,
|
|
37869
|
-
getCapabilities: () => createSQLiteCapabilities(useNative),
|
|
37870
|
-
getStatus: () => createSQLiteStatus(dimensions, nativeDiagnostics, useNative)
|
|
35841
|
+
retrievalCollection,
|
|
35842
|
+
upsert
|
|
37871
35843
|
};
|
|
37872
35844
|
};
|
|
37873
35845
|
export {
|
|
@@ -37889,8 +35861,6 @@ export {
|
|
|
37889
35861
|
resolveRAGQueryTransform,
|
|
37890
35862
|
resolveRAGHybridSearchOptions,
|
|
37891
35863
|
resolveRAGEmbeddingProvider,
|
|
37892
|
-
resolveAbsoluteSQLiteVecExtensionPath,
|
|
37893
|
-
resolveAbsoluteSQLiteVec,
|
|
37894
35864
|
reorderRAGEvaluationSuiteCases,
|
|
37895
35865
|
removeRAGEvaluationSuiteCaseHardNegative,
|
|
37896
35866
|
removeRAGEvaluationSuiteCase,
|
|
@@ -37977,7 +35947,7 @@ export {
|
|
|
37977
35947
|
deepseekEmbeddings,
|
|
37978
35948
|
createVoyageRAGReranker,
|
|
37979
35949
|
createTextFileExtractor,
|
|
37980
|
-
|
|
35950
|
+
createSyncRAGStore,
|
|
37981
35951
|
createRAGVector,
|
|
37982
35952
|
createRAGUrlSyncSource,
|
|
37983
35953
|
createRAGSyncScheduler,
|
|
@@ -38069,7 +36039,6 @@ export {
|
|
|
38069
36039
|
createRAGAdaptiveNativePlannerBenchmarkSuite,
|
|
38070
36040
|
createRAGAdaptiveNativePlannerBenchmarkSnapshot,
|
|
38071
36041
|
createRAGAccessControl,
|
|
38072
|
-
createPostgresRAGStore,
|
|
38073
36042
|
createPDFFileExtractor,
|
|
38074
36043
|
createOfficeDocumentExtractor,
|
|
38075
36044
|
createLegacyDocumentExtractor,
|
|
@@ -38164,5 +36133,5 @@ export {
|
|
|
38164
36133
|
addRAGEvaluationSuiteCase
|
|
38165
36134
|
};
|
|
38166
36135
|
|
|
38167
|
-
//# debugId=
|
|
36136
|
+
//# debugId=7661BF2D41E88F6A64756E2164756E21
|
|
38168
36137
|
//# sourceMappingURL=index.js.map
|