@absolutejs/rag 0.0.19 → 0.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -35670,2206 +35670,6 @@ var createInMemoryRAGStore = (options = {}) => {
35670
35670
  getStatus: () => createInMemoryStatus(dimensions)
35671
35671
  };
35672
35672
  };
35673
- // src/adapters/queryPlanning.ts
35674
- var planNativeCandidateSearchK = (input) => {
35675
- const base = Math.min(Math.max(input.topK * input.queryMultiplier, input.topK), input.candidateLimit);
35676
- if (input.filteredCandidateCount === undefined || !Number.isFinite(input.filteredCandidateCount)) {
35677
- return base;
35678
- }
35679
- const filtered = Math.max(0, Math.floor(input.filteredCandidateCount));
35680
- if (filtered === 0) {
35681
- return 0;
35682
- }
35683
- return Math.min(base, filtered);
35684
- };
35685
- var resolveAdaptiveNativeCandidateLimit = (input) => {
35686
- const clamp = (value) => Math.min(input.defaultCandidateLimit, Math.max(1, Math.floor(value)));
35687
- const filteredCap = typeof input.filteredCandidateCount === "number" && Number.isFinite(input.filteredCandidateCount) ? Math.max(0, Math.floor(input.filteredCandidateCount)) : undefined;
35688
- if (typeof input.explicitCandidateLimit === "number" && Number.isFinite(input.explicitCandidateLimit)) {
35689
- return filteredCap === undefined ? clamp(input.explicitCandidateLimit) : Math.min(clamp(input.explicitCandidateLimit), filteredCap);
35690
- }
35691
- const baseFloor = Math.max(input.topK, input.topK * Math.max(1, Math.floor(input.queryMultiplier)));
35692
- let tuned = input.plannerProfile === "latency" ? clamp(Math.max(input.topK * 2, baseFloor)) : input.plannerProfile === "recall" ? clamp(Math.max(input.topK * 12, baseFloor * 4)) : clamp(Math.max(input.topK * 6, baseFloor * 2));
35693
- if (filteredCap !== undefined) {
35694
- if (filteredCap === 0) {
35695
- return 0;
35696
- }
35697
- tuned = Math.min(tuned, filteredCap);
35698
- }
35699
- return tuned;
35700
- };
35701
- var planNativeCandidateSearchBackfillK = (input) => {
35702
- if (typeof input.maxBackfills === "number" && Number.isFinite(input.maxBackfills) && (input.backfillCount ?? 0) >= Math.max(0, Math.floor(input.maxBackfills))) {
35703
- return input.currentSearchK;
35704
- }
35705
- const cappedLimit = input.filteredCandidateCount === undefined || !Number.isFinite(input.filteredCandidateCount) ? input.candidateLimit : Math.min(input.candidateLimit, Math.max(0, Math.floor(input.filteredCandidateCount)));
35706
- if (input.currentSearchK >= cappedLimit) {
35707
- return input.currentSearchK;
35708
- }
35709
- return Math.min(cappedLimit, Math.max(input.currentSearchK + 1, input.currentSearchK * 2));
35710
- };
35711
- var summarizeSQLiteCandidateCoverage = (input) => {
35712
- const basis = typeof input.filteredCandidateCount === "number" && Number.isFinite(input.filteredCandidateCount) ? Math.max(0, Math.floor(input.filteredCandidateCount)) : typeof input.returnedCount === "number" && Number.isFinite(input.returnedCount) ? Math.max(0, Math.floor(input.returnedCount)) : 0;
35713
- if (basis === 0) {
35714
- return "empty";
35715
- }
35716
- if (basis < input.topK) {
35717
- return "under_target";
35718
- }
35719
- if (basis >= input.topK * 3) {
35720
- return "broad";
35721
- }
35722
- return "target_sized";
35723
- };
35724
-
35725
- // src/adapters/postgres.ts
35726
- var DEFAULT_DIMENSIONS = RAG_VECTOR_DIMENSIONS_DEFAULT;
35727
- var DEFAULT_TABLE_NAME = "rag_chunks";
35728
- var DEFAULT_SCHEMA_NAME = "public";
35729
- var DEFAULT_QUERY_MULTIPLIER = 4;
35730
- var MAX_QUERY_MULTIPLIER = 16;
35731
- var DEFAULT_POSTGRES_INDEX_TYPE = "hnsw";
35732
- var DEFAULT_POSTGRES_IVFFLAT_LISTS = 100;
35733
- var DEFAULT_POSTGRES_HNSW_M = 16;
35734
- var DEFAULT_POSTGRES_HNSW_EF_CONSTRUCTION = 64;
35735
- var IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
35736
- var FILTER_PATH_SEGMENT_RE = /^[a-zA-Z0-9_]+$/;
35737
- var isObjectFilterRecord = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
35738
- var isNestedFilterArray2 = (value) => Array.isArray(value) && value.every((entry) => isObjectFilterRecord(entry));
35739
- var isOperatorFilterRecord = (value) => isObjectFilterRecord(value) && Object.keys(value).some((key) => key.startsWith("$"));
35740
- var countFilterClauses = (filter) => {
35741
- if (!filter) {
35742
- return 0;
35743
- }
35744
- let count = 0;
35745
- for (const [key, value] of Object.entries(filter)) {
35746
- if (key === "$and" || key === "$or") {
35747
- if (isNestedFilterArray2(value)) {
35748
- count += value.reduce((total, entry) => total + countFilterClauses(entry), 0);
35749
- }
35750
- continue;
35751
- }
35752
- if (key === "$not") {
35753
- if (isObjectFilterRecord(value)) {
35754
- count += countFilterClauses(value);
35755
- }
35756
- continue;
35757
- }
35758
- count += 1;
35759
- }
35760
- return count;
35761
- };
35762
- var toPostgresJsonPath = (key) => {
35763
- const segments = key.split(".").filter(Boolean);
35764
- if (segments.length === 0 || !segments.every((segment) => FILTER_PATH_SEGMENT_RE.test(segment))) {
35765
- return null;
35766
- }
35767
- return segments;
35768
- };
35769
- var toPostgresFilterBinding = (value) => {
35770
- if (typeof value === "string" || typeof value === "number" || typeof value === "boolean" || value === null) {
35771
- return value;
35772
- }
35773
- return;
35774
- };
35775
- var buildPostgresJsonbScalarEquality = (input) => {
35776
- const comparison = input.comparison ?? "=";
35777
- return comparison === "=" ? `jsonb_typeof(${input.valueSql}) = 'null'` : `coalesce(jsonb_typeof(${input.valueSql}), 'missing') <> 'null'`;
35778
- };
35779
- var buildPostgresMetadataScalarEquality = (input) => input.value === null ? buildPostgresJsonbScalarEquality({
35780
- comparison: input.comparison,
35781
- valueSql: input.valueSql
35782
- }) : `${input.actualSql} ${input.comparison ?? "="} ${input.bind(String(input.value))}`;
35783
- var buildPostgresFilterPlan = (filter, startIndex = 0) => {
35784
- if (!filter) {
35785
- return { clause: "", params: [] };
35786
- }
35787
- const params = [];
35788
- const bind = (value) => {
35789
- params.push(value);
35790
- return `$${params.length + startIndex}`;
35791
- };
35792
- const build = (entry) => {
35793
- const clauses = [];
35794
- for (const [key, value] of Object.entries(entry)) {
35795
- if (key === "$and" || key === "$or") {
35796
- if (!isNestedFilterArray2(value) || value.length === 0) {
35797
- return null;
35798
- }
35799
- const nested = value.map((item) => build(item));
35800
- if (nested.some((item) => item === null)) {
35801
- return null;
35802
- }
35803
- clauses.push(`(${nested.filter((item) => Boolean(item)).join(key === "$and" ? " AND " : " OR ")})`);
35804
- continue;
35805
- }
35806
- if (key === "$not") {
35807
- if (!isObjectFilterRecord(value)) {
35808
- return null;
35809
- }
35810
- const nested = build(value);
35811
- if (!nested) {
35812
- return null;
35813
- }
35814
- clauses.push(`NOT (${nested})`);
35815
- continue;
35816
- }
35817
- const isScalarField = key === "chunkId" || key === "source" || key === "title";
35818
- const jsonPath = isScalarField ? null : toPostgresJsonPath(key);
35819
- if (!isScalarField && !jsonPath) {
35820
- return null;
35821
- }
35822
- let actualSql;
35823
- let metadataPathSegments = [];
35824
- let metadataValueSql;
35825
- if (isScalarField) {
35826
- actualSql = key === "chunkId" ? "chunk_id" : key;
35827
- } else {
35828
- metadataPathSegments = jsonPath ?? [];
35829
- actualSql = `jsonb_extract_path_text(metadata, ${metadataPathSegments.map((segment) => `'${segment}'`).join(", ")})`;
35830
- metadataValueSql = `metadata #> '{${metadataPathSegments.join(",")}}'`;
35831
- }
35832
- if (!isOperatorFilterRecord(value)) {
35833
- const binding = toPostgresFilterBinding(value);
35834
- if (binding === undefined) {
35835
- return null;
35836
- }
35837
- clauses.push(isScalarField ? `${actualSql} = ${bind(String(binding))}` : buildPostgresMetadataScalarEquality({
35838
- actualSql,
35839
- bind,
35840
- value: binding,
35841
- valueSql: metadataValueSql
35842
- }));
35843
- continue;
35844
- }
35845
- const operatorClauses = Object.entries(value).map(([operator, expected]) => {
35846
- switch (operator) {
35847
- case "$exists":
35848
- return isScalarField ? expected ? `${actualSql} IS NOT NULL` : `${actualSql} IS NULL` : expected ? `${metadataValueSql} IS NOT NULL` : `${metadataValueSql} IS NULL`;
35849
- case "$in": {
35850
- if (!Array.isArray(expected) || expected.length === 0) {
35851
- return null;
35852
- }
35853
- const bindings = expected.map((entry2) => toPostgresFilterBinding(entry2)).filter((entry2) => entry2 !== undefined);
35854
- if (bindings.length !== expected.length) {
35855
- return null;
35856
- }
35857
- return isScalarField ? `${actualSql} IN (${bindings.map((entry2) => bind(String(entry2))).join(", ")})` : `(${bindings.map((entry2) => buildPostgresMetadataScalarEquality({
35858
- actualSql,
35859
- bind,
35860
- value: entry2,
35861
- valueSql: metadataValueSql
35862
- })).join(" OR ")})`;
35863
- }
35864
- case "$ne": {
35865
- const binding = toPostgresFilterBinding(expected);
35866
- return binding === undefined ? null : isScalarField ? `${actualSql} <> ${bind(String(binding))}` : buildPostgresMetadataScalarEquality({
35867
- actualSql,
35868
- bind,
35869
- comparison: "<>",
35870
- value: binding,
35871
- valueSql: metadataValueSql
35872
- });
35873
- }
35874
- case "$gt":
35875
- case "$gte":
35876
- case "$lt":
35877
- case "$lte": {
35878
- if (typeof expected !== "number" || !Number.isFinite(expected)) {
35879
- return null;
35880
- }
35881
- const comparison = operator === "$gt" ? ">" : operator === "$gte" ? ">=" : operator === "$lt" ? "<" : "<=";
35882
- return `((${actualSql}) ~ '^-?[0-9]+(\\.[0-9]+)?$' AND (${actualSql})::double precision ${comparison} ${bind(expected)})`;
35883
- }
35884
- case "$contains":
35885
- if (isScalarField) {
35886
- return null;
35887
- }
35888
- if (toPostgresFilterBinding(expected) === undefined) {
35889
- return null;
35890
- }
35891
- return `(${metadataValueSql} IS NOT NULL AND ${metadataValueSql} ? ${bind(String(expected))})`;
35892
- case "$containsAny":
35893
- case "$containsAll": {
35894
- if (isScalarField || !Array.isArray(expected)) {
35895
- return null;
35896
- }
35897
- const values = expected.map((entry2) => toPostgresFilterBinding(entry2)).filter((entry2) => entry2 !== undefined);
35898
- if (values.length === 0 || values.length !== expected.length) {
35899
- return null;
35900
- }
35901
- const sqlArray = `ARRAY[${values.map((value2) => bind(String(value2))).join(", ")}]::text[]`;
35902
- return `(${metadataValueSql} IS NOT NULL AND ${metadataValueSql} ${operator === "$containsAny" ? "?|" : "?&"} ${sqlArray})`;
35903
- }
35904
- default:
35905
- return null;
35906
- }
35907
- });
35908
- if (operatorClauses.some((clause2) => clause2 === null)) {
35909
- return null;
35910
- }
35911
- clauses.push(operatorClauses.filter((clause2) => Boolean(clause2)).map((clause2) => `(${clause2})`).join(" AND "));
35912
- }
35913
- return clauses.length > 0 ? clauses.map((clause2) => `(${clause2})`).join(" AND ") : "";
35914
- };
35915
- const clause = build(filter);
35916
- return clause === null || clause.trim().length === 0 ? null : { clause, params };
35917
- };
35918
- var buildPostgresPushdownFilter = (filter) => {
35919
- if (!filter) {
35920
- return;
35921
- }
35922
- const hasPushdownFilterPlan = (entry) => {
35923
- const plan = buildPostgresFilterPlan(entry);
35924
- return plan !== null && Boolean(plan.clause) && plan.clause.trim().length > 0;
35925
- };
35926
- const hasPushdownFilterPlanEntry = (entry) => {
35927
- if (!isObjectFilterRecord(entry)) {
35928
- return false;
35929
- }
35930
- return hasPushdownFilterPlan(entry);
35931
- };
35932
- const nextEntries = [];
35933
- for (const [key, value] of Object.entries(filter)) {
35934
- if (key === "$and" || key === "$or") {
35935
- if (!isNestedFilterArray2(value)) {
35936
- continue;
35937
- }
35938
- const nested = value.map((entry) => buildPostgresPushdownFilter(entry)).filter((entry) => hasPushdownFilterPlanEntry(entry));
35939
- if (nested.length > 0) {
35940
- nextEntries.push([key, nested]);
35941
- }
35942
- continue;
35943
- }
35944
- if (key === "$not") {
35945
- if (!isObjectFilterRecord(value)) {
35946
- continue;
35947
- }
35948
- const nested = buildPostgresPushdownFilter(value);
35949
- if (hasPushdownFilterPlanEntry(nested)) {
35950
- nextEntries.push([key, nested]);
35951
- }
35952
- continue;
35953
- }
35954
- if (Array.isArray(value) || isOperatorFilterRecord(value) && Object.keys(value).some((operator) => !(operator === "$exists" || operator === "$in" || operator === "$contains" || operator === "$containsAny" || operator === "$containsAll" || operator === "$ne" || operator === "$gt" || operator === "$gte" || operator === "$lt" || operator === "$lte"))) {
35955
- continue;
35956
- }
35957
- const isScalarColumnKey = ["chunkId", "source", "title"].includes(key);
35958
- const jsonPath = isScalarColumnKey ? null : toPostgresJsonPath(key);
35959
- if (!isScalarColumnKey && !jsonPath) {
35960
- continue;
35961
- }
35962
- if (!hasPushdownFilterPlan({ [key]: value })) {
35963
- continue;
35964
- }
35965
- nextEntries.push([key, value]);
35966
- }
35967
- return nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined;
35968
- };
35969
- var resolvePostgresPushdownMode = (input) => {
35970
- const totalFilterClauseCount = countFilterClauses(input.filter);
35971
- const pushdownClauseCount = countFilterClauses(input.pushdownFilter);
35972
- const jsRemainderClauseCount = Math.max(0, totalFilterClauseCount - pushdownClauseCount);
35973
- const pushdownMode = pushdownClauseCount === 0 ? "none" : pushdownClauseCount >= totalFilterClauseCount ? "full" : "partial";
35974
- return {
35975
- jsRemainderClauseCount,
35976
- jsRemainderRatio: totalFilterClauseCount > 0 ? jsRemainderClauseCount / totalFilterClauseCount : undefined,
35977
- pushdownClauseCount,
35978
- pushdownCoverageRatio: totalFilterClauseCount > 0 ? pushdownClauseCount / totalFilterClauseCount : undefined,
35979
- pushdownMode,
35980
- totalFilterClauseCount
35981
- };
35982
- };
35983
- var assertSupportedIdentifier2 = (name) => {
35984
- if (!IDENTIFIER_RE.test(name)) {
35985
- throw new Error(`Invalid identifier "${name}". Only alphanumeric and underscore names are allowed.`);
35986
- }
35987
- };
35988
- var normalizePostgresIndexType = (value) => {
35989
- if (value === undefined) {
35990
- return DEFAULT_POSTGRES_INDEX_TYPE;
35991
- }
35992
- if (value === "none" || value === "hnsw" || value === "ivfflat") {
35993
- return value;
35994
- }
35995
- throw new Error(`Invalid postgres index type "${String(value)}". Expected "none", "hnsw", or "ivfflat".`);
35996
- };
35997
- var normalizePositiveInteger = (value, fallback) => {
35998
- if (value === undefined || !Number.isFinite(value)) {
35999
- return fallback;
36000
- }
36001
- return Math.max(1, Math.floor(value));
36002
- };
36003
- var getPostgresIndexOperatorClass = (metric) => metric === "cosine" ? "vector_cosine_ops" : metric === "inner_product" ? "vector_ip_ops" : "vector_l2_ops";
36004
- var getPostgresIndexName = (qualifiedTableName, indexType) => indexType === "none" ? undefined : `${qualifiedTableName.replace(".", "_")}_embedding_${indexType}_idx`;
36005
- var buildPostgresIndexSql = (input) => {
36006
- if (input.indexType === "none") {
36007
- return;
36008
- }
36009
- const opclass = getPostgresIndexOperatorClass(input.distanceMetric);
36010
- const indexName = getPostgresIndexName(input.qualifiedTableName, input.indexType);
36011
- const optionsSql = input.indexType === "hnsw" ? ` with (m = ${input.hnswM}, ef_construction = ${input.hnswEfConstruction})` : ` with (lists = ${input.indexLists})`;
36012
- const createPrefix = input.ifNotExists === false ? "create index" : "create index if not exists";
36013
- return `${createPrefix} ${indexName} on ${input.qualifiedTableName} using ${input.indexType} (embedding ${opclass})${optionsSql}`;
36014
- };
36015
- var normalizeQueryMultiplier = (value) => {
36016
- if (value === undefined || !Number.isFinite(value)) {
36017
- return DEFAULT_QUERY_MULTIPLIER;
36018
- }
36019
- return Math.min(MAX_QUERY_MULTIPLIER, Math.max(1, Math.floor(value)));
36020
- };
36021
- var normalizeMaxBackfills = (value) => {
36022
- if (value === undefined || !Number.isFinite(value)) {
36023
- return;
36024
- }
36025
- return Math.max(0, Math.floor(value));
36026
- };
36027
- var normalizeMinResults = (value, topK) => {
36028
- if (value === undefined || !Number.isFinite(value)) {
36029
- return topK;
36030
- }
36031
- return Math.min(topK, Math.max(1, Math.floor(value)));
36032
- };
36033
- var resolveFillTarget = (input) => {
36034
- const fillPolicy = input.fillPolicy ?? "satisfy_min_results";
36035
- return {
36036
- fillPolicy,
36037
- targetResults: fillPolicy === "strict_topk" ? input.topK : input.minResults
36038
- };
36039
- };
36040
- var toQualifiedTableName = (schemaName, tableName) => `${schemaName}.${tableName}`;
36041
- var toVectorLiteral = (vector) => `[${vector.join(",")}]`;
36042
- var parseMetadata = (value) => {
36043
- if (typeof value === "string") {
36044
- try {
36045
- const parsed = JSON.parse(value);
36046
- if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
36047
- return parsed;
36048
- }
36049
- } catch {
36050
- return;
36051
- }
36052
- }
36053
- if (!value || typeof value !== "object" || Array.isArray(value)) {
36054
- return;
36055
- }
36056
- return value;
36057
- };
36058
- var parseVectorText = (value) => {
36059
- if (!value) {
36060
- return [];
36061
- }
36062
- const normalized = value.trim();
36063
- const wrapped = normalized.startsWith("[") ? normalized : `[${normalized.replace(/[()]/g, "")}]`;
36064
- try {
36065
- const parsed = JSON.parse(wrapped);
36066
- return Array.isArray(parsed) ? parsed.filter((entry) => typeof entry === "number" && Number.isFinite(entry)) : [];
36067
- } catch {
36068
- return [];
36069
- }
36070
- };
36071
- var parseCountValue = (value) => {
36072
- if (typeof value === "number" && Number.isFinite(value)) {
36073
- return value;
36074
- }
36075
- if (typeof value === "bigint") {
36076
- return Number(value);
36077
- }
36078
- if (typeof value === "string") {
36079
- const parsed = Number(value);
36080
- return Number.isFinite(parsed) ? parsed : 0;
36081
- }
36082
- return 0;
36083
- };
36084
- var parseBooleanValue = (value) => {
36085
- if (typeof value === "boolean") {
36086
- return value;
36087
- }
36088
- if (typeof value === "number") {
36089
- return value !== 0;
36090
- }
36091
- if (typeof value === "bigint") {
36092
- return value !== 0n;
36093
- }
36094
- if (typeof value === "string") {
36095
- const normalized = value.trim().toLowerCase();
36096
- return normalized === "true" || normalized === "t" || normalized === "1";
36097
- }
36098
- return false;
36099
- };
36100
- var refreshPostgresRuntimeDiagnostics = async (db, nativeDiagnostics, input) => {
36101
- try {
36102
- const rows = await db.unsafe(`select
36103
- c.reltuples::bigint as estimated_row_count,
36104
- pg_relation_size($1::regclass) as table_bytes,
36105
- pg_indexes_size($1::regclass) as index_bytes,
36106
- pg_total_relation_size($1::regclass) as total_bytes,
36107
- exists(
36108
- select 1
36109
- from pg_indexes
36110
- where schemaname = $2
36111
- and tablename = $3
36112
- and indexname = $4
36113
- ) as index_present
36114
- from pg_class c
36115
- join pg_namespace n on n.oid = c.relnamespace
36116
- where n.nspname = $2
36117
- and c.relname = $3
36118
- limit 1`, [
36119
- input.qualifiedTableName,
36120
- input.schemaName,
36121
- input.tableName,
36122
- input.indexName ?? ""
36123
- ]);
36124
- const row = rows[0];
36125
- nativeDiagnostics.indexName = input.indexName;
36126
- nativeDiagnostics.indexPresent = input.indexName ? parseBooleanValue(row?.index_present) : undefined;
36127
- nativeDiagnostics.estimatedRowCount = parseCountValue(row?.estimated_row_count);
36128
- nativeDiagnostics.tableBytes = parseCountValue(row?.table_bytes);
36129
- nativeDiagnostics.indexBytes = parseCountValue(row?.index_bytes);
36130
- nativeDiagnostics.totalBytes = parseCountValue(row?.total_bytes);
36131
- nativeDiagnostics.lastHealthCheckAt = Date.now();
36132
- nativeDiagnostics.lastHealthError = undefined;
36133
- } catch (error) {
36134
- nativeDiagnostics.lastHealthCheckAt = Date.now();
36135
- nativeDiagnostics.lastHealthError = error instanceof Error ? error.message : String(error);
36136
- }
36137
- };
36138
- var analyzePostgresTable = async (db, nativeDiagnostics, input) => {
36139
- try {
36140
- await db.unsafe(`analyze ${input.qualifiedTableName}`);
36141
- nativeDiagnostics.lastAnalyzeAt = Date.now();
36142
- nativeDiagnostics.lastAnalyzeError = undefined;
36143
- await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, input);
36144
- } catch (error) {
36145
- nativeDiagnostics.lastAnalyzeAt = Date.now();
36146
- nativeDiagnostics.lastAnalyzeError = error instanceof Error ? error.message : String(error);
36147
- throw error;
36148
- }
36149
- };
36150
- var rebuildPostgresNativeIndex = async (db, nativeDiagnostics, input) => {
36151
- if (!input.indexName || input.indexType === "none") {
36152
- throw new Error("Postgres native index rebuild is not configured");
36153
- }
36154
- try {
36155
- await db.unsafe(`drop index if exists ${input.indexName}`);
36156
- await db.unsafe(buildPostgresIndexSql({
36157
- distanceMetric: input.distanceMetric,
36158
- hnswEfConstruction: input.hnswEfConstruction,
36159
- hnswM: input.hnswM,
36160
- ifNotExists: false,
36161
- indexLists: input.indexLists,
36162
- indexType: input.indexType,
36163
- qualifiedTableName: input.qualifiedTableName
36164
- }));
36165
- nativeDiagnostics.lastReindexAt = Date.now();
36166
- nativeDiagnostics.lastReindexError = undefined;
36167
- await analyzePostgresTable(db, nativeDiagnostics, input);
36168
- } catch (error) {
36169
- nativeDiagnostics.lastReindexAt = Date.now();
36170
- nativeDiagnostics.lastReindexError = error instanceof Error ? error.message : String(error);
36171
- throw error;
36172
- }
36173
- };
36174
- var getPostgresChunkIdsByChunkIds = async (db, qualifiedTableName, chunkIds) => {
36175
- const normalized = [...new Set(chunkIds)].filter((chunkId) => chunkId.length > 0);
36176
- if (normalized.length === 0) {
36177
- return [];
36178
- }
36179
- const placeholders = normalized.map((_, index) => `$${index + 1}`).join(", ");
36180
- const rows = await db.unsafe(`select chunk_id from ${qualifiedTableName} where chunk_id in (${placeholders})`, normalized);
36181
- return rows.map((row) => row.chunk_id).filter((chunkId) => typeof chunkId === "string");
36182
- };
36183
- var getPostgresCandidateChunkIdsByFilter = async (db, qualifiedTableName, filter) => {
36184
- if (!filter || Object.keys(filter).length === 0) {
36185
- return [];
36186
- }
36187
- const pushdownFilter = buildPostgresPushdownFilter(filter);
36188
- const filterPlan = buildPostgresFilterPlan(pushdownFilter);
36189
- const rowsSql = filterPlan?.clause ? `select chunk_id, text, title, source, metadata from ${qualifiedTableName} where ${filterPlan.clause}` : `select chunk_id, text, title, source, metadata from ${qualifiedTableName}`;
36190
- const rows = await db.unsafe(rowsSql, filterPlan?.clause ? filterPlan.params ?? [] : []);
36191
- const chunks = rows.map((row) => mapRowToChunk(row)).filter((chunk) => matchesFilter(chunk, filter));
36192
- return chunks.map((chunk) => chunk.chunkId);
36193
- };
36194
- var getPostgresCandidateChunkIds = async (db, qualifiedTableName, input) => {
36195
- const chunkIdSet = new Set;
36196
- if (input.filter && Object.keys(input.filter).length > 0) {
36197
- for (const chunkId of await getPostgresCandidateChunkIdsByFilter(db, qualifiedTableName, input.filter)) {
36198
- chunkIdSet.add(chunkId);
36199
- }
36200
- }
36201
- if (input.chunkIds && input.chunkIds.length > 0) {
36202
- for (const chunkId of await getPostgresChunkIdsByChunkIds(db, qualifiedTableName, input.chunkIds)) {
36203
- chunkIdSet.add(chunkId);
36204
- }
36205
- }
36206
- return [...chunkIdSet];
36207
- };
36208
- var normalizeDistance = (distance, metric) => {
36209
- if (!Number.isFinite(distance)) {
36210
- return 0;
36211
- }
36212
- if (metric === "cosine") {
36213
- return Math.min(1, Math.max(0, 1 - distance));
36214
- }
36215
- if (metric === "inner_product") {
36216
- return Math.max(0, -distance);
36217
- }
36218
- return Math.max(0, 1 / (1 + Math.abs(distance)));
36219
- };
36220
- var getDistanceOperator = (metric) => metric === "cosine" ? "<=>" : metric === "inner_product" ? "<#>" : "<->";
36221
- var createPostgresStatus = (dimensions, nativeDiagnostics) => ({
36222
- backend: "postgres",
36223
- dimensions,
36224
- native: nativeDiagnostics,
36225
- vectorMode: "native_pgvector"
36226
- });
36227
- var createPostgresCapabilities = () => ({
36228
- backend: "postgres",
36229
- nativeVectorSearch: true,
36230
- persistence: "external",
36231
- serverSideFiltering: true,
36232
- streamingIngestStatus: false
36233
- });
36234
- var updatePostgresLastQueryPlan = (input) => {
36235
- const pushdown = resolvePostgresPushdownMode({
36236
- filter: input.filter,
36237
- pushdownFilter: input.pushdownFilter
36238
- });
36239
- input.nativeDiagnostics.lastQueryPlan = {
36240
- backfillCount: input.backfillCount,
36241
- candidateBudgetExhausted: input.candidateBudgetExhausted,
36242
- candidateCoverage: summarizeSQLiteCandidateCoverage({
36243
- filteredCandidateCount: input.filteredCandidateCount,
36244
- returnedCount: input.returnedCount,
36245
- topK: input.topK
36246
- }),
36247
- filteredCandidateCount: input.filteredCandidateCount,
36248
- finalSearchK: input.finalSearchK,
36249
- initialSearchK: input.initialSearchK,
36250
- searchExpansionRatio: typeof input.initialSearchK === "number" && typeof input.finalSearchK === "number" && input.initialSearchK > 0 ? input.finalSearchK / input.initialSearchK : undefined,
36251
- candidateLimitUsed: input.candidateLimitUsed,
36252
- maxBackfillsUsed: input.maxBackfillsUsed,
36253
- minResultsUsed: input.minResultsUsed,
36254
- fillPolicyUsed: input.fillPolicyUsed,
36255
- plannerProfileUsed: input.plannerProfileUsed,
36256
- jsRemainderClauseCount: pushdown.jsRemainderClauseCount,
36257
- queryMultiplierUsed: input.queryMultiplierUsed,
36258
- jsRemainderRatio: pushdown.jsRemainderRatio,
36259
- pushdownApplied: pushdown.pushdownClauseCount > 0,
36260
- pushdownClauseCount: pushdown.pushdownClauseCount,
36261
- pushdownCoverageRatio: pushdown.pushdownCoverageRatio,
36262
- pushdownMode: pushdown.pushdownMode,
36263
- queryMode: "native_pgvector",
36264
- candidateYieldRatio: typeof input.returnedCount === "number" && typeof input.finalSearchK === "number" && input.finalSearchK > 0 ? input.returnedCount / input.finalSearchK : undefined,
36265
- returnedCount: input.returnedCount,
36266
- backfillLimitReached: input.backfillLimitReached,
36267
- minResultsSatisfied: input.minResultsSatisfied,
36268
- topKFillRatio: typeof input.returnedCount === "number" && input.topK > 0 ? input.returnedCount / input.topK : undefined,
36269
- totalFilterClauseCount: pushdown.totalFilterClauseCount,
36270
- underfilledTopK: input.underfilledTopK
36271
- };
36272
- };
36273
- var matchesFilter = (record, filter) => matchesMetadataFilterRecord({
36274
- chunkId: record.chunkId,
36275
- metadata: record.metadata,
36276
- source: record.source,
36277
- title: record.title,
36278
- ...record.metadata ?? {}
36279
- }, filter);
36280
- var mapRowToChunk = (row) => ({
36281
- chunkId: row.chunk_id,
36282
- metadata: parseMetadata(row.metadata),
36283
- source: row.source ?? undefined,
36284
- text: row.text,
36285
- title: row.title ?? undefined,
36286
- vector: parseVectorText(row.embedding)
36287
- });
36288
- var ensurePostgresSchema = async (db, input) => {
36289
- await db.unsafe("create extension if not exists vector");
36290
- const [schemaName] = input.qualifiedTableName.split(".");
36291
- if (schemaName) {
36292
- await db.unsafe(`create schema if not exists ${schemaName}`);
36293
- }
36294
- await db.unsafe(`
36295
- create table if not exists ${input.qualifiedTableName} (
36296
- chunk_id text primary key,
36297
- text text not null,
36298
- title text,
36299
- source text,
36300
- metadata jsonb,
36301
- embedding vector(${input.dimensions}) not null
36302
- )
36303
- `);
36304
- const indexSql = buildPostgresIndexSql(input);
36305
- if (indexSql) {
36306
- await db.unsafe(indexSql);
36307
- }
36308
- };
36309
- var createPostgresRAGStore = (options = {}) => {
36310
- const dimensions = options.dimensions ?? DEFAULT_DIMENSIONS;
36311
- const distanceMetric = options.distanceMetric ?? "cosine";
36312
- const queryMultiplier = normalizeQueryMultiplier(options.queryMultiplier);
36313
- const indexType = normalizePostgresIndexType(options.indexType);
36314
- const indexLists = normalizePositiveInteger(options.indexLists, DEFAULT_POSTGRES_IVFFLAT_LISTS);
36315
- const hnswM = normalizePositiveInteger(options.hnswM, DEFAULT_POSTGRES_HNSW_M);
36316
- const hnswEfConstruction = normalizePositiveInteger(options.hnswEfConstruction, DEFAULT_POSTGRES_HNSW_EF_CONSTRUCTION);
36317
- const tableName = options.tableName ?? DEFAULT_TABLE_NAME;
36318
- const schemaName = options.schemaName ?? DEFAULT_SCHEMA_NAME;
36319
- assertSupportedIdentifier2(tableName);
36320
- assertSupportedIdentifier2(schemaName);
36321
- const qualifiedTableName = toQualifiedTableName(schemaName, tableName);
36322
- const indexName = getPostgresIndexName(qualifiedTableName, indexType);
36323
- const db = options.sql ?? new Bun.SQL(options.connectionString ?? process.env.RAG_POSTGRES_URL ?? process.env.DATABASE_URL ?? "postgres://postgres:postgres@localhost:55433/absolute_rag_demo");
36324
- const nativeDiagnostics = {
36325
- active: true,
36326
- available: true,
36327
- distanceMetric,
36328
- extensionName: "vector",
36329
- indexName,
36330
- indexType,
36331
- mode: "pgvector",
36332
- requested: true,
36333
- schemaName,
36334
- tableName
36335
- };
36336
- const capabilities = createPostgresCapabilities();
36337
- const distanceOperator = getDistanceOperator(distanceMetric);
36338
- let initialized;
36339
- const init = () => {
36340
- initialized ??= ensurePostgresSchema(db, {
36341
- dimensions,
36342
- distanceMetric,
36343
- hnswEfConstruction,
36344
- hnswM,
36345
- indexLists,
36346
- indexType,
36347
- qualifiedTableName
36348
- }).then(() => refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
36349
- indexName,
36350
- qualifiedTableName,
36351
- schemaName,
36352
- tableName
36353
- })).catch((error) => {
36354
- nativeDiagnostics.active = false;
36355
- nativeDiagnostics.available = false;
36356
- nativeDiagnostics.lastInitError = error instanceof Error ? error.message : String(error);
36357
- nativeDiagnostics.lastMigrationError = error instanceof Error ? error.message : String(error);
36358
- nativeDiagnostics.fallbackReason = nativeDiagnostics.lastInitError;
36359
- throw error;
36360
- });
36361
- return initialized;
36362
- };
36363
- const embed = async (input) => {
36364
- input.model;
36365
- input.signal;
36366
- if (options.mockEmbedding) {
36367
- return options.mockEmbedding(input.text);
36368
- }
36369
- return normalizeVector(createRAGVector(input.text, dimensions));
36370
- };
36371
- const query = async (input) => {
36372
- await init();
36373
- const queryVector = normalizeVector(input.queryVector);
36374
- const queryMultiplier2 = normalizeQueryMultiplier(input.queryMultiplier ?? options.queryMultiplier);
36375
- const maxBackfills = normalizeMaxBackfills(input.maxBackfills);
36376
- const minResults = normalizeMinResults(input.minResults, input.topK);
36377
- const fillTarget = resolveFillTarget({
36378
- fillPolicy: input.fillPolicy,
36379
- minResults,
36380
- topK: input.topK
36381
- });
36382
- const queryVectorLiteral = toVectorLiteral(queryVector);
36383
- const pushdownFilter = buildPostgresPushdownFilter(input.filter);
36384
- const queryFilterPlan = buildPostgresFilterPlan(pushdownFilter);
36385
- const effectivePushdownFilter = queryFilterPlan ? pushdownFilter : undefined;
36386
- const countFilterPlan = queryFilterPlan;
36387
- const countSql = countFilterPlan?.clause ? `select count(*)::int as count from ${qualifiedTableName} where ${countFilterPlan.clause}` : `select count(*)::int as count from ${qualifiedTableName}`;
36388
- const totalRowsResult = await db.unsafe(countSql, countFilterPlan?.params ?? []);
36389
- nativeDiagnostics.lastFilterDebug = {
36390
- countParams: countFilterPlan?.params ?? [],
36391
- countResultRaw: totalRowsResult?.[0],
36392
- countSql,
36393
- filter: input.filter,
36394
- pushdownFilter: effectivePushdownFilter
36395
- };
36396
- const totalRows = parseCountValue(totalRowsResult?.[0]?.count);
36397
- const candidateLimit = resolveAdaptiveNativeCandidateLimit({
36398
- defaultCandidateLimit: RAG_NATIVE_QUERY_CANDIDATE_LIMIT,
36399
- explicitCandidateLimit: input.candidateLimit,
36400
- filteredCandidateCount: totalRows,
36401
- plannerProfile: input.plannerProfile,
36402
- queryMultiplier: queryMultiplier2,
36403
- topK: input.topK
36404
- });
36405
- const hasPushdownFilter = Boolean(effectivePushdownFilter);
36406
- const plannedFilteredCandidateCount = hasPushdownFilter && totalRows === 0 ? undefined : totalRows;
36407
- const initialSearchK = planNativeCandidateSearchK({
36408
- candidateLimit,
36409
- filteredCandidateCount: plannedFilteredCandidateCount,
36410
- queryMultiplier: queryMultiplier2,
36411
- topK: input.topK
36412
- });
36413
- if (initialSearchK === 0) {
36414
- return [];
36415
- }
36416
- let currentSearchK = initialSearchK;
36417
- let backfillCount = 0;
36418
- let candidateBudgetExhausted = false;
36419
- let backfillLimitReached = false;
36420
- let effectiveFilteredCandidateCount = plannedFilteredCandidateCount;
36421
- let mapped = [];
36422
- for (;; ) {
36423
- const rowsSql = queryFilterPlan?.clause ? `select chunk_id, text, title, source, metadata, embedding::text as embedding, embedding ${distanceOperator} '${queryVectorLiteral}'::vector as distance from ${qualifiedTableName} where ${queryFilterPlan.clause} order by embedding ${distanceOperator} '${queryVectorLiteral}'::vector limit $${queryFilterPlan.params.length + 1}` : `select chunk_id, text, title, source, metadata, embedding::text as embedding, embedding ${distanceOperator} '${queryVectorLiteral}'::vector as distance from ${qualifiedTableName} order by embedding ${distanceOperator} '${queryVectorLiteral}'::vector limit $1`;
36424
- const rows = await db.unsafe(rowsSql, queryFilterPlan?.clause ? [...queryFilterPlan.params ?? [], currentSearchK] : [currentSearchK]);
36425
- nativeDiagnostics.lastFilterDebug = {
36426
- ...nativeDiagnostics.lastFilterDebug,
36427
- queryParams: queryFilterPlan?.clause ? [...queryFilterPlan.params ?? [], currentSearchK] : [currentSearchK],
36428
- queryRowCount: rows.length,
36429
- querySql: rowsSql
36430
- };
36431
- if (hasPushdownFilter && effectiveFilteredCandidateCount === undefined && rows.length <= currentSearchK) {
36432
- effectiveFilteredCandidateCount = rows.length;
36433
- }
36434
- mapped = rows.map((row) => {
36435
- const chunk = mapRowToChunk(row);
36436
- return {
36437
- chunk,
36438
- score: normalizeDistance(Number(row.distance ?? 0), distanceMetric)
36439
- };
36440
- }).filter(({ chunk }) => matchesFilter(chunk, input.filter)).map((entry) => ({
36441
- chunkId: entry.chunk.chunkId,
36442
- chunkText: entry.chunk.text,
36443
- embedding: entry.chunk.vector,
36444
- metadata: entry.chunk.metadata,
36445
- score: entry.score,
36446
- source: entry.chunk.source,
36447
- title: entry.chunk.title
36448
- })).sort((left, right) => right.score - left.score);
36449
- if (mapped.length >= fillTarget.targetResults) {
36450
- break;
36451
- }
36452
- const nextSearchK = planNativeCandidateSearchBackfillK({
36453
- backfillCount,
36454
- candidateLimit,
36455
- currentSearchK,
36456
- filteredCandidateCount: effectiveFilteredCandidateCount,
36457
- maxBackfills
36458
- });
36459
- if (nextSearchK <= currentSearchK) {
36460
- backfillLimitReached = typeof maxBackfills === "number" && backfillCount >= maxBackfills && mapped.length < fillTarget.targetResults;
36461
- candidateBudgetExhausted = mapped.length < fillTarget.targetResults;
36462
- break;
36463
- }
36464
- currentSearchK = nextSearchK;
36465
- backfillCount += 1;
36466
- }
36467
- nativeDiagnostics.lastQueryError = undefined;
36468
- const returned = mapped.slice(0, input.topK);
36469
- updatePostgresLastQueryPlan({
36470
- backfillCount,
36471
- backfillLimitReached,
36472
- candidateBudgetExhausted,
36473
- candidateLimitUsed: candidateLimit,
36474
- maxBackfillsUsed: maxBackfills,
36475
- minResultsUsed: minResults,
36476
- fillPolicyUsed: fillTarget.fillPolicy,
36477
- plannerProfileUsed: input.plannerProfile,
36478
- filter: input.filter,
36479
- pushdownFilter: effectivePushdownFilter,
36480
- queryMultiplierUsed: queryMultiplier2,
36481
- filteredCandidateCount: effectiveFilteredCandidateCount,
36482
- finalSearchK: currentSearchK,
36483
- initialSearchK,
36484
- nativeDiagnostics,
36485
- minResultsSatisfied: returned.length >= minResults,
36486
- returnedCount: returned.length,
36487
- topK: input.topK,
36488
- underfilledTopK: returned.length < input.topK
36489
- });
36490
- return returned;
36491
- };
36492
- const queryLexical = async (input) => {
36493
- await init();
36494
- const pushdownFilter = buildPostgresPushdownFilter(input.filter);
36495
- const lexicalFilterPlan = buildPostgresFilterPlan(pushdownFilter);
36496
- const rowsSql = lexicalFilterPlan?.clause ? `select chunk_id, text, title, source, metadata from ${qualifiedTableName} where ${lexicalFilterPlan.clause}` : `select chunk_id, text, title, source, metadata from ${qualifiedTableName}`;
36497
- const rows = await db.unsafe(rowsSql, lexicalFilterPlan?.params ?? []);
36498
- const chunks = rows.map((row) => mapRowToChunk(row)).filter((chunk) => matchesFilter(chunk, input.filter));
36499
- const ranked = rankRAGLexicalMatches(input.query, chunks);
36500
- return ranked.slice(0, input.topK).map(({ result, score }) => ({
36501
- chunkId: result.chunkId,
36502
- chunkText: result.text,
36503
- metadata: result.metadata,
36504
- score,
36505
- source: result.source,
36506
- title: result.title
36507
- }));
36508
- };
36509
- const upsert = async (input) => {
36510
- await init();
36511
- const chunks = input.chunks.length > 0 ? await Promise.all(input.chunks.map(async (chunk) => ({
36512
- chunkId: chunk.chunkId,
36513
- metadata: chunk.metadata,
36514
- source: chunk.source,
36515
- text: chunk.text,
36516
- title: chunk.title,
36517
- vector: chunk.embedding ? normalizeVector(chunk.embedding) : normalizeVector(await embed({ text: chunk.text }))
36518
- }))) : [];
36519
- for (const chunk of chunks) {
36520
- await db.unsafe(`insert into ${qualifiedTableName} (chunk_id, text, title, source, metadata, embedding)
36521
- values ($1, $2, $3, $4, $5::jsonb, $6::vector)
36522
- on conflict (chunk_id) do update set
36523
- text = excluded.text,
36524
- title = excluded.title,
36525
- source = excluded.source,
36526
- metadata = excluded.metadata,
36527
- embedding = excluded.embedding`, [
36528
- chunk.chunkId,
36529
- chunk.text,
36530
- chunk.title ?? null,
36531
- chunk.source ?? null,
36532
- chunk.metadata ?? null,
36533
- toVectorLiteral(chunk.vector)
36534
- ]);
36535
- }
36536
- await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
36537
- indexName,
36538
- qualifiedTableName,
36539
- schemaName,
36540
- tableName
36541
- });
36542
- };
36543
- const count = async (input = {}) => {
36544
- await init();
36545
- const filter = input.filter;
36546
- const chunkIds = input.chunkIds;
36547
- const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
36548
- const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
36549
- if (!hasFilter && !hasChunkIds) {
36550
- const countResult = await db.unsafe(`select count(*)::int as count from ${qualifiedTableName}`);
36551
- return parseCountValue(countResult[0]?.count);
36552
- }
36553
- return (await getPostgresCandidateChunkIds(db, qualifiedTableName, {
36554
- filter,
36555
- chunkIds
36556
- })).length;
36557
- };
36558
- const remove = async (input = {}) => {
36559
- await init();
36560
- const filter = input.filter;
36561
- const chunkIds = input.chunkIds;
36562
- const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
36563
- const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
36564
- if (!hasFilter && !hasChunkIds) {
36565
- return 0;
36566
- }
36567
- const ids = await getPostgresCandidateChunkIds(db, qualifiedTableName, {
36568
- filter,
36569
- chunkIds
36570
- });
36571
- if (ids.length === 0) {
36572
- return 0;
36573
- }
36574
- const placeholders = ids.map((_, index) => `$${index + 1}`).join(", ");
36575
- await db.unsafe(`delete from ${qualifiedTableName} where chunk_id in (${placeholders})`, ids);
36576
- await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
36577
- indexName,
36578
- qualifiedTableName,
36579
- schemaName,
36580
- tableName
36581
- });
36582
- return ids.length;
36583
- };
36584
- const clear = async () => {
36585
- await init();
36586
- await db.unsafe(`truncate table ${qualifiedTableName}`);
36587
- await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
36588
- indexName,
36589
- qualifiedTableName,
36590
- schemaName,
36591
- tableName
36592
- });
36593
- };
36594
- const analyze = async () => {
36595
- await init();
36596
- await analyzePostgresTable(db, nativeDiagnostics, {
36597
- indexName,
36598
- qualifiedTableName,
36599
- schemaName,
36600
- tableName
36601
- });
36602
- };
36603
- const rebuildNativeIndex = async () => {
36604
- await init();
36605
- await rebuildPostgresNativeIndex(db, nativeDiagnostics, {
36606
- distanceMetric,
36607
- hnswEfConstruction,
36608
- hnswM,
36609
- indexLists,
36610
- indexName,
36611
- indexType,
36612
- qualifiedTableName,
36613
- schemaName,
36614
- tableName
36615
- });
36616
- };
36617
- const close = async () => {
36618
- await db.close?.();
36619
- };
36620
- return {
36621
- analyze,
36622
- clear,
36623
- close,
36624
- embed,
36625
- getCapabilities: () => capabilities,
36626
- getStatus: () => createPostgresStatus(dimensions, nativeDiagnostics),
36627
- query,
36628
- queryLexical,
36629
- rebuildNativeIndex: indexName ? rebuildNativeIndex : undefined,
36630
- count,
36631
- delete: remove,
36632
- upsert
36633
- };
36634
- };
36635
- // src/adapters/sqlite.ts
36636
- import { Database } from "bun:sqlite";
36637
- import { existsSync as existsSync2 } from "fs";
36638
-
36639
- // src/internal/resolveAbsoluteSQLiteVec.ts
36640
- import { existsSync, readFileSync } from "fs";
36641
- import { arch, platform } from "os";
36642
- import { dirname as dirname5, join as join3 } from "path";
36643
- var PLATFORM_PACKAGE_MAP = {
36644
- "darwin-arm64": {
36645
- libraryFile: "vec0.dylib",
36646
- packageName: "@absolutejs/absolute-rag-sqlite-darwin-arm64"
36647
- },
36648
- "darwin-x64": {
36649
- libraryFile: "vec0.dylib",
36650
- packageName: "@absolutejs/absolute-rag-sqlite-darwin-x64"
36651
- },
36652
- "linux-arm64": {
36653
- libraryFile: "vec0.so",
36654
- packageName: "@absolutejs/absolute-rag-sqlite-linux-arm64"
36655
- },
36656
- "linux-x64": {
36657
- libraryFile: "vec0.so",
36658
- packageName: "@absolutejs/absolute-rag-sqlite-linux-x64"
36659
- },
36660
- "win32-x64": {
36661
- libraryFile: "vec0.dll",
36662
- packageName: "@absolutejs/absolute-rag-sqlite-windows-x64"
36663
- }
36664
- };
36665
- var currentPlatformKey = () => `${platform()}-${arch()}`;
36666
- var getErrorMessage = (error) => error instanceof Error ? error.message : String(error);
36667
- var isPackageJsonShape = (value) => Boolean(value) && typeof value === "object";
36668
- var readPackageVersion = (packageJsonPath) => {
36669
- try {
36670
- const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8"));
36671
- if (!isPackageJsonShape(packageJson)) {
36672
- return;
36673
- }
36674
- return typeof packageJson.version === "string" ? packageJson.version : undefined;
36675
- } catch {
36676
- return;
36677
- }
36678
- };
36679
- var resolveAbsoluteSQLiteVec = () => {
36680
- const platformKey = currentPlatformKey();
36681
- const packageInfo = PLATFORM_PACKAGE_MAP[platformKey];
36682
- if (!packageInfo) {
36683
- return {
36684
- platformKey,
36685
- reason: `No AbsoluteJS sqlite-vec package is defined for ${platformKey}.`,
36686
- source: "absolute-package",
36687
- status: "unsupported_platform"
36688
- };
36689
- }
36690
- try {
36691
- const resolve4 = import.meta.resolve;
36692
- if (typeof resolve4 !== "function") {
36693
- throw new Error("AbsoluteJS sqlite-vec package resolution requires import.meta.resolve support.");
36694
- }
36695
- const packageJsonPath = new URL(resolve4(`${packageInfo.packageName}/package.json`)).pathname;
36696
- const packageRoot = dirname5(packageJsonPath);
36697
- const libraryPath = join3(packageRoot, packageInfo.libraryFile);
36698
- const packageVersion = readPackageVersion(packageJsonPath);
36699
- if (!existsSync(libraryPath)) {
36700
- return {
36701
- libraryFile: packageInfo.libraryFile,
36702
- libraryPath,
36703
- packageName: packageInfo.packageName,
36704
- packageRoot,
36705
- packageVersion,
36706
- platformKey,
36707
- reason: `Resolved ${packageInfo.packageName} but ${packageInfo.libraryFile} was not found.`,
36708
- source: "absolute-package",
36709
- status: "binary_missing"
36710
- };
36711
- }
36712
- return {
36713
- libraryFile: packageInfo.libraryFile,
36714
- libraryPath,
36715
- packageName: packageInfo.packageName,
36716
- packageRoot,
36717
- packageVersion,
36718
- platformKey,
36719
- source: "absolute-package",
36720
- status: "resolved"
36721
- };
36722
- } catch (error) {
36723
- return {
36724
- libraryFile: packageInfo.libraryFile,
36725
- packageName: packageInfo.packageName,
36726
- platformKey,
36727
- reason: getErrorMessage(error),
36728
- source: "absolute-package",
36729
- status: "package_not_installed"
36730
- };
36731
- }
36732
- };
36733
- var resolveAbsoluteSQLiteVecExtensionPath = () => {
36734
- const resolution = resolveAbsoluteSQLiteVec();
36735
- return resolution.status === "resolved" ? resolution.libraryPath ?? null : null;
36736
- };
36737
-
36738
- // src/adapters/sqlite.ts
36739
- var DEFAULT_DIMENSIONS2 = RAG_VECTOR_DIMENSIONS_DEFAULT;
36740
- var DEFAULT_TABLE_NAME2 = "rag_chunks";
36741
- var DEFAULT_NATIVE_TABLE_SUFFIX = "_vec0";
36742
- var DEFAULT_QUERY_MULTIPLIER2 = 4;
36743
- var MAX_QUERY_MULTIPLIER2 = 16;
36744
- var IDENTIFIER_RE2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
36745
- var isParsedMetadata = (value) => Boolean(value) && typeof value === "object";
36746
- var isObjectRecord5 = (value) => Boolean(value) && typeof value === "object";
36747
- var isStoredRow = (value) => isObjectRecord5(value) && typeof value.chunk_id === "string" && typeof value.text === "string" && (typeof value.title === "string" || value.title === null) && (typeof value.source === "string" || value.source === null) && (typeof value.metadata === "string" || value.metadata === null) && typeof value.embedding === "string";
36748
- var isNativeStoredRow = (value) => isObjectRecord5(value) && typeof value.chunk_id === "string" && typeof value.chunk_text === "string" && (typeof value.title === "string" || value.title === null) && (typeof value.source === "string" || value.source === null) && (typeof value.metadata === "string" || value.metadata === null) && typeof value.embedding === "string" && typeof value.distance === "number";
36749
- var toStoredRows = (value) => Array.isArray(value) ? value.filter((row) => isStoredRow(row)) : [];
36750
- var toNativeStoredRows = (value) => Array.isArray(value) ? value.filter((row) => isNativeStoredRow(row)) : [];
36751
- var createSQLiteStatus = (dimensions, nativeDiagnostics, useNative) => ({
36752
- backend: "sqlite",
36753
- dimensions,
36754
- native: nativeDiagnostics,
36755
- vectorMode: useNative ? "native_vec0" : "json_fallback"
36756
- });
36757
- var createSQLiteCapabilities = (useNative) => ({
36758
- backend: "sqlite",
36759
- nativeVectorSearch: useNative,
36760
- persistence: "embedded",
36761
- serverSideFiltering: useNative,
36762
- streamingIngestStatus: false
36763
- });
36764
- var assertSupportedIdentifier3 = (name) => {
36765
- if (!IDENTIFIER_RE2.test(name)) {
36766
- throw new Error(`Invalid table name "${name}". Only alphanumeric and underscore names are allowed.`);
36767
- }
36768
- };
36769
- var normalizeQueryMultiplier2 = (value) => {
36770
- if (value === undefined || !Number.isFinite(value)) {
36771
- return DEFAULT_QUERY_MULTIPLIER2;
36772
- }
36773
- const minMultiplier = Math.max(1, Math.floor(value));
36774
- return Math.min(minMultiplier, MAX_QUERY_MULTIPLIER2);
36775
- };
36776
- var normalizeMaxBackfills2 = (value) => {
36777
- if (value === undefined || !Number.isFinite(value)) {
36778
- return;
36779
- }
36780
- return Math.max(0, Math.floor(value));
36781
- };
36782
- var normalizeMinResults2 = (value, topK) => {
36783
- if (value === undefined || !Number.isFinite(value)) {
36784
- return topK;
36785
- }
36786
- return Math.min(topK, Math.max(1, Math.floor(value)));
36787
- };
36788
- var resolveFillTarget2 = (input) => {
36789
- const fillPolicy = input.fillPolicy ?? "satisfy_min_results";
36790
- return {
36791
- fillPolicy,
36792
- targetResults: fillPolicy === "strict_topk" ? input.topK : input.minResults
36793
- };
36794
- };
36795
- var toJSONString = (metadata) => metadata === undefined ? null : JSON.stringify(metadata);
36796
- var parseMetadata2 = (value) => {
36797
- if (value === null) {
36798
- return;
36799
- }
36800
- try {
36801
- const parsed = JSON.parse(value);
36802
- if (isParsedMetadata(parsed)) {
36803
- return parsed;
36804
- }
36805
- } catch {}
36806
- return;
36807
- };
36808
- var parseVector = (value) => {
36809
- try {
36810
- const parsed = JSON.parse(value);
36811
- if (Array.isArray(parsed)) {
36812
- return parsed.filter((element) => typeof element === "number" && Number.isFinite(element));
36813
- }
36814
- } catch {}
36815
- return [];
36816
- };
36817
- var toSQLiteFilterBinding = (value) => value === null || typeof value === "string" || typeof value === "number" || typeof value === "bigint" || typeof value === "boolean" ? value : undefined;
36818
- var FILTER_PATH_SEGMENT_RE2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
36819
- var isObjectFilterRecord2 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
36820
- var isOperatorFilterRecord2 = (value) => isObjectFilterRecord2(value) && Object.keys(value).some((key) => key.startsWith("$"));
36821
- var isNestedFilterArray3 = (value) => Array.isArray(value) && value.every((entry) => isObjectFilterRecord2(entry));
36822
- var toSQLiteJsonPath = (key) => {
36823
- const segments = key.split(".").filter(Boolean);
36824
- if (segments.length === 0 || !segments.every((segment) => FILTER_PATH_SEGMENT_RE2.test(segment))) {
36825
- return null;
36826
- }
36827
- return `$.${segments.join(".")}`;
36828
- };
36829
- var resolveSQLiteFilterColumn = (key, aliases) => {
36830
- if (key === "chunkId")
36831
- return { actualSql: aliases.chunkId, kind: "scalar" };
36832
- if (key === "source")
36833
- return { actualSql: aliases.source, kind: "scalar" };
36834
- if (key === "title")
36835
- return { actualSql: aliases.title, kind: "scalar" };
36836
- const path = toSQLiteJsonPath(key);
36837
- if (!path) {
36838
- return null;
36839
- }
36840
- return {
36841
- actualSql: `json_extract(${aliases.metadata}, ?)`,
36842
- kind: "metadata",
36843
- path
36844
- };
36845
- };
36846
- var buildSQLiteScalarClause = (actualSql, expected, params) => {
36847
- if (!isOperatorFilterRecord2(expected)) {
36848
- const binding = toSQLiteFilterBinding(expected);
36849
- if (binding === undefined) {
36850
- return null;
36851
- }
36852
- params.push(binding);
36853
- return `${actualSql} = ?`;
36854
- }
36855
- const clauses = [];
36856
- for (const [operator, value] of Object.entries(expected)) {
36857
- switch (operator) {
36858
- case "$exists":
36859
- clauses.push(Boolean(value) ? `${actualSql} IS NOT NULL` : `${actualSql} IS NULL`);
36860
- break;
36861
- case "$ne":
36862
- const neBinding = toSQLiteFilterBinding(value);
36863
- if (neBinding === undefined) {
36864
- return null;
36865
- }
36866
- params.push(neBinding);
36867
- clauses.push(`(${actualSql} IS NULL OR ${actualSql} != ?)`);
36868
- break;
36869
- case "$in":
36870
- if (!Array.isArray(value) || value.length === 0 || value.some((entry) => toSQLiteFilterBinding(entry) === undefined)) {
36871
- return null;
36872
- }
36873
- params.push(...value);
36874
- clauses.push(`${actualSql} IN (${value.map(() => "?").join(", ")})`);
36875
- break;
36876
- case "$gt":
36877
- case "$gte":
36878
- case "$lt":
36879
- case "$lte": {
36880
- if (typeof value !== "number" || !Number.isFinite(value)) {
36881
- return null;
36882
- }
36883
- params.push(value);
36884
- const comparison = operator === "$gt" ? ">" : operator === "$gte" ? ">=" : operator === "$lt" ? "<" : "<=";
36885
- clauses.push(`${actualSql} ${comparison} ?`);
36886
- break;
36887
- }
36888
- default:
36889
- return null;
36890
- }
36891
- }
36892
- return clauses.length > 0 ? clauses.map((clause) => `(${clause})`).join(" AND ") : null;
36893
- };
36894
- var buildSQLiteArrayClause = (metadataSql, path, expected, params) => {
36895
- if (!isOperatorFilterRecord2(expected)) {
36896
- return null;
36897
- }
36898
- const arrayTypeClause = `json_type(${metadataSql}, ?) = 'array'`;
36899
- const clauses = [];
36900
- for (const [operator, value] of Object.entries(expected)) {
36901
- switch (operator) {
36902
- case "$contains":
36903
- const containsBinding = toSQLiteFilterBinding(value);
36904
- if (containsBinding === undefined) {
36905
- return null;
36906
- }
36907
- params.push(path, path, containsBinding);
36908
- clauses.push(`(${arrayTypeClause} AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value = ?))`);
36909
- break;
36910
- case "$containsAny":
36911
- if (!Array.isArray(value) || value.length === 0 || value.some((entry) => toSQLiteFilterBinding(entry) === undefined)) {
36912
- return null;
36913
- }
36914
- params.push(path, path, ...value);
36915
- clauses.push(`(${arrayTypeClause} AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value IN (${value.map(() => "?").join(", ")})))`);
36916
- break;
36917
- case "$containsAll":
36918
- if (!Array.isArray(value) || value.length === 0 || value.some((entry) => toSQLiteFilterBinding(entry) === undefined)) {
36919
- return null;
36920
- }
36921
- clauses.push(...value.map((entry) => {
36922
- params.push(path, path, entry);
36923
- return `(${arrayTypeClause} AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value = ?))`;
36924
- }));
36925
- break;
36926
- default:
36927
- return null;
36928
- }
36929
- }
36930
- return clauses.length > 0 ? clauses.map((clause) => `(${clause})`).join(" AND ") : null;
36931
- };
36932
- var buildSQLiteMetadataScalarMatchClause = (metadataSql, actualSql, path, expected, params) => {
36933
- const binding = toSQLiteFilterBinding(expected);
36934
- if (binding === undefined) {
36935
- return null;
36936
- }
36937
- params.push(path, binding, path, path, binding);
36938
- return `(${actualSql} = ? OR (json_type(${metadataSql}, ?) = 'array' AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value = ?)))`;
36939
- };
36940
- var buildSQLiteFilterPlan = (filter, aliases) => {
36941
- if (!filter) {
36942
- return { clause: "", params: [] };
36943
- }
36944
- const params = [];
36945
- const build = (entry) => {
36946
- const clauses = [];
36947
- for (const [key, value] of Object.entries(entry)) {
36948
- if (key === "$and" || key === "$or") {
36949
- if (!isNestedFilterArray3(value) || value.length === 0) {
36950
- return null;
36951
- }
36952
- const nested = value.map((item) => build(item));
36953
- if (nested.some((item) => item === null)) {
36954
- return null;
36955
- }
36956
- clauses.push(`(${nested.filter((item) => Boolean(item)).join(key === "$and" ? " AND " : " OR ")})`);
36957
- continue;
36958
- }
36959
- if (key === "$not") {
36960
- if (!isObjectFilterRecord2(value)) {
36961
- return null;
36962
- }
36963
- const nested = build(value);
36964
- if (!nested) {
36965
- return null;
36966
- }
36967
- clauses.push(`NOT (${nested})`);
36968
- continue;
36969
- }
36970
- const resolved = resolveSQLiteFilterColumn(key, aliases);
36971
- if (!resolved) {
36972
- return null;
36973
- }
36974
- if (resolved.kind === "metadata") {
36975
- const metadataScalarClause = buildSQLiteMetadataScalarMatchClause(aliases.metadata, resolved.actualSql, resolved.path, value, params);
36976
- if (metadataScalarClause) {
36977
- clauses.push(metadataScalarClause);
36978
- continue;
36979
- }
36980
- const arrayClause = buildSQLiteArrayClause(aliases.metadata, resolved.path, value, params);
36981
- if (arrayClause) {
36982
- clauses.push(arrayClause);
36983
- continue;
36984
- }
36985
- params.push(resolved.path);
36986
- }
36987
- const scalarClause = buildSQLiteScalarClause(resolved.actualSql, value, params);
36988
- if (!scalarClause) {
36989
- return null;
36990
- }
36991
- clauses.push(scalarClause);
36992
- }
36993
- return clauses.length > 0 ? clauses.map((clause2) => `(${clause2})`).join(" AND ") : "";
36994
- };
36995
- const clause = build(filter);
36996
- return clause === null ? null : { clause, params };
36997
- };
36998
- var buildSQLitePushdownFilter = (filter) => {
36999
- if (!filter) {
37000
- return;
37001
- }
37002
- const nextEntries = [];
37003
- for (const [key, value] of Object.entries(filter)) {
37004
- if (key === "$and" || key === "$or") {
37005
- if (!isNestedFilterArray3(value)) {
37006
- continue;
37007
- }
37008
- const nested = value.map((entry) => buildSQLitePushdownFilter(entry)).filter((entry) => Boolean(entry));
37009
- if (nested.length > 0) {
37010
- nextEntries.push([key, nested]);
37011
- }
37012
- continue;
37013
- }
37014
- if (key === "$not") {
37015
- if (!isObjectFilterRecord2(value)) {
37016
- continue;
37017
- }
37018
- const nested = buildSQLitePushdownFilter(value);
37019
- if (nested) {
37020
- nextEntries.push([key, nested]);
37021
- }
37022
- continue;
37023
- }
37024
- if (Array.isArray(value) || isOperatorFilterRecord2(value) && Object.keys(value).some((operator) => operator === "$contains" || operator === "$containsAny" || operator === "$containsAll")) {
37025
- continue;
37026
- }
37027
- nextEntries.push([key, value]);
37028
- }
37029
- return nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined;
37030
- };
37031
- var normalizeDistance2 = (distance, metric) => {
37032
- if (!Number.isFinite(distance)) {
37033
- return 0;
37034
- }
37035
- if (metric === "cosine") {
37036
- return Math.min(1, Math.max(0, 1 - distance));
37037
- }
37038
- return Math.max(0, 1 / (1 + Math.abs(distance)));
37039
- };
37040
- var matchesFilter2 = (record, filter) => matchesMetadataFilterRecord({
37041
- chunkId: record.chunkId,
37042
- metadata: record.metadata,
37043
- source: record.source,
37044
- title: record.title,
37045
- ...record.metadata ?? {}
37046
- }, filter);
37047
- var mapFilterToRows = (rows) => rows.map((row) => ({
37048
- chunkId: row.chunk_id,
37049
- metadata: parseMetadata2(row.metadata),
37050
- source: row.source ?? undefined,
37051
- text: row.text,
37052
- title: row.title ?? undefined,
37053
- vector: parseVector(row.embedding)
37054
- }));
37055
- var buildJsonQuerySql = (tableName, whereClause) => `
37056
- SELECT chunk_id, text, title, source, metadata, embedding FROM ${tableName}
37057
- ${whereClause ? `WHERE ${whereClause}` : ""}
37058
- `;
37059
- var buildJsonCountSql = (tableName, whereClause) => `
37060
- SELECT COUNT(*) AS count FROM ${tableName}
37061
- ${whereClause ? `WHERE ${whereClause}` : ""}
37062
- `;
37063
- var getChunkCountFromSql = (db, sql, params = []) => {
37064
- const result = db.prepare(sql).get(...params);
37065
- const count = result?.count;
37066
- return typeof count === "number" && Number.isFinite(count) ? count : 0;
37067
- };
37068
- var getPragmaNumericValue = (db, pragma) => {
37069
- const row = db.prepare(`PRAGMA ${pragma}`).get();
37070
- const value = row ? row[Object.keys(row)[0] ?? ""] : undefined;
37071
- return typeof value === "number" && Number.isFinite(value) ? value : 0;
37072
- };
37073
- var refreshSQLiteRuntimeDiagnostics = (db, nativeDiagnostics, diagnosticTableName) => {
37074
- if (!nativeDiagnostics) {
37075
- return;
37076
- }
37077
- try {
37078
- const rowCount = getChunkCountFromSql(db, buildJsonCountSql(diagnosticTableName));
37079
- const pageCount = getPragmaNumericValue(db, "page_count");
37080
- const pageSize = getPragmaNumericValue(db, "page_size");
37081
- const freelistCount = getPragmaNumericValue(db, "freelist_count");
37082
- nativeDiagnostics.rowCount = rowCount;
37083
- nativeDiagnostics.pageCount = pageCount;
37084
- nativeDiagnostics.freelistCount = freelistCount;
37085
- nativeDiagnostics.databaseBytes = pageCount > 0 && pageSize > 0 ? pageCount * pageSize : 0;
37086
- nativeDiagnostics.lastHealthCheckAt = Date.now();
37087
- nativeDiagnostics.lastHealthError = undefined;
37088
- } catch (error) {
37089
- nativeDiagnostics.lastHealthCheckAt = Date.now();
37090
- nativeDiagnostics.lastHealthError = getErrorMessage2(error);
37091
- }
37092
- };
37093
- var buildNativeQuerySql = (tableName, whereClause) => `
37094
- SELECT
37095
- chunk_id,
37096
- embedding,
37097
- chunk_text,
37098
- title,
37099
- source,
37100
- metadata,
37101
- distance
37102
- FROM ${tableName}
37103
- WHERE embedding MATCH vec_f32(?)
37104
- AND k = ?
37105
- ${whereClause ? `AND (${whereClause})` : ""}
37106
- ORDER BY distance
37107
- `;
37108
- var getFilteredSQLiteCandidateCount = (db, tableName, filterPlan) => {
37109
- if (!filterPlan) {
37110
- return;
37111
- }
37112
- const result = db.prepare(buildJsonCountSql(tableName, filterPlan.clause)).get(...filterPlan.params);
37113
- const count = result?.count;
37114
- return typeof count === "number" && Number.isFinite(count) ? count : undefined;
37115
- };
37116
- var createJsonStatements = (db, tableName) => {
37117
- const insertSql = `
37118
- INSERT INTO ${tableName} (
37119
- chunk_id,
37120
- text,
37121
- title,
37122
- source,
37123
- metadata,
37124
- embedding
37125
- ) VALUES (?, ?, ?, ?, ?, ?)
37126
- ON CONFLICT(chunk_id) DO UPDATE SET
37127
- text = excluded.text,
37128
- title = excluded.title,
37129
- source = excluded.source,
37130
- metadata = excluded.metadata,
37131
- embedding = excluded.embedding
37132
- `;
37133
- const querySql = `
37134
- SELECT chunk_id, text, title, source, metadata, embedding FROM ${tableName}
37135
- `;
37136
- const clearSql = `DELETE FROM ${tableName}`;
37137
- const deleteSql = `DELETE FROM ${tableName} WHERE chunk_id = ?`;
37138
- const init = () => db.exec(`
37139
- CREATE TABLE IF NOT EXISTS ${tableName} (
37140
- chunk_id TEXT PRIMARY KEY,
37141
- text TEXT NOT NULL,
37142
- title TEXT,
37143
- source TEXT,
37144
- metadata TEXT,
37145
- embedding TEXT NOT NULL
37146
- )
37147
- `);
37148
- init();
37149
- return {
37150
- clear: db.prepare(clearSql),
37151
- delete: db.prepare(deleteSql),
37152
- init,
37153
- insert: db.prepare(insertSql),
37154
- query: db.prepare(querySql)
37155
- };
37156
- };
37157
- var getSQLiteChunkIdsByChunkIds = (db, tableName, chunkIds) => {
37158
- const uniqueChunkIds = [...new Set(chunkIds)];
37159
- if (uniqueChunkIds.length === 0) {
37160
- return [];
37161
- }
37162
- const whereClause = uniqueChunkIds.map(() => "?").join(", ");
37163
- const rows = db.prepare(`SELECT chunk_id FROM ${tableName} WHERE chunk_id IN (${whereClause})`).all(...uniqueChunkIds);
37164
- return rows.map((row) => row.chunk_id).filter((chunkId) => typeof chunkId === "string");
37165
- };
37166
- var getSQLiteCandidateChunkIdsByFilter = (db, tableName, filter, jsonStatements) => {
37167
- if (!filter || Object.keys(filter).length === 0) {
37168
- return [];
37169
- }
37170
- const pushdownFilter = buildSQLitePushdownFilter(filter);
37171
- const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
37172
- chunkId: "chunk_id",
37173
- metadata: "metadata",
37174
- source: "source",
37175
- title: "title"
37176
- });
37177
- const rawRows = toStoredRows(filterPlan ? db.prepare(buildJsonQuerySql(tableName, filterPlan.clause)).all(...filterPlan.params) : jsonStatements.query.all());
37178
- const chunks = mapFilterToRows(rawRows);
37179
- return chunks.filter((chunk) => matchesFilter2(chunk, filter)).map((chunk) => chunk.chunkId);
37180
- };
37181
- var getSQLiteCandidateChunkIds = (db, tableName, jsonStatements, input) => {
37182
- const chunkIdSet = new Set;
37183
- if (input.filter && Object.keys(input.filter).length > 0) {
37184
- for (const chunkId of getSQLiteCandidateChunkIdsByFilter(db, tableName, input.filter, jsonStatements)) {
37185
- chunkIdSet.add(chunkId);
37186
- }
37187
- }
37188
- if (input.chunkIds && input.chunkIds.length > 0) {
37189
- for (const chunkId of getSQLiteChunkIdsByChunkIds(db, tableName, input.chunkIds)) {
37190
- chunkIdSet.add(chunkId);
37191
- }
37192
- }
37193
- return [...chunkIdSet];
37194
- };
37195
- var toVectorText = (vector) => JSON.stringify(vector);
37196
- var createNativeVec0Table = (db, tableName, dimensions, metric) => {
37197
- const metricSuffix = metric === "cosine" ? " distance_metric=cosine" : "";
37198
- db.exec(`
37199
- CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName} USING vec0(
37200
- chunk_id TEXT,
37201
- embedding float[${dimensions}]${metricSuffix},
37202
- +chunk_text TEXT,
37203
- title TEXT,
37204
- source TEXT,
37205
- metadata TEXT
37206
- )
37207
- `);
37208
- };
37209
- var createNativeVec0Statements = (db, tableName) => {
37210
- const upsertSql = `
37211
- INSERT INTO ${tableName} (
37212
- chunk_id,
37213
- embedding,
37214
- chunk_text,
37215
- title,
37216
- source,
37217
- metadata
37218
- ) VALUES (?, vec_f32(?), ?, ?, ?, ?)
37219
- `;
37220
- const deleteSql = `DELETE FROM ${tableName} WHERE chunk_id = ?`;
37221
- const querySql = `
37222
- SELECT
37223
- chunk_id,
37224
- embedding,
37225
- chunk_text,
37226
- title,
37227
- source,
37228
- metadata,
37229
- distance
37230
- FROM ${tableName}
37231
- WHERE embedding MATCH vec_f32(?)
37232
- AND k = ?
37233
- ORDER BY distance
37234
- `;
37235
- return {
37236
- clear: db.prepare(`DELETE FROM ${tableName}`),
37237
- delete: db.prepare(deleteSql),
37238
- insert: db.prepare(upsertSql),
37239
- query: db.prepare(querySql)
37240
- };
37241
- };
37242
- var mapToRows = (vector, chunks, filter) => chunks.map((chunk) => ({
37243
- chunk,
37244
- score: querySimilarity(vector, normalizeVector(chunk.vector))
37245
- })).filter(({ chunk }) => matchesFilter2(chunk, filter)).sort((left, right) => right.score - left.score);
37246
- var executeNativeInitSql = (db, initSql) => {
37247
- if (!initSql) {
37248
- return;
37249
- }
37250
- if (typeof initSql === "string") {
37251
- db.exec(initSql);
37252
- return;
37253
- }
37254
- for (const command of initSql) {
37255
- db.exec(command);
37256
- }
37257
- };
37258
- var getErrorMessage2 = (error) => error instanceof Error ? error.message : String(error);
37259
- var resolveConfiguredNativeExtension = (nativeConfig) => {
37260
- const platformKey = `${process.platform}-${process.arch}`;
37261
- if (nativeConfig?.extensionPath) {
37262
- return existsSync2(nativeConfig.extensionPath) ? {
37263
- libraryPath: nativeConfig.extensionPath,
37264
- platformKey,
37265
- source: "explicit",
37266
- status: "resolved"
37267
- } : {
37268
- libraryPath: nativeConfig.extensionPath,
37269
- platformKey,
37270
- reason: `Configured native.extensionPath was not found: ${nativeConfig.extensionPath}`,
37271
- source: "explicit",
37272
- status: "binary_missing"
37273
- };
37274
- }
37275
- if (nativeConfig?.resolveFromAbsolutePackages !== false) {
37276
- return resolveAbsolutePackageNativeExtension(platformKey);
37277
- }
37278
- const envResolution = resolveNativeExtensionFromEnv(platformKey);
37279
- if (envResolution)
37280
- return envResolution;
37281
- return {
37282
- platformKey,
37283
- reason: "No native sqlite-vec path was configured. AbsoluteJS will still attempt vec0 initialization in case the extension is already registered on the Database connection.",
37284
- source: "database",
37285
- status: "not_configured"
37286
- };
37287
- };
37288
- var describeNativeFallbackReason = (resolution) => {
37289
- if (!resolution) {
37290
- return "Native sqlite vec0 was not configured.";
37291
- }
37292
- switch (resolution.status) {
37293
- case "resolved":
37294
- return;
37295
- case "package_not_installed":
37296
- return `Install ${resolution.packageName ?? "@absolutejs/absolute-rag-sqlite"} for ${resolution.platformKey}, or provide native.extensionPath.`;
37297
- case "binary_missing":
37298
- return resolution.reason ?? "Resolved sqlite-vec binary was missing.";
37299
- case "unsupported_platform":
37300
- return resolution.reason ?? "This platform is not yet supported by AbsoluteJS sqlite-vec packages.";
37301
- case "not_configured":
37302
- return resolution.reason ?? "No sqlite-vec binary path was configured.";
37303
- case "package_invalid":
37304
- return resolution.reason ?? "The sqlite-vec package manifest was invalid.";
37305
- default:
37306
- return "Native sqlite vec0 could not be initialized.";
37307
- }
37308
- };
37309
- var resolveNativeExtensionFromEnv = (platformKey) => {
37310
- const envPath = process.env.SQLITE_VEC_EXTENSION_PATH;
37311
- if (!envPath) {
37312
- return null;
37313
- }
37314
- if (existsSync2(envPath)) {
37315
- return {
37316
- libraryPath: envPath,
37317
- platformKey,
37318
- source: "env",
37319
- status: "resolved"
37320
- };
37321
- }
37322
- return {
37323
- libraryPath: envPath,
37324
- platformKey,
37325
- reason: `SQLITE_VEC_EXTENSION_PATH was set but not found: ${envPath}`,
37326
- source: "env",
37327
- status: "binary_missing"
37328
- };
37329
- };
37330
- var shouldResolveNativeFromEnv = (resolution) => resolution.status === "binary_missing" || resolution.status === "package_not_installed" || resolution.status === "unsupported_platform";
37331
- var resolveAbsolutePackageNativeExtension = (platformKey) => {
37332
- const packageResolution = resolveAbsoluteSQLiteVec();
37333
- if (!shouldResolveNativeFromEnv(packageResolution)) {
37334
- return packageResolution;
37335
- }
37336
- const envResolution = resolveNativeExtensionFromEnv(platformKey);
37337
- if (envResolution) {
37338
- return envResolution;
37339
- }
37340
- return packageResolution;
37341
- };
37342
- var activateNativeDiagnostics = (nativeDiagnostics) => {
37343
- if (!nativeDiagnostics) {
37344
- return;
37345
- }
37346
- nativeDiagnostics.available = true;
37347
- nativeDiagnostics.active = true;
37348
- nativeDiagnostics.fallbackReason = undefined;
37349
- if (nativeDiagnostics.resolution && nativeDiagnostics.resolution.status === "resolved") {
37350
- return;
37351
- }
37352
- nativeDiagnostics.resolution = {
37353
- platformKey: `${process.platform}-${process.arch}`,
37354
- reason: "sqlite-vec was already available on the Database connection or loaded by native.extensionInitSql.",
37355
- source: "database",
37356
- status: "resolved"
37357
- };
37358
- };
37359
- var loadNativeExtension = (db, nativeResolution) => {
37360
- if (nativeResolution?.status !== "resolved") {
37361
- return;
37362
- }
37363
- if (!nativeResolution.libraryPath) {
37364
- return;
37365
- }
37366
- db.loadExtension(nativeResolution.libraryPath);
37367
- };
37368
- var markNativeLoadFailure = (nativeDiagnostics, error, nativeResolution) => {
37369
- if (!nativeDiagnostics) {
37370
- return;
37371
- }
37372
- nativeDiagnostics.available = false;
37373
- nativeDiagnostics.active = false;
37374
- nativeDiagnostics.lastLoadError = getErrorMessage2(error);
37375
- nativeDiagnostics.fallbackReason = describeNativeFallbackReason(nativeResolution) ?? nativeDiagnostics.lastLoadError;
37376
- };
37377
- var markNativeQueryFailure = (nativeDiagnostics, error) => {
37378
- if (!nativeDiagnostics) {
37379
- return;
37380
- }
37381
- nativeDiagnostics.lastQueryError = getErrorMessage2(error);
37382
- nativeDiagnostics.active = false;
37383
- nativeDiagnostics.fallbackReason = nativeDiagnostics.lastQueryError;
37384
- };
37385
- var countFilterClauses2 = (filter) => {
37386
- if (!filter) {
37387
- return 0;
37388
- }
37389
- let count = 0;
37390
- for (const [key, value] of Object.entries(filter)) {
37391
- if (key === "$and" || key === "$or") {
37392
- if (isNestedFilterArray3(value)) {
37393
- count += value.reduce((total, entry) => total + countFilterClauses2(entry), 0);
37394
- }
37395
- continue;
37396
- }
37397
- if (key === "$not") {
37398
- if (isObjectFilterRecord2(value)) {
37399
- count += countFilterClauses2(value);
37400
- }
37401
- continue;
37402
- }
37403
- count += 1;
37404
- }
37405
- return count;
37406
- };
37407
- var resolveSQLitePushdownMode = (input) => {
37408
- const totalFilterClauseCount = countFilterClauses2(input.filter);
37409
- const pushdownClauseCount = countFilterClauses2(input.pushdownFilter);
37410
- const jsRemainderClauseCount = Math.max(0, totalFilterClauseCount - pushdownClauseCount);
37411
- const pushdownMode = pushdownClauseCount === 0 ? "none" : pushdownClauseCount >= totalFilterClauseCount ? "full" : "partial";
37412
- return {
37413
- jsRemainderClauseCount,
37414
- jsRemainderRatio: totalFilterClauseCount > 0 ? jsRemainderClauseCount / totalFilterClauseCount : undefined,
37415
- pushdownClauseCount,
37416
- pushdownCoverageRatio: totalFilterClauseCount > 0 ? pushdownClauseCount / totalFilterClauseCount : undefined,
37417
- pushdownMode,
37418
- totalFilterClauseCount
37419
- };
37420
- };
37421
- var updateSQLiteLastQueryPlan = (input) => {
37422
- if (!input.nativeDiagnostics) {
37423
- return;
37424
- }
37425
- const pushdown = resolveSQLitePushdownMode({
37426
- filter: input.filter,
37427
- pushdownFilter: input.pushdownFilter
37428
- });
37429
- input.nativeDiagnostics.lastQueryPlan = {
37430
- backfillCount: input.backfillCount,
37431
- candidateBudgetExhausted: input.candidateBudgetExhausted,
37432
- candidateCoverage: summarizeSQLiteCandidateCoverage({
37433
- filteredCandidateCount: input.filteredCandidateCount,
37434
- returnedCount: input.returnedCount,
37435
- topK: input.topK
37436
- }),
37437
- filteredCandidateCount: input.filteredCandidateCount,
37438
- finalSearchK: input.finalSearchK,
37439
- initialSearchK: input.initialSearchK,
37440
- searchExpansionRatio: typeof input.initialSearchK === "number" && typeof input.finalSearchK === "number" && input.initialSearchK > 0 ? input.finalSearchK / input.initialSearchK : undefined,
37441
- jsRemainderClauseCount: pushdown.jsRemainderClauseCount,
37442
- plannerProfileUsed: input.plannerProfileUsed,
37443
- candidateLimitUsed: input.candidateLimitUsed,
37444
- maxBackfillsUsed: input.maxBackfillsUsed,
37445
- minResultsUsed: input.minResultsUsed,
37446
- fillPolicyUsed: input.fillPolicyUsed,
37447
- queryMultiplierUsed: input.queryMultiplierUsed,
37448
- jsRemainderRatio: pushdown.jsRemainderRatio,
37449
- pushdownApplied: pushdown.pushdownClauseCount > 0,
37450
- pushdownClauseCount: pushdown.pushdownClauseCount,
37451
- pushdownCoverageRatio: pushdown.pushdownCoverageRatio,
37452
- pushdownMode: pushdown.pushdownMode,
37453
- queryMode: input.queryMode,
37454
- candidateYieldRatio: typeof input.returnedCount === "number" && typeof input.finalSearchK === "number" && input.finalSearchK > 0 ? input.returnedCount / input.finalSearchK : undefined,
37455
- returnedCount: input.returnedCount,
37456
- backfillLimitReached: input.backfillLimitReached,
37457
- minResultsSatisfied: input.minResultsSatisfied,
37458
- topKFillRatio: typeof input.returnedCount === "number" && input.topK > 0 ? input.returnedCount / input.topK : undefined,
37459
- totalFilterClauseCount: pushdown.totalFilterClauseCount,
37460
- underfilledTopK: input.underfilledTopK
37461
- };
37462
- };
37463
- var markNativeUpsertFailure = (nativeDiagnostics, error) => {
37464
- if (!nativeDiagnostics) {
37465
- return;
37466
- }
37467
- nativeDiagnostics.lastUpsertError = getErrorMessage2(error);
37468
- nativeDiagnostics.active = false;
37469
- nativeDiagnostics.fallbackReason = nativeDiagnostics.lastUpsertError;
37470
- };
37471
- var analyzeSQLiteBackend = (input) => {
37472
- try {
37473
- input.db.exec("PRAGMA optimize");
37474
- input.db.exec(`ANALYZE ${input.tableName}`);
37475
- input.nativeDiagnostics && (input.nativeDiagnostics.lastAnalyzeAt = Date.now());
37476
- if (input.nativeDiagnostics) {
37477
- input.nativeDiagnostics.lastAnalyzeError = undefined;
37478
- }
37479
- refreshSQLiteRuntimeDiagnostics(input.db, input.nativeDiagnostics, input.diagnosticTableName);
37480
- } catch (error) {
37481
- if (input.nativeDiagnostics) {
37482
- input.nativeDiagnostics.lastAnalyzeAt = Date.now();
37483
- input.nativeDiagnostics.lastAnalyzeError = getErrorMessage2(error);
37484
- }
37485
- throw error;
37486
- }
37487
- };
37488
- var initializeNativeBackend = (input) => {
37489
- const {
37490
- db,
37491
- dimensions,
37492
- nativeConfig,
37493
- nativeDiagnostics,
37494
- nativeDistanceMetric,
37495
- nativeResolution,
37496
- nativeTableName
37497
- } = input;
37498
- loadNativeExtension(db, nativeResolution);
37499
- executeNativeInitSql(db, nativeConfig.extensionInitSql);
37500
- createNativeVec0Table(db, nativeTableName, dimensions, nativeDistanceMetric);
37501
- const nativeStatements = createNativeVec0Statements(db, nativeTableName);
37502
- activateNativeDiagnostics(nativeDiagnostics);
37503
- return nativeStatements;
37504
- };
37505
- var createNativeInitializationError = (error, nativeTableName) => new Error(`Failed to initialize sqlite vec0 backend for table "${nativeTableName}". ` + `Install @absolutejs/absolute-rag-sqlite for your platform, set native.extensionPath, or pre-register the sqlite-vec extension in the Database connection. ` + `Details: ${getErrorMessage2(error)}`);
37506
- var initializeNativeBackendSafely = (input) => {
37507
- const { nativeConfig, nativeDiagnostics, nativeResolution, nativeTableName } = input;
37508
- try {
37509
- return initializeNativeBackend(input);
37510
- } catch (error) {
37511
- markNativeLoadFailure(nativeDiagnostics, error, nativeResolution);
37512
- if (nativeConfig.requireAvailable) {
37513
- throw createNativeInitializationError(error, nativeTableName);
37514
- }
37515
- return;
37516
- }
37517
- };
37518
- var fallbackToJsonUpsert = (chunks, jsonStatements) => {
37519
- for (const chunk of chunks) {
37520
- jsonStatements.insert.run(chunk.chunkId, chunk.text, chunk.title ?? null, chunk.source ?? null, toJSONString(chunk.metadata), toVectorText(chunk.vector));
37521
- }
37522
- };
37523
- var upsertNativeChunks = (chunks, nativeStatements) => {
37524
- if (!nativeStatements) {
37525
- throw new Error("Native vector statements unavailable");
37526
- }
37527
- for (const chunk of chunks) {
37528
- nativeStatements.delete.run(chunk.chunkId);
37529
- nativeStatements.insert.run(chunk.chunkId, toVectorText(chunk.vector), chunk.text, chunk.title ?? null, chunk.source ?? null, toJSONString(chunk.metadata));
37530
- }
37531
- };
37532
- var createSQLiteRAGStore = (options = {}) => {
37533
- const dimensions = options.dimensions ?? DEFAULT_DIMENSIONS2;
37534
- const tableName = options.tableName ?? DEFAULT_TABLE_NAME2;
37535
- assertSupportedIdentifier3(tableName);
37536
- const nativeConfig = options.native;
37537
- const nativeTableName = nativeConfig?.tableName ?? `${tableName}${DEFAULT_NATIVE_TABLE_SUFFIX}`;
37538
- if (nativeConfig?.mode === "vec0" && nativeConfig.tableName) {
37539
- assertSupportedIdentifier3(nativeConfig.tableName);
37540
- }
37541
- if (!Number.isInteger(dimensions) || dimensions <= 0) {
37542
- throw new Error(`Invalid dimension "${dimensions}". dimensions must be a positive integer.`);
37543
- }
37544
- const db = options.db ?? new Database(options.path ?? ":memory:");
37545
- const nativeDistanceMetric = nativeConfig?.distanceMetric === "l2" ? "l2" : "cosine";
37546
- const nativeQueryMultiplier = normalizeQueryMultiplier2(nativeConfig?.queryMultiplier);
37547
- const nativeResolution = nativeConfig?.mode === "vec0" ? resolveConfiguredNativeExtension(nativeConfig) : undefined;
37548
- const nativeDiagnostics = nativeConfig?.mode === "vec0" ? {
37549
- active: false,
37550
- available: false,
37551
- distanceMetric: nativeDistanceMetric,
37552
- fallbackReason: describeNativeFallbackReason(nativeResolution),
37553
- mode: nativeConfig.mode,
37554
- requested: true,
37555
- resolution: nativeResolution,
37556
- tableName: nativeTableName
37557
- } : undefined;
37558
- const jsonStatements = createJsonStatements(db, tableName);
37559
- jsonStatements.init();
37560
- let useNative = false;
37561
- let nativeStatements;
37562
- if (nativeConfig?.mode === "vec0") {
37563
- nativeStatements = initializeNativeBackendSafely({
37564
- db,
37565
- dimensions,
37566
- nativeConfig,
37567
- nativeDiagnostics,
37568
- nativeDistanceMetric,
37569
- nativeResolution,
37570
- nativeTableName
37571
- });
37572
- useNative = nativeStatements !== undefined;
37573
- }
37574
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
37575
- const embed = async (input) => {
37576
- input.model;
37577
- if (input.signal?.aborted) {
37578
- throw new DOMException("Aborted", "AbortError");
37579
- }
37580
- if (options.mockEmbedding) {
37581
- return options.mockEmbedding(input.text).then(normalizeVector);
37582
- }
37583
- return normalizeVector([...createRAGVector(input.text, dimensions)]);
37584
- };
37585
- const queryFallback = async (input) => {
37586
- const queryVector = normalizeVector(input.queryVector);
37587
- const pushdownFilter = buildSQLitePushdownFilter(input.filter);
37588
- const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
37589
- chunkId: "chunk_id",
37590
- metadata: "metadata",
37591
- source: "source",
37592
- title: "title"
37593
- });
37594
- const rawRows = toStoredRows(filterPlan ? db.prepare(buildJsonQuerySql(tableName, filterPlan.clause)).all(...filterPlan.params) : jsonStatements.query.all());
37595
- const chunks = mapFilterToRows(rawRows);
37596
- const filtered = mapToRows(queryVector, chunks, input.filter);
37597
- const limited = filtered.slice(0, input.topK);
37598
- updateSQLiteLastQueryPlan({
37599
- backfillCount: 0,
37600
- candidateBudgetExhausted: false,
37601
- filter: input.filter,
37602
- filteredCandidateCount: rawRows.length,
37603
- finalSearchK: rawRows.length,
37604
- initialSearchK: rawRows.length,
37605
- nativeDiagnostics,
37606
- pushdownFilter,
37607
- plannerProfileUsed: input.plannerProfile,
37608
- queryMultiplierUsed: input.queryMultiplier,
37609
- queryMode: "json_fallback",
37610
- returnedCount: limited.length,
37611
- topK: input.topK,
37612
- underfilledTopK: limited.length < input.topK
37613
- });
37614
- return limited.map(({ chunk, score }) => ({
37615
- chunkId: chunk.chunkId,
37616
- chunkText: chunk.text,
37617
- embedding: chunk.vector,
37618
- metadata: chunk.metadata,
37619
- score,
37620
- source: chunk.source,
37621
- title: chunk.title
37622
- }));
37623
- };
37624
- const queryNative = async (input) => {
37625
- if (!nativeStatements) {
37626
- throw new Error("Native vector backend is not available");
37627
- }
37628
- const queryMultiplier = normalizeQueryMultiplier2(input.queryMultiplier ?? nativeConfig?.queryMultiplier);
37629
- const maxBackfills = normalizeMaxBackfills2(input.maxBackfills);
37630
- const minResults = normalizeMinResults2(input.minResults, input.topK);
37631
- const fillTarget = resolveFillTarget2({
37632
- fillPolicy: input.fillPolicy,
37633
- minResults,
37634
- topK: input.topK
37635
- });
37636
- const queryVector = normalizeVector(input.queryVector);
37637
- const queryVectorText = toVectorText(queryVector);
37638
- const pushdownFilter = buildSQLitePushdownFilter(input.filter);
37639
- const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
37640
- chunkId: "chunk_id",
37641
- metadata: "metadata",
37642
- source: "source",
37643
- title: "title"
37644
- });
37645
- const filteredCandidateCount = getFilteredSQLiteCandidateCount(db, tableName, filterPlan);
37646
- const candidateLimit = resolveAdaptiveNativeCandidateLimit({
37647
- defaultCandidateLimit: RAG_NATIVE_QUERY_CANDIDATE_LIMIT,
37648
- explicitCandidateLimit: input.candidateLimit,
37649
- filteredCandidateCount,
37650
- plannerProfile: input.plannerProfile,
37651
- queryMultiplier,
37652
- topK: input.topK
37653
- });
37654
- const searchK = planNativeCandidateSearchK({
37655
- candidateLimit,
37656
- filteredCandidateCount,
37657
- queryMultiplier,
37658
- topK: input.topK
37659
- });
37660
- if (searchK === 0) {
37661
- return [];
37662
- }
37663
- const runNativeQuery = (candidateK) => toNativeStoredRows(filterPlan ? db.prepare(buildNativeQuerySql(nativeTableName, filterPlan.clause)).all(queryVectorText, candidateK, ...filterPlan.params) : nativeStatements.query.all(queryVectorText, candidateK));
37664
- let currentSearchK = searchK;
37665
- let backfillCount = 0;
37666
- let candidateBudgetExhausted = false;
37667
- let backfillLimitReached = false;
37668
- let mapped = [];
37669
- for (;; ) {
37670
- const rawRows = runNativeQuery(currentSearchK);
37671
- mapped = rawRows.map((row) => {
37672
- const chunk = {
37673
- chunkId: row.chunk_id,
37674
- metadata: parseMetadata2(row.metadata),
37675
- source: row.source ?? undefined,
37676
- text: row.chunk_text,
37677
- title: row.title ?? undefined,
37678
- vector: parseVector(row.embedding)
37679
- };
37680
- return {
37681
- chunk,
37682
- score: normalizeDistance2(row.distance, nativeDistanceMetric)
37683
- };
37684
- }).filter(({ chunk }) => matchesFilter2(chunk, input.filter)).map((entry) => ({
37685
- chunkId: entry.chunk.chunkId,
37686
- chunkText: entry.chunk.text,
37687
- embedding: entry.chunk.vector,
37688
- metadata: entry.chunk.metadata,
37689
- score: entry.score,
37690
- source: entry.chunk.source,
37691
- title: entry.chunk.title
37692
- })).sort((left, right) => right.score - left.score);
37693
- if (mapped.length >= fillTarget.targetResults) {
37694
- break;
37695
- }
37696
- const nextSearchK = planNativeCandidateSearchBackfillK({
37697
- backfillCount,
37698
- candidateLimit,
37699
- currentSearchK,
37700
- filteredCandidateCount,
37701
- maxBackfills
37702
- });
37703
- if (nextSearchK <= currentSearchK) {
37704
- backfillLimitReached = typeof maxBackfills === "number" && backfillCount >= maxBackfills && mapped.length < fillTarget.targetResults;
37705
- candidateBudgetExhausted = mapped.length < fillTarget.targetResults;
37706
- break;
37707
- }
37708
- currentSearchK = nextSearchK;
37709
- backfillCount += 1;
37710
- }
37711
- updateSQLiteLastQueryPlan({
37712
- backfillCount,
37713
- backfillLimitReached,
37714
- candidateBudgetExhausted,
37715
- filter: input.filter,
37716
- filteredCandidateCount,
37717
- finalSearchK: currentSearchK,
37718
- initialSearchK: searchK,
37719
- nativeDiagnostics,
37720
- pushdownFilter,
37721
- plannerProfileUsed: input.plannerProfile,
37722
- candidateLimitUsed: candidateLimit,
37723
- maxBackfillsUsed: maxBackfills,
37724
- minResultsUsed: minResults,
37725
- fillPolicyUsed: fillTarget.fillPolicy,
37726
- queryMultiplierUsed: queryMultiplier,
37727
- queryMode: "native_vec0",
37728
- returnedCount: Math.min(mapped.length, input.topK),
37729
- minResultsSatisfied: mapped.length >= minResults,
37730
- topK: input.topK,
37731
- underfilledTopK: mapped.length < input.topK
37732
- });
37733
- return mapped.slice(0, input.topK);
37734
- };
37735
- const query = async (input) => {
37736
- if (!useNative) {
37737
- return queryFallback(input);
37738
- }
37739
- try {
37740
- return await queryNative(input);
37741
- } catch (error) {
37742
- markNativeQueryFailure(nativeDiagnostics, error);
37743
- if (nativeConfig?.requireAvailable) {
37744
- throw new Error(`Native vector query failed for table "${nativeTableName}". ${getErrorMessage2(error)}`, { cause: error });
37745
- }
37746
- return queryFallback(input);
37747
- }
37748
- };
37749
- const queryLexical = async (input) => {
37750
- const pushdownFilter = buildSQLitePushdownFilter(input.filter);
37751
- const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
37752
- chunkId: "chunk_id",
37753
- metadata: "metadata",
37754
- source: "source",
37755
- title: "title"
37756
- });
37757
- const rawRows = toStoredRows(filterPlan ? db.prepare(buildJsonQuerySql(tableName, filterPlan.clause)).all(...filterPlan.params) : jsonStatements.query.all());
37758
- const chunks = mapFilterToRows(rawRows).filter((chunk) => matchesFilter2(chunk, input.filter));
37759
- const ranked = rankRAGLexicalMatches(input.query, chunks);
37760
- return ranked.slice(0, input.topK).map(({ result, score }) => ({
37761
- chunkId: result.chunkId,
37762
- chunkText: result.text,
37763
- metadata: result.metadata,
37764
- score,
37765
- source: result.source,
37766
- title: result.title
37767
- }));
37768
- };
37769
- const upsert = async (input) => {
37770
- const chunks = input.chunks.length > 0 ? await Promise.all(input.chunks.map(async (chunk) => ({
37771
- chunkId: chunk.chunkId,
37772
- metadata: chunk.metadata,
37773
- source: chunk.source,
37774
- text: chunk.text,
37775
- title: chunk.title,
37776
- vector: chunk.embedding ? normalizeVector(chunk.embedding) : normalizeVector(await embed({ text: chunk.text }))
37777
- }))) : [];
37778
- if (!useNative) {
37779
- fallbackToJsonUpsert(chunks, jsonStatements);
37780
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, tableName);
37781
- return;
37782
- }
37783
- try {
37784
- upsertNativeChunks(chunks, nativeStatements);
37785
- } catch (error) {
37786
- markNativeUpsertFailure(nativeDiagnostics, error);
37787
- if (nativeConfig?.requireAvailable) {
37788
- throw new Error(`Native vector upsert failed for table "${nativeTableName}". ${getErrorMessage2(error)}`, { cause: error });
37789
- }
37790
- useNative = false;
37791
- fallbackToJsonUpsert(chunks, jsonStatements);
37792
- }
37793
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
37794
- };
37795
- const count = async (input = {}) => {
37796
- const filter = input.filter;
37797
- const chunkIds = input.chunkIds;
37798
- const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
37799
- const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
37800
- if (!hasFilter && !hasChunkIds) {
37801
- return getChunkCountFromSql(db, buildJsonCountSql(tableName), []);
37802
- }
37803
- if (hasFilter && !hasChunkIds) {
37804
- return getSQLiteCandidateChunkIdsByFilter(db, tableName, filter, jsonStatements).length;
37805
- }
37806
- return getSQLiteCandidateChunkIds(db, tableName, jsonStatements, {
37807
- chunkIds,
37808
- filter
37809
- }).length;
37810
- };
37811
- const remove = async (input = {}) => {
37812
- const filter = input.filter;
37813
- const chunkIds = input.chunkIds;
37814
- const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
37815
- const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
37816
- if (!hasFilter && !hasChunkIds) {
37817
- return 0;
37818
- }
37819
- const toDelete = getSQLiteCandidateChunkIds(db, tableName, jsonStatements, {
37820
- chunkIds,
37821
- filter
37822
- });
37823
- if (toDelete.length === 0) {
37824
- return 0;
37825
- }
37826
- for (const chunkId of toDelete) {
37827
- jsonStatements.delete.run(chunkId);
37828
- }
37829
- if (!useNative || !nativeStatements) {
37830
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, tableName);
37831
- return toDelete.length;
37832
- }
37833
- for (const chunkId of toDelete) {
37834
- nativeStatements.delete.run(chunkId);
37835
- }
37836
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
37837
- return toDelete.length;
37838
- };
37839
- const clear = () => {
37840
- jsonStatements.clear.run();
37841
- if (!useNative || !nativeStatements) {
37842
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, tableName);
37843
- return;
37844
- }
37845
- try {
37846
- nativeStatements.clear.run();
37847
- } catch {
37848
- jsonStatements.clear.run();
37849
- }
37850
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
37851
- };
37852
- const analyze = () => {
37853
- analyzeSQLiteBackend({
37854
- db,
37855
- diagnosticTableName: useNative ? nativeTableName : tableName,
37856
- nativeDiagnostics,
37857
- tableName
37858
- });
37859
- };
37860
- return {
37861
- analyze,
37862
- clear,
37863
- embed,
37864
- query,
37865
- queryLexical,
37866
- count,
37867
- delete: remove,
37868
- upsert,
37869
- getCapabilities: () => createSQLiteCapabilities(useNative),
37870
- getStatus: () => createSQLiteStatus(dimensions, nativeDiagnostics, useNative)
37871
- };
37872
- };
37873
35673
  export {
37874
35674
  xaiEmbeddings,
37875
35675
  validateRAGEmbeddingDimensions,
@@ -37889,8 +35689,6 @@ export {
37889
35689
  resolveRAGQueryTransform,
37890
35690
  resolveRAGHybridSearchOptions,
37891
35691
  resolveRAGEmbeddingProvider,
37892
- resolveAbsoluteSQLiteVecExtensionPath,
37893
- resolveAbsoluteSQLiteVec,
37894
35692
  reorderRAGEvaluationSuiteCases,
37895
35693
  removeRAGEvaluationSuiteCaseHardNegative,
37896
35694
  removeRAGEvaluationSuiteCase,
@@ -37977,7 +35775,6 @@ export {
37977
35775
  deepseekEmbeddings,
37978
35776
  createVoyageRAGReranker,
37979
35777
  createTextFileExtractor,
37980
- createSQLiteRAGStore,
37981
35778
  createRAGVector,
37982
35779
  createRAGUrlSyncSource,
37983
35780
  createRAGSyncScheduler,
@@ -38069,7 +35866,6 @@ export {
38069
35866
  createRAGAdaptiveNativePlannerBenchmarkSuite,
38070
35867
  createRAGAdaptiveNativePlannerBenchmarkSnapshot,
38071
35868
  createRAGAccessControl,
38072
- createPostgresRAGStore,
38073
35869
  createPDFFileExtractor,
38074
35870
  createOfficeDocumentExtractor,
38075
35871
  createLegacyDocumentExtractor,
@@ -38164,5 +35960,5 @@ export {
38164
35960
  addRAGEvaluationSuiteCase
38165
35961
  };
38166
35962
 
38167
- //# debugId=BD3325EB562F864664756E2164756E21
35963
+ //# debugId=28A2EA681F0DA49464756E2164756E21
38168
35964
  //# sourceMappingURL=index.js.map