@absolutejs/rag 0.0.19 → 0.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -35670,2204 +35670,176 @@ var createInMemoryRAGStore = (options = {}) => {
35670
35670
  getStatus: () => createInMemoryStatus(dimensions)
35671
35671
  };
35672
35672
  };
35673
- // src/adapters/queryPlanning.ts
35674
- var planNativeCandidateSearchK = (input) => {
35675
- const base = Math.min(Math.max(input.topK * input.queryMultiplier, input.topK), input.candidateLimit);
35676
- if (input.filteredCandidateCount === undefined || !Number.isFinite(input.filteredCandidateCount)) {
35677
- return base;
35678
- }
35679
- const filtered = Math.max(0, Math.floor(input.filteredCandidateCount));
35680
- if (filtered === 0) {
35681
- return 0;
35682
- }
35683
- return Math.min(base, filtered);
35684
- };
35685
- var resolveAdaptiveNativeCandidateLimit = (input) => {
35686
- const clamp = (value) => Math.min(input.defaultCandidateLimit, Math.max(1, Math.floor(value)));
35687
- const filteredCap = typeof input.filteredCandidateCount === "number" && Number.isFinite(input.filteredCandidateCount) ? Math.max(0, Math.floor(input.filteredCandidateCount)) : undefined;
35688
- if (typeof input.explicitCandidateLimit === "number" && Number.isFinite(input.explicitCandidateLimit)) {
35689
- return filteredCap === undefined ? clamp(input.explicitCandidateLimit) : Math.min(clamp(input.explicitCandidateLimit), filteredCap);
35690
- }
35691
- const baseFloor = Math.max(input.topK, input.topK * Math.max(1, Math.floor(input.queryMultiplier)));
35692
- let tuned = input.plannerProfile === "latency" ? clamp(Math.max(input.topK * 2, baseFloor)) : input.plannerProfile === "recall" ? clamp(Math.max(input.topK * 12, baseFloor * 4)) : clamp(Math.max(input.topK * 6, baseFloor * 2));
35693
- if (filteredCap !== undefined) {
35694
- if (filteredCap === 0) {
35695
- return 0;
35696
- }
35697
- tuned = Math.min(tuned, filteredCap);
35698
- }
35699
- return tuned;
35700
- };
35701
- var planNativeCandidateSearchBackfillK = (input) => {
35702
- if (typeof input.maxBackfills === "number" && Number.isFinite(input.maxBackfills) && (input.backfillCount ?? 0) >= Math.max(0, Math.floor(input.maxBackfills))) {
35703
- return input.currentSearchK;
35704
- }
35705
- const cappedLimit = input.filteredCandidateCount === undefined || !Number.isFinite(input.filteredCandidateCount) ? input.candidateLimit : Math.min(input.candidateLimit, Math.max(0, Math.floor(input.filteredCandidateCount)));
35706
- if (input.currentSearchK >= cappedLimit) {
35707
- return input.currentSearchK;
35708
- }
35709
- return Math.min(cappedLimit, Math.max(input.currentSearchK + 1, input.currentSearchK * 2));
35710
- };
35711
- var summarizeSQLiteCandidateCoverage = (input) => {
35712
- const basis = typeof input.filteredCandidateCount === "number" && Number.isFinite(input.filteredCandidateCount) ? Math.max(0, Math.floor(input.filteredCandidateCount)) : typeof input.returnedCount === "number" && Number.isFinite(input.returnedCount) ? Math.max(0, Math.floor(input.returnedCount)) : 0;
35713
- if (basis === 0) {
35714
- return "empty";
35715
- }
35716
- if (basis < input.topK) {
35717
- return "under_target";
35718
- }
35719
- if (basis >= input.topK * 3) {
35720
- return "broad";
35721
- }
35722
- return "target_sized";
35723
- };
35724
-
35725
- // src/adapters/postgres.ts
35726
- var DEFAULT_DIMENSIONS = RAG_VECTOR_DIMENSIONS_DEFAULT;
35727
- var DEFAULT_TABLE_NAME = "rag_chunks";
35728
- var DEFAULT_SCHEMA_NAME = "public";
35729
- var DEFAULT_QUERY_MULTIPLIER = 4;
35730
- var MAX_QUERY_MULTIPLIER = 16;
35731
- var DEFAULT_POSTGRES_INDEX_TYPE = "hnsw";
35732
- var DEFAULT_POSTGRES_IVFFLAT_LISTS = 100;
35733
- var DEFAULT_POSTGRES_HNSW_M = 16;
35734
- var DEFAULT_POSTGRES_HNSW_EF_CONSTRUCTION = 64;
35735
- var IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
35736
- var FILTER_PATH_SEGMENT_RE = /^[a-zA-Z0-9_]+$/;
35737
- var isObjectFilterRecord = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
35738
- var isNestedFilterArray2 = (value) => Array.isArray(value) && value.every((entry) => isObjectFilterRecord(entry));
35739
- var isOperatorFilterRecord = (value) => isObjectFilterRecord(value) && Object.keys(value).some((key) => key.startsWith("$"));
35740
- var countFilterClauses = (filter) => {
35741
- if (!filter) {
35742
- return 0;
35743
- }
35744
- let count = 0;
35745
- for (const [key, value] of Object.entries(filter)) {
35746
- if (key === "$and" || key === "$or") {
35747
- if (isNestedFilterArray2(value)) {
35748
- count += value.reduce((total, entry) => total + countFilterClauses(entry), 0);
35749
- }
35750
- continue;
35751
- }
35752
- if (key === "$not") {
35753
- if (isObjectFilterRecord(value)) {
35754
- count += countFilterClauses(value);
35755
- }
35756
- continue;
35757
- }
35758
- count += 1;
35759
- }
35760
- return count;
35761
- };
35762
- var toPostgresJsonPath = (key) => {
35763
- const segments = key.split(".").filter(Boolean);
35764
- if (segments.length === 0 || !segments.every((segment) => FILTER_PATH_SEGMENT_RE.test(segment))) {
35765
- return null;
35766
- }
35767
- return segments;
35768
- };
35769
- var toPostgresFilterBinding = (value) => {
35770
- if (typeof value === "string" || typeof value === "number" || typeof value === "boolean" || value === null) {
35771
- return value;
35772
- }
35773
- return;
35774
- };
35775
- var buildPostgresJsonbScalarEquality = (input) => {
35776
- const comparison = input.comparison ?? "=";
35777
- return comparison === "=" ? `jsonb_typeof(${input.valueSql}) = 'null'` : `coalesce(jsonb_typeof(${input.valueSql}), 'missing') <> 'null'`;
35778
- };
35779
- var buildPostgresMetadataScalarEquality = (input) => input.value === null ? buildPostgresJsonbScalarEquality({
35780
- comparison: input.comparison,
35781
- valueSql: input.valueSql
35782
- }) : `${input.actualSql} ${input.comparison ?? "="} ${input.bind(String(input.value))}`;
35783
- var buildPostgresFilterPlan = (filter, startIndex = 0) => {
35784
- if (!filter) {
35785
- return { clause: "", params: [] };
35786
- }
35787
- const params = [];
35788
- const bind = (value) => {
35789
- params.push(value);
35790
- return `$${params.length + startIndex}`;
35791
- };
35792
- const build = (entry) => {
35793
- const clauses = [];
35794
- for (const [key, value] of Object.entries(entry)) {
35795
- if (key === "$and" || key === "$or") {
35796
- if (!isNestedFilterArray2(value) || value.length === 0) {
35797
- return null;
35798
- }
35799
- const nested = value.map((item) => build(item));
35800
- if (nested.some((item) => item === null)) {
35801
- return null;
35802
- }
35803
- clauses.push(`(${nested.filter((item) => Boolean(item)).join(key === "$and" ? " AND " : " OR ")})`);
35804
- continue;
35805
- }
35806
- if (key === "$not") {
35807
- if (!isObjectFilterRecord(value)) {
35808
- return null;
35809
- }
35810
- const nested = build(value);
35811
- if (!nested) {
35812
- return null;
35813
- }
35814
- clauses.push(`NOT (${nested})`);
35815
- continue;
35816
- }
35817
- const isScalarField = key === "chunkId" || key === "source" || key === "title";
35818
- const jsonPath = isScalarField ? null : toPostgresJsonPath(key);
35819
- if (!isScalarField && !jsonPath) {
35820
- return null;
35821
- }
35822
- let actualSql;
35823
- let metadataPathSegments = [];
35824
- let metadataValueSql;
35825
- if (isScalarField) {
35826
- actualSql = key === "chunkId" ? "chunk_id" : key;
35827
- } else {
35828
- metadataPathSegments = jsonPath ?? [];
35829
- actualSql = `jsonb_extract_path_text(metadata, ${metadataPathSegments.map((segment) => `'${segment}'`).join(", ")})`;
35830
- metadataValueSql = `metadata #> '{${metadataPathSegments.join(",")}}'`;
35831
- }
35832
- if (!isOperatorFilterRecord(value)) {
35833
- const binding = toPostgresFilterBinding(value);
35834
- if (binding === undefined) {
35835
- return null;
35836
- }
35837
- clauses.push(isScalarField ? `${actualSql} = ${bind(String(binding))}` : buildPostgresMetadataScalarEquality({
35838
- actualSql,
35839
- bind,
35840
- value: binding,
35841
- valueSql: metadataValueSql
35842
- }));
35843
- continue;
35844
- }
35845
- const operatorClauses = Object.entries(value).map(([operator, expected]) => {
35846
- switch (operator) {
35847
- case "$exists":
35848
- return isScalarField ? expected ? `${actualSql} IS NOT NULL` : `${actualSql} IS NULL` : expected ? `${metadataValueSql} IS NOT NULL` : `${metadataValueSql} IS NULL`;
35849
- case "$in": {
35850
- if (!Array.isArray(expected) || expected.length === 0) {
35851
- return null;
35852
- }
35853
- const bindings = expected.map((entry2) => toPostgresFilterBinding(entry2)).filter((entry2) => entry2 !== undefined);
35854
- if (bindings.length !== expected.length) {
35855
- return null;
35856
- }
35857
- return isScalarField ? `${actualSql} IN (${bindings.map((entry2) => bind(String(entry2))).join(", ")})` : `(${bindings.map((entry2) => buildPostgresMetadataScalarEquality({
35858
- actualSql,
35859
- bind,
35860
- value: entry2,
35861
- valueSql: metadataValueSql
35862
- })).join(" OR ")})`;
35863
- }
35864
- case "$ne": {
35865
- const binding = toPostgresFilterBinding(expected);
35866
- return binding === undefined ? null : isScalarField ? `${actualSql} <> ${bind(String(binding))}` : buildPostgresMetadataScalarEquality({
35867
- actualSql,
35868
- bind,
35869
- comparison: "<>",
35870
- value: binding,
35871
- valueSql: metadataValueSql
35872
- });
35873
- }
35874
- case "$gt":
35875
- case "$gte":
35876
- case "$lt":
35877
- case "$lte": {
35878
- if (typeof expected !== "number" || !Number.isFinite(expected)) {
35879
- return null;
35880
- }
35881
- const comparison = operator === "$gt" ? ">" : operator === "$gte" ? ">=" : operator === "$lt" ? "<" : "<=";
35882
- return `((${actualSql}) ~ '^-?[0-9]+(\\.[0-9]+)?$' AND (${actualSql})::double precision ${comparison} ${bind(expected)})`;
35883
- }
35884
- case "$contains":
35885
- if (isScalarField) {
35886
- return null;
35887
- }
35888
- if (toPostgresFilterBinding(expected) === undefined) {
35889
- return null;
35890
- }
35891
- return `(${metadataValueSql} IS NOT NULL AND ${metadataValueSql} ? ${bind(String(expected))})`;
35892
- case "$containsAny":
35893
- case "$containsAll": {
35894
- if (isScalarField || !Array.isArray(expected)) {
35895
- return null;
35896
- }
35897
- const values = expected.map((entry2) => toPostgresFilterBinding(entry2)).filter((entry2) => entry2 !== undefined);
35898
- if (values.length === 0 || values.length !== expected.length) {
35899
- return null;
35900
- }
35901
- const sqlArray = `ARRAY[${values.map((value2) => bind(String(value2))).join(", ")}]::text[]`;
35902
- return `(${metadataValueSql} IS NOT NULL AND ${metadataValueSql} ${operator === "$containsAny" ? "?|" : "?&"} ${sqlArray})`;
35903
- }
35904
- default:
35905
- return null;
35906
- }
35907
- });
35908
- if (operatorClauses.some((clause2) => clause2 === null)) {
35909
- return null;
35910
- }
35911
- clauses.push(operatorClauses.filter((clause2) => Boolean(clause2)).map((clause2) => `(${clause2})`).join(" AND "));
35912
- }
35913
- return clauses.length > 0 ? clauses.map((clause2) => `(${clause2})`).join(" AND ") : "";
35914
- };
35915
- const clause = build(filter);
35916
- return clause === null || clause.trim().length === 0 ? null : { clause, params };
35917
- };
35918
- var buildPostgresPushdownFilter = (filter) => {
35919
- if (!filter) {
35920
- return;
35921
- }
35922
- const hasPushdownFilterPlan = (entry) => {
35923
- const plan = buildPostgresFilterPlan(entry);
35924
- return plan !== null && Boolean(plan.clause) && plan.clause.trim().length > 0;
35925
- };
35926
- const hasPushdownFilterPlanEntry = (entry) => {
35927
- if (!isObjectFilterRecord(entry)) {
35928
- return false;
35929
- }
35930
- return hasPushdownFilterPlan(entry);
35931
- };
35932
- const nextEntries = [];
35933
- for (const [key, value] of Object.entries(filter)) {
35934
- if (key === "$and" || key === "$or") {
35935
- if (!isNestedFilterArray2(value)) {
35936
- continue;
35937
- }
35938
- const nested = value.map((entry) => buildPostgresPushdownFilter(entry)).filter((entry) => hasPushdownFilterPlanEntry(entry));
35939
- if (nested.length > 0) {
35940
- nextEntries.push([key, nested]);
35941
- }
35942
- continue;
35943
- }
35944
- if (key === "$not") {
35945
- if (!isObjectFilterRecord(value)) {
35946
- continue;
35947
- }
35948
- const nested = buildPostgresPushdownFilter(value);
35949
- if (hasPushdownFilterPlanEntry(nested)) {
35950
- nextEntries.push([key, nested]);
35951
- }
35952
- continue;
35953
- }
35954
- if (Array.isArray(value) || isOperatorFilterRecord(value) && Object.keys(value).some((operator) => !(operator === "$exists" || operator === "$in" || operator === "$contains" || operator === "$containsAny" || operator === "$containsAll" || operator === "$ne" || operator === "$gt" || operator === "$gte" || operator === "$lt" || operator === "$lte"))) {
35955
- continue;
35956
- }
35957
- const isScalarColumnKey = ["chunkId", "source", "title"].includes(key);
35958
- const jsonPath = isScalarColumnKey ? null : toPostgresJsonPath(key);
35959
- if (!isScalarColumnKey && !jsonPath) {
35960
- continue;
35961
- }
35962
- if (!hasPushdownFilterPlan({ [key]: value })) {
35963
- continue;
35964
- }
35965
- nextEntries.push([key, value]);
35966
- }
35967
- return nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined;
35968
- };
35969
- var resolvePostgresPushdownMode = (input) => {
35970
- const totalFilterClauseCount = countFilterClauses(input.filter);
35971
- const pushdownClauseCount = countFilterClauses(input.pushdownFilter);
35972
- const jsRemainderClauseCount = Math.max(0, totalFilterClauseCount - pushdownClauseCount);
35973
- const pushdownMode = pushdownClauseCount === 0 ? "none" : pushdownClauseCount >= totalFilterClauseCount ? "full" : "partial";
35974
- return {
35975
- jsRemainderClauseCount,
35976
- jsRemainderRatio: totalFilterClauseCount > 0 ? jsRemainderClauseCount / totalFilterClauseCount : undefined,
35977
- pushdownClauseCount,
35978
- pushdownCoverageRatio: totalFilterClauseCount > 0 ? pushdownClauseCount / totalFilterClauseCount : undefined,
35979
- pushdownMode,
35980
- totalFilterClauseCount
35981
- };
35982
- };
35983
- var assertSupportedIdentifier2 = (name) => {
35984
- if (!IDENTIFIER_RE.test(name)) {
35985
- throw new Error(`Invalid identifier "${name}". Only alphanumeric and underscore names are allowed.`);
35986
- }
35987
- };
35988
- var normalizePostgresIndexType = (value) => {
35989
- if (value === undefined) {
35990
- return DEFAULT_POSTGRES_INDEX_TYPE;
35991
- }
35992
- if (value === "none" || value === "hnsw" || value === "ivfflat") {
35993
- return value;
35994
- }
35995
- throw new Error(`Invalid postgres index type "${String(value)}". Expected "none", "hnsw", or "ivfflat".`);
35996
- };
35997
- var normalizePositiveInteger = (value, fallback) => {
35998
- if (value === undefined || !Number.isFinite(value)) {
35999
- return fallback;
36000
- }
36001
- return Math.max(1, Math.floor(value));
36002
- };
36003
- var getPostgresIndexOperatorClass = (metric) => metric === "cosine" ? "vector_cosine_ops" : metric === "inner_product" ? "vector_ip_ops" : "vector_l2_ops";
36004
- var getPostgresIndexName = (qualifiedTableName, indexType) => indexType === "none" ? undefined : `${qualifiedTableName.replace(".", "_")}_embedding_${indexType}_idx`;
36005
- var buildPostgresIndexSql = (input) => {
36006
- if (input.indexType === "none") {
36007
- return;
36008
- }
36009
- const opclass = getPostgresIndexOperatorClass(input.distanceMetric);
36010
- const indexName = getPostgresIndexName(input.qualifiedTableName, input.indexType);
36011
- const optionsSql = input.indexType === "hnsw" ? ` with (m = ${input.hnswM}, ef_construction = ${input.hnswEfConstruction})` : ` with (lists = ${input.indexLists})`;
36012
- const createPrefix = input.ifNotExists === false ? "create index" : "create index if not exists";
36013
- return `${createPrefix} ${indexName} on ${input.qualifiedTableName} using ${input.indexType} (embedding ${opclass})${optionsSql}`;
36014
- };
36015
- var normalizeQueryMultiplier = (value) => {
36016
- if (value === undefined || !Number.isFinite(value)) {
36017
- return DEFAULT_QUERY_MULTIPLIER;
36018
- }
36019
- return Math.min(MAX_QUERY_MULTIPLIER, Math.max(1, Math.floor(value)));
36020
- };
36021
- var normalizeMaxBackfills = (value) => {
36022
- if (value === undefined || !Number.isFinite(value)) {
36023
- return;
36024
- }
36025
- return Math.max(0, Math.floor(value));
36026
- };
36027
- var normalizeMinResults = (value, topK) => {
36028
- if (value === undefined || !Number.isFinite(value)) {
36029
- return topK;
36030
- }
36031
- return Math.min(topK, Math.max(1, Math.floor(value)));
36032
- };
36033
- var resolveFillTarget = (input) => {
36034
- const fillPolicy = input.fillPolicy ?? "satisfy_min_results";
36035
- return {
36036
- fillPolicy,
36037
- targetResults: fillPolicy === "strict_topk" ? input.topK : input.minResults
36038
- };
36039
- };
36040
- var toQualifiedTableName = (schemaName, tableName) => `${schemaName}.${tableName}`;
36041
- var toVectorLiteral = (vector) => `[${vector.join(",")}]`;
36042
- var parseMetadata = (value) => {
36043
- if (typeof value === "string") {
36044
- try {
36045
- const parsed = JSON.parse(value);
36046
- if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
36047
- return parsed;
36048
- }
36049
- } catch {
36050
- return;
36051
- }
36052
- }
36053
- if (!value || typeof value !== "object" || Array.isArray(value)) {
36054
- return;
36055
- }
36056
- return value;
36057
- };
36058
- var parseVectorText = (value) => {
36059
- if (!value) {
36060
- return [];
36061
- }
36062
- const normalized = value.trim();
36063
- const wrapped = normalized.startsWith("[") ? normalized : `[${normalized.replace(/[()]/g, "")}]`;
36064
- try {
36065
- const parsed = JSON.parse(wrapped);
36066
- return Array.isArray(parsed) ? parsed.filter((entry) => typeof entry === "number" && Number.isFinite(entry)) : [];
36067
- } catch {
36068
- return [];
36069
- }
36070
- };
36071
- var parseCountValue = (value) => {
36072
- if (typeof value === "number" && Number.isFinite(value)) {
36073
- return value;
36074
- }
36075
- if (typeof value === "bigint") {
36076
- return Number(value);
36077
- }
36078
- if (typeof value === "string") {
36079
- const parsed = Number(value);
36080
- return Number.isFinite(parsed) ? parsed : 0;
36081
- }
36082
- return 0;
36083
- };
36084
- var parseBooleanValue = (value) => {
36085
- if (typeof value === "boolean") {
36086
- return value;
36087
- }
36088
- if (typeof value === "number") {
36089
- return value !== 0;
36090
- }
36091
- if (typeof value === "bigint") {
36092
- return value !== 0n;
36093
- }
36094
- if (typeof value === "string") {
36095
- const normalized = value.trim().toLowerCase();
36096
- return normalized === "true" || normalized === "t" || normalized === "1";
36097
- }
36098
- return false;
36099
- };
36100
- var refreshPostgresRuntimeDiagnostics = async (db, nativeDiagnostics, input) => {
36101
- try {
36102
- const rows = await db.unsafe(`select
36103
- c.reltuples::bigint as estimated_row_count,
36104
- pg_relation_size($1::regclass) as table_bytes,
36105
- pg_indexes_size($1::regclass) as index_bytes,
36106
- pg_total_relation_size($1::regclass) as total_bytes,
36107
- exists(
36108
- select 1
36109
- from pg_indexes
36110
- where schemaname = $2
36111
- and tablename = $3
36112
- and indexname = $4
36113
- ) as index_present
36114
- from pg_class c
36115
- join pg_namespace n on n.oid = c.relnamespace
36116
- where n.nspname = $2
36117
- and c.relname = $3
36118
- limit 1`, [
36119
- input.qualifiedTableName,
36120
- input.schemaName,
36121
- input.tableName,
36122
- input.indexName ?? ""
36123
- ]);
36124
- const row = rows[0];
36125
- nativeDiagnostics.indexName = input.indexName;
36126
- nativeDiagnostics.indexPresent = input.indexName ? parseBooleanValue(row?.index_present) : undefined;
36127
- nativeDiagnostics.estimatedRowCount = parseCountValue(row?.estimated_row_count);
36128
- nativeDiagnostics.tableBytes = parseCountValue(row?.table_bytes);
36129
- nativeDiagnostics.indexBytes = parseCountValue(row?.index_bytes);
36130
- nativeDiagnostics.totalBytes = parseCountValue(row?.total_bytes);
36131
- nativeDiagnostics.lastHealthCheckAt = Date.now();
36132
- nativeDiagnostics.lastHealthError = undefined;
36133
- } catch (error) {
36134
- nativeDiagnostics.lastHealthCheckAt = Date.now();
36135
- nativeDiagnostics.lastHealthError = error instanceof Error ? error.message : String(error);
36136
- }
36137
- };
36138
- var analyzePostgresTable = async (db, nativeDiagnostics, input) => {
36139
- try {
36140
- await db.unsafe(`analyze ${input.qualifiedTableName}`);
36141
- nativeDiagnostics.lastAnalyzeAt = Date.now();
36142
- nativeDiagnostics.lastAnalyzeError = undefined;
36143
- await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, input);
36144
- } catch (error) {
36145
- nativeDiagnostics.lastAnalyzeAt = Date.now();
36146
- nativeDiagnostics.lastAnalyzeError = error instanceof Error ? error.message : String(error);
36147
- throw error;
36148
- }
36149
- };
36150
- var rebuildPostgresNativeIndex = async (db, nativeDiagnostics, input) => {
36151
- if (!input.indexName || input.indexType === "none") {
36152
- throw new Error("Postgres native index rebuild is not configured");
36153
- }
36154
- try {
36155
- await db.unsafe(`drop index if exists ${input.indexName}`);
36156
- await db.unsafe(buildPostgresIndexSql({
36157
- distanceMetric: input.distanceMetric,
36158
- hnswEfConstruction: input.hnswEfConstruction,
36159
- hnswM: input.hnswM,
36160
- ifNotExists: false,
36161
- indexLists: input.indexLists,
36162
- indexType: input.indexType,
36163
- qualifiedTableName: input.qualifiedTableName
36164
- }));
36165
- nativeDiagnostics.lastReindexAt = Date.now();
36166
- nativeDiagnostics.lastReindexError = undefined;
36167
- await analyzePostgresTable(db, nativeDiagnostics, input);
36168
- } catch (error) {
36169
- nativeDiagnostics.lastReindexAt = Date.now();
36170
- nativeDiagnostics.lastReindexError = error instanceof Error ? error.message : String(error);
36171
- throw error;
36172
- }
36173
- };
36174
- var getPostgresChunkIdsByChunkIds = async (db, qualifiedTableName, chunkIds) => {
36175
- const normalized = [...new Set(chunkIds)].filter((chunkId) => chunkId.length > 0);
36176
- if (normalized.length === 0) {
36177
- return [];
36178
- }
36179
- const placeholders = normalized.map((_, index) => `$${index + 1}`).join(", ");
36180
- const rows = await db.unsafe(`select chunk_id from ${qualifiedTableName} where chunk_id in (${placeholders})`, normalized);
36181
- return rows.map((row) => row.chunk_id).filter((chunkId) => typeof chunkId === "string");
36182
- };
36183
- var getPostgresCandidateChunkIdsByFilter = async (db, qualifiedTableName, filter) => {
36184
- if (!filter || Object.keys(filter).length === 0) {
36185
- return [];
36186
- }
36187
- const pushdownFilter = buildPostgresPushdownFilter(filter);
36188
- const filterPlan = buildPostgresFilterPlan(pushdownFilter);
36189
- const rowsSql = filterPlan?.clause ? `select chunk_id, text, title, source, metadata from ${qualifiedTableName} where ${filterPlan.clause}` : `select chunk_id, text, title, source, metadata from ${qualifiedTableName}`;
36190
- const rows = await db.unsafe(rowsSql, filterPlan?.clause ? filterPlan.params ?? [] : []);
36191
- const chunks = rows.map((row) => mapRowToChunk(row)).filter((chunk) => matchesFilter(chunk, filter));
36192
- return chunks.map((chunk) => chunk.chunkId);
36193
- };
36194
- var getPostgresCandidateChunkIds = async (db, qualifiedTableName, input) => {
36195
- const chunkIdSet = new Set;
36196
- if (input.filter && Object.keys(input.filter).length > 0) {
36197
- for (const chunkId of await getPostgresCandidateChunkIdsByFilter(db, qualifiedTableName, input.filter)) {
36198
- chunkIdSet.add(chunkId);
36199
- }
36200
- }
36201
- if (input.chunkIds && input.chunkIds.length > 0) {
36202
- for (const chunkId of await getPostgresChunkIdsByChunkIds(db, qualifiedTableName, input.chunkIds)) {
36203
- chunkIdSet.add(chunkId);
36204
- }
36205
- }
36206
- return [...chunkIdSet];
36207
- };
36208
- var normalizeDistance = (distance, metric) => {
36209
- if (!Number.isFinite(distance)) {
36210
- return 0;
36211
- }
36212
- if (metric === "cosine") {
36213
- return Math.min(1, Math.max(0, 1 - distance));
36214
- }
36215
- if (metric === "inner_product") {
36216
- return Math.max(0, -distance);
36217
- }
36218
- return Math.max(0, 1 / (1 + Math.abs(distance)));
36219
- };
36220
- var getDistanceOperator = (metric) => metric === "cosine" ? "<=>" : metric === "inner_product" ? "<#>" : "<->";
36221
- var createPostgresStatus = (dimensions, nativeDiagnostics) => ({
36222
- backend: "postgres",
36223
- dimensions,
36224
- native: nativeDiagnostics,
36225
- vectorMode: "native_pgvector"
36226
- });
36227
- var createPostgresCapabilities = () => ({
36228
- backend: "postgres",
36229
- nativeVectorSearch: true,
36230
- persistence: "external",
36231
- serverSideFiltering: true,
36232
- streamingIngestStatus: false
36233
- });
36234
- var updatePostgresLastQueryPlan = (input) => {
36235
- const pushdown = resolvePostgresPushdownMode({
36236
- filter: input.filter,
36237
- pushdownFilter: input.pushdownFilter
36238
- });
36239
- input.nativeDiagnostics.lastQueryPlan = {
36240
- backfillCount: input.backfillCount,
36241
- candidateBudgetExhausted: input.candidateBudgetExhausted,
36242
- candidateCoverage: summarizeSQLiteCandidateCoverage({
36243
- filteredCandidateCount: input.filteredCandidateCount,
36244
- returnedCount: input.returnedCount,
36245
- topK: input.topK
36246
- }),
36247
- filteredCandidateCount: input.filteredCandidateCount,
36248
- finalSearchK: input.finalSearchK,
36249
- initialSearchK: input.initialSearchK,
36250
- searchExpansionRatio: typeof input.initialSearchK === "number" && typeof input.finalSearchK === "number" && input.initialSearchK > 0 ? input.finalSearchK / input.initialSearchK : undefined,
36251
- candidateLimitUsed: input.candidateLimitUsed,
36252
- maxBackfillsUsed: input.maxBackfillsUsed,
36253
- minResultsUsed: input.minResultsUsed,
36254
- fillPolicyUsed: input.fillPolicyUsed,
36255
- plannerProfileUsed: input.plannerProfileUsed,
36256
- jsRemainderClauseCount: pushdown.jsRemainderClauseCount,
36257
- queryMultiplierUsed: input.queryMultiplierUsed,
36258
- jsRemainderRatio: pushdown.jsRemainderRatio,
36259
- pushdownApplied: pushdown.pushdownClauseCount > 0,
36260
- pushdownClauseCount: pushdown.pushdownClauseCount,
36261
- pushdownCoverageRatio: pushdown.pushdownCoverageRatio,
36262
- pushdownMode: pushdown.pushdownMode,
36263
- queryMode: "native_pgvector",
36264
- candidateYieldRatio: typeof input.returnedCount === "number" && typeof input.finalSearchK === "number" && input.finalSearchK > 0 ? input.returnedCount / input.finalSearchK : undefined,
36265
- returnedCount: input.returnedCount,
36266
- backfillLimitReached: input.backfillLimitReached,
36267
- minResultsSatisfied: input.minResultsSatisfied,
36268
- topKFillRatio: typeof input.returnedCount === "number" && input.topK > 0 ? input.returnedCount / input.topK : undefined,
36269
- totalFilterClauseCount: pushdown.totalFilterClauseCount,
36270
- underfilledTopK: input.underfilledTopK
36271
- };
36272
- };
36273
- var matchesFilter = (record, filter) => matchesMetadataFilterRecord({
36274
- chunkId: record.chunkId,
36275
- metadata: record.metadata,
36276
- source: record.source,
36277
- title: record.title,
36278
- ...record.metadata ?? {}
36279
- }, filter);
36280
- var mapRowToChunk = (row) => ({
36281
- chunkId: row.chunk_id,
36282
- metadata: parseMetadata(row.metadata),
36283
- source: row.source ?? undefined,
36284
- text: row.text,
36285
- title: row.title ?? undefined,
36286
- vector: parseVectorText(row.embedding)
36287
- });
36288
- var ensurePostgresSchema = async (db, input) => {
36289
- await db.unsafe("create extension if not exists vector");
36290
- const [schemaName] = input.qualifiedTableName.split(".");
36291
- if (schemaName) {
36292
- await db.unsafe(`create schema if not exists ${schemaName}`);
36293
- }
36294
- await db.unsafe(`
36295
- create table if not exists ${input.qualifiedTableName} (
36296
- chunk_id text primary key,
36297
- text text not null,
36298
- title text,
36299
- source text,
36300
- metadata jsonb,
36301
- embedding vector(${input.dimensions}) not null
36302
- )
36303
- `);
36304
- const indexSql = buildPostgresIndexSql(input);
36305
- if (indexSql) {
36306
- await db.unsafe(indexSql);
36307
- }
36308
- };
36309
- var createPostgresRAGStore = (options = {}) => {
36310
- const dimensions = options.dimensions ?? DEFAULT_DIMENSIONS;
36311
- const distanceMetric = options.distanceMetric ?? "cosine";
36312
- const queryMultiplier = normalizeQueryMultiplier(options.queryMultiplier);
36313
- const indexType = normalizePostgresIndexType(options.indexType);
36314
- const indexLists = normalizePositiveInteger(options.indexLists, DEFAULT_POSTGRES_IVFFLAT_LISTS);
36315
- const hnswM = normalizePositiveInteger(options.hnswM, DEFAULT_POSTGRES_HNSW_M);
36316
- const hnswEfConstruction = normalizePositiveInteger(options.hnswEfConstruction, DEFAULT_POSTGRES_HNSW_EF_CONSTRUCTION);
36317
- const tableName = options.tableName ?? DEFAULT_TABLE_NAME;
36318
- const schemaName = options.schemaName ?? DEFAULT_SCHEMA_NAME;
36319
- assertSupportedIdentifier2(tableName);
36320
- assertSupportedIdentifier2(schemaName);
36321
- const qualifiedTableName = toQualifiedTableName(schemaName, tableName);
36322
- const indexName = getPostgresIndexName(qualifiedTableName, indexType);
36323
- const db = options.sql ?? new Bun.SQL(options.connectionString ?? process.env.RAG_POSTGRES_URL ?? process.env.DATABASE_URL ?? "postgres://postgres:postgres@localhost:55433/absolute_rag_demo");
36324
- const nativeDiagnostics = {
36325
- active: true,
36326
- available: true,
36327
- distanceMetric,
36328
- extensionName: "vector",
36329
- indexName,
36330
- indexType,
36331
- mode: "pgvector",
36332
- requested: true,
36333
- schemaName,
36334
- tableName
36335
- };
36336
- const capabilities = createPostgresCapabilities();
36337
- const distanceOperator = getDistanceOperator(distanceMetric);
36338
- let initialized;
36339
- const init = () => {
36340
- initialized ??= ensurePostgresSchema(db, {
36341
- dimensions,
36342
- distanceMetric,
36343
- hnswEfConstruction,
36344
- hnswM,
36345
- indexLists,
36346
- indexType,
36347
- qualifiedTableName
36348
- }).then(() => refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
36349
- indexName,
36350
- qualifiedTableName,
36351
- schemaName,
36352
- tableName
36353
- })).catch((error) => {
36354
- nativeDiagnostics.active = false;
36355
- nativeDiagnostics.available = false;
36356
- nativeDiagnostics.lastInitError = error instanceof Error ? error.message : String(error);
36357
- nativeDiagnostics.lastMigrationError = error instanceof Error ? error.message : String(error);
36358
- nativeDiagnostics.fallbackReason = nativeDiagnostics.lastInitError;
36359
- throw error;
36360
- });
36361
- return initialized;
36362
- };
36363
- const embed = async (input) => {
36364
- input.model;
36365
- input.signal;
36366
- if (options.mockEmbedding) {
36367
- return options.mockEmbedding(input.text);
36368
- }
36369
- return normalizeVector(createRAGVector(input.text, dimensions));
36370
- };
36371
- const query = async (input) => {
36372
- await init();
36373
- const queryVector = normalizeVector(input.queryVector);
36374
- const queryMultiplier2 = normalizeQueryMultiplier(input.queryMultiplier ?? options.queryMultiplier);
36375
- const maxBackfills = normalizeMaxBackfills(input.maxBackfills);
36376
- const minResults = normalizeMinResults(input.minResults, input.topK);
36377
- const fillTarget = resolveFillTarget({
36378
- fillPolicy: input.fillPolicy,
36379
- minResults,
36380
- topK: input.topK
36381
- });
36382
- const queryVectorLiteral = toVectorLiteral(queryVector);
36383
- const pushdownFilter = buildPostgresPushdownFilter(input.filter);
36384
- const queryFilterPlan = buildPostgresFilterPlan(pushdownFilter);
36385
- const effectivePushdownFilter = queryFilterPlan ? pushdownFilter : undefined;
36386
- const countFilterPlan = queryFilterPlan;
36387
- const countSql = countFilterPlan?.clause ? `select count(*)::int as count from ${qualifiedTableName} where ${countFilterPlan.clause}` : `select count(*)::int as count from ${qualifiedTableName}`;
36388
- const totalRowsResult = await db.unsafe(countSql, countFilterPlan?.params ?? []);
36389
- nativeDiagnostics.lastFilterDebug = {
36390
- countParams: countFilterPlan?.params ?? [],
36391
- countResultRaw: totalRowsResult?.[0],
36392
- countSql,
36393
- filter: input.filter,
36394
- pushdownFilter: effectivePushdownFilter
36395
- };
36396
- const totalRows = parseCountValue(totalRowsResult?.[0]?.count);
36397
- const candidateLimit = resolveAdaptiveNativeCandidateLimit({
36398
- defaultCandidateLimit: RAG_NATIVE_QUERY_CANDIDATE_LIMIT,
36399
- explicitCandidateLimit: input.candidateLimit,
36400
- filteredCandidateCount: totalRows,
36401
- plannerProfile: input.plannerProfile,
36402
- queryMultiplier: queryMultiplier2,
36403
- topK: input.topK
36404
- });
36405
- const hasPushdownFilter = Boolean(effectivePushdownFilter);
36406
- const plannedFilteredCandidateCount = hasPushdownFilter && totalRows === 0 ? undefined : totalRows;
36407
- const initialSearchK = planNativeCandidateSearchK({
36408
- candidateLimit,
36409
- filteredCandidateCount: plannedFilteredCandidateCount,
36410
- queryMultiplier: queryMultiplier2,
36411
- topK: input.topK
36412
- });
36413
- if (initialSearchK === 0) {
36414
- return [];
36415
- }
36416
- let currentSearchK = initialSearchK;
36417
- let backfillCount = 0;
36418
- let candidateBudgetExhausted = false;
36419
- let backfillLimitReached = false;
36420
- let effectiveFilteredCandidateCount = plannedFilteredCandidateCount;
36421
- let mapped = [];
36422
- for (;; ) {
36423
- const rowsSql = queryFilterPlan?.clause ? `select chunk_id, text, title, source, metadata, embedding::text as embedding, embedding ${distanceOperator} '${queryVectorLiteral}'::vector as distance from ${qualifiedTableName} where ${queryFilterPlan.clause} order by embedding ${distanceOperator} '${queryVectorLiteral}'::vector limit $${queryFilterPlan.params.length + 1}` : `select chunk_id, text, title, source, metadata, embedding::text as embedding, embedding ${distanceOperator} '${queryVectorLiteral}'::vector as distance from ${qualifiedTableName} order by embedding ${distanceOperator} '${queryVectorLiteral}'::vector limit $1`;
36424
- const rows = await db.unsafe(rowsSql, queryFilterPlan?.clause ? [...queryFilterPlan.params ?? [], currentSearchK] : [currentSearchK]);
36425
- nativeDiagnostics.lastFilterDebug = {
36426
- ...nativeDiagnostics.lastFilterDebug,
36427
- queryParams: queryFilterPlan?.clause ? [...queryFilterPlan.params ?? [], currentSearchK] : [currentSearchK],
36428
- queryRowCount: rows.length,
36429
- querySql: rowsSql
36430
- };
36431
- if (hasPushdownFilter && effectiveFilteredCandidateCount === undefined && rows.length <= currentSearchK) {
36432
- effectiveFilteredCandidateCount = rows.length;
36433
- }
36434
- mapped = rows.map((row) => {
36435
- const chunk = mapRowToChunk(row);
36436
- return {
36437
- chunk,
36438
- score: normalizeDistance(Number(row.distance ?? 0), distanceMetric)
36439
- };
36440
- }).filter(({ chunk }) => matchesFilter(chunk, input.filter)).map((entry) => ({
36441
- chunkId: entry.chunk.chunkId,
36442
- chunkText: entry.chunk.text,
36443
- embedding: entry.chunk.vector,
36444
- metadata: entry.chunk.metadata,
36445
- score: entry.score,
36446
- source: entry.chunk.source,
36447
- title: entry.chunk.title
36448
- })).sort((left, right) => right.score - left.score);
36449
- if (mapped.length >= fillTarget.targetResults) {
36450
- break;
36451
- }
36452
- const nextSearchK = planNativeCandidateSearchBackfillK({
36453
- backfillCount,
36454
- candidateLimit,
36455
- currentSearchK,
36456
- filteredCandidateCount: effectiveFilteredCandidateCount,
36457
- maxBackfills
36458
- });
36459
- if (nextSearchK <= currentSearchK) {
36460
- backfillLimitReached = typeof maxBackfills === "number" && backfillCount >= maxBackfills && mapped.length < fillTarget.targetResults;
36461
- candidateBudgetExhausted = mapped.length < fillTarget.targetResults;
36462
- break;
36463
- }
36464
- currentSearchK = nextSearchK;
36465
- backfillCount += 1;
36466
- }
36467
- nativeDiagnostics.lastQueryError = undefined;
36468
- const returned = mapped.slice(0, input.topK);
36469
- updatePostgresLastQueryPlan({
36470
- backfillCount,
36471
- backfillLimitReached,
36472
- candidateBudgetExhausted,
36473
- candidateLimitUsed: candidateLimit,
36474
- maxBackfillsUsed: maxBackfills,
36475
- minResultsUsed: minResults,
36476
- fillPolicyUsed: fillTarget.fillPolicy,
36477
- plannerProfileUsed: input.plannerProfile,
36478
- filter: input.filter,
36479
- pushdownFilter: effectivePushdownFilter,
36480
- queryMultiplierUsed: queryMultiplier2,
36481
- filteredCandidateCount: effectiveFilteredCandidateCount,
36482
- finalSearchK: currentSearchK,
36483
- initialSearchK,
36484
- nativeDiagnostics,
36485
- minResultsSatisfied: returned.length >= minResults,
36486
- returnedCount: returned.length,
36487
- topK: input.topK,
36488
- underfilledTopK: returned.length < input.topK
36489
- });
36490
- return returned;
36491
- };
36492
- const queryLexical = async (input) => {
36493
- await init();
36494
- const pushdownFilter = buildPostgresPushdownFilter(input.filter);
36495
- const lexicalFilterPlan = buildPostgresFilterPlan(pushdownFilter);
36496
- const rowsSql = lexicalFilterPlan?.clause ? `select chunk_id, text, title, source, metadata from ${qualifiedTableName} where ${lexicalFilterPlan.clause}` : `select chunk_id, text, title, source, metadata from ${qualifiedTableName}`;
36497
- const rows = await db.unsafe(rowsSql, lexicalFilterPlan?.params ?? []);
36498
- const chunks = rows.map((row) => mapRowToChunk(row)).filter((chunk) => matchesFilter(chunk, input.filter));
36499
- const ranked = rankRAGLexicalMatches(input.query, chunks);
36500
- return ranked.slice(0, input.topK).map(({ result, score }) => ({
36501
- chunkId: result.chunkId,
36502
- chunkText: result.text,
36503
- metadata: result.metadata,
36504
- score,
36505
- source: result.source,
36506
- title: result.title
36507
- }));
36508
- };
36509
- const upsert = async (input) => {
36510
- await init();
36511
- const chunks = input.chunks.length > 0 ? await Promise.all(input.chunks.map(async (chunk) => ({
36512
- chunkId: chunk.chunkId,
36513
- metadata: chunk.metadata,
36514
- source: chunk.source,
36515
- text: chunk.text,
36516
- title: chunk.title,
36517
- vector: chunk.embedding ? normalizeVector(chunk.embedding) : normalizeVector(await embed({ text: chunk.text }))
36518
- }))) : [];
36519
- for (const chunk of chunks) {
36520
- await db.unsafe(`insert into ${qualifiedTableName} (chunk_id, text, title, source, metadata, embedding)
36521
- values ($1, $2, $3, $4, $5::jsonb, $6::vector)
36522
- on conflict (chunk_id) do update set
36523
- text = excluded.text,
36524
- title = excluded.title,
36525
- source = excluded.source,
36526
- metadata = excluded.metadata,
36527
- embedding = excluded.embedding`, [
36528
- chunk.chunkId,
36529
- chunk.text,
36530
- chunk.title ?? null,
36531
- chunk.source ?? null,
36532
- chunk.metadata ?? null,
36533
- toVectorLiteral(chunk.vector)
36534
- ]);
36535
- }
36536
- await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
36537
- indexName,
36538
- qualifiedTableName,
36539
- schemaName,
36540
- tableName
36541
- });
36542
- };
36543
- const count = async (input = {}) => {
36544
- await init();
36545
- const filter = input.filter;
36546
- const chunkIds = input.chunkIds;
36547
- const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
36548
- const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
36549
- if (!hasFilter && !hasChunkIds) {
36550
- const countResult = await db.unsafe(`select count(*)::int as count from ${qualifiedTableName}`);
36551
- return parseCountValue(countResult[0]?.count);
36552
- }
36553
- return (await getPostgresCandidateChunkIds(db, qualifiedTableName, {
36554
- filter,
36555
- chunkIds
36556
- })).length;
36557
- };
36558
- const remove = async (input = {}) => {
36559
- await init();
36560
- const filter = input.filter;
36561
- const chunkIds = input.chunkIds;
36562
- const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
36563
- const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
36564
- if (!hasFilter && !hasChunkIds) {
36565
- return 0;
36566
- }
36567
- const ids = await getPostgresCandidateChunkIds(db, qualifiedTableName, {
36568
- filter,
36569
- chunkIds
36570
- });
36571
- if (ids.length === 0) {
36572
- return 0;
36573
- }
36574
- const placeholders = ids.map((_, index) => `$${index + 1}`).join(", ");
36575
- await db.unsafe(`delete from ${qualifiedTableName} where chunk_id in (${placeholders})`, ids);
36576
- await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
36577
- indexName,
36578
- qualifiedTableName,
36579
- schemaName,
36580
- tableName
36581
- });
36582
- return ids.length;
36583
- };
36584
- const clear = async () => {
36585
- await init();
36586
- await db.unsafe(`truncate table ${qualifiedTableName}`);
36587
- await refreshPostgresRuntimeDiagnostics(db, nativeDiagnostics, {
36588
- indexName,
36589
- qualifiedTableName,
36590
- schemaName,
36591
- tableName
36592
- });
36593
- };
36594
- const analyze = async () => {
36595
- await init();
36596
- await analyzePostgresTable(db, nativeDiagnostics, {
36597
- indexName,
36598
- qualifiedTableName,
36599
- schemaName,
36600
- tableName
36601
- });
36602
- };
36603
- const rebuildNativeIndex = async () => {
36604
- await init();
36605
- await rebuildPostgresNativeIndex(db, nativeDiagnostics, {
36606
- distanceMetric,
36607
- hnswEfConstruction,
36608
- hnswM,
36609
- indexLists,
36610
- indexName,
36611
- indexType,
36612
- qualifiedTableName,
36613
- schemaName,
36614
- tableName
36615
- });
36616
- };
36617
- const close = async () => {
36618
- await db.close?.();
36619
- };
36620
- return {
36621
- analyze,
36622
- clear,
36623
- close,
36624
- embed,
36625
- getCapabilities: () => capabilities,
36626
- getStatus: () => createPostgresStatus(dimensions, nativeDiagnostics),
36627
- query,
36628
- queryLexical,
36629
- rebuildNativeIndex: indexName ? rebuildNativeIndex : undefined,
36630
- count,
36631
- delete: remove,
36632
- upsert
36633
- };
36634
- };
36635
- // src/adapters/sqlite.ts
36636
- import { Database } from "bun:sqlite";
36637
- import { existsSync as existsSync2 } from "fs";
36638
-
36639
- // src/internal/resolveAbsoluteSQLiteVec.ts
36640
- import { existsSync, readFileSync } from "fs";
36641
- import { arch, platform } from "os";
36642
- import { dirname as dirname5, join as join3 } from "path";
36643
- var PLATFORM_PACKAGE_MAP = {
36644
- "darwin-arm64": {
36645
- libraryFile: "vec0.dylib",
36646
- packageName: "@absolutejs/absolute-rag-sqlite-darwin-arm64"
36647
- },
36648
- "darwin-x64": {
36649
- libraryFile: "vec0.dylib",
36650
- packageName: "@absolutejs/absolute-rag-sqlite-darwin-x64"
36651
- },
36652
- "linux-arm64": {
36653
- libraryFile: "vec0.so",
36654
- packageName: "@absolutejs/absolute-rag-sqlite-linux-arm64"
36655
- },
36656
- "linux-x64": {
36657
- libraryFile: "vec0.so",
36658
- packageName: "@absolutejs/absolute-rag-sqlite-linux-x64"
36659
- },
36660
- "win32-x64": {
36661
- libraryFile: "vec0.dll",
36662
- packageName: "@absolutejs/absolute-rag-sqlite-windows-x64"
36663
- }
36664
- };
36665
- var currentPlatformKey = () => `${platform()}-${arch()}`;
36666
- var getErrorMessage = (error) => error instanceof Error ? error.message : String(error);
36667
- var isPackageJsonShape = (value) => Boolean(value) && typeof value === "object";
36668
- var readPackageVersion = (packageJsonPath) => {
36669
- try {
36670
- const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8"));
36671
- if (!isPackageJsonShape(packageJson)) {
36672
- return;
36673
- }
36674
- return typeof packageJson.version === "string" ? packageJson.version : undefined;
36675
- } catch {
36676
- return;
36677
- }
36678
- };
36679
- var resolveAbsoluteSQLiteVec = () => {
36680
- const platformKey = currentPlatformKey();
36681
- const packageInfo = PLATFORM_PACKAGE_MAP[platformKey];
36682
- if (!packageInfo) {
36683
- return {
36684
- platformKey,
36685
- reason: `No AbsoluteJS sqlite-vec package is defined for ${platformKey}.`,
36686
- source: "absolute-package",
36687
- status: "unsupported_platform"
36688
- };
36689
- }
36690
- try {
36691
- const resolve4 = import.meta.resolve;
36692
- if (typeof resolve4 !== "function") {
36693
- throw new Error("AbsoluteJS sqlite-vec package resolution requires import.meta.resolve support.");
36694
- }
36695
- const packageJsonPath = new URL(resolve4(`${packageInfo.packageName}/package.json`)).pathname;
36696
- const packageRoot = dirname5(packageJsonPath);
36697
- const libraryPath = join3(packageRoot, packageInfo.libraryFile);
36698
- const packageVersion = readPackageVersion(packageJsonPath);
36699
- if (!existsSync(libraryPath)) {
36700
- return {
36701
- libraryFile: packageInfo.libraryFile,
36702
- libraryPath,
36703
- packageName: packageInfo.packageName,
36704
- packageRoot,
36705
- packageVersion,
36706
- platformKey,
36707
- reason: `Resolved ${packageInfo.packageName} but ${packageInfo.libraryFile} was not found.`,
36708
- source: "absolute-package",
36709
- status: "binary_missing"
36710
- };
36711
- }
36712
- return {
36713
- libraryFile: packageInfo.libraryFile,
36714
- libraryPath,
36715
- packageName: packageInfo.packageName,
36716
- packageRoot,
36717
- packageVersion,
36718
- platformKey,
36719
- source: "absolute-package",
36720
- status: "resolved"
36721
- };
36722
- } catch (error) {
36723
- return {
36724
- libraryFile: packageInfo.libraryFile,
36725
- packageName: packageInfo.packageName,
36726
- platformKey,
36727
- reason: getErrorMessage(error),
36728
- source: "absolute-package",
36729
- status: "package_not_installed"
36730
- };
36731
- }
36732
- };
36733
- var resolveAbsoluteSQLiteVecExtensionPath = () => {
36734
- const resolution = resolveAbsoluteSQLiteVec();
36735
- return resolution.status === "resolved" ? resolution.libraryPath ?? null : null;
36736
- };
36737
-
36738
- // src/adapters/sqlite.ts
36739
- var DEFAULT_DIMENSIONS2 = RAG_VECTOR_DIMENSIONS_DEFAULT;
36740
- var DEFAULT_TABLE_NAME2 = "rag_chunks";
36741
- var DEFAULT_NATIVE_TABLE_SUFFIX = "_vec0";
36742
- var DEFAULT_QUERY_MULTIPLIER2 = 4;
36743
- var MAX_QUERY_MULTIPLIER2 = 16;
36744
- var IDENTIFIER_RE2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
36745
- var isParsedMetadata = (value) => Boolean(value) && typeof value === "object";
36746
- var isObjectRecord5 = (value) => Boolean(value) && typeof value === "object";
36747
- var isStoredRow = (value) => isObjectRecord5(value) && typeof value.chunk_id === "string" && typeof value.text === "string" && (typeof value.title === "string" || value.title === null) && (typeof value.source === "string" || value.source === null) && (typeof value.metadata === "string" || value.metadata === null) && typeof value.embedding === "string";
36748
- var isNativeStoredRow = (value) => isObjectRecord5(value) && typeof value.chunk_id === "string" && typeof value.chunk_text === "string" && (typeof value.title === "string" || value.title === null) && (typeof value.source === "string" || value.source === null) && (typeof value.metadata === "string" || value.metadata === null) && typeof value.embedding === "string" && typeof value.distance === "number";
36749
- var toStoredRows = (value) => Array.isArray(value) ? value.filter((row) => isStoredRow(row)) : [];
36750
- var toNativeStoredRows = (value) => Array.isArray(value) ? value.filter((row) => isNativeStoredRow(row)) : [];
36751
- var createSQLiteStatus = (dimensions, nativeDiagnostics, useNative) => ({
36752
- backend: "sqlite",
36753
- dimensions,
36754
- native: nativeDiagnostics,
36755
- vectorMode: useNative ? "native_vec0" : "json_fallback"
36756
- });
36757
- var createSQLiteCapabilities = (useNative) => ({
36758
- backend: "sqlite",
36759
- nativeVectorSearch: useNative,
36760
- persistence: "embedded",
36761
- serverSideFiltering: useNative,
36762
- streamingIngestStatus: false
36763
- });
36764
- var assertSupportedIdentifier3 = (name) => {
36765
- if (!IDENTIFIER_RE2.test(name)) {
36766
- throw new Error(`Invalid table name "${name}". Only alphanumeric and underscore names are allowed.`);
36767
- }
36768
- };
36769
- var normalizeQueryMultiplier2 = (value) => {
36770
- if (value === undefined || !Number.isFinite(value)) {
36771
- return DEFAULT_QUERY_MULTIPLIER2;
36772
- }
36773
- const minMultiplier = Math.max(1, Math.floor(value));
36774
- return Math.min(minMultiplier, MAX_QUERY_MULTIPLIER2);
36775
- };
36776
- var normalizeMaxBackfills2 = (value) => {
36777
- if (value === undefined || !Number.isFinite(value)) {
36778
- return;
36779
- }
36780
- return Math.max(0, Math.floor(value));
36781
- };
36782
- var normalizeMinResults2 = (value, topK) => {
36783
- if (value === undefined || !Number.isFinite(value)) {
36784
- return topK;
36785
- }
36786
- return Math.min(topK, Math.max(1, Math.floor(value)));
36787
- };
36788
- var resolveFillTarget2 = (input) => {
36789
- const fillPolicy = input.fillPolicy ?? "satisfy_min_results";
36790
- return {
36791
- fillPolicy,
36792
- targetResults: fillPolicy === "strict_topk" ? input.topK : input.minResults
36793
- };
36794
- };
36795
- var toJSONString = (metadata) => metadata === undefined ? null : JSON.stringify(metadata);
36796
- var parseMetadata2 = (value) => {
36797
- if (value === null) {
36798
- return;
36799
- }
36800
- try {
36801
- const parsed = JSON.parse(value);
36802
- if (isParsedMetadata(parsed)) {
36803
- return parsed;
36804
- }
36805
- } catch {}
36806
- return;
36807
- };
36808
- var parseVector = (value) => {
36809
- try {
36810
- const parsed = JSON.parse(value);
36811
- if (Array.isArray(parsed)) {
36812
- return parsed.filter((element) => typeof element === "number" && Number.isFinite(element));
36813
- }
36814
- } catch {}
36815
- return [];
36816
- };
36817
- var toSQLiteFilterBinding = (value) => value === null || typeof value === "string" || typeof value === "number" || typeof value === "bigint" || typeof value === "boolean" ? value : undefined;
36818
- var FILTER_PATH_SEGMENT_RE2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
36819
- var isObjectFilterRecord2 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
36820
- var isOperatorFilterRecord2 = (value) => isObjectFilterRecord2(value) && Object.keys(value).some((key) => key.startsWith("$"));
36821
- var isNestedFilterArray3 = (value) => Array.isArray(value) && value.every((entry) => isObjectFilterRecord2(entry));
36822
- var toSQLiteJsonPath = (key) => {
36823
- const segments = key.split(".").filter(Boolean);
36824
- if (segments.length === 0 || !segments.every((segment) => FILTER_PATH_SEGMENT_RE2.test(segment))) {
36825
- return null;
36826
- }
36827
- return `$.${segments.join(".")}`;
36828
- };
36829
- var resolveSQLiteFilterColumn = (key, aliases) => {
36830
- if (key === "chunkId")
36831
- return { actualSql: aliases.chunkId, kind: "scalar" };
36832
- if (key === "source")
36833
- return { actualSql: aliases.source, kind: "scalar" };
36834
- if (key === "title")
36835
- return { actualSql: aliases.title, kind: "scalar" };
36836
- const path = toSQLiteJsonPath(key);
36837
- if (!path) {
36838
- return null;
36839
- }
36840
- return {
36841
- actualSql: `json_extract(${aliases.metadata}, ?)`,
36842
- kind: "metadata",
36843
- path
36844
- };
36845
- };
36846
- var buildSQLiteScalarClause = (actualSql, expected, params) => {
36847
- if (!isOperatorFilterRecord2(expected)) {
36848
- const binding = toSQLiteFilterBinding(expected);
36849
- if (binding === undefined) {
36850
- return null;
36851
- }
36852
- params.push(binding);
36853
- return `${actualSql} = ?`;
36854
- }
36855
- const clauses = [];
36856
- for (const [operator, value] of Object.entries(expected)) {
36857
- switch (operator) {
36858
- case "$exists":
36859
- clauses.push(Boolean(value) ? `${actualSql} IS NOT NULL` : `${actualSql} IS NULL`);
36860
- break;
36861
- case "$ne":
36862
- const neBinding = toSQLiteFilterBinding(value);
36863
- if (neBinding === undefined) {
36864
- return null;
36865
- }
36866
- params.push(neBinding);
36867
- clauses.push(`(${actualSql} IS NULL OR ${actualSql} != ?)`);
36868
- break;
36869
- case "$in":
36870
- if (!Array.isArray(value) || value.length === 0 || value.some((entry) => toSQLiteFilterBinding(entry) === undefined)) {
36871
- return null;
36872
- }
36873
- params.push(...value);
36874
- clauses.push(`${actualSql} IN (${value.map(() => "?").join(", ")})`);
36875
- break;
36876
- case "$gt":
36877
- case "$gte":
36878
- case "$lt":
36879
- case "$lte": {
36880
- if (typeof value !== "number" || !Number.isFinite(value)) {
36881
- return null;
36882
- }
36883
- params.push(value);
36884
- const comparison = operator === "$gt" ? ">" : operator === "$gte" ? ">=" : operator === "$lt" ? "<" : "<=";
36885
- clauses.push(`${actualSql} ${comparison} ?`);
36886
- break;
36887
- }
36888
- default:
36889
- return null;
36890
- }
36891
- }
36892
- return clauses.length > 0 ? clauses.map((clause) => `(${clause})`).join(" AND ") : null;
36893
- };
36894
- var buildSQLiteArrayClause = (metadataSql, path, expected, params) => {
36895
- if (!isOperatorFilterRecord2(expected)) {
36896
- return null;
36897
- }
36898
- const arrayTypeClause = `json_type(${metadataSql}, ?) = 'array'`;
36899
- const clauses = [];
36900
- for (const [operator, value] of Object.entries(expected)) {
36901
- switch (operator) {
36902
- case "$contains":
36903
- const containsBinding = toSQLiteFilterBinding(value);
36904
- if (containsBinding === undefined) {
36905
- return null;
36906
- }
36907
- params.push(path, path, containsBinding);
36908
- clauses.push(`(${arrayTypeClause} AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value = ?))`);
36909
- break;
36910
- case "$containsAny":
36911
- if (!Array.isArray(value) || value.length === 0 || value.some((entry) => toSQLiteFilterBinding(entry) === undefined)) {
36912
- return null;
36913
- }
36914
- params.push(path, path, ...value);
36915
- clauses.push(`(${arrayTypeClause} AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value IN (${value.map(() => "?").join(", ")})))`);
36916
- break;
36917
- case "$containsAll":
36918
- if (!Array.isArray(value) || value.length === 0 || value.some((entry) => toSQLiteFilterBinding(entry) === undefined)) {
36919
- return null;
36920
- }
36921
- clauses.push(...value.map((entry) => {
36922
- params.push(path, path, entry);
36923
- return `(${arrayTypeClause} AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value = ?))`;
36924
- }));
36925
- break;
36926
- default:
36927
- return null;
36928
- }
36929
- }
36930
- return clauses.length > 0 ? clauses.map((clause) => `(${clause})`).join(" AND ") : null;
36931
- };
36932
- var buildSQLiteMetadataScalarMatchClause = (metadataSql, actualSql, path, expected, params) => {
36933
- const binding = toSQLiteFilterBinding(expected);
36934
- if (binding === undefined) {
36935
- return null;
36936
- }
36937
- params.push(path, binding, path, path, binding);
36938
- return `(${actualSql} = ? OR (json_type(${metadataSql}, ?) = 'array' AND EXISTS (SELECT 1 FROM json_each(json_extract(${metadataSql}, ?)) WHERE json_each.value = ?)))`;
36939
- };
36940
- var buildSQLiteFilterPlan = (filter, aliases) => {
36941
- if (!filter) {
36942
- return { clause: "", params: [] };
36943
- }
36944
- const params = [];
36945
- const build = (entry) => {
36946
- const clauses = [];
36947
- for (const [key, value] of Object.entries(entry)) {
36948
- if (key === "$and" || key === "$or") {
36949
- if (!isNestedFilterArray3(value) || value.length === 0) {
36950
- return null;
36951
- }
36952
- const nested = value.map((item) => build(item));
36953
- if (nested.some((item) => item === null)) {
36954
- return null;
36955
- }
36956
- clauses.push(`(${nested.filter((item) => Boolean(item)).join(key === "$and" ? " AND " : " OR ")})`);
36957
- continue;
36958
- }
36959
- if (key === "$not") {
36960
- if (!isObjectFilterRecord2(value)) {
36961
- return null;
36962
- }
36963
- const nested = build(value);
36964
- if (!nested) {
36965
- return null;
36966
- }
36967
- clauses.push(`NOT (${nested})`);
36968
- continue;
36969
- }
36970
- const resolved = resolveSQLiteFilterColumn(key, aliases);
36971
- if (!resolved) {
36972
- return null;
36973
- }
36974
- if (resolved.kind === "metadata") {
36975
- const metadataScalarClause = buildSQLiteMetadataScalarMatchClause(aliases.metadata, resolved.actualSql, resolved.path, value, params);
36976
- if (metadataScalarClause) {
36977
- clauses.push(metadataScalarClause);
36978
- continue;
36979
- }
36980
- const arrayClause = buildSQLiteArrayClause(aliases.metadata, resolved.path, value, params);
36981
- if (arrayClause) {
36982
- clauses.push(arrayClause);
36983
- continue;
36984
- }
36985
- params.push(resolved.path);
36986
- }
36987
- const scalarClause = buildSQLiteScalarClause(resolved.actualSql, value, params);
36988
- if (!scalarClause) {
36989
- return null;
36990
- }
36991
- clauses.push(scalarClause);
36992
- }
36993
- return clauses.length > 0 ? clauses.map((clause2) => `(${clause2})`).join(" AND ") : "";
36994
- };
36995
- const clause = build(filter);
36996
- return clause === null ? null : { clause, params };
36997
- };
36998
- var buildSQLitePushdownFilter = (filter) => {
36999
- if (!filter) {
37000
- return;
37001
- }
37002
- const nextEntries = [];
37003
- for (const [key, value] of Object.entries(filter)) {
37004
- if (key === "$and" || key === "$or") {
37005
- if (!isNestedFilterArray3(value)) {
37006
- continue;
37007
- }
37008
- const nested = value.map((entry) => buildSQLitePushdownFilter(entry)).filter((entry) => Boolean(entry));
37009
- if (nested.length > 0) {
37010
- nextEntries.push([key, nested]);
37011
- }
37012
- continue;
37013
- }
37014
- if (key === "$not") {
37015
- if (!isObjectFilterRecord2(value)) {
37016
- continue;
37017
- }
37018
- const nested = buildSQLitePushdownFilter(value);
37019
- if (nested) {
37020
- nextEntries.push([key, nested]);
37021
- }
37022
- continue;
37023
- }
37024
- if (Array.isArray(value) || isOperatorFilterRecord2(value) && Object.keys(value).some((operator) => operator === "$contains" || operator === "$containsAny" || operator === "$containsAll")) {
37025
- continue;
37026
- }
37027
- nextEntries.push([key, value]);
37028
- }
37029
- return nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined;
37030
- };
37031
- var normalizeDistance2 = (distance, metric) => {
37032
- if (!Number.isFinite(distance)) {
37033
- return 0;
37034
- }
37035
- if (metric === "cosine") {
37036
- return Math.min(1, Math.max(0, 1 - distance));
37037
- }
37038
- return Math.max(0, 1 / (1 + Math.abs(distance)));
37039
- };
37040
- var matchesFilter2 = (record, filter) => matchesMetadataFilterRecord({
37041
- chunkId: record.chunkId,
37042
- metadata: record.metadata,
37043
- source: record.source,
37044
- title: record.title,
37045
- ...record.metadata ?? {}
37046
- }, filter);
37047
- var mapFilterToRows = (rows) => rows.map((row) => ({
37048
- chunkId: row.chunk_id,
37049
- metadata: parseMetadata2(row.metadata),
37050
- source: row.source ?? undefined,
37051
- text: row.text,
37052
- title: row.title ?? undefined,
37053
- vector: parseVector(row.embedding)
37054
- }));
37055
- var buildJsonQuerySql = (tableName, whereClause) => `
37056
- SELECT chunk_id, text, title, source, metadata, embedding FROM ${tableName}
37057
- ${whereClause ? `WHERE ${whereClause}` : ""}
37058
- `;
37059
- var buildJsonCountSql = (tableName, whereClause) => `
37060
- SELECT COUNT(*) AS count FROM ${tableName}
37061
- ${whereClause ? `WHERE ${whereClause}` : ""}
37062
- `;
37063
- var getChunkCountFromSql = (db, sql, params = []) => {
37064
- const result = db.prepare(sql).get(...params);
37065
- const count = result?.count;
37066
- return typeof count === "number" && Number.isFinite(count) ? count : 0;
37067
- };
37068
- var getPragmaNumericValue = (db, pragma) => {
37069
- const row = db.prepare(`PRAGMA ${pragma}`).get();
37070
- const value = row ? row[Object.keys(row)[0] ?? ""] : undefined;
37071
- return typeof value === "number" && Number.isFinite(value) ? value : 0;
37072
- };
37073
- var refreshSQLiteRuntimeDiagnostics = (db, nativeDiagnostics, diagnosticTableName) => {
37074
- if (!nativeDiagnostics) {
37075
- return;
37076
- }
37077
- try {
37078
- const rowCount = getChunkCountFromSql(db, buildJsonCountSql(diagnosticTableName));
37079
- const pageCount = getPragmaNumericValue(db, "page_count");
37080
- const pageSize = getPragmaNumericValue(db, "page_size");
37081
- const freelistCount = getPragmaNumericValue(db, "freelist_count");
37082
- nativeDiagnostics.rowCount = rowCount;
37083
- nativeDiagnostics.pageCount = pageCount;
37084
- nativeDiagnostics.freelistCount = freelistCount;
37085
- nativeDiagnostics.databaseBytes = pageCount > 0 && pageSize > 0 ? pageCount * pageSize : 0;
37086
- nativeDiagnostics.lastHealthCheckAt = Date.now();
37087
- nativeDiagnostics.lastHealthError = undefined;
37088
- } catch (error) {
37089
- nativeDiagnostics.lastHealthCheckAt = Date.now();
37090
- nativeDiagnostics.lastHealthError = getErrorMessage2(error);
37091
- }
37092
- };
37093
- var buildNativeQuerySql = (tableName, whereClause) => `
37094
- SELECT
37095
- chunk_id,
37096
- embedding,
37097
- chunk_text,
37098
- title,
37099
- source,
37100
- metadata,
37101
- distance
37102
- FROM ${tableName}
37103
- WHERE embedding MATCH vec_f32(?)
37104
- AND k = ?
37105
- ${whereClause ? `AND (${whereClause})` : ""}
37106
- ORDER BY distance
37107
- `;
37108
- var getFilteredSQLiteCandidateCount = (db, tableName, filterPlan) => {
37109
- if (!filterPlan) {
37110
- return;
37111
- }
37112
- const result = db.prepare(buildJsonCountSql(tableName, filterPlan.clause)).get(...filterPlan.params);
37113
- const count = result?.count;
37114
- return typeof count === "number" && Number.isFinite(count) ? count : undefined;
37115
- };
37116
- var createJsonStatements = (db, tableName) => {
37117
- const insertSql = `
37118
- INSERT INTO ${tableName} (
37119
- chunk_id,
37120
- text,
37121
- title,
37122
- source,
37123
- metadata,
37124
- embedding
37125
- ) VALUES (?, ?, ?, ?, ?, ?)
37126
- ON CONFLICT(chunk_id) DO UPDATE SET
37127
- text = excluded.text,
37128
- title = excluded.title,
37129
- source = excluded.source,
37130
- metadata = excluded.metadata,
37131
- embedding = excluded.embedding
37132
- `;
37133
- const querySql = `
37134
- SELECT chunk_id, text, title, source, metadata, embedding FROM ${tableName}
37135
- `;
37136
- const clearSql = `DELETE FROM ${tableName}`;
37137
- const deleteSql = `DELETE FROM ${tableName} WHERE chunk_id = ?`;
37138
- const init = () => db.exec(`
37139
- CREATE TABLE IF NOT EXISTS ${tableName} (
37140
- chunk_id TEXT PRIMARY KEY,
37141
- text TEXT NOT NULL,
37142
- title TEXT,
37143
- source TEXT,
37144
- metadata TEXT,
37145
- embedding TEXT NOT NULL
37146
- )
37147
- `);
37148
- init();
37149
- return {
37150
- clear: db.prepare(clearSql),
37151
- delete: db.prepare(deleteSql),
37152
- init,
37153
- insert: db.prepare(insertSql),
37154
- query: db.prepare(querySql)
37155
- };
37156
- };
37157
- var getSQLiteChunkIdsByChunkIds = (db, tableName, chunkIds) => {
37158
- const uniqueChunkIds = [...new Set(chunkIds)];
37159
- if (uniqueChunkIds.length === 0) {
37160
- return [];
37161
- }
37162
- const whereClause = uniqueChunkIds.map(() => "?").join(", ");
37163
- const rows = db.prepare(`SELECT chunk_id FROM ${tableName} WHERE chunk_id IN (${whereClause})`).all(...uniqueChunkIds);
37164
- return rows.map((row) => row.chunk_id).filter((chunkId) => typeof chunkId === "string");
37165
- };
37166
- var getSQLiteCandidateChunkIdsByFilter = (db, tableName, filter, jsonStatements) => {
37167
- if (!filter || Object.keys(filter).length === 0) {
37168
- return [];
37169
- }
37170
- const pushdownFilter = buildSQLitePushdownFilter(filter);
37171
- const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
37172
- chunkId: "chunk_id",
37173
- metadata: "metadata",
37174
- source: "source",
37175
- title: "title"
35673
+ // src/adapters/sync.ts
35674
+ import {
35675
+ createSyncEngine,
35676
+ createTextIndex,
35677
+ createVectorIndex,
35678
+ defineSearchCollection
35679
+ } from "@absolutejs/sync/engine";
35680
+ var createSyncRAGStore = (options = {}) => {
35681
+ const dimensions = options.dimensions ?? RAG_VECTOR_DIMENSIONS_DEFAULT;
35682
+ const engine = options.engine ?? createSyncEngine();
35683
+ const retrievalCollection = options.collection ?? "ragRetrieval";
35684
+ const table = options.table ?? "ragChunks";
35685
+ const records = new Map;
35686
+ const toLight = (record) => ({
35687
+ chunkId: record.chunkId,
35688
+ chunkText: record.text,
35689
+ source: record.source,
35690
+ title: record.title
37176
35691
  });
37177
- const rawRows = toStoredRows(filterPlan ? db.prepare(buildJsonQuerySql(tableName, filterPlan.clause)).all(...filterPlan.params) : jsonStatements.query.all());
37178
- const chunks = mapFilterToRows(rawRows);
37179
- return chunks.filter((chunk) => matchesFilter2(chunk, filter)).map((chunk) => chunk.chunkId);
37180
- };
37181
- var getSQLiteCandidateChunkIds = (db, tableName, jsonStatements, input) => {
37182
- const chunkIdSet = new Set;
37183
- if (input.filter && Object.keys(input.filter).length > 0) {
37184
- for (const chunkId of getSQLiteCandidateChunkIdsByFilter(db, tableName, input.filter, jsonStatements)) {
37185
- chunkIdSet.add(chunkId);
37186
- }
37187
- }
37188
- if (input.chunkIds && input.chunkIds.length > 0) {
37189
- for (const chunkId of getSQLiteChunkIdsByChunkIds(db, tableName, input.chunkIds)) {
37190
- chunkIdSet.add(chunkId);
37191
- }
37192
- }
37193
- return [...chunkIdSet];
37194
- };
37195
- var toVectorText = (vector) => JSON.stringify(vector);
37196
- var createNativeVec0Table = (db, tableName, dimensions, metric) => {
37197
- const metricSuffix = metric === "cosine" ? " distance_metric=cosine" : "";
37198
- db.exec(`
37199
- CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName} USING vec0(
37200
- chunk_id TEXT,
37201
- embedding float[${dimensions}]${metricSuffix},
37202
- +chunk_text TEXT,
37203
- title TEXT,
37204
- source TEXT,
37205
- metadata TEXT
37206
- )
37207
- `);
37208
- };
37209
- var createNativeVec0Statements = (db, tableName) => {
37210
- const upsertSql = `
37211
- INSERT INTO ${tableName} (
37212
- chunk_id,
37213
- embedding,
37214
- chunk_text,
37215
- title,
37216
- source,
37217
- metadata
37218
- ) VALUES (?, vec_f32(?), ?, ?, ?, ?)
37219
- `;
37220
- const deleteSql = `DELETE FROM ${tableName} WHERE chunk_id = ?`;
37221
- const querySql = `
37222
- SELECT
37223
- chunk_id,
37224
- embedding,
37225
- chunk_text,
37226
- title,
37227
- source,
37228
- metadata,
37229
- distance
37230
- FROM ${tableName}
37231
- WHERE embedding MATCH vec_f32(?)
37232
- AND k = ?
37233
- ORDER BY distance
37234
- `;
37235
- return {
37236
- clear: db.prepare(`DELETE FROM ${tableName}`),
37237
- delete: db.prepare(deleteSql),
37238
- insert: db.prepare(upsertSql),
37239
- query: db.prepare(querySql)
37240
- };
37241
- };
37242
- var mapToRows = (vector, chunks, filter) => chunks.map((chunk) => ({
37243
- chunk,
37244
- score: querySimilarity(vector, normalizeVector(chunk.vector))
37245
- })).filter(({ chunk }) => matchesFilter2(chunk, filter)).sort((left, right) => right.score - left.score);
37246
- var executeNativeInitSql = (db, initSql) => {
37247
- if (!initSql) {
37248
- return;
37249
- }
37250
- if (typeof initSql === "string") {
37251
- db.exec(initSql);
37252
- return;
37253
- }
37254
- for (const command of initSql) {
37255
- db.exec(command);
37256
- }
37257
- };
37258
- var getErrorMessage2 = (error) => error instanceof Error ? error.message : String(error);
37259
- var resolveConfiguredNativeExtension = (nativeConfig) => {
37260
- const platformKey = `${process.platform}-${process.arch}`;
37261
- if (nativeConfig?.extensionPath) {
37262
- return existsSync2(nativeConfig.extensionPath) ? {
37263
- libraryPath: nativeConfig.extensionPath,
37264
- platformKey,
37265
- source: "explicit",
37266
- status: "resolved"
37267
- } : {
37268
- libraryPath: nativeConfig.extensionPath,
37269
- platformKey,
37270
- reason: `Configured native.extensionPath was not found: ${nativeConfig.extensionPath}`,
37271
- source: "explicit",
37272
- status: "binary_missing"
37273
- };
37274
- }
37275
- if (nativeConfig?.resolveFromAbsolutePackages !== false) {
37276
- return resolveAbsolutePackageNativeExtension(platformKey);
37277
- }
37278
- const envResolution = resolveNativeExtensionFromEnv(platformKey);
37279
- if (envResolution)
37280
- return envResolution;
37281
- return {
37282
- platformKey,
37283
- reason: "No native sqlite-vec path was configured. AbsoluteJS will still attempt vec0 initialization in case the extension is already registered on the Database connection.",
37284
- source: "database",
37285
- status: "not_configured"
37286
- };
37287
- };
37288
- var describeNativeFallbackReason = (resolution) => {
37289
- if (!resolution) {
37290
- return "Native sqlite vec0 was not configured.";
37291
- }
37292
- switch (resolution.status) {
37293
- case "resolved":
37294
- return;
37295
- case "package_not_installed":
37296
- return `Install ${resolution.packageName ?? "@absolutejs/absolute-rag-sqlite"} for ${resolution.platformKey}, or provide native.extensionPath.`;
37297
- case "binary_missing":
37298
- return resolution.reason ?? "Resolved sqlite-vec binary was missing.";
37299
- case "unsupported_platform":
37300
- return resolution.reason ?? "This platform is not yet supported by AbsoluteJS sqlite-vec packages.";
37301
- case "not_configured":
37302
- return resolution.reason ?? "No sqlite-vec binary path was configured.";
37303
- case "package_invalid":
37304
- return resolution.reason ?? "The sqlite-vec package manifest was invalid.";
37305
- default:
37306
- return "Native sqlite vec0 could not be initialized.";
37307
- }
37308
- };
37309
- var resolveNativeExtensionFromEnv = (platformKey) => {
37310
- const envPath = process.env.SQLITE_VEC_EXTENSION_PATH;
37311
- if (!envPath) {
37312
- return null;
37313
- }
37314
- if (existsSync2(envPath)) {
37315
- return {
37316
- libraryPath: envPath,
37317
- platformKey,
37318
- source: "env",
37319
- status: "resolved"
37320
- };
37321
- }
37322
- return {
37323
- libraryPath: envPath,
37324
- platformKey,
37325
- reason: `SQLITE_VEC_EXTENSION_PATH was set but not found: ${envPath}`,
37326
- source: "env",
37327
- status: "binary_missing"
37328
- };
37329
- };
37330
- var shouldResolveNativeFromEnv = (resolution) => resolution.status === "binary_missing" || resolution.status === "package_not_installed" || resolution.status === "unsupported_platform";
37331
- var resolveAbsolutePackageNativeExtension = (platformKey) => {
37332
- const packageResolution = resolveAbsoluteSQLiteVec();
37333
- if (!shouldResolveNativeFromEnv(packageResolution)) {
37334
- return packageResolution;
37335
- }
37336
- const envResolution = resolveNativeExtensionFromEnv(platformKey);
37337
- if (envResolution) {
37338
- return envResolution;
37339
- }
37340
- return packageResolution;
37341
- };
37342
- var activateNativeDiagnostics = (nativeDiagnostics) => {
37343
- if (!nativeDiagnostics) {
37344
- return;
37345
- }
37346
- nativeDiagnostics.available = true;
37347
- nativeDiagnostics.active = true;
37348
- nativeDiagnostics.fallbackReason = undefined;
37349
- if (nativeDiagnostics.resolution && nativeDiagnostics.resolution.status === "resolved") {
37350
- return;
37351
- }
37352
- nativeDiagnostics.resolution = {
37353
- platformKey: `${process.platform}-${process.arch}`,
37354
- reason: "sqlite-vec was already available on the Database connection or loaded by native.extensionInitSql.",
37355
- source: "database",
37356
- status: "resolved"
37357
- };
37358
- };
37359
- var loadNativeExtension = (db, nativeResolution) => {
37360
- if (nativeResolution?.status !== "resolved") {
37361
- return;
37362
- }
37363
- if (!nativeResolution.libraryPath) {
37364
- return;
37365
- }
37366
- db.loadExtension(nativeResolution.libraryPath);
37367
- };
37368
- var markNativeLoadFailure = (nativeDiagnostics, error, nativeResolution) => {
37369
- if (!nativeDiagnostics) {
37370
- return;
37371
- }
37372
- nativeDiagnostics.available = false;
37373
- nativeDiagnostics.active = false;
37374
- nativeDiagnostics.lastLoadError = getErrorMessage2(error);
37375
- nativeDiagnostics.fallbackReason = describeNativeFallbackReason(nativeResolution) ?? nativeDiagnostics.lastLoadError;
37376
- };
37377
- var markNativeQueryFailure = (nativeDiagnostics, error) => {
37378
- if (!nativeDiagnostics) {
37379
- return;
37380
- }
37381
- nativeDiagnostics.lastQueryError = getErrorMessage2(error);
37382
- nativeDiagnostics.active = false;
37383
- nativeDiagnostics.fallbackReason = nativeDiagnostics.lastQueryError;
37384
- };
37385
- var countFilterClauses2 = (filter) => {
37386
- if (!filter) {
37387
- return 0;
37388
- }
37389
- let count = 0;
37390
- for (const [key, value] of Object.entries(filter)) {
37391
- if (key === "$and" || key === "$or") {
37392
- if (isNestedFilterArray3(value)) {
37393
- count += value.reduce((total, entry) => total + countFilterClauses2(entry), 0);
37394
- }
37395
- continue;
37396
- }
37397
- if (key === "$not") {
37398
- if (isObjectFilterRecord2(value)) {
37399
- count += countFilterClauses2(value);
37400
- }
37401
- continue;
37402
- }
37403
- count += 1;
37404
- }
37405
- return count;
37406
- };
37407
- var resolveSQLitePushdownMode = (input) => {
37408
- const totalFilterClauseCount = countFilterClauses2(input.filter);
37409
- const pushdownClauseCount = countFilterClauses2(input.pushdownFilter);
37410
- const jsRemainderClauseCount = Math.max(0, totalFilterClauseCount - pushdownClauseCount);
37411
- const pushdownMode = pushdownClauseCount === 0 ? "none" : pushdownClauseCount >= totalFilterClauseCount ? "full" : "partial";
37412
- return {
37413
- jsRemainderClauseCount,
37414
- jsRemainderRatio: totalFilterClauseCount > 0 ? jsRemainderClauseCount / totalFilterClauseCount : undefined,
37415
- pushdownClauseCount,
37416
- pushdownCoverageRatio: totalFilterClauseCount > 0 ? pushdownClauseCount / totalFilterClauseCount : undefined,
37417
- pushdownMode,
37418
- totalFilterClauseCount
37419
- };
37420
- };
37421
- var updateSQLiteLastQueryPlan = (input) => {
37422
- if (!input.nativeDiagnostics) {
37423
- return;
37424
- }
37425
- const pushdown = resolveSQLitePushdownMode({
37426
- filter: input.filter,
37427
- pushdownFilter: input.pushdownFilter
35692
+ const textIndex = createTextIndex({
35693
+ fields: ["chunkText"],
35694
+ key: (chunk) => chunk.chunkId
37428
35695
  });
37429
- input.nativeDiagnostics.lastQueryPlan = {
37430
- backfillCount: input.backfillCount,
37431
- candidateBudgetExhausted: input.candidateBudgetExhausted,
37432
- candidateCoverage: summarizeSQLiteCandidateCoverage({
37433
- filteredCandidateCount: input.filteredCandidateCount,
37434
- returnedCount: input.returnedCount,
37435
- topK: input.topK
37436
- }),
37437
- filteredCandidateCount: input.filteredCandidateCount,
37438
- finalSearchK: input.finalSearchK,
37439
- initialSearchK: input.initialSearchK,
37440
- searchExpansionRatio: typeof input.initialSearchK === "number" && typeof input.finalSearchK === "number" && input.initialSearchK > 0 ? input.finalSearchK / input.initialSearchK : undefined,
37441
- jsRemainderClauseCount: pushdown.jsRemainderClauseCount,
37442
- plannerProfileUsed: input.plannerProfileUsed,
37443
- candidateLimitUsed: input.candidateLimitUsed,
37444
- maxBackfillsUsed: input.maxBackfillsUsed,
37445
- minResultsUsed: input.minResultsUsed,
37446
- fillPolicyUsed: input.fillPolicyUsed,
37447
- queryMultiplierUsed: input.queryMultiplierUsed,
37448
- jsRemainderRatio: pushdown.jsRemainderRatio,
37449
- pushdownApplied: pushdown.pushdownClauseCount > 0,
37450
- pushdownClauseCount: pushdown.pushdownClauseCount,
37451
- pushdownCoverageRatio: pushdown.pushdownCoverageRatio,
37452
- pushdownMode: pushdown.pushdownMode,
37453
- queryMode: input.queryMode,
37454
- candidateYieldRatio: typeof input.returnedCount === "number" && typeof input.finalSearchK === "number" && input.finalSearchK > 0 ? input.returnedCount / input.finalSearchK : undefined,
37455
- returnedCount: input.returnedCount,
37456
- backfillLimitReached: input.backfillLimitReached,
37457
- minResultsSatisfied: input.minResultsSatisfied,
37458
- topKFillRatio: typeof input.returnedCount === "number" && input.topK > 0 ? input.returnedCount / input.topK : undefined,
37459
- totalFilterClauseCount: pushdown.totalFilterClauseCount,
37460
- underfilledTopK: input.underfilledTopK
37461
- };
37462
- };
37463
- var markNativeUpsertFailure = (nativeDiagnostics, error) => {
37464
- if (!nativeDiagnostics) {
37465
- return;
37466
- }
37467
- nativeDiagnostics.lastUpsertError = getErrorMessage2(error);
37468
- nativeDiagnostics.active = false;
37469
- nativeDiagnostics.fallbackReason = nativeDiagnostics.lastUpsertError;
37470
- };
37471
- var analyzeSQLiteBackend = (input) => {
37472
- try {
37473
- input.db.exec("PRAGMA optimize");
37474
- input.db.exec(`ANALYZE ${input.tableName}`);
37475
- input.nativeDiagnostics && (input.nativeDiagnostics.lastAnalyzeAt = Date.now());
37476
- if (input.nativeDiagnostics) {
37477
- input.nativeDiagnostics.lastAnalyzeError = undefined;
37478
- }
37479
- refreshSQLiteRuntimeDiagnostics(input.db, input.nativeDiagnostics, input.diagnosticTableName);
37480
- } catch (error) {
37481
- if (input.nativeDiagnostics) {
37482
- input.nativeDiagnostics.lastAnalyzeAt = Date.now();
37483
- input.nativeDiagnostics.lastAnalyzeError = getErrorMessage2(error);
37484
- }
37485
- throw error;
37486
- }
37487
- };
37488
- var initializeNativeBackend = (input) => {
37489
- const {
37490
- db,
37491
- dimensions,
37492
- nativeConfig,
37493
- nativeDiagnostics,
37494
- nativeDistanceMetric,
37495
- nativeResolution,
37496
- nativeTableName
37497
- } = input;
37498
- loadNativeExtension(db, nativeResolution);
37499
- executeNativeInitSql(db, nativeConfig.extensionInitSql);
37500
- createNativeVec0Table(db, nativeTableName, dimensions, nativeDistanceMetric);
37501
- const nativeStatements = createNativeVec0Statements(db, nativeTableName);
37502
- activateNativeDiagnostics(nativeDiagnostics);
37503
- return nativeStatements;
37504
- };
37505
- var createNativeInitializationError = (error, nativeTableName) => new Error(`Failed to initialize sqlite vec0 backend for table "${nativeTableName}". ` + `Install @absolutejs/absolute-rag-sqlite for your platform, set native.extensionPath, or pre-register the sqlite-vec extension in the Database connection. ` + `Details: ${getErrorMessage2(error)}`);
37506
- var initializeNativeBackendSafely = (input) => {
37507
- const { nativeConfig, nativeDiagnostics, nativeResolution, nativeTableName } = input;
37508
- try {
37509
- return initializeNativeBackend(input);
37510
- } catch (error) {
37511
- markNativeLoadFailure(nativeDiagnostics, error, nativeResolution);
37512
- if (nativeConfig.requireAvailable) {
37513
- throw createNativeInitializationError(error, nativeTableName);
37514
- }
37515
- return;
37516
- }
37517
- };
37518
- var fallbackToJsonUpsert = (chunks, jsonStatements) => {
37519
- for (const chunk of chunks) {
37520
- jsonStatements.insert.run(chunk.chunkId, chunk.text, chunk.title ?? null, chunk.source ?? null, toJSONString(chunk.metadata), toVectorText(chunk.vector));
37521
- }
37522
- };
37523
- var upsertNativeChunks = (chunks, nativeStatements) => {
37524
- if (!nativeStatements) {
37525
- throw new Error("Native vector statements unavailable");
37526
- }
37527
- for (const chunk of chunks) {
37528
- nativeStatements.delete.run(chunk.chunkId);
37529
- nativeStatements.insert.run(chunk.chunkId, toVectorText(chunk.vector), chunk.text, chunk.title ?? null, chunk.source ?? null, toJSONString(chunk.metadata));
37530
- }
37531
- };
37532
- var createSQLiteRAGStore = (options = {}) => {
37533
- const dimensions = options.dimensions ?? DEFAULT_DIMENSIONS2;
37534
- const tableName = options.tableName ?? DEFAULT_TABLE_NAME2;
37535
- assertSupportedIdentifier3(tableName);
37536
- const nativeConfig = options.native;
37537
- const nativeTableName = nativeConfig?.tableName ?? `${tableName}${DEFAULT_NATIVE_TABLE_SUFFIX}`;
37538
- if (nativeConfig?.mode === "vec0" && nativeConfig.tableName) {
37539
- assertSupportedIdentifier3(nativeConfig.tableName);
37540
- }
37541
- if (!Number.isInteger(dimensions) || dimensions <= 0) {
37542
- throw new Error(`Invalid dimension "${dimensions}". dimensions must be a positive integer.`);
37543
- }
37544
- const db = options.db ?? new Database(options.path ?? ":memory:");
37545
- const nativeDistanceMetric = nativeConfig?.distanceMetric === "l2" ? "l2" : "cosine";
37546
- const nativeQueryMultiplier = normalizeQueryMultiplier2(nativeConfig?.queryMultiplier);
37547
- const nativeResolution = nativeConfig?.mode === "vec0" ? resolveConfiguredNativeExtension(nativeConfig) : undefined;
37548
- const nativeDiagnostics = nativeConfig?.mode === "vec0" ? {
37549
- active: false,
37550
- available: false,
37551
- distanceMetric: nativeDistanceMetric,
37552
- fallbackReason: describeNativeFallbackReason(nativeResolution),
37553
- mode: nativeConfig.mode,
37554
- requested: true,
37555
- resolution: nativeResolution,
37556
- tableName: nativeTableName
37557
- } : undefined;
37558
- const jsonStatements = createJsonStatements(db, tableName);
37559
- jsonStatements.init();
37560
- let useNative = false;
37561
- let nativeStatements;
37562
- if (nativeConfig?.mode === "vec0") {
37563
- nativeStatements = initializeNativeBackendSafely({
37564
- db,
37565
- dimensions,
37566
- nativeConfig,
37567
- nativeDiagnostics,
37568
- nativeDistanceMetric,
37569
- nativeResolution,
37570
- nativeTableName
37571
- });
37572
- useNative = nativeStatements !== undefined;
37573
- }
37574
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
35696
+ const vectorIndex = createVectorIndex({
35697
+ embedding: (chunk) => records.get(chunk.chunkId)?.vector ?? [],
35698
+ key: (chunk) => chunk.chunkId
35699
+ });
35700
+ engine.registerSearch(defineSearchCollection({
35701
+ index: () => textIndex,
35702
+ key: (chunk) => chunk.chunkId,
35703
+ name: retrievalCollection,
35704
+ source: () => [...records.values()].map(toLight),
35705
+ table
35706
+ }));
37575
35707
  const embed = async (input) => {
37576
- input.model;
37577
- if (input.signal?.aborted) {
37578
- throw new DOMException("Aborted", "AbortError");
35708
+ if (options.embedding) {
35709
+ return options.embedding(input);
37579
35710
  }
37580
35711
  if (options.mockEmbedding) {
37581
- return options.mockEmbedding(input.text).then(normalizeVector);
35712
+ return options.mockEmbedding(input.text);
37582
35713
  }
37583
- return normalizeVector([...createRAGVector(input.text, dimensions)]);
37584
- };
37585
- const queryFallback = async (input) => {
37586
- const queryVector = normalizeVector(input.queryVector);
37587
- const pushdownFilter = buildSQLitePushdownFilter(input.filter);
37588
- const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
37589
- chunkId: "chunk_id",
37590
- metadata: "metadata",
37591
- source: "source",
37592
- title: "title"
37593
- });
37594
- const rawRows = toStoredRows(filterPlan ? db.prepare(buildJsonQuerySql(tableName, filterPlan.clause)).all(...filterPlan.params) : jsonStatements.query.all());
37595
- const chunks = mapFilterToRows(rawRows);
37596
- const filtered = mapToRows(queryVector, chunks, input.filter);
37597
- const limited = filtered.slice(0, input.topK);
37598
- updateSQLiteLastQueryPlan({
37599
- backfillCount: 0,
37600
- candidateBudgetExhausted: false,
37601
- filter: input.filter,
37602
- filteredCandidateCount: rawRows.length,
37603
- finalSearchK: rawRows.length,
37604
- initialSearchK: rawRows.length,
37605
- nativeDiagnostics,
37606
- pushdownFilter,
37607
- plannerProfileUsed: input.plannerProfile,
37608
- queryMultiplierUsed: input.queryMultiplier,
37609
- queryMode: "json_fallback",
37610
- returnedCount: limited.length,
37611
- topK: input.topK,
37612
- underfilledTopK: limited.length < input.topK
37613
- });
37614
- return limited.map(({ chunk, score }) => ({
37615
- chunkId: chunk.chunkId,
37616
- chunkText: chunk.text,
37617
- embedding: chunk.vector,
37618
- metadata: chunk.metadata,
37619
- score,
37620
- source: chunk.source,
37621
- title: chunk.title
37622
- }));
35714
+ return normalizeVector(createRAGVector(input.text, dimensions));
37623
35715
  };
37624
- const queryNative = async (input) => {
37625
- if (!nativeStatements) {
37626
- throw new Error("Native vector backend is not available");
37627
- }
37628
- const queryMultiplier = normalizeQueryMultiplier2(input.queryMultiplier ?? nativeConfig?.queryMultiplier);
37629
- const maxBackfills = normalizeMaxBackfills2(input.maxBackfills);
37630
- const minResults = normalizeMinResults2(input.minResults, input.topK);
37631
- const fillTarget = resolveFillTarget2({
37632
- fillPolicy: input.fillPolicy,
37633
- minResults,
37634
- topK: input.topK
37635
- });
37636
- const queryVector = normalizeVector(input.queryVector);
37637
- const queryVectorText = toVectorText(queryVector);
37638
- const pushdownFilter = buildSQLitePushdownFilter(input.filter);
37639
- const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
37640
- chunkId: "chunk_id",
37641
- metadata: "metadata",
37642
- source: "source",
37643
- title: "title"
37644
- });
37645
- const filteredCandidateCount = getFilteredSQLiteCandidateCount(db, tableName, filterPlan);
37646
- const candidateLimit = resolveAdaptiveNativeCandidateLimit({
37647
- defaultCandidateLimit: RAG_NATIVE_QUERY_CANDIDATE_LIMIT,
37648
- explicitCandidateLimit: input.candidateLimit,
37649
- filteredCandidateCount,
37650
- plannerProfile: input.plannerProfile,
37651
- queryMultiplier,
37652
- topK: input.topK
37653
- });
37654
- const searchK = planNativeCandidateSearchK({
37655
- candidateLimit,
37656
- filteredCandidateCount,
37657
- queryMultiplier,
37658
- topK: input.topK
37659
- });
37660
- if (searchK === 0) {
37661
- return [];
37662
- }
37663
- const runNativeQuery = (candidateK) => toNativeStoredRows(filterPlan ? db.prepare(buildNativeQuerySql(nativeTableName, filterPlan.clause)).all(queryVectorText, candidateK, ...filterPlan.params) : nativeStatements.query.all(queryVectorText, candidateK));
37664
- let currentSearchK = searchK;
37665
- let backfillCount = 0;
37666
- let candidateBudgetExhausted = false;
37667
- let backfillLimitReached = false;
37668
- let mapped = [];
37669
- for (;; ) {
37670
- const rawRows = runNativeQuery(currentSearchK);
37671
- mapped = rawRows.map((row) => {
37672
- const chunk = {
37673
- chunkId: row.chunk_id,
37674
- metadata: parseMetadata2(row.metadata),
37675
- source: row.source ?? undefined,
37676
- text: row.chunk_text,
37677
- title: row.title ?? undefined,
37678
- vector: parseVector(row.embedding)
37679
- };
37680
- return {
37681
- chunk,
37682
- score: normalizeDistance2(row.distance, nativeDistanceMetric)
37683
- };
37684
- }).filter(({ chunk }) => matchesFilter2(chunk, input.filter)).map((entry) => ({
37685
- chunkId: entry.chunk.chunkId,
37686
- chunkText: entry.chunk.text,
37687
- embedding: entry.chunk.vector,
37688
- metadata: entry.chunk.metadata,
37689
- score: entry.score,
37690
- source: entry.chunk.source,
37691
- title: entry.chunk.title
37692
- })).sort((left, right) => right.score - left.score);
37693
- if (mapped.length >= fillTarget.targetResults) {
37694
- break;
37695
- }
37696
- const nextSearchK = planNativeCandidateSearchBackfillK({
37697
- backfillCount,
37698
- candidateLimit,
37699
- currentSearchK,
37700
- filteredCandidateCount,
37701
- maxBackfills
37702
- });
37703
- if (nextSearchK <= currentSearchK) {
37704
- backfillLimitReached = typeof maxBackfills === "number" && backfillCount >= maxBackfills && mapped.length < fillTarget.targetResults;
37705
- candidateBudgetExhausted = mapped.length < fillTarget.targetResults;
37706
- break;
37707
- }
37708
- currentSearchK = nextSearchK;
37709
- backfillCount += 1;
35716
+ const matchesFilter = (record, filter) => matchesMetadataFilterRecord({
35717
+ chunkId: record.chunkId,
35718
+ metadata: record.metadata,
35719
+ source: record.source,
35720
+ title: record.title,
35721
+ ...record.metadata ?? {}
35722
+ }, filter);
35723
+ const toResult = (record, score) => ({
35724
+ chunkId: record.chunkId,
35725
+ chunkText: record.text,
35726
+ embedding: record.vector,
35727
+ metadata: record.metadata,
35728
+ score,
35729
+ source: record.source,
35730
+ title: record.title
35731
+ });
35732
+ const upsert = async (input) => {
35733
+ for (const chunk of input.chunks) {
35734
+ const vector = chunk.embedding ? normalizeVector(chunk.embedding) : normalizeVector(await embed({ text: chunk.text }));
35735
+ const record = {
35736
+ chunkId: chunk.chunkId,
35737
+ metadata: chunk.metadata,
35738
+ source: chunk.source,
35739
+ text: chunk.text,
35740
+ title: chunk.title,
35741
+ vector
35742
+ };
35743
+ records.set(record.chunkId, record);
35744
+ const light = toLight(record);
35745
+ vectorIndex.add(light);
35746
+ textIndex.add(light);
35747
+ await engine.applyChange(table, { op: "insert", row: light });
37710
35748
  }
37711
- updateSQLiteLastQueryPlan({
37712
- backfillCount,
37713
- backfillLimitReached,
37714
- candidateBudgetExhausted,
37715
- filter: input.filter,
37716
- filteredCandidateCount,
37717
- finalSearchK: currentSearchK,
37718
- initialSearchK: searchK,
37719
- nativeDiagnostics,
37720
- pushdownFilter,
37721
- plannerProfileUsed: input.plannerProfile,
37722
- candidateLimitUsed: candidateLimit,
37723
- maxBackfillsUsed: maxBackfills,
37724
- minResultsUsed: minResults,
37725
- fillPolicyUsed: fillTarget.fillPolicy,
37726
- queryMultiplierUsed: queryMultiplier,
37727
- queryMode: "native_vec0",
37728
- returnedCount: Math.min(mapped.length, input.topK),
37729
- minResultsSatisfied: mapped.length >= minResults,
37730
- topK: input.topK,
37731
- underfilledTopK: mapped.length < input.topK
37732
- });
37733
- return mapped.slice(0, input.topK);
37734
35749
  };
37735
35750
  const query = async (input) => {
37736
- if (!useNative) {
37737
- return queryFallback(input);
37738
- }
37739
- try {
37740
- return await queryNative(input);
37741
- } catch (error) {
37742
- markNativeQueryFailure(nativeDiagnostics, error);
37743
- if (nativeConfig?.requireAvailable) {
37744
- throw new Error(`Native vector query failed for table "${nativeTableName}". ${getErrorMessage2(error)}`, { cause: error });
35751
+ const queryVector = normalizeVector(input.queryVector);
35752
+ const results = [];
35753
+ for (const record of records.values()) {
35754
+ if (!matchesFilter(record, input.filter)) {
35755
+ continue;
35756
+ }
35757
+ const score = querySimilarity(queryVector, normalizeVector(record.vector));
35758
+ if (Number.isFinite(score)) {
35759
+ results.push(toResult(record, score));
37745
35760
  }
37746
- return queryFallback(input);
37747
35761
  }
35762
+ results.sort((first, second) => second.score - first.score);
35763
+ return results.slice(0, input.topK);
37748
35764
  };
37749
35765
  const queryLexical = async (input) => {
37750
- const pushdownFilter = buildSQLitePushdownFilter(input.filter);
37751
- const filterPlan = buildSQLiteFilterPlan(pushdownFilter, {
37752
- chunkId: "chunk_id",
37753
- metadata: "metadata",
37754
- source: "source",
37755
- title: "title"
37756
- });
37757
- const rawRows = toStoredRows(filterPlan ? db.prepare(buildJsonQuerySql(tableName, filterPlan.clause)).all(...filterPlan.params) : jsonStatements.query.all());
37758
- const chunks = mapFilterToRows(rawRows).filter((chunk) => matchesFilter2(chunk, input.filter));
37759
- const ranked = rankRAGLexicalMatches(input.query, chunks);
37760
- return ranked.slice(0, input.topK).map(({ result, score }) => ({
37761
- chunkId: result.chunkId,
37762
- chunkText: result.text,
37763
- metadata: result.metadata,
37764
- score,
37765
- source: result.source,
37766
- title: result.title
37767
- }));
37768
- };
37769
- const upsert = async (input) => {
37770
- const chunks = input.chunks.length > 0 ? await Promise.all(input.chunks.map(async (chunk) => ({
37771
- chunkId: chunk.chunkId,
37772
- metadata: chunk.metadata,
37773
- source: chunk.source,
37774
- text: chunk.text,
37775
- title: chunk.title,
37776
- vector: chunk.embedding ? normalizeVector(chunk.embedding) : normalizeVector(await embed({ text: chunk.text }))
37777
- }))) : [];
37778
- if (!useNative) {
37779
- fallbackToJsonUpsert(chunks, jsonStatements);
37780
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, tableName);
37781
- return;
37782
- }
37783
- try {
37784
- upsertNativeChunks(chunks, nativeStatements);
37785
- } catch (error) {
37786
- markNativeUpsertFailure(nativeDiagnostics, error);
37787
- if (nativeConfig?.requireAvailable) {
37788
- throw new Error(`Native vector upsert failed for table "${nativeTableName}". ${getErrorMessage2(error)}`, { cause: error });
35766
+ const hits = textIndex.search(input.query, input.topK * 5);
35767
+ const results = [];
35768
+ for (const hit of hits) {
35769
+ const record = records.get(hit.row.chunkId);
35770
+ if (record && matchesFilter(record, input.filter)) {
35771
+ results.push(toResult(record, hit.score));
37789
35772
  }
37790
- useNative = false;
37791
- fallbackToJsonUpsert(chunks, jsonStatements);
37792
35773
  }
37793
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
35774
+ return results.slice(0, input.topK);
37794
35775
  };
35776
+ const hasFilters = (chunkIds, filter) => ({
35777
+ filtered: Boolean(filter && Object.keys(filter).length > 0),
35778
+ ided: chunkIds.size > 0
35779
+ });
37795
35780
  const count = async (input = {}) => {
37796
- const filter = input.filter;
37797
- const chunkIds = input.chunkIds;
37798
- const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
37799
- const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
37800
- if (!hasFilter && !hasChunkIds) {
37801
- return getChunkCountFromSql(db, buildJsonCountSql(tableName), []);
35781
+ const chunkIds = new Set(input.chunkIds ?? []);
35782
+ const { filtered, ided } = hasFilters(chunkIds, input.filter);
35783
+ if (!filtered && !ided) {
35784
+ return records.size;
37802
35785
  }
37803
- if (hasFilter && !hasChunkIds) {
37804
- return getSQLiteCandidateChunkIdsByFilter(db, tableName, filter, jsonStatements).length;
37805
- }
37806
- return getSQLiteCandidateChunkIds(db, tableName, jsonStatements, {
37807
- chunkIds,
37808
- filter
37809
- }).length;
35786
+ return [...records.values()].filter((record) => ided && chunkIds.has(record.chunkId) || filtered && matchesFilter(record, input.filter)).length;
37810
35787
  };
37811
35788
  const remove = async (input = {}) => {
37812
- const filter = input.filter;
37813
- const chunkIds = input.chunkIds;
37814
- const hasFilter = Boolean(filter && Object.keys(filter).length > 0);
37815
- const hasChunkIds = Boolean(chunkIds && chunkIds.length > 0);
37816
- if (!hasFilter && !hasChunkIds) {
37817
- return 0;
37818
- }
37819
- const toDelete = getSQLiteCandidateChunkIds(db, tableName, jsonStatements, {
37820
- chunkIds,
37821
- filter
37822
- });
37823
- if (toDelete.length === 0) {
35789
+ const chunkIds = new Set(input.chunkIds ?? []);
35790
+ const { filtered, ided } = hasFilters(chunkIds, input.filter);
35791
+ if (!filtered && !ided) {
37824
35792
  return 0;
37825
35793
  }
37826
- for (const chunkId of toDelete) {
37827
- jsonStatements.delete.run(chunkId);
37828
- }
37829
- if (!useNative || !nativeStatements) {
37830
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, tableName);
37831
- return toDelete.length;
37832
- }
37833
- for (const chunkId of toDelete) {
37834
- nativeStatements.delete.run(chunkId);
35794
+ let removed = 0;
35795
+ for (const record of [...records.values()]) {
35796
+ const matches = ided && chunkIds.has(record.chunkId) || filtered && matchesFilter(record, input.filter);
35797
+ if (!matches) {
35798
+ continue;
35799
+ }
35800
+ records.delete(record.chunkId);
35801
+ vectorIndex.remove(record.chunkId);
35802
+ textIndex.remove(record.chunkId);
35803
+ await engine.applyChange(table, {
35804
+ op: "delete",
35805
+ row: toLight(record)
35806
+ });
35807
+ removed += 1;
37835
35808
  }
37836
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
37837
- return toDelete.length;
35809
+ return removed;
37838
35810
  };
37839
- const clear = () => {
37840
- jsonStatements.clear.run();
37841
- if (!useNative || !nativeStatements) {
37842
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, tableName);
37843
- return;
37844
- }
37845
- try {
37846
- nativeStatements.clear.run();
37847
- } catch {
37848
- jsonStatements.clear.run();
35811
+ const clear = async () => {
35812
+ for (const record of [...records.values()]) {
35813
+ records.delete(record.chunkId);
35814
+ vectorIndex.remove(record.chunkId);
35815
+ textIndex.remove(record.chunkId);
35816
+ await engine.applyChange(table, { op: "delete", row: toLight(record) });
37849
35817
  }
37850
- refreshSQLiteRuntimeDiagnostics(db, nativeDiagnostics, useNative ? nativeTableName : tableName);
37851
35818
  };
37852
- const analyze = () => {
37853
- analyzeSQLiteBackend({
37854
- db,
37855
- diagnosticTableName: useNative ? nativeTableName : tableName,
37856
- nativeDiagnostics,
37857
- tableName
37858
- });
35819
+ const status = {
35820
+ backend: "in_memory",
35821
+ dimensions,
35822
+ vectorMode: "in_memory"
35823
+ };
35824
+ const capabilities = {
35825
+ backend: "in_memory",
35826
+ nativeVectorSearch: false,
35827
+ persistence: "memory_only",
35828
+ serverSideFiltering: false,
35829
+ streamingIngestStatus: false
37859
35830
  };
37860
35831
  return {
37861
- analyze,
37862
35832
  clear,
35833
+ count,
35834
+ delete: remove,
37863
35835
  embed,
35836
+ engine,
35837
+ getCapabilities: () => capabilities,
35838
+ getStatus: () => status,
37864
35839
  query,
37865
35840
  queryLexical,
37866
- count,
37867
- delete: remove,
37868
- upsert,
37869
- getCapabilities: () => createSQLiteCapabilities(useNative),
37870
- getStatus: () => createSQLiteStatus(dimensions, nativeDiagnostics, useNative)
35841
+ retrievalCollection,
35842
+ upsert
37871
35843
  };
37872
35844
  };
37873
35845
  export {
@@ -37889,8 +35861,6 @@ export {
37889
35861
  resolveRAGQueryTransform,
37890
35862
  resolveRAGHybridSearchOptions,
37891
35863
  resolveRAGEmbeddingProvider,
37892
- resolveAbsoluteSQLiteVecExtensionPath,
37893
- resolveAbsoluteSQLiteVec,
37894
35864
  reorderRAGEvaluationSuiteCases,
37895
35865
  removeRAGEvaluationSuiteCaseHardNegative,
37896
35866
  removeRAGEvaluationSuiteCase,
@@ -37977,7 +35947,7 @@ export {
37977
35947
  deepseekEmbeddings,
37978
35948
  createVoyageRAGReranker,
37979
35949
  createTextFileExtractor,
37980
- createSQLiteRAGStore,
35950
+ createSyncRAGStore,
37981
35951
  createRAGVector,
37982
35952
  createRAGUrlSyncSource,
37983
35953
  createRAGSyncScheduler,
@@ -38069,7 +36039,6 @@ export {
38069
36039
  createRAGAdaptiveNativePlannerBenchmarkSuite,
38070
36040
  createRAGAdaptiveNativePlannerBenchmarkSnapshot,
38071
36041
  createRAGAccessControl,
38072
- createPostgresRAGStore,
38073
36042
  createPDFFileExtractor,
38074
36043
  createOfficeDocumentExtractor,
38075
36044
  createLegacyDocumentExtractor,
@@ -38164,5 +36133,5 @@ export {
38164
36133
  addRAGEvaluationSuiteCase
38165
36134
  };
38166
36135
 
38167
- //# debugId=BD3325EB562F864664756E2164756E21
36136
+ //# debugId=7661BF2D41E88F6A64756E2164756E21
38168
36137
  //# sourceMappingURL=index.js.map