@jcode.labs/ragmir 2.17.2 → 2.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -6
- package/dist/access-log.d.ts +10 -0
- package/dist/access-log.d.ts.map +1 -1
- package/dist/access-log.js +119 -49
- package/dist/access-log.js.map +1 -1
- package/dist/chunking.d.ts +5 -1
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +102 -16
- package/dist/chunking.js.map +1 -1
- package/dist/citation.d.ts +18 -0
- package/dist/citation.d.ts.map +1 -0
- package/dist/citation.js +54 -0
- package/dist/citation.js.map +1 -0
- package/dist/cli-entry.d.ts +3 -0
- package/dist/cli-entry.d.ts.map +1 -0
- package/dist/cli-entry.js +16 -0
- package/dist/cli-entry.js.map +1 -0
- package/dist/cli-fast.d.ts +13 -0
- package/dist/cli-fast.d.ts.map +1 -0
- package/dist/cli-fast.js +145 -0
- package/dist/cli-fast.js.map +1 -0
- package/dist/cli-options.d.ts +3 -0
- package/dist/cli-options.d.ts.map +1 -1
- package/dist/cli-options.js +7 -0
- package/dist/cli-options.js.map +1 -1
- package/dist/cli-stdin.d.ts +3 -0
- package/dist/cli-stdin.d.ts.map +1 -0
- package/dist/cli-stdin.js +15 -0
- package/dist/cli-stdin.js.map +1 -0
- package/dist/cli.js +251 -44
- package/dist/cli.js.map +1 -1
- package/dist/client.d.ts +13 -2
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +155 -10
- package/dist/client.js.map +1 -1
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +227 -49
- package/dist/config.js.map +1 -1
- package/dist/context-resources.d.ts +4 -2
- package/dist/context-resources.d.ts.map +1 -1
- package/dist/context-resources.js +55 -24
- package/dist/context-resources.js.map +1 -1
- package/dist/defaults.d.ts +27 -0
- package/dist/defaults.d.ts.map +1 -1
- package/dist/defaults.js +42 -2
- package/dist/defaults.js.map +1 -1
- package/dist/destroy.d.ts +2 -2
- package/dist/destroy.d.ts.map +1 -1
- package/dist/destroy.js +34 -20
- package/dist/destroy.js.map +1 -1
- package/dist/doctor.d.ts +3 -2
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +86 -41
- package/dist/doctor.js.map +1 -1
- package/dist/durable-file.d.ts +13 -0
- package/dist/durable-file.d.ts.map +1 -0
- package/dist/durable-file.js +56 -0
- package/dist/durable-file.js.map +1 -0
- package/dist/embeddings.d.ts +14 -3
- package/dist/embeddings.d.ts.map +1 -1
- package/dist/embeddings.js +233 -95
- package/dist/embeddings.js.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/evaluate.d.ts +4 -3
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +611 -110
- package/dist/evaluate.js.map +1 -1
- package/dist/files.d.ts +4 -1
- package/dist/files.d.ts.map +1 -1
- package/dist/files.js +119 -33
- package/dist/files.js.map +1 -1
- package/dist/generation-retention.d.ts +52 -0
- package/dist/generation-retention.d.ts.map +1 -0
- package/dist/generation-retention.js +318 -0
- package/dist/generation-retention.js.map +1 -0
- package/dist/index-diagnostics.d.ts +3 -2
- package/dist/index-diagnostics.d.ts.map +1 -1
- package/dist/index-diagnostics.js +32 -3
- package/dist/index-diagnostics.js.map +1 -1
- package/dist/index-policy.d.ts.map +1 -1
- package/dist/index-policy.js +3 -0
- package/dist/index-policy.js.map +1 -1
- package/dist/index-write-lock.d.ts +16 -1
- package/dist/index-write-lock.d.ts.map +1 -1
- package/dist/index-write-lock.js +392 -13
- package/dist/index-write-lock.js.map +1 -1
- package/dist/index.d.ts +12 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -2
- package/dist/index.js.map +1 -1
- package/dist/ingest.d.ts +11 -2
- package/dist/ingest.d.ts.map +1 -1
- package/dist/ingest.js +680 -165
- package/dist/ingest.js.map +1 -1
- package/dist/ingestion-metrics.d.ts +58 -0
- package/dist/ingestion-metrics.d.ts.map +1 -0
- package/dist/ingestion-metrics.js +216 -0
- package/dist/ingestion-metrics.js.map +1 -0
- package/dist/ingestion-state.d.ts +11 -2
- package/dist/ingestion-state.d.ts.map +1 -1
- package/dist/ingestion-state.js +480 -49
- package/dist/ingestion-state.js.map +1 -1
- package/dist/limits.d.ts.map +1 -1
- package/dist/limits.js +11 -1
- package/dist/limits.js.map +1 -1
- package/dist/mcp-output.d.ts +44 -4
- package/dist/mcp-output.d.ts.map +1 -1
- package/dist/mcp-output.js +132 -159
- package/dist/mcp-output.js.map +1 -1
- package/dist/mcp-summaries.d.ts +32 -0
- package/dist/mcp-summaries.d.ts.map +1 -0
- package/dist/mcp-summaries.js +227 -0
- package/dist/mcp-summaries.js.map +1 -0
- package/dist/mcp.d.ts +1 -1
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +85 -46
- package/dist/mcp.js.map +1 -1
- package/dist/ocr-cache.d.ts +15 -0
- package/dist/ocr-cache.d.ts.map +1 -0
- package/dist/ocr-cache.js +90 -0
- package/dist/ocr-cache.js.map +1 -0
- package/dist/ocr.d.ts +31 -0
- package/dist/ocr.d.ts.map +1 -1
- package/dist/ocr.js +193 -17
- package/dist/ocr.js.map +1 -1
- package/dist/parsing.d.ts.map +1 -1
- package/dist/parsing.js +482 -85
- package/dist/parsing.js.map +1 -1
- package/dist/permissions.js +2 -2
- package/dist/permissions.js.map +1 -1
- package/dist/preview.d.ts.map +1 -1
- package/dist/preview.js +10 -11
- package/dist/preview.js.map +1 -1
- package/dist/quality-report.d.ts +12 -0
- package/dist/quality-report.d.ts.map +1 -0
- package/dist/quality-report.js +146 -0
- package/dist/quality-report.js.map +1 -0
- package/dist/query.d.ts +7 -4
- package/dist/query.d.ts.map +1 -1
- package/dist/query.js +371 -284
- package/dist/query.js.map +1 -1
- package/dist/ranking.d.ts +38 -0
- package/dist/ranking.d.ts.map +1 -0
- package/dist/ranking.js +278 -0
- package/dist/ranking.js.map +1 -0
- package/dist/redaction.d.ts +6 -1
- package/dist/redaction.d.ts.map +1 -1
- package/dist/redaction.js +118 -4
- package/dist/redaction.js.map +1 -1
- package/dist/research.d.ts +6 -1
- package/dist/research.d.ts.map +1 -1
- package/dist/research.js +330 -130
- package/dist/research.js.map +1 -1
- package/dist/runtime-info.d.ts +3 -0
- package/dist/runtime-info.d.ts.map +1 -0
- package/dist/runtime-info.js +81 -0
- package/dist/runtime-info.js.map +1 -0
- package/dist/security.d.ts +3 -2
- package/dist/security.d.ts.map +1 -1
- package/dist/security.js +242 -45
- package/dist/security.js.map +1 -1
- package/dist/semantic-config.d.ts +7 -1
- package/dist/semantic-config.d.ts.map +1 -1
- package/dist/semantic-config.js +30 -4
- package/dist/semantic-config.js.map +1 -1
- package/dist/setup.js +6 -3
- package/dist/setup.js.map +1 -1
- package/dist/skill.js +34 -12
- package/dist/skill.js.map +1 -1
- package/dist/source-fingerprint-cache.d.ts +20 -0
- package/dist/source-fingerprint-cache.d.ts.map +1 -0
- package/dist/source-fingerprint-cache.js +133 -0
- package/dist/source-fingerprint-cache.js.map +1 -0
- package/dist/storage-maintenance.d.ts +46 -0
- package/dist/storage-maintenance.d.ts.map +1 -0
- package/dist/storage-maintenance.js +329 -0
- package/dist/storage-maintenance.js.map +1 -0
- package/dist/store.d.ts +39 -2
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +559 -24
- package/dist/store.js.map +1 -1
- package/dist/text.d.ts +1 -1
- package/dist/text.d.ts.map +1 -1
- package/dist/text.js +12 -2
- package/dist/text.js.map +1 -1
- package/dist/types.d.ts +414 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/vector-index.d.ts +42 -0
- package/dist/vector-index.d.ts.map +1 -0
- package/dist/vector-index.js +249 -0
- package/dist/vector-index.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/workload.d.ts +22 -0
- package/dist/workload.d.ts.map +1 -0
- package/dist/workload.js +181 -0
- package/dist/workload.js.map +1 -0
- package/examples/document-evidence-benchmark/README.md +3 -2
- package/examples/document-evidence-benchmark/golden-queries.json +1 -1
- package/package.json +36 -11
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import * as lancedb from "@lancedb/lancedb";
|
|
5
|
+
import { loadConfig } from "./config.js";
|
|
6
|
+
import { isRecord } from "./guards.js";
|
|
7
|
+
import { withIndexWriteLock } from "./index-write-lock.js";
|
|
8
|
+
import { operationSignal } from "./operation.js";
|
|
9
|
+
import { ensurePrivateDirectory, hardenPrivateFile } from "./permissions.js";
|
|
10
|
+
import { activeIndexTableName, openRowsTableByName } from "./store.js";
|
|
11
|
+
import { maintainAdaptiveIndices } from "./vector-index.js";
|
|
12
|
+
const FULL_TEXT_INDEX_NAME = "searchText_idx";
|
|
13
|
+
const MAINTENANCE_STATE_FILENAME = "storage-maintenance.json";
|
|
14
|
+
const MAINTENANCE_STATE_SCHEMA_VERSION = 1;
|
|
15
|
+
const MUTATION_COMPACTION_THRESHOLD = 20;
|
|
16
|
+
const MINIMUM_FRAGMENT_COUNT = 8;
|
|
17
|
+
const SMALL_FRAGMENT_RATIO_THRESHOLD = 0.25;
|
|
18
|
+
const OLD_VERSION_RETENTION_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
19
|
+
export async function optimizeStorage(options = {}) {
|
|
20
|
+
const config = await loadConfig(options.cwd ?? process.cwd());
|
|
21
|
+
const signal = operationSignal(options);
|
|
22
|
+
return withIndexWriteLock(config.storageDir, signal, async () => {
|
|
23
|
+
const tableName = await activeIndexTableName(config);
|
|
24
|
+
return maintainStorageTable(tableName, config, undefined, {
|
|
25
|
+
...(options.dryRun === undefined ? {} : { dryRun: options.dryRun }),
|
|
26
|
+
force: true,
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
export async function maintainStorageTable(tableName, config, connection, options = {}) {
|
|
31
|
+
try {
|
|
32
|
+
const table = await openRowsTableByName(tableName, config, connection);
|
|
33
|
+
if (!table) {
|
|
34
|
+
return missingTableReport(tableName, options);
|
|
35
|
+
}
|
|
36
|
+
return maintainOpenStorageTable(table, tableName, config, options);
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
return warningReport(tableName, options, error);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export async function maintainOpenStorageTable(table, tableName, config, options = {}) {
|
|
43
|
+
const dryRun = options.dryRun === true;
|
|
44
|
+
const forced = options.force === true;
|
|
45
|
+
const previousState = await readMaintenanceState(config);
|
|
46
|
+
const previousMutations = previousState?.tableName === tableName ? previousState.mutationsSinceOptimization : 0;
|
|
47
|
+
const mutationsSinceOptimization = previousMutations + Math.max(0, Math.floor(options.additionalMutations ?? 0));
|
|
48
|
+
const before = await inspectTableHealth(table);
|
|
49
|
+
const reasons = maintenanceReasons(before, mutationsSinceOptimization, forced);
|
|
50
|
+
const plannedActions = maintenanceActions(before, mutationsSinceOptimization, forced);
|
|
51
|
+
if (dryRun) {
|
|
52
|
+
const adaptiveIndices = await maintainAdaptiveIndices(table, config, {
|
|
53
|
+
dryRun: true,
|
|
54
|
+
...(previousState?.tableName === tableName && previousState.vectorIndexPolicySignature
|
|
55
|
+
? { previousPolicySignature: previousState.vectorIndexPolicySignature }
|
|
56
|
+
: {}),
|
|
57
|
+
...(options.vectorDimension === undefined
|
|
58
|
+
? {}
|
|
59
|
+
: { vectorDimension: options.vectorDimension }),
|
|
60
|
+
});
|
|
61
|
+
return reportForHealth({
|
|
62
|
+
tableName,
|
|
63
|
+
status: plannedActions.length > 0 || adaptiveIndices.plannedActions.length > 0
|
|
64
|
+
? "needed"
|
|
65
|
+
: "healthy",
|
|
66
|
+
dryRun,
|
|
67
|
+
forced,
|
|
68
|
+
health: before,
|
|
69
|
+
mutationsSinceOptimization,
|
|
70
|
+
reasons,
|
|
71
|
+
plannedActions,
|
|
72
|
+
completedActions: [],
|
|
73
|
+
optimizeStats: null,
|
|
74
|
+
adaptiveIndices,
|
|
75
|
+
warning: null,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
const warnings = [];
|
|
79
|
+
const completedActions = [];
|
|
80
|
+
let optimizeStats = null;
|
|
81
|
+
let optimized = false;
|
|
82
|
+
if (plannedActions.includes("compact-fragments")) {
|
|
83
|
+
try {
|
|
84
|
+
optimizeStats = await table.optimize({
|
|
85
|
+
cleanupOlderThan: new Date(Date.now() - OLD_VERSION_RETENTION_MS),
|
|
86
|
+
deleteUnverified: false,
|
|
87
|
+
});
|
|
88
|
+
optimized = true;
|
|
89
|
+
completedActions.push("compact-fragments", "prune-old-versions");
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
warnings.push(`LanceDB compaction failed (${errorDetail(error)}).`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
if (plannedActions.includes("refresh-full-text-index")) {
|
|
96
|
+
try {
|
|
97
|
+
await table.createIndex("searchText", {
|
|
98
|
+
config: lancedb.Index.fts({ asciiFolding: true, lowercase: true, withPosition: true }),
|
|
99
|
+
replace: true,
|
|
100
|
+
});
|
|
101
|
+
const coverage = await fullTextIndexCoverage(table, before.totalRows);
|
|
102
|
+
if (!coverage.complete) {
|
|
103
|
+
warnings.push(`Full-text index refresh left ${coverage.unindexedRows} unindexed row(s).`);
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
completedActions.push("refresh-full-text-index");
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
warnings.push(`Full-text index refresh failed (${errorDetail(error)}). Bounded lexical scans remain available.`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const adaptiveIndices = await maintainAdaptiveIndices(table, config, {
|
|
114
|
+
...(previousState?.tableName === tableName && previousState.vectorIndexPolicySignature
|
|
115
|
+
? { previousPolicySignature: previousState.vectorIndexPolicySignature }
|
|
116
|
+
: {}),
|
|
117
|
+
...(options.vectorDimension === undefined ? {} : { vectorDimension: options.vectorDimension }),
|
|
118
|
+
});
|
|
119
|
+
const after = await inspectTableHealth(table);
|
|
120
|
+
const remainingMutations = optimized ? 0 : mutationsSinceOptimization;
|
|
121
|
+
const vectorIndexPolicySignature = adaptiveIndices.vectorIndex.strategy === adaptiveIndices.desiredVectorStrategy
|
|
122
|
+
? adaptiveIndices.policySignature
|
|
123
|
+
: previousState?.tableName === tableName
|
|
124
|
+
? previousState.vectorIndexPolicySignature
|
|
125
|
+
: undefined;
|
|
126
|
+
try {
|
|
127
|
+
await writeMaintenanceState({
|
|
128
|
+
schemaVersion: MAINTENANCE_STATE_SCHEMA_VERSION,
|
|
129
|
+
tableName,
|
|
130
|
+
mutationsSinceOptimization: remainingMutations,
|
|
131
|
+
tableVersion: after.tableVersion,
|
|
132
|
+
updatedAt: new Date().toISOString(),
|
|
133
|
+
...(vectorIndexPolicySignature ? { vectorIndexPolicySignature } : {}),
|
|
134
|
+
...(optimized ? { lastOptimizedAt: new Date().toISOString() } : {}),
|
|
135
|
+
}, config);
|
|
136
|
+
}
|
|
137
|
+
catch (error) {
|
|
138
|
+
warnings.push(`Storage maintenance state could not be saved (${errorDetail(error)}).`);
|
|
139
|
+
}
|
|
140
|
+
return reportForHealth({
|
|
141
|
+
tableName,
|
|
142
|
+
status: warnings.length > 0 || adaptiveIndices.warning
|
|
143
|
+
? "warning"
|
|
144
|
+
: plannedActions.length > 0 || adaptiveIndices.completedActions.length > 0
|
|
145
|
+
? "completed"
|
|
146
|
+
: "healthy",
|
|
147
|
+
dryRun,
|
|
148
|
+
forced,
|
|
149
|
+
health: after,
|
|
150
|
+
mutationsSinceOptimization: remainingMutations,
|
|
151
|
+
reasons,
|
|
152
|
+
plannedActions,
|
|
153
|
+
completedActions,
|
|
154
|
+
optimizeStats,
|
|
155
|
+
adaptiveIndices,
|
|
156
|
+
warning: warnings.length > 0 ? warnings.join(" ") : null,
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
async function inspectTableHealth(table) {
|
|
160
|
+
const [stats, tableVersion] = await Promise.all([table.stats(), table.version()]);
|
|
161
|
+
return {
|
|
162
|
+
totalRows: stats.numRows,
|
|
163
|
+
tableVersion,
|
|
164
|
+
fragments: {
|
|
165
|
+
total: stats.fragmentStats.numFragments,
|
|
166
|
+
small: stats.fragmentStats.numSmallFragments,
|
|
167
|
+
smallRatio: stats.fragmentStats.numFragments === 0
|
|
168
|
+
? 0
|
|
169
|
+
: stats.fragmentStats.numSmallFragments / stats.fragmentStats.numFragments,
|
|
170
|
+
},
|
|
171
|
+
fullTextIndex: await fullTextIndexCoverage(table, stats.numRows),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
async function fullTextIndexCoverage(table, totalRows) {
|
|
175
|
+
const index = (await table.listIndices()).find((candidate) => candidate.name === FULL_TEXT_INDEX_NAME);
|
|
176
|
+
if (!index) {
|
|
177
|
+
return {
|
|
178
|
+
present: false,
|
|
179
|
+
indexedRows: 0,
|
|
180
|
+
unindexedRows: totalRows,
|
|
181
|
+
complete: totalRows === 0,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
const stats = await table.indexStats(FULL_TEXT_INDEX_NAME);
|
|
185
|
+
if (!stats) {
|
|
186
|
+
return {
|
|
187
|
+
present: false,
|
|
188
|
+
indexedRows: 0,
|
|
189
|
+
unindexedRows: totalRows,
|
|
190
|
+
complete: totalRows === 0,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
return {
|
|
194
|
+
present: true,
|
|
195
|
+
indexedRows: stats.numIndexedRows,
|
|
196
|
+
unindexedRows: stats.numUnindexedRows,
|
|
197
|
+
complete: stats.numUnindexedRows === 0 && stats.numIndexedRows === totalRows,
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
function maintenanceReasons(health, mutationsSinceOptimization, forced) {
|
|
201
|
+
const reasons = [];
|
|
202
|
+
if (forced) {
|
|
203
|
+
reasons.push("forced");
|
|
204
|
+
}
|
|
205
|
+
if (!health.fullTextIndex.present && health.totalRows > 0) {
|
|
206
|
+
reasons.push("missing-full-text-index");
|
|
207
|
+
}
|
|
208
|
+
else if (health.fullTextIndex.unindexedRows > 0) {
|
|
209
|
+
reasons.push("unindexed-full-text-rows");
|
|
210
|
+
}
|
|
211
|
+
if (mutationsSinceOptimization >= MUTATION_COMPACTION_THRESHOLD) {
|
|
212
|
+
reasons.push("mutation-threshold");
|
|
213
|
+
}
|
|
214
|
+
if (health.fragments.total >= MINIMUM_FRAGMENT_COUNT &&
|
|
215
|
+
health.fragments.smallRatio >= SMALL_FRAGMENT_RATIO_THRESHOLD) {
|
|
216
|
+
reasons.push("fragmentation-threshold");
|
|
217
|
+
}
|
|
218
|
+
return reasons;
|
|
219
|
+
}
|
|
220
|
+
function maintenanceActions(health, mutationsSinceOptimization, forced) {
|
|
221
|
+
const actions = [];
|
|
222
|
+
const compact = forced ||
|
|
223
|
+
mutationsSinceOptimization >= MUTATION_COMPACTION_THRESHOLD ||
|
|
224
|
+
(health.fragments.total >= MINIMUM_FRAGMENT_COUNT &&
|
|
225
|
+
health.fragments.smallRatio >= SMALL_FRAGMENT_RATIO_THRESHOLD);
|
|
226
|
+
if (compact) {
|
|
227
|
+
actions.push("compact-fragments", "prune-old-versions");
|
|
228
|
+
}
|
|
229
|
+
if (!health.fullTextIndex.complete) {
|
|
230
|
+
actions.push("refresh-full-text-index");
|
|
231
|
+
}
|
|
232
|
+
return actions;
|
|
233
|
+
}
|
|
234
|
+
function reportForHealth(input) {
|
|
235
|
+
return {
|
|
236
|
+
schemaVersion: MAINTENANCE_STATE_SCHEMA_VERSION,
|
|
237
|
+
tableName: input.tableName,
|
|
238
|
+
status: input.status,
|
|
239
|
+
dryRun: input.dryRun,
|
|
240
|
+
forced: input.forced,
|
|
241
|
+
totalRows: input.health.totalRows,
|
|
242
|
+
tableVersion: input.health.tableVersion,
|
|
243
|
+
mutationsSinceOptimization: input.mutationsSinceOptimization,
|
|
244
|
+
fragments: input.health.fragments,
|
|
245
|
+
fullTextIndex: input.health.fullTextIndex,
|
|
246
|
+
reasons: input.reasons,
|
|
247
|
+
plannedActions: input.plannedActions,
|
|
248
|
+
completedActions: input.completedActions,
|
|
249
|
+
optimizeStats: input.optimizeStats,
|
|
250
|
+
adaptiveIndices: input.adaptiveIndices,
|
|
251
|
+
warning: input.warning,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
function missingTableReport(tableName, options) {
|
|
255
|
+
return {
|
|
256
|
+
schemaVersion: MAINTENANCE_STATE_SCHEMA_VERSION,
|
|
257
|
+
tableName,
|
|
258
|
+
status: "missing",
|
|
259
|
+
dryRun: options.dryRun === true,
|
|
260
|
+
forced: options.force === true,
|
|
261
|
+
totalRows: 0,
|
|
262
|
+
tableVersion: null,
|
|
263
|
+
mutationsSinceOptimization: 0,
|
|
264
|
+
fragments: { total: 0, small: 0, smallRatio: 0 },
|
|
265
|
+
fullTextIndex: { present: false, indexedRows: 0, unindexedRows: 0, complete: false },
|
|
266
|
+
adaptiveIndices: null,
|
|
267
|
+
reasons: [],
|
|
268
|
+
plannedActions: [],
|
|
269
|
+
completedActions: [],
|
|
270
|
+
optimizeStats: null,
|
|
271
|
+
warning: null,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
function warningReport(tableName, options, error) {
|
|
275
|
+
return {
|
|
276
|
+
...missingTableReport(tableName, options),
|
|
277
|
+
status: "warning",
|
|
278
|
+
warning: `Storage maintenance inspection failed (${errorDetail(error)}). The active index remains available.`,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
async function readMaintenanceState(config) {
|
|
282
|
+
try {
|
|
283
|
+
const value = JSON.parse(await readFile(path.join(config.storageDir, MAINTENANCE_STATE_FILENAME), "utf8"));
|
|
284
|
+
return isMaintenanceState(value) ? value : null;
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
if (isNodeError(error) && error.code === "ENOENT") {
|
|
288
|
+
return null;
|
|
289
|
+
}
|
|
290
|
+
if (error instanceof SyntaxError) {
|
|
291
|
+
return null;
|
|
292
|
+
}
|
|
293
|
+
throw error;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
function isMaintenanceState(value) {
|
|
297
|
+
return (isRecord(value) &&
|
|
298
|
+
value.schemaVersion === MAINTENANCE_STATE_SCHEMA_VERSION &&
|
|
299
|
+
typeof value.tableName === "string" &&
|
|
300
|
+
typeof value.mutationsSinceOptimization === "number" &&
|
|
301
|
+
Number.isInteger(value.mutationsSinceOptimization) &&
|
|
302
|
+
value.mutationsSinceOptimization >= 0 &&
|
|
303
|
+
typeof value.tableVersion === "number" &&
|
|
304
|
+
Number.isInteger(value.tableVersion) &&
|
|
305
|
+
typeof value.updatedAt === "string" &&
|
|
306
|
+
(!("lastOptimizedAt" in value) || typeof value.lastOptimizedAt === "string") &&
|
|
307
|
+
(!("vectorIndexPolicySignature" in value) ||
|
|
308
|
+
typeof value.vectorIndexPolicySignature === "string"));
|
|
309
|
+
}
|
|
310
|
+
async function writeMaintenanceState(state, config) {
|
|
311
|
+
await ensurePrivateDirectory(config.storageDir);
|
|
312
|
+
const target = path.join(config.storageDir, MAINTENANCE_STATE_FILENAME);
|
|
313
|
+
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
314
|
+
try {
|
|
315
|
+
await writeFile(temporary, JSON.stringify(state, null, 2), "utf8");
|
|
316
|
+
await hardenPrivateFile(temporary);
|
|
317
|
+
await rename(temporary, target);
|
|
318
|
+
}
|
|
319
|
+
finally {
|
|
320
|
+
await rm(temporary, { force: true });
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
function errorDetail(error) {
|
|
324
|
+
return error instanceof Error ? error.message : String(error);
|
|
325
|
+
}
|
|
326
|
+
function isNodeError(error) {
|
|
327
|
+
return error instanceof Error && "code" in error;
|
|
328
|
+
}
|
|
329
|
+
//# sourceMappingURL=storage-maintenance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage-maintenance.js","sourceRoot":"","sources":["../src/storage-maintenance.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAClE,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAC5E,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAEtE,OAAO,EAAuC,uBAAuB,EAAE,MAAM,mBAAmB,CAAA;AAEhG,MAAM,oBAAoB,GAAG,gBAAgB,CAAA;AAC7C,MAAM,0BAA0B,GAAG,0BAA0B,CAAA;AAC7D,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAC1C,MAAM,6BAA6B,GAAG,EAAE,CAAA;AACxC,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAChC,MAAM,8BAA8B,GAAG,IAAI,CAAA;AAC3C,MAAM,wBAAwB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAA;AAuEzD,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAAkC,EAAE;IAEpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IAC7D,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;IACvC,OAAO,kBAAkB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAA;QACpD,OAAO,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE;YACxD,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YACnE,KAAK,EAAE,IAAI;SACZ,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,SAAiB,EACjB,MAAc,EACd,UAAuB,EACvB,UAAqC,EAAE;IAEvC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,CAAA;QACtE,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,kBAAkB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC/C,CAAC;QACD,OAAO,wBAAwB,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;IACpE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAA;IACjD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,KAAY,EACZ,SAAiB,EACjB,MAAc,EACd,UAAqC,EAAE;IAEvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,KAAK,IAAI,CAAA;IACrC,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAA;IACxD,MAAM,iBAAiB,GACrB,aAAa,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAA;IACvF,MAAM,0BAA0B,GAC9B,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,mBAAmB,IAAI,CAAC,CAAC,CAAC,CAAA;IAC/E,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAA;IAC9C,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,EAAE,0BAA0B,EAAE,MAAM,CAAC,CAAA;IAC9E,MAAM,cAAc,GAAG,kBAAkB,CAAC,MAAM,EAAE,0BAA0B,EAAE,MAAM,CAAC,CAAA;IAErF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,eAAe,GAAG,MAAM,uBAAuB,CAAC,KAAK,EAAE,MAAM,EAAE;YACnE,MAAM,EAAE,IAAI;YACZ,GAAG,CAAC,aAAa,EAAE,SAAS,KAAK,SAAS,IAAI,aAAa,CAAC,0BAA0B;gBACpF,CAAC,CAAC,EAAE,uBAAuB,EAAE,aAAa,CAAC,0BAA0B,EAAE;gBACvE,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,OAAO,CAAC,eAAe,KAAK,SAAS;gBACvC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC;SAClD,CAAC,CAAA;QACF,OAAO,eAAe,CAAC;YACrB,SAAS;YACT,MAAM,EACJ,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;gBACpE,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,SAAS;YACf,MAAM;YACN,MAAM;YACN,MAAM,EAAE,MAAM;YACd,0BAA0B;YAC1B,OAAO;YACP,cAAc;YACd,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,IAAI;YACnB,eAAe;YACf,OAAO,EAAE,IAAI;SACd,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,gBAAgB,GAA+B,EAAE,CAAA;IACvD,IAAI,aAAa,GAAyB,IAAI,CAAA;IAC9C,IAAI,SAAS,GAAG,KAAK,CAAA;IAErB,IAAI,cAAc,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC;YACH,aAAa,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC;gBACnC,gBAAgB,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,wBAAwB,CAAC;gBACjE,gBAAgB,EAAE,KAAK;aACxB,CAAC,CAAA;YACF,SAAS,GAAG,IAAI,CAAA;YAChB,gBAAgB,CAAC,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAA;QAClE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,8BAA8B,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACrE,CAAC;IACH,CAAC;IAED,IAAI,cAAc,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE;gBACpC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;gBACtF,OAAO,EAAE,IAAI;aACd,CAAC,CAAA;YACF,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;YACrE,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBACvB,QAAQ,CAAC,IAAI,CAAC,gCAAgC,QAAQ,CAAC,aAAa,oBAAoB,CAAC,CAAA;YAC3F,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA;YAClD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CACX,mCAAmC,WAAW,CAAC,KAAK,CAAC,4CAA4C,CAClG,CAAA;QACH,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,uBAAuB,CAAC,KAAK,EAAE,MAAM,EAAE;QACnE,GAAG,CAAC,aAAa,EAAE,SAAS,KAAK,SAAS,IAAI,aAAa,CAAC,0BAA0B;YACpF,CAAC,CAAC,EAAE,uBAAuB,EAAE,aAAa,CAAC,0BAA0B,EAAE;YACvE,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC;KAC/F,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAA;IAC7C,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAA;IACrE,MAAM,0BAA0B,GAC9B,eAAe,CAAC,WAAW,CAAC,QAAQ,KAAK,eAAe,CAAC,qBAAqB;QAC5E,CAAC,CAAC,eAAe,CAAC,eAAe;QACjC,CAAC,CAAC,aAAa,EAAE,SAAS,KAAK,SAAS;YACtC,CAAC,CAAC,aAAa,CAAC,0BAA0B;YAC1C,CAAC,CAAC,SAAS,CAAA;IACjB,IAAI,CAAC;QACH,MAAM,qBAAqB,CACzB;YACE,aAAa,EAAE,gCAAgC;YAC/C,SAAS;YACT,0BAA0B,EAAE,kBAAkB;YAC9C,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,GAAG,CAAC,0BAA0B,CAAC,CAAC,CAAC,EAAE,0BAA0B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,EACD,MAAM,CACP,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,iDAAiD,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACxF,CAAC;IAED,OAAO,eAAe,CAAC;QACrB,SAAS;QACT,MAAM,EACJ,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,OAAO;YAC5C,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;gBACxE,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,SAAS;QACjB,MAAM;QACN,MAAM;QACN,MAAM,EAAE,KAAK;QACb,0BAA0B,EAAE,kBAAkB;QAC9C,OAAO;QACP,cAAc;QACd,gBAAgB;QAChB,aAAa;QACb,eAAe;QACf,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;KACzD,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,KAAY;IAC5C,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACjF,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,OAAO;QACxB,YAAY;QACZ,SAAS,EAAE;YACT,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,YAAY;YACvC,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,iBAAiB;YAC5C,UAAU,EACR,KAAK,CAAC,aAAa,CAAC,YAAY,KAAK,CAAC;gBACpC,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,iBAAiB,GAAG,KAAK,CAAC,aAAa,CAAC,YAAY;SAC/E;QACD,aAAa,EAAE,MAAM,qBAAqB,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC;KACjE,CAAA;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,KAAY,EACZ,SAAiB;IAEjB,MAAM,KAAK,GAAG,CAAC,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAC5C,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,oBAAoB,CACvD,CAAA;IACD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,SAAS;YACxB,QAAQ,EAAE,SAAS,KAAK,CAAC;SAC1B,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAA;IAC1D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,SAAS;YACxB,QAAQ,EAAE,SAAS,KAAK,CAAC;SAC1B,CAAA;IACH,CAAC;IACD,OAAO;QACL,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,KAAK,CAAC,cAAc;QACjC,aAAa,EAAE,KAAK,CAAC,gBAAgB;QACrC,QAAQ,EAAE,KAAK,CAAC,gBAAgB,KAAK,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS;KAC7E,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAmB,EACnB,0BAAkC,EAClC,MAAe;IAEf,MAAM,OAAO,GAA+B,EAAE,CAAA;IAC9C,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACxB,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA;IACzC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAA;IAC1C,CAAC;IACD,IAAI,0BAA0B,IAAI,6BAA6B,EAAE,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;IACpC,CAAC;IACD,IACE,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,sBAAsB;QAChD,MAAM,CAAC,SAAS,CAAC,UAAU,IAAI,8BAA8B,EAC7D,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA;IACzC,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAmB,EACnB,0BAAkC,EAClC,MAAe;IAEf,MAAM,OAAO,GAA+B,EAAE,CAAA;IAC9C,MAAM,OAAO,GACX,MAAM;QACN,0BAA0B,IAAI,6BAA6B;QAC3D,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,sBAAsB;YAC/C,MAAM,CAAC,SAAS,CAAC,UAAU,IAAI,8BAA8B,CAAC,CAAA;IAClE,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAA;IACzD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA;IACzC,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,KAaxB;IACC,OAAO;QACL,aAAa,EAAE,gCAAgC;QAC/C,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;QACjC,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,YAAY;QACvC,0BAA0B,EAAE,KAAK,CAAC,0BAA0B;QAC5D,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;QACjC,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,aAAa;QACzC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,OAAO,EAAE,KAAK,CAAC,OAAO;KACvB,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CACzB,SAAiB,EACjB,OAAkC;IAElC,OAAO;QACL,aAAa,EAAE,gCAAgC;QAC/C,SAAS;QACT,MAAM,EAAE,SAAS;QACjB,MAAM,EAAE,OAAO,CAAC,MAAM,KAAK,IAAI;QAC/B,MAAM,EAAE,OAAO,CAAC,KAAK,KAAK,IAAI;QAC9B,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,IAAI;QAClB,0BAA0B,EAAE,CAAC;QAC7B,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;QAChD,aAAa,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE;QACpF,eAAe,EAAE,IAAI;QACrB,OAAO,EAAE,EAAE;QACX,cAAc,EAAE,EAAE;QAClB,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,IAAI;KACd,CAAA;AACH,CAAC;AAED,SAAS,aAAa,CACpB,SAAiB,EACjB,OAAkC,EAClC,KAAc;IAEd,OAAO;QACL,GAAG,kBAAkB,CAAC,SAAS,EAAE,OAAO,CAAC;QACzC,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,0CAA0C,WAAW,CAAC,KAAK,CAAC,wCAAwC;KAC9G,CAAA;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,MAAc;IAChD,IAAI,CAAC;QACH,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAC/B,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,0BAA0B,CAAC,EAAE,MAAM,CAAC,CACjF,CAAA;QACD,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,IAAI,CAAA;QACb,CAAC;QACD,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,KAAK,CAAC,aAAa,KAAK,gCAAgC;QACxD,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,OAAO,KAAK,CAAC,0BAA0B,KAAK,QAAQ;QACpD,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,0BAA0B,CAAC;QAClD,KAAK,CAAC,0BAA0B,IAAI,CAAC;QACrC,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;QACtC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC;QACpC,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,CAAC,CAAC,CAAC,iBAAiB,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,eAAe,KAAK,QAAQ,CAAC;QAC5E,CAAC,CAAC,CAAC,4BAA4B,IAAI,KAAK,CAAC;YACvC,OAAO,KAAK,CAAC,0BAA0B,KAAK,QAAQ,CAAC,CACxD,CAAA;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,KAA8B,EAC9B,MAAc;IAEd,MAAM,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAA;IACvE,MAAM,SAAS,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,MAAM,CAAA;IAChE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;QAClE,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAA;QAClC,MAAM,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IACtC,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC"}
|
package/dist/store.d.ts
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
import * as lancedb from "@lancedb/lancedb";
|
|
2
|
-
import type { Config, IndexManifest, VectorRow } from "./types.js";
|
|
2
|
+
import type { Config, IndexManifest, IndexManifestFile, VectorRow } from "./types.js";
|
|
3
|
+
export declare const INDEX_READ_DIAGNOSTICS_CHANNEL = "ragmir:index-read";
|
|
4
|
+
export interface IndexReadDiagnosticsEvent {
|
|
5
|
+
kind: "connection-open" | "connection-close" | "manifest-read" | "manifest-recovery" | "table-open" | "table-close";
|
|
6
|
+
projectRoot: string;
|
|
7
|
+
recoveryReason?: "canonical-invalid" | "canonical-missing";
|
|
8
|
+
tableName?: string;
|
|
9
|
+
}
|
|
10
|
+
type ManifestCandidateStatus = "valid" | "missing" | "invalid";
|
|
11
|
+
export interface IndexManifestRecoveryDiagnostic {
|
|
12
|
+
canonicalStatus: ManifestCandidateStatus;
|
|
13
|
+
previousStatus: ManifestCandidateStatus | "not-checked";
|
|
14
|
+
selected: "canonical" | "previous" | null;
|
|
15
|
+
warning: string | null;
|
|
16
|
+
}
|
|
3
17
|
export interface EmptyTextFileRecord {
|
|
4
18
|
relativePath: string;
|
|
5
19
|
checksum: string;
|
|
@@ -10,13 +24,35 @@ export interface IndexWriteResult {
|
|
|
10
24
|
vectorIndexWarning: string | null;
|
|
11
25
|
lexicalIndexWarning: string | null;
|
|
12
26
|
}
|
|
27
|
+
export interface IndexReadSnapshot {
|
|
28
|
+
manifest: IndexManifest | null;
|
|
29
|
+
manifestFingerprint: string | null;
|
|
30
|
+
tableName: string;
|
|
31
|
+
table: lancedb.Table | null;
|
|
32
|
+
}
|
|
33
|
+
export interface IndexManifestFilePage {
|
|
34
|
+
files: IndexManifestFile[];
|
|
35
|
+
total: number;
|
|
36
|
+
offset: number;
|
|
37
|
+
limit: number;
|
|
38
|
+
nextOffset: number | null;
|
|
39
|
+
}
|
|
13
40
|
export declare function connectStore(config: Config): Promise<lancedb.Connection>;
|
|
41
|
+
export declare function closeStoreConnection(connection: lancedb.Connection, config: Config): void;
|
|
42
|
+
export declare function closeIndexReadSnapshot(snapshot: IndexReadSnapshot, config: Config): void;
|
|
43
|
+
export declare function closeRowsTable(table: lancedb.Table, config: Config): void;
|
|
14
44
|
export declare function writeRows(rows: VectorRow[], config: Config, connection?: lancedb.Connection): Promise<IndexWriteResult>;
|
|
15
45
|
export declare function writeRowsToTable(rows: VectorRow[], tableName: string, config: Config, connection?: lancedb.Connection): Promise<IndexWriteResult>;
|
|
16
46
|
export declare function updateRows(rows: VectorRow[], replacePaths: string[], config: Config, connection?: lancedb.Connection): Promise<IndexWriteResult>;
|
|
17
47
|
export declare function updateRowsInTable(rows: VectorRow[], replacePaths: string[], tableName: string, config: Config, connection?: lancedb.Connection): Promise<IndexWriteResult>;
|
|
18
|
-
export declare function writeIndexManifest(manifest: IndexManifest, config: Config): Promise<void>;
|
|
48
|
+
export declare function writeIndexManifest(manifest: IndexManifest, config: Config, indexedFiles?: Iterable<IndexManifestFile> | undefined): Promise<void>;
|
|
19
49
|
export declare function readIndexManifest(config: Config): Promise<IndexManifest | null>;
|
|
50
|
+
export declare function readIndexManifestHeader(config: Config): Promise<IndexManifest | null>;
|
|
51
|
+
export declare function readIndexManifestFilePage(config: Config, offset: number, limit: number): Promise<IndexManifestFilePage | null>;
|
|
52
|
+
export declare function loadIndexReadSnapshot(config: Config, connection?: lancedb.Connection): Promise<IndexReadSnapshot>;
|
|
53
|
+
export declare function indexReadSnapshotCurrent(snapshot: IndexReadSnapshot, config: Config): Promise<boolean>;
|
|
54
|
+
export declare function indexManifestRecoveryDiagnostic(config: Config): IndexManifestRecoveryDiagnostic | null;
|
|
55
|
+
export declare function indexManifestRecoveryWarning(config: Config): string | null;
|
|
20
56
|
export declare function writeEmptyTextFiles(records: EmptyTextFileRecord[], config: Config): Promise<void>;
|
|
21
57
|
export declare function readEmptyTextFiles(config: Config): Promise<EmptyTextFileRecord[]>;
|
|
22
58
|
export declare function openRowsTable(config: Config, connection?: lancedb.Connection): Promise<lancedb.Table | null>;
|
|
@@ -25,4 +61,5 @@ export declare function activeIndexTableName(config: Config): Promise<string>;
|
|
|
25
61
|
export declare function dropRowsTable(tableName: string, config: Config, connection?: lancedb.Connection): Promise<void>;
|
|
26
62
|
export declare function readRows(config: Config): Promise<VectorRow[]>;
|
|
27
63
|
export declare function countRows(config: Config): Promise<number>;
|
|
64
|
+
export {};
|
|
28
65
|
//# sourceMappingURL=store.d.ts.map
|
package/dist/store.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAM3C,OAAO,KAAK,EACV,MAAM,EAGN,aAAa,EACb,iBAAiB,EAEjB,SAAS,EACV,MAAM,YAAY,CAAA;AAQnB,eAAO,MAAM,8BAA8B,sBAAsB,CAAA;AAEjE,MAAM,WAAW,yBAAyB;IACxC,IAAI,EACA,iBAAiB,GACjB,kBAAkB,GAClB,eAAe,GACf,mBAAmB,GACnB,YAAY,GACZ,aAAa,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,CAAC,EAAE,mBAAmB,GAAG,mBAAmB,CAAA;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAUD,KAAK,uBAAuB,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,CAAA;AAE9D,MAAM,WAAW,+BAA+B;IAC9C,eAAe,EAAE,uBAAuB,CAAA;IACxC,cAAc,EAAE,uBAAuB,GAAG,aAAa,CAAA;IACvD,QAAQ,EAAE,WAAW,GAAG,UAAU,GAAG,IAAI,CAAA;IACzC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAcD,MAAM,WAAW,mBAAmB;IAClC,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAA;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAA;IAC9B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAA;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,OAAO,CAAC,KAAK,GAAG,IAAI,CAAA;CAC5B;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,iBAAiB,EAAE,CAAA;IAC1B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;CAC1B;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAO9E;AAED,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAMzF;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAKxF;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAUzE;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,SAAS,EAAE,EACjB,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAO3B;AAED,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,SAAS,EAAE,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAqB3B;AAED,wBAAsB,UAAU,CAC9B,IAAI,EAAE,SAAS,EAAE,EACjB,YAAY,EAAE,MAAM,EAAE,EACtB,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAG3B;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,SAAS,EAAE,EACjB,YAAY,EAAE,MAAM,EAAE,EACtB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAuB3B;AA4BD,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,aAAa,EACvB,MAAM,EAAE,MAAM,EACd,YAAY,GAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAiC,GAC5E,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAcrF;AAED,wBAAsB,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAQ3F;AAED,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAqBvC;AAED,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,GAC9B,OAAO,CAAC,iBAAiB,CAAC,CAsB5B;AAED,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,OAAO,CAAC,CAElB;AAmBD,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,GACb,+BAA+B,GAAG,IAAI,CAExC;AAED,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAE1E;AAiSD,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,mBAAmB,EAAE,EAC9B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAevF;AAED,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,GAC9B,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAE/B;AAED,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,GAC9B,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAgB/B;AAQD,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAE1E;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,GAC9B,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAMnE;AAED,wBAAsB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG/D"}
|