@solidactions/cli 3.38.0 → 3.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -5
- package/dist/command-manifest.json +29 -4
- package/dist/commands/database-export.js +4 -2
- package/dist/commands/database.js +615 -49
- package/dist/index.js +19 -6
- package/dist/utils/database-kind-refusal.js +27 -0
- package/dist/utils/database-pull-guidance.js +20 -0
- package/dist/utils/database-replica-cache.js +432 -0
- package/dist/utils/examples-ref.js +4 -4
- package/package.json +2 -2
|
@@ -86,6 +86,7 @@ const url_1 = require("url");
|
|
|
86
86
|
const api_1 = require("../utils/api");
|
|
87
87
|
const database_data_plane_1 = require("../utils/database-data-plane");
|
|
88
88
|
const database_client_support_1 = require("../utils/database-client-support");
|
|
89
|
+
const database_kind_refusal_1 = require("../utils/database-kind-refusal");
|
|
89
90
|
const database_sql_import_1 = require("../utils/database-sql-import");
|
|
90
91
|
var database_sql_import_2 = require("../utils/database-sql-import");
|
|
91
92
|
Object.defineProperty(exports, "parseDatabaseImportSql", { enumerable: true, get: function () { return database_sql_import_2.parseDatabaseImportSql; } });
|
|
@@ -93,6 +94,8 @@ const database_sql_import_runner_1 = require("../utils/database-sql-import-runne
|
|
|
93
94
|
const table_1 = require("../utils/table");
|
|
94
95
|
const database_download_1 = require("../utils/database-download");
|
|
95
96
|
const database_ingest_digest_1 = require("../utils/database-ingest-digest");
|
|
97
|
+
const database_pull_guidance_1 = require("../utils/database-pull-guidance");
|
|
98
|
+
const database_replica_cache_1 = require("../utils/database-replica-cache");
|
|
96
99
|
function defaultConfirmation(message) {
|
|
97
100
|
const input = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
98
101
|
return new Promise((resolve) => {
|
|
@@ -132,6 +135,8 @@ function resolveDependencies(dependencies) {
|
|
|
132
135
|
setTimeout(resolve, milliseconds);
|
|
133
136
|
})),
|
|
134
137
|
zstdAvailable: dependencies.zstdAvailable ?? (0, database_download_1.zstdDecompressAvailable)(),
|
|
138
|
+
replicaCacheRoot: dependencies.replicaCacheRoot,
|
|
139
|
+
env: dependencies.env ?? process.env,
|
|
135
140
|
put: dependencies.put ?? axios_1.default.put,
|
|
136
141
|
};
|
|
137
142
|
}
|
|
@@ -476,6 +481,43 @@ function renderAnalyticalSchema(schema) {
|
|
|
476
481
|
return `${table.name} — ${table.row_count} rows, last loaded ${table.last_loaded_at ?? 'never'}\n ${columns}`;
|
|
477
482
|
}).join('\n\n');
|
|
478
483
|
}
|
|
484
|
+
function stableDuckdbFileSchemaColumn(column) {
|
|
485
|
+
if (!column || typeof column.name !== 'string' || typeof column.type !== 'string') {
|
|
486
|
+
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'The database response was invalid.');
|
|
487
|
+
}
|
|
488
|
+
return { name: column.name, type: column.type };
|
|
489
|
+
}
|
|
490
|
+
function stableDuckdbFileSchema(data) {
|
|
491
|
+
if (!data || !Array.isArray(data.tables)) {
|
|
492
|
+
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'The database response was invalid.');
|
|
493
|
+
}
|
|
494
|
+
const tables = data.tables.map((table) => {
|
|
495
|
+
if (!table || typeof table.name !== 'string' || !Array.isArray(table.columns)) {
|
|
496
|
+
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'The database response was invalid.');
|
|
497
|
+
}
|
|
498
|
+
return {
|
|
499
|
+
name: table.name,
|
|
500
|
+
columns: table.columns.map(stableDuckdbFileSchemaColumn),
|
|
501
|
+
};
|
|
502
|
+
});
|
|
503
|
+
if (data.note === undefined) {
|
|
504
|
+
return { tables };
|
|
505
|
+
}
|
|
506
|
+
if (typeof data.note !== 'string') {
|
|
507
|
+
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'The database response was invalid.');
|
|
508
|
+
}
|
|
509
|
+
return { tables, note: data.note };
|
|
510
|
+
}
|
|
511
|
+
function renderDuckdbFileSchema(schema) {
|
|
512
|
+
if (schema.tables.length === 0) {
|
|
513
|
+
return 'No tables.';
|
|
514
|
+
}
|
|
515
|
+
const rendered = schema.tables.map((table) => {
|
|
516
|
+
const columns = table.columns.map((column) => `${column.name} (${column.type})`).join(', ');
|
|
517
|
+
return `${table.name}\n ${columns}`;
|
|
518
|
+
}).join('\n\n');
|
|
519
|
+
return schema.note === undefined ? rendered : `${rendered}\n\n${schema.note}`;
|
|
520
|
+
}
|
|
479
521
|
function databaseRecordSizeBytes(database) {
|
|
480
522
|
return database.kind === 'duckdb_file' ? database.bytes : database.size_bytes;
|
|
481
523
|
}
|
|
@@ -626,7 +668,10 @@ async function removeOwnedFiles(filesystem, paths) {
|
|
|
626
668
|
}
|
|
627
669
|
}
|
|
628
670
|
function pullSidecars(temp) {
|
|
629
|
-
|
|
671
|
+
// The real database host's sync client also leaves `${temp}-info` (68 bytes) beside the
|
|
672
|
+
// replica; it must be owned here or `finalizeReplica` renames only the main file and the
|
|
673
|
+
// hidden `-info` stays beside the user's copy (cli#266).
|
|
674
|
+
return [`${temp}-wal`, `${temp}-shm`, `${temp}-journal`, `${temp}-client_wal_index`, `${temp}-info`];
|
|
630
675
|
}
|
|
631
676
|
async function anyPathExists(filesystem, candidates) {
|
|
632
677
|
for (const candidate of candidates) {
|
|
@@ -888,17 +933,26 @@ async function databaseListWithConfig(options, config, dependencies = {}) {
|
|
|
888
933
|
throw new database_data_plane_1.DatabaseOperationError('invalid_kind', `--kind must be "libsql", "duckdb" or "duckdb_file" (received "${options.kind}").`);
|
|
889
934
|
}
|
|
890
935
|
const data = stableListResponse(await (0, database_data_plane_1.requestDatabaseOperation)(config, { operation: 'list' }, requestDependencies(io)));
|
|
936
|
+
// The server returns soft-deleted rows (`deleted_at` set) for every kind;
|
|
937
|
+
// the default view hides them in the table and in `--json`, and
|
|
938
|
+
// `--include-deleted` shows them with their DELETED AT and PURGE AT.
|
|
939
|
+
const visible = options.includeDeleted === true
|
|
940
|
+
? data.databases
|
|
941
|
+
: data.databases.filter((database) => database.deleted_at === null);
|
|
891
942
|
const databases = options.kind
|
|
892
|
-
?
|
|
893
|
-
:
|
|
943
|
+
? visible.filter((database) => database.kind === options.kind)
|
|
944
|
+
: visible;
|
|
894
945
|
if (options.json) {
|
|
895
|
-
writeJson(io.stdout,
|
|
946
|
+
writeJson(io.stdout, { ...data, databases });
|
|
896
947
|
return;
|
|
897
948
|
}
|
|
898
949
|
io.stdout(`${renderDatabaseTable(databases)}\n${renderQuota(data.quota)}`);
|
|
899
950
|
}
|
|
900
951
|
const CREATE_POLL_INTERVAL_MS = 2_000;
|
|
901
952
|
const CREATE_POLL_TIMEOUT_MS = 5 * 60_000;
|
|
953
|
+
// Frozen contract 4 (wave cli-duckdb, Task 4): printed after the table,
|
|
954
|
+
// non-JSON only, for `--kind duckdb_file`.
|
|
955
|
+
const DUCKDB_FILE_CREATE_HINT = 'Open it from a workflow with openAnalytics from @solidactions/sdk 0.11.0 or newer; add @duckdb/node-api as a normal dependency (npm install @duckdb/node-api, not --save-dev). Run `solidactions database --help` for the rest.';
|
|
902
956
|
async function databaseCreateWithConfig(name, options, config, dependencies = {}) {
|
|
903
957
|
const io = resolveDependencies(dependencies);
|
|
904
958
|
if (!isKnownKindOption(options.kind)) {
|
|
@@ -977,6 +1031,9 @@ async function databaseCreateWithConfig(name, options, config, dependencies = {}
|
|
|
977
1031
|
return;
|
|
978
1032
|
}
|
|
979
1033
|
io.stdout(renderMutation('create', data.database));
|
|
1034
|
+
if (data.database.kind === 'duckdb_file') {
|
|
1035
|
+
io.stdout(DUCKDB_FILE_CREATE_HINT);
|
|
1036
|
+
}
|
|
980
1037
|
}
|
|
981
1038
|
// The "call show first, branch on `.kind`" helper every later analytical verb
|
|
982
1039
|
// (schema/query/exec/dump/pull/push/import/ingest) reuses to learn a
|
|
@@ -1075,19 +1132,17 @@ async function databaseUndeleteWithConfig(name, options, config, dependencies =
|
|
|
1075
1132
|
}
|
|
1076
1133
|
io.stdout(renderMutation('undelete', data.database));
|
|
1077
1134
|
}
|
|
1078
|
-
// `query` and `schema` on a DuckDB database file are the app's to answer (it
|
|
1079
|
-
// says `not_available` until queries ship). The answer is relayed as the
|
|
1080
|
-
// error; a success body would be a contract the CLI does not know yet, so it
|
|
1081
|
-
// is refused as an invalid response rather than guessed at.
|
|
1082
|
-
async function requestDuckdbFileNotAvailable(config, body, io) {
|
|
1083
|
-
await (0, database_data_plane_1.requestDatabaseOperation)(config, body, requestDependencies(io));
|
|
1084
|
-
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'The database response was invalid.');
|
|
1085
|
-
}
|
|
1086
1135
|
async function databaseSchemaWithConfig(name, options, config, dependencies = {}) {
|
|
1087
1136
|
const io = resolveDependencies(dependencies);
|
|
1088
1137
|
const row = await requestDatabaseRecord(name, config, dependencies);
|
|
1089
1138
|
if (row.kind === 'duckdb_file') {
|
|
1090
|
-
await
|
|
1139
|
+
const schema = stableDuckdbFileSchema(await (0, database_data_plane_1.requestDatabaseOperation)(config, { operation: 'schema', name }, requestDependencies(io)));
|
|
1140
|
+
if (options.json) {
|
|
1141
|
+
writeJson(io.stdout, schema);
|
|
1142
|
+
return;
|
|
1143
|
+
}
|
|
1144
|
+
io.stdout(renderDuckdbFileSchema(schema));
|
|
1145
|
+
return;
|
|
1091
1146
|
}
|
|
1092
1147
|
if (row.kind === 'duckdb') {
|
|
1093
1148
|
// Read-intent, served from the cached catalog — never wakes a
|
|
@@ -1128,6 +1183,23 @@ function stableAnalyticalQueryResult(data) {
|
|
|
1128
1183
|
elapsed_ms: data.elapsed_ms,
|
|
1129
1184
|
};
|
|
1130
1185
|
}
|
|
1186
|
+
function stableDuckdbFileQueryResult(data) {
|
|
1187
|
+
if (!data
|
|
1188
|
+
|| !Array.isArray(data.columns)
|
|
1189
|
+
|| !data.columns.every((column) => typeof column === 'string')
|
|
1190
|
+
|| !Array.isArray(data.rows)
|
|
1191
|
+
|| !data.rows.every((row) => Array.isArray(row))
|
|
1192
|
+
|| typeof data.row_count !== 'number'
|
|
1193
|
+
|| typeof data.truncated !== 'boolean') {
|
|
1194
|
+
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'The database response was invalid.');
|
|
1195
|
+
}
|
|
1196
|
+
return {
|
|
1197
|
+
columns: [...data.columns],
|
|
1198
|
+
rows: data.rows.map((row) => [...row]),
|
|
1199
|
+
row_count: data.row_count,
|
|
1200
|
+
truncated: data.truncated,
|
|
1201
|
+
};
|
|
1202
|
+
}
|
|
1131
1203
|
// CLI-surface cap (`services.analytical.query_row_limit_cli` on the server;
|
|
1132
1204
|
// `config/services.php`) — the server clamps to the same value when
|
|
1133
1205
|
// `row_limit` is omitted, so this is both the client-side validation bound
|
|
@@ -1178,7 +1250,27 @@ async function databaseQueryWithConfig(name, sql, options, config, dependencies
|
|
|
1178
1250
|
const io = resolveDependencies(dependencies);
|
|
1179
1251
|
const row = await requestDatabaseRecord(name, config, dependencies);
|
|
1180
1252
|
if (row.kind === 'duckdb_file') {
|
|
1181
|
-
|
|
1253
|
+
// The server clamps the rows it returns (`max(1, min($rowLimit ?? $rowCap, $rowCap))`),
|
|
1254
|
+
// so the CLI validates `>= 1` and integer only — never the 10,000 analytical bound.
|
|
1255
|
+
if (options.limit !== undefined && (Number.isNaN(options.limit) || !Number.isInteger(options.limit) || options.limit < 1)) {
|
|
1256
|
+
throw new database_data_plane_1.DatabaseOperationError('invalid_limit', `--limit must be a whole number of 1 or more (received "${options.limit}").`);
|
|
1257
|
+
}
|
|
1258
|
+
const body = { operation: 'query', name, sql };
|
|
1259
|
+
if (options.limit !== undefined) {
|
|
1260
|
+
body.row_limit = options.limit;
|
|
1261
|
+
}
|
|
1262
|
+
const result = stableDuckdbFileQueryResult(await (0, database_data_plane_1.requestDatabaseOperation)(config, body, requestDependencies(io)));
|
|
1263
|
+
if (options.json) {
|
|
1264
|
+
writeJson(io.stdout, result);
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
io.stdout(result.columns.length > 0
|
|
1268
|
+
? (0, table_1.renderTable)(result.columns, result.rows.map((resultRow) => resultRow.map(formatAnalyticalQueryValue))).join('\n')
|
|
1269
|
+
: 'No rows returned.');
|
|
1270
|
+
if (result.truncated) {
|
|
1271
|
+
io.stdout(`(truncated at ${result.row_count} rows)`);
|
|
1272
|
+
}
|
|
1273
|
+
return;
|
|
1182
1274
|
}
|
|
1183
1275
|
if (row.kind === 'duckdb') {
|
|
1184
1276
|
if (options.limit !== undefined && (Number.isNaN(options.limit) || options.limit < 1 || options.limit > ANALYTICAL_QUERY_ROW_LIMIT_CLI_MAX)) {
|
|
@@ -1278,12 +1370,16 @@ async function databaseDumpWithConfig(name, file, options, config, dependencies
|
|
|
1278
1370
|
throw freshSafeFileError(error);
|
|
1279
1371
|
}
|
|
1280
1372
|
}
|
|
1281
|
-
async function databaseWritablePullWithConfig(name, destination, options, config, io) {
|
|
1373
|
+
async function databaseWritablePullWithConfig(name, destination, options, config, io, row) {
|
|
1282
1374
|
const target = resolveDatabaseDestination(io, destination, path_1.default.join('.solidactions', 'databases', `${safeDatabaseStem(name)}.db`));
|
|
1283
1375
|
let temp;
|
|
1284
1376
|
let reservationHandle;
|
|
1285
1377
|
let ownsTemp = false;
|
|
1286
1378
|
let ownsSidecars = false;
|
|
1379
|
+
// With a cache (cli#262) the libSQL replica lives in the cache and `temp` stays the open reservation.
|
|
1380
|
+
let cache;
|
|
1381
|
+
let cacheOrigin = null;
|
|
1382
|
+
let reserved;
|
|
1287
1383
|
let client;
|
|
1288
1384
|
let localCreateClient;
|
|
1289
1385
|
let credentialDeadline;
|
|
@@ -1304,42 +1400,59 @@ async function databaseWritablePullWithConfig(name, destination, options, config
|
|
|
1304
1400
|
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'Database file operation failed.');
|
|
1305
1401
|
}
|
|
1306
1402
|
};
|
|
1307
|
-
const openAttachedClient = async (initial) => {
|
|
1403
|
+
const openAttachedClient = async (initial, preloaded) => {
|
|
1404
|
+
const replicaPath = cache?.replicaPath ?? temp;
|
|
1405
|
+
// The cache module checks its own folders (`<root>/<key>`, `data`); a symlinked ancestor of the
|
|
1406
|
+
// cache root (a linked home) is the user's setup, so only the replica file itself is checked there (I3).
|
|
1407
|
+
const assertSafeReplicaAncestors = async () => {
|
|
1408
|
+
if (!cache)
|
|
1409
|
+
await assertNoSymlinkComponents(io.filesystem, replicaPath);
|
|
1410
|
+
};
|
|
1308
1411
|
if (!initial) {
|
|
1309
|
-
await
|
|
1310
|
-
const beforeMint = await lstatIfPresent(io.filesystem,
|
|
1412
|
+
await assertSafeReplicaAncestors();
|
|
1413
|
+
const beforeMint = await lstatIfPresent(io.filesystem, replicaPath);
|
|
1311
1414
|
if (!beforeMint || beforeMint.isSymbolicLink() || !beforeMint.isFile()) {
|
|
1312
1415
|
throw new database_data_plane_1.DatabaseOperationError('unsafe_destination', 'The database replica temporary path is unsafe.');
|
|
1313
1416
|
}
|
|
1314
1417
|
}
|
|
1315
|
-
let loaded;
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1418
|
+
let loaded = preloaded;
|
|
1419
|
+
if (!loaded) {
|
|
1420
|
+
try {
|
|
1421
|
+
loaded = await (0, database_client_support_1.loadDatabaseClientBeforeMint)(() => (0, database_data_plane_1.requestDatabaseAccess)(config, name, 'write', requestDependencies(io)), { loadClient: io.loadClient });
|
|
1422
|
+
}
|
|
1423
|
+
catch (error) {
|
|
1424
|
+
const safeError = freshSafeFileError(error);
|
|
1425
|
+
if (!initial && error instanceof database_data_plane_1.DatabaseOperationError) {
|
|
1426
|
+
throw new WritableRenewalAttemptError(safeError, true);
|
|
1427
|
+
}
|
|
1428
|
+
throw safeError;
|
|
1323
1429
|
}
|
|
1324
|
-
throw safeError;
|
|
1325
1430
|
}
|
|
1326
1431
|
const access = validateWritableAccess(loaded.access, io.now(), io.monotonicNow());
|
|
1327
1432
|
const createClient = loaded.createClient;
|
|
1328
|
-
|
|
1433
|
+
// Another server answered the renewal mint: the cache replica is not its copy (plan D3, I6).
|
|
1434
|
+
if (cache && !initial && (0, database_replica_cache_1.syncOriginOf)(access.url) !== cacheOrigin) {
|
|
1435
|
+
throw new WritableRenewalAttemptError(new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'Database file operation failed.'), true);
|
|
1436
|
+
}
|
|
1437
|
+
if (initial && !cache) {
|
|
1329
1438
|
await handoffReplicaReservation(io, temp, reservationHandle);
|
|
1330
1439
|
reservationHandle = undefined;
|
|
1331
1440
|
ownsTemp = false;
|
|
1332
1441
|
}
|
|
1333
|
-
await
|
|
1334
|
-
const beforeCreate = await lstatIfPresent(io.filesystem,
|
|
1335
|
-
if ((initial && beforeCreate !== null)
|
|
1442
|
+
await assertSafeReplicaAncestors();
|
|
1443
|
+
const beforeCreate = await lstatIfPresent(io.filesystem, replicaPath);
|
|
1444
|
+
if ((initial && !cache && beforeCreate !== null)
|
|
1445
|
+
// A warm cache replica is a regular file the session continues (review finding 8b). A non-regular
|
|
1446
|
+
// one was wiped before this open; one appearing since is a race and is refused.
|
|
1447
|
+
|| (initial && cache !== undefined && beforeCreate !== null
|
|
1448
|
+
&& (beforeCreate.isSymbolicLink() || !beforeCreate.isFile()))
|
|
1336
1449
|
|| (!initial && (!beforeCreate || beforeCreate.isSymbolicLink() || !beforeCreate.isFile()))) {
|
|
1337
1450
|
throw new database_data_plane_1.DatabaseOperationError('unsafe_destination', 'The database replica temporary path is unsafe.');
|
|
1338
1451
|
}
|
|
1339
1452
|
let opened;
|
|
1340
1453
|
try {
|
|
1341
1454
|
opened = createClient({
|
|
1342
|
-
url: (0, url_1.pathToFileURL)(
|
|
1455
|
+
url: (0, url_1.pathToFileURL)(replicaPath).href,
|
|
1343
1456
|
syncUrl: access.url,
|
|
1344
1457
|
authToken: access.token,
|
|
1345
1458
|
intMode: 'string',
|
|
@@ -1367,7 +1480,7 @@ async function databaseWritablePullWithConfig(name, destination, options, config
|
|
|
1367
1480
|
catch {
|
|
1368
1481
|
syncFailed = true;
|
|
1369
1482
|
}
|
|
1370
|
-
const replicaStat = await lstatIfPresent(io.filesystem,
|
|
1483
|
+
const replicaStat = await lstatIfPresent(io.filesystem, replicaPath);
|
|
1371
1484
|
if (replicaStat?.isFile() && !replicaStat.isSymbolicLink())
|
|
1372
1485
|
ownsTemp = true;
|
|
1373
1486
|
if (syncFailed
|
|
@@ -1387,6 +1500,28 @@ async function databaseWritablePullWithConfig(name, destination, options, config
|
|
|
1387
1500
|
localCreateClient = createClient;
|
|
1388
1501
|
credentialDeadline = access.deadline;
|
|
1389
1502
|
};
|
|
1503
|
+
const discardReservation = async () => {
|
|
1504
|
+
if (reservationHandle) {
|
|
1505
|
+
try {
|
|
1506
|
+
await reservationHandle.close();
|
|
1507
|
+
}
|
|
1508
|
+
catch {
|
|
1509
|
+
// Cleanup below remains scoped to the exclusively reserved path.
|
|
1510
|
+
}
|
|
1511
|
+
reservationHandle = undefined;
|
|
1512
|
+
}
|
|
1513
|
+
// With a cache the temp is ours only while it is still the reserved inode.
|
|
1514
|
+
if (temp && ownsTemp && reserved)
|
|
1515
|
+
ownsTemp = await isReservedFile(io, temp, reserved);
|
|
1516
|
+
if (temp) {
|
|
1517
|
+
await removeOwnedFiles(io.filesystem, [
|
|
1518
|
+
...(ownsTemp ? [temp] : []),
|
|
1519
|
+
...(ownsSidecars ? pullSidecars(temp) : []),
|
|
1520
|
+
]);
|
|
1521
|
+
}
|
|
1522
|
+
ownsTemp = false;
|
|
1523
|
+
ownsSidecars = false;
|
|
1524
|
+
};
|
|
1390
1525
|
try {
|
|
1391
1526
|
if (!await confirmFileOverwrite(target, options, io))
|
|
1392
1527
|
return;
|
|
@@ -1398,7 +1533,77 @@ async function databaseWritablePullWithConfig(name, destination, options, config
|
|
|
1398
1533
|
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'Database replica temporary files already exist.');
|
|
1399
1534
|
}
|
|
1400
1535
|
ownsSidecars = true;
|
|
1401
|
-
|
|
1536
|
+
let first;
|
|
1537
|
+
if (row && cacheEligible(io, options)) {
|
|
1538
|
+
// The first mint comes before the cache: its URL host is part of the cache identity (review finding 1).
|
|
1539
|
+
const minted = await (0, database_client_support_1.loadDatabaseClientBeforeMint)(() => (0, database_data_plane_1.requestDatabaseAccess)(config, name, 'write', requestDependencies(io)), { loadClient: io.loadClient });
|
|
1540
|
+
const outcome = await (0, database_replica_cache_1.acquireReplicaCache)({
|
|
1541
|
+
root: io.replicaCacheRoot,
|
|
1542
|
+
identity: cacheIdentity(config, name, row, minted.access),
|
|
1543
|
+
now: io.now,
|
|
1544
|
+
});
|
|
1545
|
+
if (outcome.kind === 'skipped') {
|
|
1546
|
+
const note = cacheSkipNote(outcome.reason);
|
|
1547
|
+
if (note !== undefined)
|
|
1548
|
+
io.stderr(note);
|
|
1549
|
+
}
|
|
1550
|
+
else {
|
|
1551
|
+
cache = outcome.cache;
|
|
1552
|
+
cacheOrigin = (0, database_replica_cache_1.syncOriginOf)(minted.access.url);
|
|
1553
|
+
first = minted;
|
|
1554
|
+
const reservedStat = await reservationHandle.stat();
|
|
1555
|
+
reserved = { dev: reservedStat.dev, ino: reservedStat.ino };
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
/** Wipes the cache; when the wipe itself fails, drops the cache (lock released) so the session runs uncached (I3). */
|
|
1559
|
+
const wipeOrDropCache = async () => {
|
|
1560
|
+
try {
|
|
1561
|
+
await cache.wipe();
|
|
1562
|
+
return true;
|
|
1563
|
+
}
|
|
1564
|
+
catch {
|
|
1565
|
+
await cache.release();
|
|
1566
|
+
cache = undefined;
|
|
1567
|
+
reserved = undefined;
|
|
1568
|
+
return false;
|
|
1569
|
+
}
|
|
1570
|
+
};
|
|
1571
|
+
if (cache) {
|
|
1572
|
+
// A replica that is a symlink or not a regular file is a cache fault: wipe and open cold (I3).
|
|
1573
|
+
const existingReplica = await lstatIfPresent(io.filesystem, cache.replicaPath);
|
|
1574
|
+
if (existingReplica && (existingReplica.isSymbolicLink() || !existingReplica.isFile())) {
|
|
1575
|
+
await wipeOrDropCache();
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
if (!cache) {
|
|
1579
|
+
await openAttachedClient(true);
|
|
1580
|
+
}
|
|
1581
|
+
else {
|
|
1582
|
+
const startedWarm = cache.warm;
|
|
1583
|
+
try {
|
|
1584
|
+
await openAttachedClient(true, first);
|
|
1585
|
+
}
|
|
1586
|
+
catch (error) {
|
|
1587
|
+
if (!(error instanceof database_data_plane_1.DatabaseOperationError) || error.code !== 'upstream_unavailable')
|
|
1588
|
+
throw error;
|
|
1589
|
+
if (!startedWarm) {
|
|
1590
|
+
// A cold cache that cannot open or sync (a full disk, an unopenable cache folder) must not fail
|
|
1591
|
+
// a session the uncached path would open: drop the cache and open uncached, once (I3).
|
|
1592
|
+
await cache.wipe().catch(() => undefined);
|
|
1593
|
+
await cache.release();
|
|
1594
|
+
cache = undefined;
|
|
1595
|
+
reserved = undefined;
|
|
1596
|
+
await openAttachedClient(true);
|
|
1597
|
+
}
|
|
1598
|
+
else if (await wipeOrDropCache()) {
|
|
1599
|
+
// A warm replica the server will not continue: wipe it and open cold, once (review finding 8b).
|
|
1600
|
+
await openAttachedClient(true, first);
|
|
1601
|
+
}
|
|
1602
|
+
else {
|
|
1603
|
+
await openAttachedClient(true);
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1402
1607
|
io.stdout(WRITABLE_WARNING);
|
|
1403
1608
|
inputSession = createWritableInput(io);
|
|
1404
1609
|
const accumulator = new database_sql_import_1.DatabaseSqlStatementAccumulator();
|
|
@@ -1512,9 +1717,39 @@ async function databaseWritablePullWithConfig(name, destination, options, config
|
|
|
1512
1717
|
}
|
|
1513
1718
|
if (sessionFailure && !publishAfterFailure)
|
|
1514
1719
|
throw sessionFailure;
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1720
|
+
if (cache) {
|
|
1721
|
+
try {
|
|
1722
|
+
await snapshotCacheReplica(io, localCreateClient, cache, { temp, handle: reservationHandle });
|
|
1723
|
+
}
|
|
1724
|
+
catch {
|
|
1725
|
+
// The session's writes are on the server already (readYourWrites, offline: false):
|
|
1726
|
+
// replay nothing, publish a read-only copy through the uncached pull (review finding 8a).
|
|
1727
|
+
await cache.wipe().catch(() => undefined);
|
|
1728
|
+
await discardReservation();
|
|
1729
|
+
io.stderr("The live session's writes are saved. Publishing a read-only copy instead.");
|
|
1730
|
+
// `io` is the resolved dependency object, which satisfies DatabaseCommandDependencies.
|
|
1731
|
+
await databasePullWithConfig(name, destination, { yes: true, cache: false, skipWorkflowWarning: true }, config, io);
|
|
1732
|
+
if (sessionFailure)
|
|
1733
|
+
throw sessionFailure;
|
|
1734
|
+
return;
|
|
1735
|
+
}
|
|
1736
|
+
reservationHandle = undefined;
|
|
1737
|
+
await publishSnapshot(io, temp, target, reserved);
|
|
1738
|
+
ownsTemp = false;
|
|
1739
|
+
ownsSidecars = false;
|
|
1740
|
+
try {
|
|
1741
|
+
await cache.markSynced();
|
|
1742
|
+
}
|
|
1743
|
+
catch {
|
|
1744
|
+
// The file is published; a cache whose meta.json could not be written is not trusted again (D5).
|
|
1745
|
+
await cache.wipe().catch(() => undefined);
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
else {
|
|
1749
|
+
await finalizeReplica(io, temp, target, localCreateClient);
|
|
1750
|
+
ownsTemp = false;
|
|
1751
|
+
ownsSidecars = false;
|
|
1752
|
+
}
|
|
1518
1753
|
if (sessionFailure)
|
|
1519
1754
|
throw sessionFailure;
|
|
1520
1755
|
io.stdout(`Database replica saved to ${displayDestination(io, target)}.`);
|
|
@@ -1536,6 +1771,167 @@ async function databaseWritablePullWithConfig(name, destination, options, config
|
|
|
1536
1771
|
// Cleanup below remains scoped to this invocation's replica.
|
|
1537
1772
|
}
|
|
1538
1773
|
}
|
|
1774
|
+
await discardReservation();
|
|
1775
|
+
throw freshSafeFileError(error);
|
|
1776
|
+
}
|
|
1777
|
+
finally {
|
|
1778
|
+
await cache?.release();
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
/** A cache problem inside a cached pull: the cache is wiped and the uncached path runs (plan D5, invariant I3). */
|
|
1782
|
+
class ReplicaCacheFallback extends Error {
|
|
1783
|
+
}
|
|
1784
|
+
const SNAPSHOT_COPY_CHUNK_BYTES = 1024 * 1024;
|
|
1785
|
+
function cacheEligible(io, options) {
|
|
1786
|
+
return options.cache !== false && typeof io.replicaCacheRoot === 'string' && io.replicaCacheRoot !== '';
|
|
1787
|
+
}
|
|
1788
|
+
function cacheSkipNote(reason) {
|
|
1789
|
+
if (reason === 'busy') {
|
|
1790
|
+
return 'Another pull of this database is using the local cache; downloading a full copy instead.';
|
|
1791
|
+
}
|
|
1792
|
+
if (reason === 'rewritten_often') {
|
|
1793
|
+
return 'This database is replaced often, so its local cache is off for now; downloading a full copy.';
|
|
1794
|
+
}
|
|
1795
|
+
return undefined;
|
|
1796
|
+
}
|
|
1797
|
+
function cacheIdentity(config, name, row, access) {
|
|
1798
|
+
return {
|
|
1799
|
+
host: config.host,
|
|
1800
|
+
workspaceId: config.workspaceId ?? '',
|
|
1801
|
+
name,
|
|
1802
|
+
databaseId: row.id ?? null,
|
|
1803
|
+
createdAt: row.kind === 'duckdb_file' ? null : row.created_at ?? null,
|
|
1804
|
+
syncOrigin: (0, database_replica_cache_1.syncOriginOf)(access.url),
|
|
1805
|
+
};
|
|
1806
|
+
}
|
|
1807
|
+
function isQuickCheckOk(result) {
|
|
1808
|
+
return Array.isArray(result?.rows) && result.rows.length === 1 && result.rows[0]?.[0] === 'ok';
|
|
1809
|
+
}
|
|
1810
|
+
/** Runs `operation` on a credential-free local client of `file` (no `syncUrl`) and always closes it. */
|
|
1811
|
+
async function withLocalSnapshotClient(io, createClient, file, operation) {
|
|
1812
|
+
let client;
|
|
1813
|
+
let closePromise;
|
|
1814
|
+
const closeClient = () => {
|
|
1815
|
+
closePromise ??= (0, database_data_plane_1.runDatabaseClientOperationWithDeadline)(async () => { await client?.close(); }, undefined, io.dataPlaneTimeoutMs);
|
|
1816
|
+
return closePromise;
|
|
1817
|
+
};
|
|
1818
|
+
try {
|
|
1819
|
+
client = createClient({ url: (0, url_1.pathToFileURL)(file).href, intMode: 'string' });
|
|
1820
|
+
await operation((sql) => (0, database_data_plane_1.runDatabaseClientOperationWithDeadline)(() => client.execute(sql), closeClient, io.dataPlaneTimeoutMs));
|
|
1821
|
+
}
|
|
1822
|
+
finally {
|
|
1823
|
+
if (client)
|
|
1824
|
+
await closeClient();
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
/** Copies the snapshot's bytes into the still-open reservation; never unlinks or recreates it by path. */
|
|
1828
|
+
async function copySnapshotIntoReservation(io, snapshotPath, handle) {
|
|
1829
|
+
const source = await io.filesystem.open(snapshotPath, 'r');
|
|
1830
|
+
let snapshotSize;
|
|
1831
|
+
let copied = 0;
|
|
1832
|
+
try {
|
|
1833
|
+
snapshotSize = (await source.stat()).size;
|
|
1834
|
+
await handle.truncate(0);
|
|
1835
|
+
const buffer = Buffer.alloc(SNAPSHOT_COPY_CHUNK_BYTES);
|
|
1836
|
+
for (;;) {
|
|
1837
|
+
const { bytesRead } = await source.read(buffer, 0, buffer.byteLength, copied);
|
|
1838
|
+
if (bytesRead === 0)
|
|
1839
|
+
break;
|
|
1840
|
+
let offset = 0;
|
|
1841
|
+
while (offset < bytesRead) {
|
|
1842
|
+
const { bytesWritten } = await handle.write(buffer, offset, bytesRead - offset, copied + offset);
|
|
1843
|
+
if (bytesWritten <= 0)
|
|
1844
|
+
throw new ReplicaCacheFallback('The snapshot copy made no progress.');
|
|
1845
|
+
offset += bytesWritten;
|
|
1846
|
+
}
|
|
1847
|
+
copied += bytesRead;
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
finally {
|
|
1851
|
+
await source.close();
|
|
1852
|
+
}
|
|
1853
|
+
await handle.sync();
|
|
1854
|
+
if (copied !== snapshotSize || (await handle.stat()).size !== snapshotSize) {
|
|
1855
|
+
throw new ReplicaCacheFallback('The snapshot copy is incomplete.');
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
/**
|
|
1859
|
+
* Turns a synced cache replica into the reserved temp's bytes, all through the real local client:
|
|
1860
|
+
* `PRAGMA quick_check` on the replica (review N1: `VACUUM INTO` rebuilds the free list, so only
|
|
1861
|
+
* this check sees damage there), `VACUUM INTO` the cache's own `snapshotPath`, `PRAGMA quick_check`
|
|
1862
|
+
* on the snapshot (review finding 3), then a copy into the still-open reservation handle (review
|
|
1863
|
+
* finding 12), which it closes. The reserved file's `{ dev, ino }` is the caller's `handle.stat()`
|
|
1864
|
+
* (the same open handle, so the same inode) and goes to `publishSnapshot`. Any failure throws
|
|
1865
|
+
* `ReplicaCacheFallback`.
|
|
1866
|
+
*/
|
|
1867
|
+
async function snapshotCacheReplica(io, createClient, cache, reservation) {
|
|
1868
|
+
try {
|
|
1869
|
+
await withLocalSnapshotClient(io, createClient, cache.replicaPath, async (execute) => {
|
|
1870
|
+
if (!isQuickCheckOk(await execute('PRAGMA quick_check'))) {
|
|
1871
|
+
throw new ReplicaCacheFallback('The cache replica failed PRAGMA quick_check.');
|
|
1872
|
+
}
|
|
1873
|
+
await execute(`VACUUM INTO '${cache.snapshotPath.replaceAll("'", "''")}'`);
|
|
1874
|
+
});
|
|
1875
|
+
await withLocalSnapshotClient(io, createClient, cache.snapshotPath, async (execute) => {
|
|
1876
|
+
if (!isQuickCheckOk(await execute('PRAGMA quick_check'))) {
|
|
1877
|
+
throw new ReplicaCacheFallback('The snapshot failed PRAGMA quick_check.');
|
|
1878
|
+
}
|
|
1879
|
+
});
|
|
1880
|
+
await copySnapshotIntoReservation(io, cache.snapshotPath, reservation.handle);
|
|
1881
|
+
await reservation.handle.close();
|
|
1882
|
+
await cache.discardSnapshot();
|
|
1883
|
+
}
|
|
1884
|
+
catch (error) {
|
|
1885
|
+
if (error instanceof ReplicaCacheFallback)
|
|
1886
|
+
throw error;
|
|
1887
|
+
throw new ReplicaCacheFallback('The cache snapshot could not be made.');
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
/** Publishes the reserved temp (the snapshot's bytes) only while it is still the reserved inode. */
|
|
1891
|
+
async function publishSnapshot(io, temp, target, expected) {
|
|
1892
|
+
await assertNoSymlinkComponents(io.filesystem, temp);
|
|
1893
|
+
const stat = await lstatIfPresent(io.filesystem, temp);
|
|
1894
|
+
if (!stat
|
|
1895
|
+
|| stat.isSymbolicLink()
|
|
1896
|
+
|| !stat.isFile()
|
|
1897
|
+
|| stat.dev !== expected.dev
|
|
1898
|
+
|| stat.ino !== expected.ino) {
|
|
1899
|
+
throw new database_data_plane_1.DatabaseOperationError('unsafe_destination', 'The database replica temporary path is unsafe.');
|
|
1900
|
+
}
|
|
1901
|
+
await removeOwnedFiles(io.filesystem, pullSidecars(temp));
|
|
1902
|
+
if (await anyPathExists(io.filesystem, pullSidecars(temp))) {
|
|
1903
|
+
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'Database file operation failed.');
|
|
1904
|
+
}
|
|
1905
|
+
await io.filesystem.chmod(temp, 0o444);
|
|
1906
|
+
await assertNoSymlinkComponents(io.filesystem, target);
|
|
1907
|
+
await io.filesystem.rename(temp, target);
|
|
1908
|
+
}
|
|
1909
|
+
async function isReservedFile(io, temp, expected) {
|
|
1910
|
+
try {
|
|
1911
|
+
const stat = await lstatIfPresent(io.filesystem, temp);
|
|
1912
|
+
return stat !== null
|
|
1913
|
+
&& stat.isFile()
|
|
1914
|
+
&& !stat.isSymbolicLink()
|
|
1915
|
+
&& stat.dev === expected.dev
|
|
1916
|
+
&& stat.ino === expected.ino;
|
|
1917
|
+
}
|
|
1918
|
+
catch {
|
|
1919
|
+
return false;
|
|
1920
|
+
}
|
|
1921
|
+
}
|
|
1922
|
+
/**
|
|
1923
|
+
* The read-only pull through the hidden replica cache (cli#262). `'handled'`: the target is
|
|
1924
|
+
* published (or the error is thrown). `'fallback'`: nothing is published, the run's own reservation
|
|
1925
|
+
* is gone, and the caller runs the uncached path (plan D5).
|
|
1926
|
+
*/
|
|
1927
|
+
async function databaseCachedReadonlyPullWithConfig(name, target, config, io, row) {
|
|
1928
|
+
let temp;
|
|
1929
|
+
let reservationHandle;
|
|
1930
|
+
let reserved;
|
|
1931
|
+
let ownsTemp = false;
|
|
1932
|
+
let ownsSidecars = false;
|
|
1933
|
+
let cache;
|
|
1934
|
+
const discardReservation = async () => {
|
|
1539
1935
|
if (reservationHandle) {
|
|
1540
1936
|
try {
|
|
1541
1937
|
await reservationHandle.close();
|
|
@@ -1543,15 +1939,165 @@ async function databaseWritablePullWithConfig(name, destination, options, config
|
|
|
1543
1939
|
catch {
|
|
1544
1940
|
// Cleanup below remains scoped to the exclusively reserved path.
|
|
1545
1941
|
}
|
|
1942
|
+
reservationHandle = undefined;
|
|
1546
1943
|
}
|
|
1944
|
+
// The temp is ours only while it is still the reserved inode.
|
|
1945
|
+
if (temp && ownsTemp && reserved)
|
|
1946
|
+
ownsTemp = await isReservedFile(io, temp, reserved);
|
|
1547
1947
|
if (temp) {
|
|
1548
1948
|
await removeOwnedFiles(io.filesystem, [
|
|
1549
1949
|
...(ownsTemp ? [temp] : []),
|
|
1550
1950
|
...(ownsSidecars ? pullSidecars(temp) : []),
|
|
1551
1951
|
]);
|
|
1552
1952
|
}
|
|
1953
|
+
ownsTemp = false;
|
|
1954
|
+
ownsSidecars = false;
|
|
1955
|
+
};
|
|
1956
|
+
try {
|
|
1957
|
+
const reservation = await reserveDestinationTemp(io, target);
|
|
1958
|
+
temp = reservation.temp;
|
|
1959
|
+
reservationHandle = reservation.handle;
|
|
1960
|
+
ownsTemp = true;
|
|
1961
|
+
const reservedStat = await reservationHandle.stat();
|
|
1962
|
+
reserved = { dev: reservedStat.dev, ino: reservedStat.ino };
|
|
1963
|
+
if (await anyPathExists(io.filesystem, pullSidecars(temp))) {
|
|
1964
|
+
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'Database replica temporary files already exist.');
|
|
1965
|
+
}
|
|
1966
|
+
ownsSidecars = true;
|
|
1967
|
+
// The first mint comes before the cache: its URL host is part of the cache identity (review finding 1).
|
|
1968
|
+
const first = await (0, database_client_support_1.loadDatabaseClientBeforeMint)(() => (0, database_data_plane_1.requestDatabaseAccess)(config, name, 'read', requestDependencies(io)), { loadClient: io.loadClient });
|
|
1969
|
+
const origin = (0, database_replica_cache_1.syncOriginOf)(first.access.url);
|
|
1970
|
+
const outcome = await (0, database_replica_cache_1.acquireReplicaCache)({
|
|
1971
|
+
root: io.replicaCacheRoot,
|
|
1972
|
+
identity: cacheIdentity(config, name, row, first.access),
|
|
1973
|
+
now: io.now,
|
|
1974
|
+
});
|
|
1975
|
+
if (outcome.kind === 'skipped') {
|
|
1976
|
+
await discardReservation();
|
|
1977
|
+
const note = cacheSkipNote(outcome.reason);
|
|
1978
|
+
if (note !== undefined)
|
|
1979
|
+
io.stderr(note);
|
|
1980
|
+
return 'fallback';
|
|
1981
|
+
}
|
|
1982
|
+
cache = outcome.cache;
|
|
1983
|
+
const replicaUrl = (0, url_1.pathToFileURL)(cache.replicaPath).href;
|
|
1984
|
+
const cacheReplicaStat = async () => {
|
|
1985
|
+
try {
|
|
1986
|
+
return await lstatIfPresent(io.filesystem, cache.replicaPath);
|
|
1987
|
+
}
|
|
1988
|
+
catch {
|
|
1989
|
+
throw new ReplicaCacheFallback('The cache replica cannot be read.');
|
|
1990
|
+
}
|
|
1991
|
+
};
|
|
1992
|
+
let createClient = first.createClient;
|
|
1993
|
+
let access = first.access;
|
|
1994
|
+
let synced = false;
|
|
1995
|
+
let credentialRemintUsed = false;
|
|
1996
|
+
for (let attempt = 0; attempt < PULL_ATTEMPTS; attempt += 1) {
|
|
1997
|
+
if (attempt > 0) {
|
|
1998
|
+
const minted = await (0, database_client_support_1.loadDatabaseClientBeforeMint)(() => (0, database_data_plane_1.requestDatabaseAccess)(config, name, 'read', requestDependencies(io)), { loadClient: io.loadClient });
|
|
1999
|
+
createClient = minted.createClient;
|
|
2000
|
+
access = minted.access;
|
|
2001
|
+
// Another server answered the re-mint: this replica is not its copy (plan D3, I6).
|
|
2002
|
+
if ((0, database_replica_cache_1.syncOriginOf)(access.url) !== origin) {
|
|
2003
|
+
throw new ReplicaCacheFallback('The sync origin changed during the pull.');
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
let client;
|
|
2007
|
+
try {
|
|
2008
|
+
client = createClient({
|
|
2009
|
+
url: replicaUrl,
|
|
2010
|
+
syncUrl: access.url,
|
|
2011
|
+
authToken: access.token,
|
|
2012
|
+
intMode: 'string',
|
|
2013
|
+
});
|
|
2014
|
+
}
|
|
2015
|
+
catch {
|
|
2016
|
+
// A replica the native client cannot open, cold or warm, is a cache fault: the uncached path runs once (D5, I3).
|
|
2017
|
+
throw new ReplicaCacheFallback('The cache replica could not be opened.');
|
|
2018
|
+
}
|
|
2019
|
+
let attemptFailure;
|
|
2020
|
+
let closePromise;
|
|
2021
|
+
const closeClient = () => {
|
|
2022
|
+
closePromise ??= (0, database_data_plane_1.runDatabaseClientOperationWithDeadline)(async () => { await client.close(); }, undefined, io.dataPlaneTimeoutMs);
|
|
2023
|
+
return closePromise;
|
|
2024
|
+
};
|
|
2025
|
+
try {
|
|
2026
|
+
if (typeof client.sync !== 'function')
|
|
2027
|
+
throw new Error('Database sync is unavailable.');
|
|
2028
|
+
await (0, database_data_plane_1.runDatabaseClientOperationWithDeadline)(() => client.sync(), closeClient, io.dataPlaneTimeoutMs);
|
|
2029
|
+
}
|
|
2030
|
+
catch (error) {
|
|
2031
|
+
attemptFailure = error;
|
|
2032
|
+
}
|
|
2033
|
+
finally {
|
|
2034
|
+
try {
|
|
2035
|
+
await closeClient();
|
|
2036
|
+
}
|
|
2037
|
+
catch (error) {
|
|
2038
|
+
if (!attemptFailure)
|
|
2039
|
+
attemptFailure = error;
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
2042
|
+
// The replica lives in the cache: anything but a regular file there is a cache fault (D5).
|
|
2043
|
+
const replicaStat = await cacheReplicaStat();
|
|
2044
|
+
if (replicaStat && (replicaStat.isSymbolicLink() || !replicaStat.isFile())) {
|
|
2045
|
+
throw new ReplicaCacheFallback('The cache replica is not a regular file.');
|
|
2046
|
+
}
|
|
2047
|
+
if (!attemptFailure) {
|
|
2048
|
+
if (!replicaStat)
|
|
2049
|
+
throw new ReplicaCacheFallback('The cache replica is missing after sync.');
|
|
2050
|
+
synced = true;
|
|
2051
|
+
break;
|
|
2052
|
+
}
|
|
2053
|
+
const failureClass = classifyPullFailure(attemptFailure);
|
|
2054
|
+
// A sync that failed for a reason retrying cannot cure, warm or cold (a replica the server will not
|
|
2055
|
+
// continue, a full disk, an unopenable cache folder): wipe, and the uncached path runs its own attempts (review 14, I3).
|
|
2056
|
+
if (failureClass === 'deterministic') {
|
|
2057
|
+
throw new ReplicaCacheFallback('The cache sync failed.');
|
|
2058
|
+
}
|
|
2059
|
+
const hasAnotherAttempt = attempt + 1 < PULL_ATTEMPTS;
|
|
2060
|
+
if (failureClass === 'credential_expiry' && !credentialRemintUsed && hasAnotherAttempt) {
|
|
2061
|
+
credentialRemintUsed = true;
|
|
2062
|
+
continue;
|
|
2063
|
+
}
|
|
2064
|
+
if (failureClass === 'transient_transport' && hasAnotherAttempt) {
|
|
2065
|
+
await io.sleep(PULL_RETRY_BACKOFF_MS[attempt] ?? PULL_RETRY_BACKOFF_MS.at(-1));
|
|
2066
|
+
continue;
|
|
2067
|
+
}
|
|
2068
|
+
break;
|
|
2069
|
+
}
|
|
2070
|
+
// I2: nothing is published from a replica whose sync did not succeed in this invocation.
|
|
2071
|
+
if (!synced) {
|
|
2072
|
+
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'Database file operation failed.');
|
|
2073
|
+
}
|
|
2074
|
+
await snapshotCacheReplica(io, createClient, cache, { temp, handle: reservationHandle });
|
|
2075
|
+
reservationHandle = undefined;
|
|
2076
|
+
await publishSnapshot(io, temp, target, reserved);
|
|
2077
|
+
ownsTemp = false;
|
|
2078
|
+
ownsSidecars = false;
|
|
2079
|
+
try {
|
|
2080
|
+
await cache.markSynced();
|
|
2081
|
+
}
|
|
2082
|
+
catch {
|
|
2083
|
+
// The file is published; a cache whose meta.json could not be written is not trusted again (D5).
|
|
2084
|
+
await cache.wipe().catch(() => undefined);
|
|
2085
|
+
}
|
|
2086
|
+
io.stdout(`Database replica saved to ${displayDestination(io, target)}.`);
|
|
2087
|
+
return 'handled';
|
|
2088
|
+
}
|
|
2089
|
+
catch (error) {
|
|
2090
|
+
const fallback = error instanceof ReplicaCacheFallback || error instanceof database_replica_cache_1.ReplicaCacheError;
|
|
2091
|
+
if (fallback)
|
|
2092
|
+
await cache?.wipe().catch(() => undefined);
|
|
2093
|
+
await discardReservation();
|
|
2094
|
+
if (fallback)
|
|
2095
|
+
return 'fallback';
|
|
1553
2096
|
throw freshSafeFileError(error);
|
|
1554
2097
|
}
|
|
2098
|
+
finally {
|
|
2099
|
+
await cache?.release();
|
|
2100
|
+
}
|
|
1555
2101
|
}
|
|
1556
2102
|
async function databasePullWithConfig(name, destination, options, config, dependencies = {}) {
|
|
1557
2103
|
const io = resolveDependencies(dependencies);
|
|
@@ -1566,17 +2112,31 @@ async function databasePullWithConfig(name, destination, options, config, depend
|
|
|
1566
2112
|
throw new database_data_plane_1.DatabaseOperationError('invalid_flag_combination', '--version is only for DuckDB databases; a standard database has no numbered versions.');
|
|
1567
2113
|
}
|
|
1568
2114
|
refuseIfAnalyticalRow(row, name, 'sqlite-only');
|
|
2115
|
+
if ((0, database_pull_guidance_1.isInsideWorkflowRun)(io.env) && options.skipWorkflowWarning !== true) {
|
|
2116
|
+
io.stderr(database_pull_guidance_1.WORKFLOW_RUN_PULL_WARNING);
|
|
2117
|
+
}
|
|
1569
2118
|
if (options.writable) {
|
|
1570
|
-
return databaseWritablePullWithConfig(name, destination, options, config, io);
|
|
2119
|
+
return databaseWritablePullWithConfig(name, destination, options, config, io, row);
|
|
1571
2120
|
}
|
|
1572
2121
|
const target = resolveDatabaseDestination(io, destination, path_1.default.join('.solidactions', 'databases', `${safeDatabaseStem(name)}.db`));
|
|
2122
|
+
// Asked once, before either path (plan D8): a cache fallback never asks again.
|
|
2123
|
+
try {
|
|
2124
|
+
if (!await confirmFileOverwrite(target, options, io))
|
|
2125
|
+
return;
|
|
2126
|
+
}
|
|
2127
|
+
catch (error) {
|
|
2128
|
+
throw freshSafeFileError(error);
|
|
2129
|
+
}
|
|
2130
|
+
const handled = cacheEligible(io, options)
|
|
2131
|
+
? await databaseCachedReadonlyPullWithConfig(name, target, config, io, row)
|
|
2132
|
+
: 'fallback';
|
|
2133
|
+
if (handled === 'handled')
|
|
2134
|
+
return;
|
|
1573
2135
|
let temp;
|
|
1574
2136
|
let reservationHandle;
|
|
1575
2137
|
let ownsTemp = false;
|
|
1576
2138
|
let ownsSidecars = false;
|
|
1577
2139
|
try {
|
|
1578
|
-
if (!await confirmFileOverwrite(target, options, io))
|
|
1579
|
-
return;
|
|
1580
2140
|
const reservation = await reserveDestinationTemp(io, target);
|
|
1581
2141
|
temp = reservation.temp;
|
|
1582
2142
|
reservationHandle = reservation.handle;
|
|
@@ -1903,7 +2463,10 @@ async function databaseDump(name, file, options, dependencies = {}) {
|
|
|
1903
2463
|
await databaseDumpWithConfig(name, file, options, await (0, api_1.requireConfigWithWorkspace)(), dependencies);
|
|
1904
2464
|
}
|
|
1905
2465
|
async function databasePull(name, destination, options, dependencies = {}) {
|
|
1906
|
-
await databasePullWithConfig(name, destination, options, await (0, api_1.requireConfigWithWorkspace)(),
|
|
2466
|
+
await databasePullWithConfig(name, destination, options, await (0, api_1.requireConfigWithWorkspace)(), {
|
|
2467
|
+
...dependencies,
|
|
2468
|
+
replicaCacheRoot: dependencies.replicaCacheRoot ?? (0, database_replica_cache_1.defaultReplicaCacheRoot)(),
|
|
2469
|
+
});
|
|
1907
2470
|
}
|
|
1908
2471
|
async function databaseVersions(name, options, dependencies = {}) {
|
|
1909
2472
|
await databaseVersionsWithConfig(name, options, await (0, api_1.requireConfigWithWorkspace)(), dependencies);
|
|
@@ -1999,8 +2562,9 @@ async function databaseIngestWithConfig(name, file, options, config, dependencie
|
|
|
1999
2562
|
}
|
|
2000
2563
|
const format = (0, database_ingest_digest_1.ingestFormatFromExtension)(file);
|
|
2001
2564
|
const row = await requestDatabaseRecord(name, config, dependencies);
|
|
2002
|
-
|
|
2003
|
-
|
|
2565
|
+
const ingestRefusal = (0, database_kind_refusal_1.analyticalVerbKindMismatch)(name, row, 'ingest');
|
|
2566
|
+
if (ingestRefusal) {
|
|
2567
|
+
throw ingestRefusal;
|
|
2004
2568
|
}
|
|
2005
2569
|
if (!row.id) {
|
|
2006
2570
|
throw new database_data_plane_1.DatabaseOperationError('upstream_unavailable', 'The database response was missing its id.');
|
|
@@ -2110,13 +2674,15 @@ async function databaseIngest(name, file, options, dependencies = {}) {
|
|
|
2110
2674
|
}
|
|
2111
2675
|
// Analytical-only guard shared by drop-table/optimize/connect/wake (mirrors
|
|
2112
2676
|
// `databaseIngestWithConfig`'s own check): unlike `refuseIfAnalytical`, which
|
|
2113
|
-
// refuses an analytical name on the SQLite-only verbs, these
|
|
2114
|
-
//
|
|
2115
|
-
//
|
|
2677
|
+
// refuses an analytical name on the SQLite-only verbs, these verbs only ever
|
|
2678
|
+
// operate on the analytical (duckdb) kind, so any other kind is the refusal —
|
|
2679
|
+
// before any operation-specific request goes out. A DuckDB database file gets
|
|
2680
|
+
// the sentence naming the kind (Frozen contract 3).
|
|
2116
2681
|
function refuseUnlessAnalytical(name, row, verb) {
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2682
|
+
const refusal = (0, database_kind_refusal_1.analyticalVerbKindMismatch)(name, row, verb);
|
|
2683
|
+
if (refusal) {
|
|
2684
|
+
throw refusal;
|
|
2685
|
+
}
|
|
2120
2686
|
}
|
|
2121
2687
|
/**
|
|
2122
2688
|
* Shared by `drop-table` and `optimize`: `admit()` on the server — the same
|